diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000000..de299a90971e --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,33 @@ +# +# An auto defined `clippy` feature was introduced, +# but it was found to clash with user defined features, +# so was renamed to `cargo-clippy`. +# +# If you want standard clippy run: +# RUSTFLAGS= cargo clippy +[target.'cfg(feature = "cargo-clippy")'] +rustflags = [ + "-Aclippy::all", + "-Dclippy::correctness", + "-Aclippy::if-same-then-else", + "-Aclippy::clone-double-ref", + "-Dclippy::complexity", + "-Aclippy::clone_on_copy", # Too common + "-Aclippy::needless_lifetimes", # Backward compat? + "-Aclippy::zero-prefixed-literal", # 00_1000_000 + "-Aclippy::type_complexity", # raison d'etre + "-Aclippy::nonminimal-bool", # maybe + "-Aclippy::borrowed-box", # Reasonable to fix this one + "-Aclippy::too-many-arguments", # (Turning this on would lead to) + "-Aclippy::unnecessary_cast", # Types may change + "-Aclippy::identity-op", # One case where we do 0 + + "-Aclippy::useless_conversion", # Types may change + "-Aclippy::unit_arg", # styalistic. + "-Aclippy::option-map-unit-fn", # styalistic + "-Aclippy::bind_instead_of_map", # styalistic + "-Aclippy::erasing_op", # E.g. 0 * DOLLARS + "-Aclippy::eq_op", # In tests we test equality. + "-Aclippy::while_immutable_condition", # false positives + "-Aclippy::needless_option_as_deref", # false positives + "-Aclippy::derivable_impls", # false positives +] diff --git a/.dockerignore b/.dockerignore index 39dbc05c97e1..c58599e3fb72 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ doc Dockerfile .dockerignore .local +.env* diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 000000000000..c99a3070231d --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,15 @@ +# You can easily exclude big automated code changes by running +# +# git config blame.ignoreRevsFile .git-blame-ignore-revs +# +# in your local repository. It will work also in IDE integrations. +# +# On versions of Git 2.20 and later comments (#), empty lines, and any leading and +# trailing whitespace is ignored. Everything but a SHA-1 per line will error out on +# older versions. +# +# You should add new commit hashes to this file when you create or find such big +# automated refactorings while reading code history. If you only know the short hash, +# use `git rev-parse 1d5abf01` to expand it to the full SHA1 hash needed in this file. + +1d5abf01abafdb6c15bcd0172f5de09fd87c5fbf # Run cargo fmt on the whole code base (#9394) diff --git a/.github/ISSUE_TEMPLATE/bug.yaml b/.github/ISSUE_TEMPLATE/bug.yaml new file mode 100644 index 000000000000..ae40df08eca7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yaml @@ -0,0 +1,34 @@ +name: Bug Report +description: Let us know about an issue you experienced with this software +# labels: ["some existing label","another one"] +body: + - type: checkboxes + attributes: + label: Is there an existing issue? + description: Please search to see if an issue already exists and leave a comment that you also experienced this issue or add your specifics that are related to an existing issue. + options: + - label: I have searched the existing issues + required: true + - type: checkboxes + attributes: + label: Experiencing problems? Have you tried our Stack Exchange first? + description: Please search to see if an post already exists, and ask if not. Please do not file support issues here. + options: + - label: This is not a support question. + required: true + - type: textarea + id: bug + attributes: + label: Description of bug + # description: What seems to be the problem? + # placeholder: Describe the problem. + validations: + required: true + - type: textarea + id: steps + attributes: + label: Steps to reproduce + description: Provide the steps that led to the discovery of the issue. + # placeholder: Describe what you were doing so we can reproduce the problem. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000000..e422e317411f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,7 @@ +blank_issues_enabled: true +contact_links: + - name: Support & Troubleshooting with the Substrate Stack Exchange Community + url: https://substrate.stackexchange.com + about: | + For general problems with Substrate or related technologies, please search here first + for solutions, by keyword and tags. If you discover no solution, please then ask and questions in our community! We highly encourage everyone also share their understanding by answering questions for others. diff --git a/.github/ISSUE_TEMPLATE/feature.yaml b/.github/ISSUE_TEMPLATE/feature.yaml new file mode 100644 index 000000000000..92b2fea3e88d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yaml @@ -0,0 +1,52 @@ +name: Feature Request +description: Submit your requests and suggestions to improve! +body: + - type: checkboxes + attributes: + label: Is there an existing issue? + description: Please search to see if an issue already exists and leave a comment that you also experienced this issue or add your specifics that are related to an existing issue. + options: + - label: I have searched the existing issues + required: true + - type: checkboxes + attributes: + label: Experiencing problems? Have you tried our Stack Exchange first? + description: Please search to see if an post already exists, and ask if not. Please do not file support issues here. + options: + - label: This is not a support question. + required: true + - type: textarea + id: content + attributes: + label: Motivation + description: Please give precedence as to what lead you to file this issue. + # placeholder: Describe ... + validations: + required: false + - type: textarea + id: content + attributes: + label: Request + description: Please describe what is needed. + # placeholder: Describe what you would like to see added or changed. + validations: + required: true + - type: textarea + id: content + attributes: + label: Solution + description: If possible, please describe what a solution could be. + # placeholder: Describe what you would like to see added or changed. + validations: + required: false + - type: dropdown + id: help + attributes: + label: Are you willing to help with this request? + multiple: true + options: + - Yes! + - No. + - Maybe (please elaborate above) + validations: + required: true diff --git a/.github/pr-custom-review.yml b/.github/pr-custom-review.yml new file mode 100644 index 000000000000..518faf15a611 --- /dev/null +++ b/.github/pr-custom-review.yml @@ -0,0 +1,16 @@ +# 🔒 PROTECTED: Changes to locks-review-team should be approved by the current locks-review-team +locks-review-team: locks-review +team-leads-team: polkadot-review +action-review-team: ci + +rules: + - name: Core developers + check_type: changed_files + condition: .* + min_approvals: 2 + teams: + - core-devs + +prevent-review-request: + teams: + - core-devs diff --git a/.github/workflows/check-labels.yml b/.github/workflows/check-labels.yml index 7180e7b50966..74fdd9b2d818 100644 --- a/.github/workflows/check-labels.yml +++ b/.github/workflows/check-labels.yml @@ -14,7 +14,7 @@ jobs: ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - name: Check labels - run: bash ${{ github.workspace }}/.maintain/github/check_labels.sh + run: bash ${{ github.workspace }}/scripts/ci/github/check_labels.sh env: GITHUB_PR: ${{ github.event.pull_request.number }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/monthly-tag.yml b/.github/workflows/monthly-tag.yml index 8736a341cecf..9fed86539601 100644 --- a/.github/workflows/monthly-tag.yml +++ b/.github/workflows/monthly-tag.yml @@ -29,7 +29,7 @@ jobs: echo "" >> Changelog.md echo "## Changes since last snapshot (${{ steps.tags.outputs.old }})" >> Changelog.md echo "" >> Changelog.md - ./.maintain/gitlab/generate_changelog.sh ${{ steps.tags.outputs.old }} >> Changelog.md + ./scripts/ci/github/generate_changelog.sh ${{ steps.tags.outputs.old }} >> Changelog.md - name: Release snapshot id: release-snapshot uses: actions/create-release@latest diff --git a/.github/workflows/polkadot-companion-labels.yml b/.github/workflows/polkadot-companion-labels.yml index 3c3987b5f4d5..0a5af0935852 100644 --- a/.github/workflows/polkadot-companion-labels.yml +++ b/.github/workflows/polkadot-companion-labels.yml @@ -14,7 +14,7 @@ jobs: with: authToken: ${{ secrets.GITHUB_TOKEN }} ref: ${{ github.event.pull_request.head.sha }} - contexts: 'continuous-integration/gitlab-check-polkadot-companion-build' + contexts: 'continuous-integration/gitlab-check-dependent-polkadot' timeout: 1800 notPresentTimeout: 3600 # It can take quite a while before the job starts on Gitlab when the CI queue is large failureStates: failure diff --git a/.github/workflows/pr-custom-review.yml b/.github/workflows/pr-custom-review.yml new file mode 100644 index 000000000000..d113d3906784 --- /dev/null +++ b/.github/workflows/pr-custom-review.yml @@ -0,0 +1,23 @@ +name: Check reviews + +on: + pull_request: + branches: + - master + - main + types: + - opened + - reopened + - synchronize + - review_requested + - review_request_removed + pull_request_review: + +jobs: + pr-custom-review: + runs-on: ubuntu-latest + steps: + - name: pr-custom-review + uses: paritytech/pr-custom-review@v2 + with: + token: ${{ secrets.PRCR_TOKEN }} diff --git a/.gitignore b/.gitignore index 4bdcdd196a75..ce8815fb83e6 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,6 @@ rls*.log .local **/hfuzz_target/ **/hfuzz_workspace/ -.cargo/ .cargo-remote.toml *.bin *.iml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fa986923708d..d196ade85a6f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,7 +18,7 @@ # script: # - echo "List of shell commands to run in your job" # - echo "You can also just specify a script here, like so:" -# - ./.maintain/gitlab/my_amazing_script.sh +# - ./scripts/ci/gitlab/my_amazing_script.sh stages: - check @@ -39,12 +39,6 @@ variables: &default-vars DOCKER_OS: "debian:stretch" ARCH: "x86_64" CI_IMAGE: "paritytech/ci-linux:production" - # FIXME set to release - CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.12" - CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder" - VAULT_SERVER_URL: "https://vault.parity-mgmt-vault.parity.io" - VAULT_AUTH_PATH: "gitlab-parity-io-jwt" - VAULT_AUTH_ROLE: "cicd_gitlab_parity_${CI_PROJECT_NAME}" default: cache: {} @@ -57,6 +51,14 @@ default: paths: - artifacts/ +.collect-artifacts-short: &collect-artifacts-short + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" + when: on_success + expire_in: 3 hours + paths: + - artifacts/ + .kubernetes-env: &kubernetes-env retry: max: 2 @@ -160,78 +162,35 @@ default: fi .cargo-check-benches-script: &cargo-check-benches-script - - mkdir -p artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA + - mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all - 'cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json - | tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json' + | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json' - 'cargo run --release -p node-bench -- ::trie::read::small --json - | tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json' + | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json' + - sccache -s + +.build-linux-substrate-script: &build-linux-substrate-script + - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose + - mv ./target/release/substrate ./artifacts/substrate/. + - echo -n "Substrate version = " + - if [ "${CI_COMMIT_TAG}" ]; then + echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION; + else + ./artifacts/substrate/substrate --version | + sed -n -E 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' | + tee ./artifacts/substrate/VERSION; + fi + - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256 + - cp -r ./scripts/ci/docker/substrate.Dockerfile ./artifacts/substrate/ - sccache -s -#### Vault secrets -.vault-secrets: &vault-secrets - secrets: - DOCKER_HUB_USER: - vault: cicd/gitlab/parity/DOCKER_HUB_USER@kv - file: false - DOCKER_HUB_PASS: - vault: cicd/gitlab/parity/DOCKER_HUB_PASS@kv - file: false - GITHUB_PR_TOKEN: - vault: cicd/gitlab/parity/GITHUB_PR_TOKEN@kv - file: false - AWS_ACCESS_KEY_ID: - vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_ACCESS_KEY_ID@kv - file: false - AWS_SECRET_ACCESS_KEY: - vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_SECRET_ACCESS_KEY@kv - file: false - AWX_TOKEN: - vault: cicd/gitlab/$CI_PROJECT_PATH/AWX_TOKEN@kv - file: false - CRATES_TOKEN: - vault: cicd/gitlab/$CI_PROJECT_PATH/CRATES_TOKEN@kv - file: false - DOCKER_CHAOS_TOKEN: - vault: cicd/gitlab/$CI_PROJECT_PATH/DOCKER_CHAOS_TOKEN@kv - file: false - DOCKER_CHAOS_USER: - vault: cicd/gitlab/$CI_PROJECT_PATH/DOCKER_CHAOS_USER@kv - file: false - GITHUB_EMAIL: - vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_EMAIL@kv - file: false - GITHUB_RELEASE_TOKEN: - vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_RELEASE_TOKEN@kv - file: false - GITHUB_TOKEN: - vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_TOKEN@kv - file: false - GITHUB_USER: - vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_USER@kv - file: false - MATRIX_ACCESS_TOKEN: - vault: cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ACCESS_TOKEN@kv - file: false - MATRIX_ROOM_ID: - vault: cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ROOM_ID@kv - file: false - PIPELINE_TOKEN: - vault: cicd/gitlab/$CI_PROJECT_PATH/PIPELINE_TOKEN@kv - file: false - VALIDATOR_KEYS: - vault: cicd/gitlab/$CI_PROJECT_PATH/VALIDATOR_KEYS@kv - file: false - VALIDATOR_KEYS_CHAOS: - vault: cicd/gitlab/$CI_PROJECT_PATH/VALIDATOR_KEYS_CHAOS@kv - file: false #### stage: .pre skip-if-draft: image: paritytech/tools:latest <<: *kubernetes-env - <<: *vault-secrets stage: .pre rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs @@ -239,7 +198,7 @@ skip-if-draft: - echo "Commit message is ${CI_COMMIT_MESSAGE}" - echo "Ref is ${CI_COMMIT_REF_NAME}" - echo "pipeline source is ${CI_PIPELINE_SOURCE}" - - ./.maintain/gitlab/skip_if_draft.sh + - ./scripts/ci/gitlab/skip_if_draft.sh #### stage: check @@ -247,7 +206,6 @@ check-runtime: stage: check image: paritytech/tools:latest <<: *kubernetes-env - <<: *vault-secrets rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs variables: @@ -255,19 +213,18 @@ check-runtime: GITLAB_API: "https://gitlab.parity.io/api/v4" GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" script: - - ./.maintain/gitlab/check_runtime.sh + - ./scripts/ci/gitlab/check_runtime.sh allow_failure: true check-signed-tag: stage: check image: paritytech/tools:latest <<: *kubernetes-env - <<: *vault-secrets rules: - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 script: - - ./.maintain/gitlab/check_signed.sh + - ./scripts/ci/gitlab/check_signed.sh test-dependency-rules: stage: check @@ -276,7 +233,7 @@ test-dependency-rules: rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs script: - - .maintain/ensure-deps.sh + - ./scripts/ci/gitlab/ensure-deps.sh test-prometheus-alerting-rules: stage: check @@ -288,11 +245,11 @@ test-prometheus-alerting-rules: - if: $CI_COMMIT_BRANCH changes: - .gitlab-ci.yml - - .maintain/monitoring/**/* + - ./scripts/ci/monitoring/**/* script: - - promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml - - cat .maintain/monitoring/alerting-rules/alerting-rules.yaml | - promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml + - promtool check rules ./scripts/ci/monitoring/alerting-rules/alerting-rules.yaml + - cat ./scripts/ci/monitoring/alerting-rules/alerting-rules.yaml | + promtool test rules ./scripts/ci/monitoring/alerting-rules/alerting-rule-tests.yaml #### stage: test @@ -301,17 +258,17 @@ cargo-deny: <<: *docker-env <<: *nightly-pipeline script: - - cargo deny check --hide-inclusion-graph -c .maintain/deny.toml + - cargo deny check --hide-inclusion-graph -c ./scripts/ci/deny.toml after_script: - echo "___The complete log is in the artifacts___" - - cargo deny check -c .maintain/deny.toml 2> deny.log + - cargo deny check -c ./scripts/ci/deny.toml 2> deny.log artifacts: name: $CI_COMMIT_SHORT_SHA expire_in: 3 days when: always paths: - deny.log - # FIXME: Temorarily allow to fail. + # FIXME: Temporarily allow to fail. allow_failure: true cargo-fmt: @@ -321,6 +278,13 @@ cargo-fmt: script: - cargo +nightly fmt --all -- --check +cargo-clippy: + stage: test + <<: *docker-env + <<: *test-refs + script: + - SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo +nightly clippy --all-targets + cargo-check-benches: stage: test <<: *docker-env @@ -332,6 +296,8 @@ cargo-check-benches: - *rust-info-script script: - *cargo-check-benches-script + tags: + - linux-docker-benches node-bench-regression-guard: # it's not belong to `build` semantically, but dag jobs can't depend on each other @@ -353,6 +319,9 @@ node-bench-regression-guard: CI_IMAGE: "paritytech/node-bench-regression-guard:latest" before_script: [""] script: + - echo "------- IMPORTANT -------" + - echo "node-bench-regression-guard depends on the results of a cargo-check-benches job" + - echo "In case of this job failure, check your pipeline's cargo-check-benches" - 'node-bench-regression-guard --reference artifacts/benches/master-* --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA' @@ -392,13 +361,13 @@ test-deterministic-wasm: # build runtime - cargo build --verbose --release -p node-runtime # make checksum - - sha256sum target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256 + - sha256sum ./target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256 # clean up – FIXME: can we reuse some of the artifacts? - cargo clean # build again - cargo build --verbose --release -p node-runtime # confirm checksum - - sha256sum -c checksum.sha256 + - sha256sum -c ./checksum.sha256 - sccache -s test-linux-stable: &test-linux @@ -414,26 +383,11 @@ test-linux-stable: &test-linux WASM_BUILD_NO_COLOR: 1 script: # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests - - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml - - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec + - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path ./bin/node/cli/Cargo.toml + - time cargo test -p frame-support-test --features=conditional-storage,no-metadata-docs --manifest-path ./frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout - sccache -s -#unleash-check: - #stage: test - #<<: *docker-env - #<<: *test-refs-no-trigger - #script: - #- cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - #- cargo unleash de-dev-deps - # Reuse build artifacts when running checks (cuts down check time by 3x) - # TODO: Implement this optimization in cargo-unleash rather than here - #- mkdir -p target/unleash - #- export CARGO_TARGET_DIR=target/unleash - #- cargo unleash check ${CARGO_UNLEASH_PKG_DEF} - # FIXME: this job must not fail, or unleash-to-crates-io will publish broken stuff - #allow_failure: true - test-frame-examples-compile-to-wasm: # into one job stage: test @@ -446,9 +400,9 @@ test-frame-examples-compile-to-wasm: RUSTFLAGS: "-Cdebug-assertions=y" RUST_BACKTRACE: 1 script: - - cd frame/example-offchain-worker/ + - cd ./frame/examples/offchain-worker/ - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - - cd ../example + - cd ../basic - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - sccache -s @@ -478,8 +432,8 @@ check-tracing: <<: *test-refs script: # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases - - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features - - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features --features=with-tracing + - time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features + - time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features --features=with-tracing - sccache -s test-full-crypto-feature: @@ -522,31 +476,43 @@ cargo-check-macos: #### stage: build -check-polkadot-companion-status: - stage: build - image: paritytech/tools:latest - <<: *kubernetes-env - <<: *vault-secrets - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - script: - - ./.maintain/gitlab/check_polkadot_companion_status.sh +# PIPELINE_SCRIPTS_TAG can be found in the project variables -check-polkadot-companion-build: +.check-dependent-project: &check-dependent-project stage: build <<: *docker-env - <<: *test-refs-no-trigger - <<: *vault-secrets - needs: - - job: test-linux-stable-int - artifacts: false + <<: *test-refs-no-trigger-prs-only script: - - ./.maintain/gitlab/check_polkadot_companion_build.sh - after_script: - - cd polkadot && git rev-parse --abbrev-ref HEAD - allow_failure: true + - git clone + --depth=1 + "--branch=$PIPELINE_SCRIPTS_TAG" + https://github.com/paritytech/pipeline-scripts + - ./pipeline-scripts/check_dependent_project.sh + paritytech + substrate + --substrate + "$DEPENDENT_REPO" + "$GITHUB_PR_TOKEN" + "$CARGO_UPDATE_CRATES" + +# Individual jobs are set up for each dependent project so that they can be ran in parallel. +# Arguably we could generate a job for each companion in the PR's description using Gitlab's +# parent-child pipelines but that's more complicated. + +check-dependent-polkadot: + <<: *check-dependent-project + variables: + DEPENDENT_REPO: polkadot + CARGO_UPDATE_CRATES: "sp-io" + +check-dependent-cumulus: + <<: *check-dependent-project + variables: + DEPENDENT_REPO: cumulus + CARGO_UPDATE_CRATES: "sp-io polkadot-runtime-common" -build-linux-substrate: &build-binary + +build-linux-substrate: stage: build <<: *collect-artifacts <<: *docker-env @@ -557,21 +523,9 @@ build-linux-substrate: &build-binary before_script: - mkdir -p ./artifacts/substrate/ script: - - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose - - mv ./target/release/substrate ./artifacts/substrate/. - - echo -n "Substrate version = " - - if [ "${CI_COMMIT_TAG}" ]; then - echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION; - else - ./artifacts/substrate/substrate --version | - sed -n -E 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' | - tee ./artifacts/substrate/VERSION; - fi - - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256 + - *build-linux-substrate-script - printf '\n# building node-template\n\n' - - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz - - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/ - - sccache -s + - ./scripts/ci/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz build-linux-subkey: &build-subkey stage: build @@ -593,7 +547,7 @@ build-linux-subkey: &build-subkey sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' | tee ./artifacts/subkey/VERSION; - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256 - - cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/ + - cp -r ./scripts/ci/docker/subkey.Dockerfile ./artifacts/subkey/ - sccache -s build-macos-subkey: @@ -601,6 +555,18 @@ build-macos-subkey: tags: - osx +check-rustdoc: + stage: test + <<: *docker-env + <<: *test-refs + variables: + <<: *default-vars + SKIP_WASM_BUILD: 1 + RUSTDOCFLAGS: "-Dwarnings" + script: + - time cargo +nightly doc --workspace --all-features --verbose --no-deps + - sccache -s + build-rustdoc: stage: build <<: *docker-env @@ -616,9 +582,7 @@ build-rustdoc: paths: - ./crate-docs/ script: - # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` - - RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html" - time cargo +nightly doc --no-deps --workspace --all-features --verbose + - time cargo +nightly doc --workspace --all-features --verbose - rm -f ./target/doc/.lock - mv ./target/doc ./crate-docs # FIXME: remove me after CI image gets nonroot @@ -631,7 +595,6 @@ build-rustdoc: .build-push-docker-image: &build-push-docker-image <<: *build-refs <<: *kubernetes-env - <<: *vault-secrets image: quay.io/buildah/stable variables: &docker-build-vars <<: *default-vars @@ -660,7 +623,6 @@ build-rustdoc: - buildah push --format=v2s2 "$IMAGE_NAME:latest" after_script: - buildah logout --all - # pass artifacts to the trigger-simnet job - echo "SUBSTRATE_IMAGE_NAME=${IMAGE_NAME}" | tee -a ./artifacts/$PRODUCT/build.env - IMAGE_TAG="$(cat ./artifacts/$PRODUCT/VERSION)" - echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}" | tee -a ./artifacts/$PRODUCT/build.env @@ -676,11 +638,6 @@ publish-docker-substrate: variables: <<: *docker-build-vars PRODUCT: substrate - artifacts: - reports: - # this artifact is used in trigger-simnet job - # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance - dotenv: ./artifacts/substrate/build.env publish-docker-subkey: stage: publish @@ -696,7 +653,6 @@ publish-s3-release: stage: publish <<: *build-refs <<: *kubernetes-env - <<: *vault-secrets needs: - job: build-linux-substrate artifacts: true @@ -718,17 +674,10 @@ publish-s3-release: publish-rustdoc: stage: publish <<: *kubernetes-env - <<: *vault-secrets image: node:16 variables: GIT_DEPTH: 100 - # --- Following variables are for rustdocs deployment --- - # Space separated values of branches and tags to generate rustdocs - RUSTDOCS_DEPLOY_REFS: "master monthly-2021-09+1 monthly-2021-08 v3.0.0" - # Location of the docs index template - INDEX_TPL: ".maintain/docs-index-tpl.ejs" - # Where the `/latest` symbolic link links to. One of the $RUSTDOCS_DEPLOY_REFS value. - LATEST: "monthly-2021-09+1" + RUSTDOCS_DEPLOY_REFS: "master" rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never @@ -747,42 +696,30 @@ publish-rustdoc: # Putting spaces at the front and back to ensure we are not matching just any substring, but the # whole space-separated value. - '[[ " ${RUSTDOCS_DEPLOY_REFS} " =~ " ${CI_COMMIT_REF_NAME} " ]] || exit 0' - - rm -rf /tmp/* + # setup ssh + - eval $(ssh-agent) + - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} + - mkdir ~/.ssh && touch ~/.ssh/known_hosts + - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts # Set git config - - rm -rf .git/config - git config user.email "devops-team@parity.io" - git config user.name "${GITHUB_USER}" - - git config remote.origin.url "https://${GITHUB_TOKEN}@github.com/paritytech/${CI_PROJECT_NAME}.git" + - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch origin gh-pages - # Install `ejs` and generate index.html based on RUSTDOCS_DEPLOY_REFS - - yarn global add ejs - - 'ejs ${INDEX_TPL} -i "{\"deploy_refs\":\"${RUSTDOCS_DEPLOY_REFS}\",\"repo_name\":\"${CI_PROJECT_NAME}\",\"latest\":\"${LATEST}\"}" > /tmp/index.html' # Save README and docs - cp -r ./crate-docs/ /tmp/doc/ - cp README.md /tmp/doc/ - - git checkout gh-pages - # Remove directories no longer necessary, as specified in $RUSTDOCS_DEPLOY_REFS. - # Also ensure $RUSTDOCS_DEPLOY_REFS is not just empty spaces. - # Even though this block spans multiple lines, they are concatenated to run as a single line - # command, so note for the semi-colons in the inner-most code block. - - if [[ ! -z ${RUSTDOCS_DEPLOY_REFS// } ]]; then - for FILE in *; do - if [[ ! " $RUSTDOCS_DEPLOY_REFS " =~ " $FILE " ]]; then - echo "Removing ${FILE}..."; - rm -rf $FILE; - fi - done - fi - # Move the index page & built back - - mv -f /tmp/index.html . + # we don't need to commit changes because we copy docs to /tmp + - git checkout gh-pages --force + # Install `index-tpl-crud` and generate index.html based on RUSTDOCS_DEPLOY_REFS + - which index-tpl-crud &> /dev/null || yarn global add @substrate/index-tpl-crud + - index-tpl-crud upsert ./index.html ${CI_COMMIT_REF_NAME} # Ensure the destination dir doesn't exist. - rm -rf ${CI_COMMIT_REF_NAME} - mv -f /tmp/doc ${CI_COMMIT_REF_NAME} - # Add the symlink - - '[[ -e "$LATEST" ]] && ln -sf "${LATEST}" latest' # Upload files - - git add --all --force + - git add --all # `git commit` has an exit code of > 0 if there is nothing to commit. # This causes GitLab to exit immediately and marks this job failed. # We don't want to mark the entire job failed if there's nothing to @@ -795,27 +732,12 @@ publish-rustdoc: publish-draft-release: stage: publish - <<: *vault-secrets image: paritytech/tools:latest rules: - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 script: - - ./.maintain/gitlab/publish_draft_release.sh - allow_failure: true - -unleash-to-crates-io: - stage: publish - <<: *docker-env - <<: *vault-secrets - rules: - - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - # FIXME: wait until https://github.com/paritytech/cargo-unleash/issues/50 is fixed, also - # remove allow_failure: true on the check job - # - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - - cargo unleash em-dragons --no-check --owner github:paritytech:core-devs ${CARGO_UNLEASH_PKG_DEF} + - ./scripts/ci/gitlab/publish_draft_release.sh allow_failure: true #### stage: deploy @@ -838,38 +760,4 @@ deploy-prometheus-alerting-rules: - if: $CI_COMMIT_REF_NAME == "master" changes: - .gitlab-ci.yml - - .maintain/monitoring/**/* - -# Runs "quick" and "long" tests on nightly schedule and on commit / merge to master -# A "quick" test is a smoke test where basic check-expect tests run by -# checking values from metrics exposed by the app. -# A "long" test is the load testing where we send 50K transactions into the -# network and check if all completed successfully -simnet-tests: - stage: deploy - image: docker.io/paritytech/simnet:${SIMNET_REF} - <<: *kubernetes-env - <<: *vault-secrets - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME == "master" - needs: - - job: publish-docker-substrate - # variables: - # `build.env` brings here `${SUBSTRATE_IMAGE_NAME}` and `${SUBSTRATE_IMAGE_TAG}` - # (`$VERSION` here, # i.e. `2643-0.8.29-5f689e0a-6b24dc54`). - # ${SIMNET_REF} is a gitlab variable - before_script: - - echo "Simnet Tests Config - docker.io/paritytech/simnet:${SIMNET_REF} - ${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IAMGE_TAG}" - script: - - /home/nonroot/simnet/gurke/scripts/run-test-environment-manager.sh - --github-remote-dir="https://github.com/paritytech/substrate/tree/master/simnet_tests" - --config="simnet_tests/configs/default_local_testnet.toml" - --image="${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}" - retry: 2 - tags: - - parity-simnet + - ./scripts/ci/monitoring/**/* diff --git a/.maintain/Dockerfile b/.maintain/Dockerfile index ba3cdcf0722d..e3e6f9118f61 100644 --- a/.maintain/Dockerfile +++ b/.maintain/Dockerfile @@ -40,14 +40,12 @@ RUN mv /usr/share/ca* /tmp && \ COPY --from=builder /substrate/target/$PROFILE/substrate /usr/local/bin COPY --from=builder /substrate/target/$PROFILE/subkey /usr/local/bin -COPY --from=builder /substrate/target/$PROFILE/node-rpc-client /usr/local/bin COPY --from=builder /substrate/target/$PROFILE/node-template /usr/local/bin COPY --from=builder /substrate/target/$PROFILE/chain-spec-builder /usr/local/bin # checks RUN ldd /usr/local/bin/substrate && \ /usr/local/bin/substrate --version - # Shrinking RUN rm -rf /usr/lib/python* && \ rm -rf /usr/bin /usr/sbin /usr/share/man diff --git a/.maintain/frame-weight-template.hbs b/.maintain/frame-weight-template.hbs index 045140d54dff..34852daf7d47 100644 --- a/.maintain/frame-weight-template.hbs +++ b/.maintain/frame-weight-template.hbs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,7 @@ //! EXECUTION: {{cmd.execution}}, WASM-EXECUTION: {{cmd.wasm_execution}}, CHAIN: {{cmd.chain}}, DB CACHE: {{cmd.db_cache}} // Executed Command: -{{#each args as |arg|~}} +{{#each args as |arg|}} // {{arg}} {{/each}} @@ -35,80 +35,80 @@ use sp_std::marker::PhantomData; /// Weight functions needed for {{pallet}}. pub trait WeightInfo { - {{~#each benchmarks as |benchmark|}} + {{#each benchmarks as |benchmark|}} fn {{benchmark.name~}} ( {{~#each benchmark.components as |c| ~}} {{c.name}}: u32, {{/each~}} ) -> Weight; - {{~/each}} + {{/each}} } /// Weights for {{pallet}} using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); -{{~#if (eq pallet "frame_system")}} +{{#if (eq pallet "frame_system")}} impl WeightInfo for SubstrateWeight { -{{~else}} +{{else}} impl WeightInfo for SubstrateWeight { -{{~/if}} - {{~#each benchmarks as |benchmark|}} - {{~#each benchmark.comments as |comment|}} +{{/if}} + {{#each benchmarks as |benchmark|}} + {{#each benchmark.comments as |comment|}} // {{comment}} - {{~/each}} + {{/each}} fn {{benchmark.name~}} ( {{~#each benchmark.components as |c| ~}} {{~#if (not c.is_used)}}_{{/if}}{{c.name}}: u32, {{/each~}} ) -> Weight { ({{underscore benchmark.base_weight}} as Weight) - {{~#each benchmark.component_weight as |cw|}} + {{#each benchmark.component_weight as |cw|}} // Standard Error: {{underscore cw.error}} .saturating_add(({{underscore cw.slope}} as Weight).saturating_mul({{cw.name}} as Weight)) - {{~/each}} - {{~#if (ne benchmark.base_reads "0")}} + {{/each}} + {{#if (ne benchmark.base_reads "0")}} .saturating_add(T::DbWeight::get().reads({{benchmark.base_reads}} as Weight)) - {{~/if}} - {{~#each benchmark.component_reads as |cr|}} + {{/if}} + {{#each benchmark.component_reads as |cr|}} .saturating_add(T::DbWeight::get().reads(({{cr.slope}} as Weight).saturating_mul({{cr.name}} as Weight))) - {{~/each}} - {{~#if (ne benchmark.base_writes "0")}} + {{/each}} + {{#if (ne benchmark.base_writes "0")}} .saturating_add(T::DbWeight::get().writes({{benchmark.base_writes}} as Weight)) - {{~/if}} - {{~#each benchmark.component_writes as |cw|}} + {{/if}} + {{#each benchmark.component_writes as |cw|}} .saturating_add(T::DbWeight::get().writes(({{cw.slope}} as Weight).saturating_mul({{cw.name}} as Weight))) - {{~/each}} + {{/each}} } - {{~/each}} + {{/each}} } // For backwards compatibility and tests impl WeightInfo for () { - {{~#each benchmarks as |benchmark|}} - {{~#each benchmark.comments as |comment|}} + {{#each benchmarks as |benchmark|}} + {{#each benchmark.comments as |comment|}} // {{comment}} - {{~/each}} + {{/each}} fn {{benchmark.name~}} ( {{~#each benchmark.components as |c| ~}} {{~#if (not c.is_used)}}_{{/if}}{{c.name}}: u32, {{/each~}} ) -> Weight { ({{underscore benchmark.base_weight}} as Weight) - {{~#each benchmark.component_weight as |cw|}} + {{#each benchmark.component_weight as |cw|}} // Standard Error: {{underscore cw.error}} .saturating_add(({{underscore cw.slope}} as Weight).saturating_mul({{cw.name}} as Weight)) - {{~/each}} - {{~#if (ne benchmark.base_reads "0")}} + {{/each}} + {{#if (ne benchmark.base_reads "0")}} .saturating_add(RocksDbWeight::get().reads({{benchmark.base_reads}} as Weight)) - {{~/if}} - {{~#each benchmark.component_reads as |cr|}} + {{/if}} + {{#each benchmark.component_reads as |cr|}} .saturating_add(RocksDbWeight::get().reads(({{cr.slope}} as Weight).saturating_mul({{cr.name}} as Weight))) - {{~/each}} - {{~#if (ne benchmark.base_writes "0")}} + {{/each}} + {{#if (ne benchmark.base_writes "0")}} .saturating_add(RocksDbWeight::get().writes({{benchmark.base_writes}} as Weight)) - {{~/if}} - {{~#each benchmark.component_writes as |cw|}} + {{/if}} + {{#each benchmark.component_writes as |cw|}} .saturating_add(RocksDbWeight::get().writes(({{cw.slope}} as Weight).saturating_mul({{cw.name}} as Weight))) - {{~/each}} + {{/each}} } - {{~/each}} + {{/each}} } diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh deleted file mode 100755 index 72bfaf715152..000000000000 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env sh -# -# check if a pr is compatible with polkadot companion pr or master if not -# available -# -# to override one that was just mentioned mark companion pr in the body of the -# polkadot pr like -# -# polkadot companion: paritytech/polkadot#567 -# - -set -e - -github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls" -# use github api v3 in order to access the data without authentication -github_header="Authorization: token ${GITHUB_PR_TOKEN}" - -boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; } -boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; } - - - -boldcat <<-EOT - - -check_polkadot_companion_build -============================== - -this job checks if there is a string in the description of the pr like - -polkadot companion: paritytech/polkadot#567 - - -it will then run cargo check from this polkadot's branch with substrate code -from this pull request. otherwise, it will uses master instead - - -EOT - -# Set the user name and email to make merging work -git config --global user.name 'CI system' -git config --global user.email '<>' - -# Merge master into our branch before building Polkadot to make sure we don't miss -# any commits that are required by Polkadot. -git fetch --depth 100 origin -git merge origin/master - -# Clone the current Polkadot master branch into ./polkadot. -# NOTE: we need to pull enough commits to be able to find a common -# ancestor for successfully performing merges below. -git clone --depth 20 https://github.com/paritytech/polkadot.git - -cd polkadot - -# either it's a pull request then check for a companion otherwise use -# polkadot:master -if expr match "${CI_COMMIT_REF_NAME}" '^[0-9]\+$' >/dev/null -then - boldprint "this is pull request no ${CI_COMMIT_REF_NAME}" - - pr_data_file="$(mktemp)" - # get the last reference to a pr in polkadot - curl -sSL -H "${github_header}" -o "${pr_data_file}" \ - "${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME}" - - pr_body="$(sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p' "${pr_data_file}")" - - pr_companion="$(echo "${pr_body}" | sed -n -r \ - -e 's;^.*[Cc]ompanion.*paritytech/polkadot#([0-9]+).*$;\1;p' \ - -e 's;^.*[Cc]ompanion.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \ - | tail -n 1)" - - if [ "${pr_companion}" ] - then - boldprint "companion pr specified/detected: #${pr_companion}" - git fetch origin refs/pull/${pr_companion}/head:pr/${pr_companion} - git checkout pr/${pr_companion} - git merge origin/master - else - boldprint "no companion branch found - building polkadot:master" - fi - rm -f "${pr_data_file}" -else - boldprint "this is not a pull request - building polkadot:master" -fi - -# Patch all Substrate crates in Polkadot -diener patch --crates-to-patch ../ --substrate --path Cargo.toml - -# We need to update specifically our patched Substrate crates so that other -# crates that depend on them (e.g. Polkadot, BEEFY) use this unified version -# NOTE: There's no way to only update patched crates, so we use a heuristic -# of updating a crucial Substrate crate (`sp-core`) to minimize the impact of -# updating unrelated dependencies -cargo update -p sp-core - -# Test Polkadot pr or master branch with this Substrate commit. -time cargo test --workspace --release --verbose --features=runtime-benchmarks diff --git a/.maintain/gitlab/check_polkadot_companion_status.sh b/.maintain/gitlab/check_polkadot_companion_status.sh deleted file mode 100755 index e0412c7b7bec..000000000000 --- a/.maintain/gitlab/check_polkadot_companion_status.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/sh -# -# check for a polkadot companion pr and ensure it has approvals and is -# mergeable -# - -github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls" -github_api_polkadot_pull_url="https://api.github.com/repos/paritytech/polkadot/pulls" -# use github api v3 in order to access the data without authentication -github_header="Authorization: token ${GITHUB_PR_TOKEN}" - -boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; } -boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; } - - - -boldcat <<-EOT - - -check_polkadot_companion_status -=============================== - -this job checks if there is a string in the description of the pr like - -polkadot companion: paritytech/polkadot#567 - -and checks its status. - - -EOT - - -if ! [ "${CI_COMMIT_REF_NAME}" -gt 0 2>/dev/null ] -then - boldprint "this doesn't seem to be a pull request" - exit 1 -fi - -boldprint "this is pull request no ${CI_COMMIT_REF_NAME}" - -pr_body="$(curl -H "${github_header}" -s ${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME} \ - | sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p')" - -# get companion if explicitly specified -pr_companion="$(echo "${pr_body}" | sed -n -r \ - -e 's;^.*[Cc]ompanion.*paritytech/polkadot#([0-9]+).*$;\1;p' \ - -e 's;^.*[Cc]ompanion.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \ - | tail -n 1)" - -if [ -z "${pr_companion}" ] -then - boldprint "no companion pr found" - exit 0 -fi - -boldprint "companion pr: #${pr_companion}" - -# check the status of that pull request - needs to be -# approved and mergable - -curl -H "${github_header}" -sS -o companion_pr.json \ - ${github_api_polkadot_pull_url}/${pr_companion} - -pr_head_sha=$(jq -r -e '.head.sha' < companion_pr.json) -boldprint "Polkadot PR's HEAD SHA: $pr_head_sha" - -curl -H "${github_header}" -sS -o companion_pr_reviews.json \ - ${github_api_polkadot_pull_url}/${pr_companion}/reviews - -# If there are any 'CHANGES_REQUESTED' reviews for the *current* review -jq -r -e '.[] | select(.state == "CHANGES_REQUESTED").commit_id' \ - < companion_pr_reviews.json > companion_pr_reviews_current.json -while IFS= read -r line; do - if [ "$line" = "$pr_head_sha" ]; then - boldprint "polkadot pr #${pr_companion} has CHANGES_REQUESTED for the latest commit" - exit 1 - fi -done < companion_pr_reviews_current.json - -# Then we check for at least 1 APPROVED -if [ -z "$(jq -r -e '.[].state | select(. == "APPROVED")' < companion_pr_reviews.json)" ]; then - boldprint "polkadot pr #${pr_companion} not APPROVED" - exit 1 -fi - -boldprint "polkadot pr #${pr_companion} state APPROVED" - -if jq -e .merged < companion_pr.json >/dev/null -then - boldprint "polkadot pr #${pr_companion} already merged" - exit 0 -fi - -if jq -e '.mergeable' < companion_pr.json >/dev/null -then - boldprint "polkadot pr #${pr_companion} mergeable" -else - boldprint "polkadot pr #${pr_companion} not mergeable" - exit 1 -fi - -exit 0 diff --git a/.maintain/rustdoc-header.html b/.maintain/rustdoc-header.html deleted file mode 100644 index a679d5e299da..000000000000 --- a/.maintain/rustdoc-header.html +++ /dev/null @@ -1,10 +0,0 @@ - - - - diff --git a/.maintain/rustdocs-release.sh b/.maintain/rustdocs-release.sh new file mode 100755 index 000000000000..2a1e141e63ad --- /dev/null +++ b/.maintain/rustdocs-release.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# set -x + +# This script manages the deployment of Substrate rustdocs to https://paritytech.github.io/substrate/. +# - With `deploy` sub-command, it will checkout the passed-in branch/tag ref, build the rustdocs +# locally (this takes some time), update the `index.html` index page, and push it to remote +# `gh-pages` branch. So users running this command need to have write access to the remote +# `gh-pages` branch. This sub-command depends on [@substrate/index-tpl-crud](https://www.npmjs.com/package/@substrate/index-tpl-crud) +# to update the DOM of index.html page. +# - With `remove` sub-command, it will remove the deployed rustdocs from `gh-pages`, and update the +# index.html page as necessary. It may remove the `latest` symbolic link. +# +# Examples: +# # Showing help text +# rustdocs-release.sh -h +# +# # Deploy rustdocs of `monthly-2021-10` tag +# rustdocs-release.sh deploy monthly-2021-10 +# +# # In addition to the above, the `latest` symlink will point to this version of rustdocs +# rustdocs-release.sh deploy -l monthly-2021-10 +# +# # Remove the rustdocs of `monthly-2021-10` from `gh-pages`. +# rustdocs-release.sh remove monthly-2021-10 +# +# Dependencies: +# - @substrate/index-tpl-crud - https://www.npmjs.com/package/@substrate/index-tpl-crud +# + +# Script setting +# The git repo http URL +REMOTE_REPO="https://github.com/paritytech/substrate.git" +TMP_PREFIX="/tmp" # tmp location that the built doc is copied to. +DOC_INDEX_PAGE="sc_service/index.html" + +# Set to `true` if using cargo `nightly` toolchain to build the doc. +# Set to `false` to use the default cargo toolchain. This is preferred if you want to build +# the rustdocs with a pinned nightly version set to your default toolchain. +CARGO_NIGHTLY=false + +# Set the git remote name. Most of the time the default is `origin`. +GIT_REMOTE="origin" +LATEST=false + +# Setting the help text +declare -A HELP_TXT +HELP_TXT["deploy"]=$(cat <<-EOH +Build and deploy the rustdocs of the specified branch/tag to \`gh-pages\` branch. + + usage: $0 deploy [-l] + example: $0 deploy -l monthly-2021-10 + + options: + -l The \`latest\` path will be sym'linked to this rustdocs version +EOH +) + +HELP_TXT["remove"]=$(cat <<-EOH +Remove the rustdocs of the specified version from \`gh-pages\` branch. + + usage: $0 remove + example: $0 remove monthly-2021-10 +EOH +) + +set_and_check_rustdoc_ref() { + [[ -z "$1" ]] && { + echo -e "git branch_ref is not specified.\n" + echo "${HELP_TXT[$2]}" + exit 1 + } + BUILD_RUSTDOC_REF=$1 +} + +check_local_change() { + # Check there is no local changes before proceeding + [[ -n $(git status --porcelain) ]] \ + && echo "Local changes exist, please either discard or commit them as this command will change the current checkout branch." \ + && exit 1 +} + +build_rustdocs() { + # Build the docs + time cargo $($CARGO_NIGHTLY && echo "+nightly") doc --workspace --all-features --verbose \ + || { echo "Generate $1 rustdocs failed" && exit 1; } + rm -f target/doc/.lock + + # Moving the built doc to the tmp location + mv target/doc "${2}" + [[ -n "${DOC_INDEX_PAGE}" ]] \ + && echo "" > "${2}/index.html" +} + +upsert_index_page() { + # Check if `index-tpl-crud` exists + which index-tpl-crud &> /dev/null || yarn global add @substrate/index-tpl-crud + index-tpl-crud upsert $($1 && echo "-l") ./index.html "$2" +} + +rm_index_page() { + which index-tpl-crud &> /dev/null || yarn global add @substrate/index-tpl-crud + index-tpl-crud rm ./index.html "$1" +} + +git_add_commit_push() { + git add --all + git commit -m "$1" || echo "Nothing to commit" + git push "${GIT_REMOTE}" gh-pages --force +} + +import_gh_key() { + [[ -n $GITHUB_SSH_PRIV_KEY ]] && { + eval $(ssh-agent) + ssh-add - <<< $GITHUB_SSH_PRIV_KEY + } + + # Adding github.com as known_hosts + ssh-keygen -F github.com &>/dev/null || { + [[ -e ~/.ssh ]] || mkdir ~/.ssh + [[ -e ~/.ssh/known_hosts ]] || touch ~/.ssh/known_hosts + ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + } +} + +deploy_main() { + check_local_change + import_gh_key + + CURRENT_GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + TMP_PROJECT_PATH="${TMP_PREFIX}/${PROJECT_NAME}" + DOC_PATH="${TMP_PROJECT_PATH}/${BUILD_RUSTDOC_REF}" + + # Build the tmp project path + rm -rf "${TMP_PROJECT_PATH}" && mkdir "${TMP_PROJECT_PATH}" + + # Copy .gitignore file to tmp + [[ -e "${PROJECT_PATH}/.gitignore" ]] && cp "${PROJECT_PATH}/.gitignore" "${TMP_PROJECT_PATH}" + + git fetch --all + git checkout -f ${BUILD_RUSTDOC_REF} || { echo "Checkout \`${BUILD_RUSTDOC_REF}\` error." && exit 1; } + build_rustdocs "${BUILD_RUSTDOC_REF}" "${DOC_PATH}" + + # git checkout `gh-pages` branch + git fetch "${GIT_REMOTE}" gh-pages + git checkout gh-pages + # Move the built back + [[ -e "${TMP_PROJECT_PATH}/.gitignore" ]] && cp -f "${TMP_PROJECT_PATH}/.gitignore" . + # Ensure the destination dir doesn't exist under current path. + rm -rf "${BUILD_RUSTDOC_REF}" + mv -f "${DOC_PATH}" "${BUILD_RUSTDOC_REF}" + + upsert_index_page $LATEST "${BUILD_RUSTDOC_REF}" + # Add the latest symlink + $LATEST && rm -rf latest && ln -sf "${BUILD_RUSTDOC_REF}" latest + + git_add_commit_push "___Deployed rustdocs of ${BUILD_RUSTDOC_REF}___" + # Clean up + # Remove the tmp asset created + rm -rf "${TMP_PROJECT_PATH}" + # Resume back previous checkout branch. + git checkout -f "$CURRENT_GIT_BRANCH" +} + +remove_main() { + check_local_change + import_gh_key + + CURRENT_GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + + # git checkout `gh-pages` branch + git fetch "${GIT_REMOTE}" gh-pages + git checkout gh-pages + + rm -rf "${BUILD_RUSTDOC_REF}" + rm_index_page "${BUILD_RUSTDOC_REF}" + # check if the destination of `latest` exists and rmove if not. + [[ -e "latest" ]] || rm latest + + git_add_commit_push "___Removed rustdocs of ${BUILD_RUSTDOC_REF}___" + + # Resume back previous checkout branch. + git checkout -f "$CURRENT_GIT_BRANCH" +} + +# ---- The script execution entry point starts here ---- + +# Arguments handling +SUBCMD=$1 +[[ $SUBCMD == "deploy" || $SUBCMD == "remove" ]] \ + || { echo "Please specify a subcommand of \`deploy\` or \`remove\`" && exit 1 ;} +shift + +# After removing the subcommand, there could only be 1 or 2 parameters afterward +[[ $# -lt 1 || $# -gt 2 ]] && { + echo "${HELP_TXT[${SUBCMD}]}" + exit 1 +} + +# Parsing options and argument for `deploy` subcommand +[[ $SUBCMD == "deploy" ]] && { + while getopts :lh opt; do + case $opt in + l) + LATEST=true + ;; + h) + echo "${HELP_TXT[$SUBCMD]}" + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac + done +} +# Parsing options and argument for `remove` subcommand +[[ $SUBCMD == "remove" ]] && { + while getopts :h opt; do + case $opt in + h) + echo "${HELP_TXT[$SUBCMD]}" + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac + done +} + +shift $(($OPTIND - 1)) +set_and_check_rustdoc_ref ${1:-''} $SUBCMD + +SCRIPT=$(realpath $0) +SCRIPT_PATH=$(dirname $SCRIPT) +PROJECT_PATH=$(dirname ${SCRIPT_PATH}) +PROJECT_NAME=$(basename "$PROJECT_PATH") + +pushd "${PROJECT_PATH}" &>/dev/null +[[ $SUBCMD == "deploy" ]] && deploy_main +[[ $SUBCMD == "remove" ]] && remove_main +popd &>/dev/null diff --git a/Cargo.lock b/Cargo.lock index 0756371a4ecf..1f565c07ff60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,15 +12,6 @@ dependencies = [ "regex", ] -[[package]] -name = "addr2line" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd" -dependencies = [ - "gimli 0.25.0", -] - [[package]] name = "addr2line" version = "0.17.0" @@ -53,7 +44,7 @@ checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ "cfg-if 1.0.0", "cipher", - "cpufeatures 0.2.1", + "cpufeatures 0.2.2", "opaque-debug 0.3.0", ] @@ -68,7 +59,7 @@ dependencies = [ "cipher", "ctr", "ghash", - "subtle 2.4.1", + "subtle", ] [[package]] @@ -77,7 +68,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.5", + "getrandom 0.2.6", "once_cell", "version_check 0.9.4", ] @@ -91,15 +82,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "ansi_term" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" -dependencies = [ - "winapi 0.3.9", -] - [[package]] name = "ansi_term" version = "0.12.1" @@ -165,9 +147,9 @@ checksum = "e22d1f4b888c298a027c99dc9048015fac177587de20fc30232a057dfbe24a21" [[package]] name = "assert_cmd" -version = "1.0.8" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c98233c6673d8601ab23e77eb38f999c51100d46c5703b17288c57fddf3a1ffe" +checksum = "93ae1ddd39efd67689deb1979d80bad3bf7f2b09c6e6117c8d1f2443b5e2f83e" dependencies = [ "bstr", "doc-comment", @@ -220,9 +202,9 @@ dependencies = [ [[package]] name = "async-global-executor" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c026b7e44f1316b567ee750fea85103f87fcb80792b860e979f221259796ca0a" +checksum = "c290043c9a95b05d45e952fb6383c67bcb61471f60cfa21e890dba6654234f43" dependencies = [ "async-channel", "async-executor", @@ -243,7 +225,7 @@ dependencies = [ "concurrent-queue", "futures-lite", "libc", - "log 0.4.14", + "log 0.4.16", "once_cell", "parking", "polling", @@ -271,43 +253,25 @@ dependencies = [ "event-listener", ] -[[package]] -name = "async-process" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83137067e3a2a6a06d67168e49e68a0957d215410473a740cea95a2425c0b7c6" -dependencies = [ - "async-io", - "blocking", - "cfg-if 1.0.0", - "event-listener", - "futures-lite", - "libc", - "once_cell", - "signal-hook", - "winapi 0.3.9", -] - [[package]] name = "async-std" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8056f1455169ab86dd47b47391e4ab0cbd25410a70e9fe675544f49bafaf952" +checksum = "52580991739c5cdb36cde8b2a516371c0a3b70dda36d916cc08b82372916808c" dependencies = [ "async-attributes", "async-channel", "async-global-executor", "async-io", "async-lock", - "async-process", - "crossbeam-utils 0.8.7", + "crossbeam-utils 0.8.8", "futures-channel", "futures-core", "futures-io", "futures-lite", "gloo-timers", "kv-log-macro", - "log 0.4.14", + "log 0.4.16", "memchr", "num_cpus", "once_cell", @@ -328,14 +292,14 @@ dependencies = [ "futures-io", "futures-util", "pin-utils", - "trust-dns-resolver", + "trust-dns-resolver 0.20.4", ] [[package]] name = "async-stream" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" dependencies = [ "async-stream-impl", "futures-core", @@ -343,9 +307,9 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" dependencies = [ "proc-macro2", "quote", @@ -360,9 +324,9 @@ checksum = "30696a84d817107fc028e049980e09d5e140e8da8f1caeb17e8e950658a3cea9" [[package]] name = "async-trait" -version = "0.1.52" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" dependencies = [ "proc-macro2", "quote", @@ -442,7 +406,7 @@ version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f" dependencies = [ - "addr2line 0.17.0", + "addr2line", "cc", "cfg-if 1.0.0", "libc", @@ -451,30 +415,23 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "bae" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b8de67cc41132507eeece2584804efcb15f85ba516e34c944b7667f480397a" -dependencies = [ - "heck 0.3.3", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "base-x" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base58" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" +checksum = "6107fe1be6682a68940da878d9e9f5e90ca5745b3dec9fd1bb393c8777d4f581" [[package]] name = "base64" @@ -507,6 +464,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "base64ct" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea908e7347a8c64e378c17e30ef880ad73e3b4498346b055c2c00ea342f3179" + [[package]] name = "beef" version = "0.5.1" @@ -523,25 +486,38 @@ dependencies = [ "beefy-primitives", "fnv", "futures 0.3.21", - "log 0.4.14", - "parity-scale-codec", - "parking_lot 0.11.2", + "futures-timer", + "hex", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", + "sc-chain-spec", "sc-client-api", + "sc-consensus", + "sc-finality-grandpa", "sc-keystore", "sc-network", "sc-network-gossip", "sc-network-test", "sc-utils", + "serde", "sp-api", "sp-application-crypto", "sp-arithmetic", "sp-blockchain", + "sp-consensus", "sp-core", + "sp-finality-grandpa", + "sp-keyring", "sp-keystore", "sp-runtime", - "strum 0.21.0", + "sp-tracing", + "strum", "substrate-prometheus-endpoint", + "substrate-test-runtime-client", + "tempfile", "thiserror", + "tokio", "wasm-timer", ] @@ -556,12 +532,17 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-pubsub", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "sc-rpc", + "sc-utils", "serde", + "serde_json", "sp-core", "sp-runtime", + "substrate-test-runtime-client", + "thiserror", ] [[package]] @@ -571,7 +552,7 @@ dependencies = [ "env_logger 0.9.0", "hex", "hex-literal", - "log 0.4.14", + "log 0.4.16", "tiny-keccak", ] @@ -579,8 +560,9 @@ dependencies = [ name = "beefy-primitives" version = "4.0.0-dev" dependencies = [ + "hex", "hex-literal", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-api", "sp-application-crypto", @@ -590,6 +572,12 @@ dependencies = [ "sp-std", ] +[[package]] +name = "bimap" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0455254eb5c6964c4545d8bac815e1a1be4f3afe0ae695ea539c12d728d44b" + [[package]] name = "bincode" version = "1.3.3" @@ -630,10 +618,22 @@ version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7774144344a4faa177370406a7ff5f1da24303817368584c6206c8303eb07848" dependencies = [ - "funty", - "radium", + "funty 1.1.0", + "radium 0.6.2", + "tap", + "wyz 0.2.0", +] + +[[package]] +name = "bitvec" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1489fcb93a5bb47da0462ca93ad252ad6af2145cce58d10d46a83931ba9f016b" +dependencies = [ + "funty 2.0.0", + "radium 0.7.0", "tap", - "wyz", + "wyz 0.5.0", ] [[package]] @@ -647,6 +647,15 @@ dependencies = [ "opaque-debug 0.3.0", ] +[[package]] +name = "blake2" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cf849ee05b2ee5fba5e36f97ff8ec2533916700fc0758d40d92136a42f3388" +dependencies = [ + "digest 0.10.3", +] + [[package]] name = "blake2-rfc" version = "0.2.18" @@ -716,6 +725,15 @@ dependencies = [ "generic-array 0.14.5", ] +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array 0.14.5", +] + [[package]] name = "block-padding" version = "0.1.5" @@ -733,9 +751,9 @@ checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" [[package]] name = "blocking" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046e47d4b2d391b1f6f8b407b1deb8dee56c1852ccd868becf2710f601b5f427" +checksum = "c6ccb65d468978a086b69884437ded69a90faab3bbe6e67f242173ea728acccc" dependencies = [ "async-channel", "async-task", @@ -790,6 +808,27 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" +[[package]] +name = "bytecheck" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "314889ea31cda264cb7c3d6e6e5c9415a987ecb0e72c17c00d36fbb881d34abe" +dependencies = [ + "bytecheck_derive", + "ptr_meta", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a2b3b92c135dae665a6f760205b89187638e83bed17ef3e44e83c712cf30600" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "byteorder" version = "1.4.3" @@ -818,6 +857,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cache-padded" version = "1.2.0" @@ -844,14 +894,13 @@ dependencies = [ [[package]] name = "cargo_metadata" -version = "0.13.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "081e3f0755c1f380c2d010481b6fa2e02973586d5f2b24eebb7a2a1d98b143d8" +checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 0.11.0", - "semver-parser 0.10.2", + "semver 1.0.7", "serde", "serde_json", ] @@ -907,6 +956,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "chacha20" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b72a433d0cf2aef113ba70f62634c56fddb0f244e6377185c56a7cadbd8f91" +dependencies = [ + "cfg-if 1.0.0", + "cipher", + "cpufeatures 0.2.2", + "zeroize", +] + [[package]] name = "chacha20poly1305" version = "0.8.0" @@ -914,7 +975,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1580317203210c517b6d44794abfbe600698276db18127e37ad3e69bf5e848e5" dependencies = [ "aead", - "chacha20", + "chacha20 0.7.1", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "chacha20poly1305" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b84ed6d1d5f7aa9bdde921a5090e0ca4d934d250ea3b402a5fab3a994e28a2a" +dependencies = [ + "aead", + "chacha20 0.8.1", "cipher", "poly1305", "zeroize", @@ -924,14 +998,14 @@ dependencies = [ name = "chain-spec-builder" version = "2.0.0" dependencies = [ - "ansi_term 0.12.1", + "ansi_term", + "clap 3.1.8", "node-cli", - "rand 0.7.3", + "rand 0.8.5", "sc-chain-spec", "sc-keystore", "sp-core", "sp-keystore", - "structopt", ] [[package]] @@ -1004,15 +1078,54 @@ version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ - "ansi_term 0.12.1", + "ansi_term", "atty", "bitflags", "strsim 0.8.0", - "textwrap", + "textwrap 0.11.0", "unicode-width", "vec_map", ] +[[package]] +name = "clap" +version = "3.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "indexmap", + "lazy_static", + "os_str_bytes", + "strsim 0.10.0", + "termcolor", + "textwrap 0.15.0", +] + +[[package]] +name = "clap_complete" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df6f3613c0a3cddfd78b41b10203eb322cb29b600cbdf808a7d3db95691b8e25" +dependencies = [ + "clap 3.1.8", +] + +[[package]] +name = "clap_derive" +version = "3.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1" +dependencies = [ + "heck 0.4.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "cloudabi" version = "0.0.3" @@ -1031,6 +1144,12 @@ dependencies = [ "cache-padded", ] +[[package]] +name = "const-oid" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4c78c047431fee22c1a7bb92e00ad095a02a983affe4d8a72e2a2c62c1b94f3" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -1059,6 +1178,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + [[package]] name = "cpp_demangle" version = "0.3.5" @@ -1079,22 +1207,13 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" dependencies = [ "libc", ] -[[package]] -name = "cranelift-bforest" -version = "0.68.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9221545c0507dc08a62b2d8b5ffe8e17ac580b0a74d1813b496b8d70b070fbd0" -dependencies = [ - "cranelift-entity 0.68.0", -] - [[package]] name = "cranelift-bforest" version = "0.76.0" @@ -1105,22 +1224,12 @@ dependencies = [ ] [[package]] -name = "cranelift-codegen" -version = "0.68.0" +name = "cranelift-bforest" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9936ea608b6cd176f107037f6adbb4deac933466fc7231154f96598b2d3ab1" +checksum = "62fc68cdb867b7d27b5f33cd65eb11376dfb41a2d09568a1a2c2bc1dc204f4ef" dependencies = [ - "byteorder", - "cranelift-bforest 0.68.0", - "cranelift-codegen-meta 0.68.0", - "cranelift-codegen-shared 0.68.0", - "cranelift-entity 0.68.0", - "gimli 0.22.0", - "log 0.4.14", - "regalloc", - "smallvec 1.8.0", - "target-lexicon 0.11.2", - "thiserror", + "cranelift-entity 0.80.1", ] [[package]] @@ -1134,21 +1243,27 @@ dependencies = [ "cranelift-codegen-shared 0.76.0", "cranelift-entity 0.76.0", "gimli 0.25.0", - "log 0.4.14", - "regalloc", - "serde", + "log 0.4.16", + "regalloc 0.0.31", "smallvec 1.8.0", - "target-lexicon 0.12.3", + "target-lexicon", ] [[package]] -name = "cranelift-codegen-meta" -version = "0.68.0" +name = "cranelift-codegen" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ef2b2768568306540f4c8db3acce9105534d34c4a1e440529c1e702d7f8c8d7" +checksum = "31253a44ab62588f8235a996cc9b0636d98a299190069ced9628b8547329b47a" dependencies = [ - "cranelift-codegen-shared 0.68.0", - "cranelift-entity 0.68.0", + "cranelift-bforest 0.80.1", + "cranelift-codegen-meta 0.80.1", + "cranelift-codegen-shared 0.80.1", + "cranelift-entity 0.80.1", + "gimli 0.26.1", + "log 0.4.16", + "regalloc 0.0.33", + "smallvec 1.8.0", + "target-lexicon", ] [[package]] @@ -1162,88 +1277,90 @@ dependencies = [ ] [[package]] -name = "cranelift-codegen-shared" -version = "0.68.0" +name = "cranelift-codegen-meta" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6759012d6d19c4caec95793f052613e9d4113e925e7f14154defbac0f1d4c938" +checksum = "7a20ab4627d30b702fb1b8a399882726d216b8164d3b3fa6189e3bf901506afe" +dependencies = [ + "cranelift-codegen-shared 0.80.1", +] [[package]] name = "cranelift-codegen-shared" version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dabb5fe66e04d4652e434195b45ae65b5c8172d520247b8f66d8df42b2b45dc" -dependencies = [ - "serde", -] [[package]] -name = "cranelift-entity" -version = "0.68.0" +name = "cranelift-codegen-shared" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86badbce14e15f52a45b666b38abe47b204969dd7f8fb7488cb55dd46b361fa6" -dependencies = [ - "serde", -] +checksum = "6687d9668dacfed4468361f7578d86bded8ca4db978f734d9b631494bebbb5b8" [[package]] name = "cranelift-entity" version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3329733e4d4b8e91c809efcaa4faee80bf66f20164e3dd16d707346bd3494799" + +[[package]] +name = "cranelift-entity" +version = "0.80.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77c5d72db97ba2cb36f69037a709edbae0d29cb25503775891e7151c5c874bf" dependencies = [ "serde", ] [[package]] name = "cranelift-frontend" -version = "0.68.0" +version = "0.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b608bb7656c554d0a4cf8f50c7a10b857e80306f6ff829ad6d468a7e2323c8d8" +checksum = "279afcc0d3e651b773f94837c3d581177b348c8d69e928104b2e9fccb226f921" dependencies = [ - "cranelift-codegen 0.68.0", - "log 0.4.14", + "cranelift-codegen 0.76.0", + "log 0.4.16", "smallvec 1.8.0", - "target-lexicon 0.11.2", + "target-lexicon", ] [[package]] name = "cranelift-frontend" -version = "0.76.0" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279afcc0d3e651b773f94837c3d581177b348c8d69e928104b2e9fccb226f921" +checksum = "426dca83f63c7c64ea459eb569aadc5e0c66536c0042ed5d693f91830e8750d0" dependencies = [ - "cranelift-codegen 0.76.0", - "log 0.4.14", + "cranelift-codegen 0.80.1", + "log 0.4.16", "smallvec 1.8.0", - "target-lexicon 0.12.3", + "target-lexicon", ] [[package]] name = "cranelift-native" -version = "0.76.0" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c04d1fe6a5abb5bb0edc78baa8ef238370fb8e389cc88b6d153f7c3e9680425" +checksum = "8007864b5d0c49b026c861a15761785a2871124e401630c03ef1426e6d0d559e" dependencies = [ - "cranelift-codegen 0.76.0", + "cranelift-codegen 0.80.1", "libc", - "target-lexicon 0.12.3", + "target-lexicon", ] [[package]] name = "cranelift-wasm" -version = "0.76.0" +version = "0.80.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d260ad44f6fd2c91f7f5097191a2a9e3edcbb36df1fb787b600dad5ea148ec" +checksum = "94cf12c071415ba261d897387ae5350c4d83c238376c8c5a96514ecfa2ea66a3" dependencies = [ - "cranelift-codegen 0.76.0", - "cranelift-entity 0.76.0", - "cranelift-frontend 0.76.0", + "cranelift-codegen 0.80.1", + "cranelift-entity 0.80.1", + "cranelift-frontend 0.80.1", "itertools", - "log 0.4.14", - "serde", + "log 0.4.16", "smallvec 1.8.0", - "thiserror", - "wasmparser 0.79.0", + "wasmparser 0.81.0", + "wasmtime-types", ] [[package]] @@ -1263,9 +1380,10 @@ checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10" dependencies = [ "atty", "cast", - "clap", + "clap 2.34.0", "criterion-plot", "csv", + "futures 0.3.21", "itertools", "lazy_static", "num-traits", @@ -1278,6 +1396,7 @@ dependencies = [ "serde_derive", "serde_json", "tinytemplate", + "tokio", "walkdir", ] @@ -1293,12 +1412,12 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.7", + "crossbeam-utils 0.8.8", ] [[package]] @@ -1309,17 +1428,17 @@ checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.8.7", + "crossbeam-utils 0.8.8", ] [[package]] name = "crossbeam-epoch" -version = "0.9.7" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.7", + "crossbeam-utils 0.8.8", "lazy_static", "memoffset", "scopeguard", @@ -1338,9 +1457,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ "cfg-if 1.0.0", "lazy_static", @@ -1353,23 +1472,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] -name = "crypto-mac" -version = "0.7.0" +name = "crypto-bigint" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" +checksum = "03c6a1d5fa1de37e071642dfa44ec552ca5b299adb128fab16138e24b548fd21" dependencies = [ - "generic-array 0.12.4", - "subtle 1.0.0", + "generic-array 0.14.5", + "rand_core 0.6.3", + "subtle", + "zeroize", ] [[package]] -name = "crypto-mac" -version = "0.8.0" +name = "crypto-common" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" dependencies = [ "generic-array 0.14.5", - "subtle 2.4.1", + "typenum", +] + +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array 0.14.5", + "subtle", ] [[package]] @@ -1379,7 +1510,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" dependencies = [ "generic-array 0.14.5", - "subtle 2.4.1", + "subtle", ] [[package]] @@ -1410,14 +1541,14 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" dependencies = [ - "sct", + "sct 0.6.1", ] [[package]] name = "ctor" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccc0a48a9b826acdf4028595adc9db92caea352f7af011a3034acd172a52a0aa" +checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c" dependencies = [ "quote", "syn", @@ -1452,7 +1583,7 @@ dependencies = [ "byteorder", "digest 0.8.1", "rand_core 0.5.1", - "subtle 2.4.1", + "subtle", "zeroize", ] @@ -1465,15 +1596,28 @@ dependencies = [ "byteorder", "digest 0.9.0", "rand_core 0.5.1", - "subtle 2.4.1", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "4.0.0-pre.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4033478fbf70d6acf2655ac70da91ee65852d69daf7a67bf7a2f518fb47aafcf" +dependencies = [ + "byteorder", + "digest 0.9.0", + "rand_core 0.6.3", + "subtle", "zeroize", ] [[package]] name = "darling" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d720b8683f8dd83c65155f0530560cba68cd2bf395f6513a483caee57ff7f4" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ "darling_core", "darling_macro", @@ -1481,23 +1625,22 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a340f241d2ceed1deb47ae36c4144b2707ec7dd0b649f894cb39bb595986324" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", "syn", ] [[package]] name = "darling_macro" -version = "0.13.1" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c41b3b7352feb3211a0d743dc5700a4e3b60f51bd2b368892d1e0f9a95f44b" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core", "quote", @@ -1530,6 +1673,15 @@ dependencies = [ "syn", ] +[[package]] +name = "der" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6919815d73839e7ad218de758883aae3a257ba6759ce7a9992501efbb53d705c" +dependencies = [ + "const-oid", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -1549,12 +1701,6 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" -[[package]] -name = "difference" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" - [[package]] name = "difflib" version = "0.4.0" @@ -1579,11 +1725,22 @@ dependencies = [ "generic-array 0.14.5", ] +[[package]] +name = "digest" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +dependencies = [ + "block-buffer 0.10.2", + "crypto-common", + "subtle", +] + [[package]] name = "directories" -version = "3.0.2" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e69600ff1703123957937708eb27f7a564e48885c537782722ed0ba3189ce1d7" +checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210" dependencies = [ "dirs-sys", ] @@ -1600,9 +1757,9 @@ dependencies = [ [[package]] name = "dirs-sys" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", "redox_users", @@ -1648,6 +1805,18 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + +[[package]] +name = "dtoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5caaa75cbd2b960ff1e5392d2cfb1f44717fffe12fc1f32b7b5d1267f99732a6" + [[package]] name = "dyn-clonable" version = "0.9.0" @@ -1671,15 +1840,15 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2626afccd7561a06cf1367e2950c4718ea04565e20fb5029b6c7d8ad09abcf" +checksum = "21e50f3adc76d6a43f5ed73b698a87d0760ca74617f60f7c3b879003536fdd28" [[package]] name = "dynasm" -version = "1.2.1" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b1801e630bd336d0bbbdbf814de6cc749c9a400c7e3d995e6adfd455d0c83c" +checksum = "add9a102807b524ec050363f09e06f1504214b0e1c7797f64261c891022dce8b" dependencies = [ "bitflags", "byteorder", @@ -1692,20 +1861,31 @@ dependencies = [ [[package]] name = "dynasmrt" -version = "1.2.1" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d428afc93ad288f6dffc1fa5f4a78201ad2eec33c5a522e51c181009eb09061" +checksum = "64fba5a42bd76a17cad4bfa00de168ee1cbfa06a5e8ce992ae880218c05641a9" dependencies = [ "byteorder", "dynasm", "memmap2 0.5.3", ] +[[package]] +name = "ecdsa" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d69ae62e0ce582d56380743515fefaf1a8c70cec685d9677636d7e30ae9dc9" +dependencies = [ + "der", + "elliptic-curve", + "signature", +] + [[package]] name = "ed25519" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed12bbf7b5312f8da1c2722bc06d8c6b12c2d86a7fb35a194c7f3e6fc2bbe39" +checksum = "3d5c4b5e5959dc2c2b89918d8e2cc40fcdd623cef026ed09d2f0ee05199dc8e4" dependencies = [ "signature", ] @@ -1730,6 +1910,24 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "elliptic-curve" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b477563c2bfed38a3b7a60964c49e058b2510ad3f12ba3483fd8f62c2306d6" +dependencies = [ + "base16ct", + "crypto-bigint", + "der", + "ff", + "generic-array 0.14.5", + "group", + "rand_core 0.6.3", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "enum-as-inner" version = "0.3.4" @@ -1742,20 +1940,52 @@ dependencies = [ "syn", ] +[[package]] +name = "enum-as-inner" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" +dependencies = [ + "heck 0.4.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "enum-iterator" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eeac5c5edb79e4e39fe8439ef35207780a11f69c52cbe424ce3dfad4cb78de6" +dependencies = [ + "enum-iterator-derive", +] + +[[package]] +name = "enum-iterator-derive" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c134c37760b27a871ba422106eedbb8247da973a09e82558bf26d619c882b159" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "enumflags2" -version = "0.6.4" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c8d82922337cd23a15f88b70d8e4ef5f11da38dd7cdb55e84dd5de99695da0" +checksum = "1b3ab37dc79652c9d85f1f7b6070d77d321d2467f5fe7b00d6b7a86c57b092ae" dependencies = [ "enumflags2_derive", ] [[package]] name = "enumflags2_derive" -version = "0.6.4" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "946ee94e3dbf58fdd324f9ce245c7b238d46a66f00e86a020b71996349e46cce" +checksum = "f58dc3c5e468259f19f2d46304a6b28f1c3d034442e14b322d2b850e36f6d5ae" dependencies = [ "proc-macro2", "quote", @@ -1764,18 +1994,18 @@ dependencies = [ [[package]] name = "enumset" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6216d2c19a6fb5f29d1ada1dc7bc4367a8cbf0fa4af5cf12e07b5bbdde6b5b2c" +checksum = "e8b6b2301b38343c1f00b2cc5116d6c28306758344db73e3347f21aa1e60e756" dependencies = [ "enumset_derive", ] [[package]] name = "enumset_derive" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6451128aa6655d880755345d085494cf7561a6bee7c8dc821e5d77e6d267ecd4" +checksum = "b201779d9a5dee6b1478eb1c6b9643b1fbca784ce98ac74e7f33cd1dad620058" dependencies = [ "darling", "proc-macro2", @@ -1789,7 +2019,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" dependencies = [ - "log 0.4.14", + "log 0.4.16", "regex", ] @@ -1801,7 +2031,7 @@ checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" dependencies = [ "atty", "humantime", - "log 0.4.14", + "log 0.4.16", "regex", "termcolor", ] @@ -1812,15 +2042,6 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68b91989ae21441195d7d9b9993a2f9295c7e1a8c96255d8b729accddc124797" -[[package]] -name = "erased-serde" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56047058e1ab118075ca22f9ecd737bcc961aa3566a3019cb71388afa280bd8a" -dependencies = [ - "serde", -] - [[package]] name = "errno" version = "0.2.8" @@ -1887,6 +2108,16 @@ dependencies = [ "libc", ] +[[package]] +name = "ff" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2958d04124b9f27f175eaeb9a9f383d026098aa837eadd8ba22c11f13a05b9e" +dependencies = [ + "rand_core 0.6.3", + "subtle", +] + [[package]] name = "file-per-thread-logger" version = "0.1.5" @@ -1894,7 +2125,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21e16290574b39ee41c71aeb90ae960c504ebaf1e2a1c87bd52aa56ed6e1a02f" dependencies = [ "env_logger 0.9.0", - "log 0.4.14", + "log 0.4.16", ] [[package]] @@ -1905,22 +2136,22 @@ checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.11", + "redox_syscall 0.2.13", "winapi 0.3.9", ] [[package]] name = "finality-grandpa" -version = "0.14.4" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ac3ff5224ef91f3c97e03eb1de2db82743427e91aaa5ac635f454f0b164f5a" +checksum = "d9def033d8505edf199f6a5d07aa7e6d2d6185b164293b77f0efd108f4f3e11d" dependencies = [ "either", "futures 0.3.21", - "futures-timer 3.0.2", - "log 0.4.14", + "futures-timer", + "log 0.4.16", "num-traits", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "parking_lot 0.11.2", "rand 0.8.5", "scale-info", @@ -1944,6 +2175,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" +[[package]] +name = "fixedbitset" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" + [[package]] name = "flate2" version = "1.0.22" @@ -1982,7 +2219,7 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" name = "fork-tree" version = "3.0.0" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", ] [[package]] @@ -2003,12 +2240,15 @@ dependencies = [ "frame-system", "hex-literal", "linregress", - "log 0.4.14", - "parity-scale-codec", - "paste 1.0.6", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "paste 1.0.7", "scale-info", + "serde", "sp-api", + "sp-application-crypto", "sp-io", + "sp-keystore", "sp-runtime", "sp-runtime-interface", "sp-std", @@ -2021,32 +2261,69 @@ version = "4.0.0-dev" dependencies = [ "Inflector", "chrono", + "clap 3.1.8", "frame-benchmarking", "frame-support", + "frame-system", "handlebars", + "hash-db", + "hex", + "itertools", + "kvdb", "linked-hash-map", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "memory-db", + "parity-scale-codec 3.1.2", + "rand 0.8.5", + "sc-block-builder", "sc-cli", + "sc-client-api", "sc-client-db", "sc-executor", "sc-service", "serde", + "serde_json", + "serde_nanos", + "sp-api", + "sp-blockchain", "sp-core", + "sp-database", "sp-externalities", + "sp-inherents", "sp-keystore", "sp-runtime", "sp-state-machine", - "structopt", + "sp-std", + "sp-storage", + "sp-trie", + "thousands", +] + +[[package]] +name = "frame-election-provider-solution-type" +version = "4.0.0-dev" +dependencies = [ + "frame-election-provider-support", + "frame-support", + "parity-scale-codec 3.1.2", + "proc-macro-crate 1.1.3", + "proc-macro2", + "quote", + "scale-info", + "sp-arithmetic", + "syn", + "trybuild", ] [[package]] name = "frame-election-provider-support" version = "4.0.0-dev" dependencies = [ + "frame-election-provider-solution-type", "frame-support", "frame-system", - "parity-scale-codec", + "parity-scale-codec 3.1.2", + "rand 0.7.3", "scale-info", "sp-arithmetic", "sp-core", @@ -2056,6 +2333,23 @@ dependencies = [ "sp-std", ] +[[package]] +name = "frame-election-solution-type-fuzzer" +version = "2.0.0-alpha.5" +dependencies = [ + "clap 3.1.8", + "frame-election-provider-solution-type", + "frame-election-provider-support", + "frame-support", + "honggfuzz", + "parity-scale-codec 3.1.2", + "rand 0.8.5", + "scale-info", + "sp-arithmetic", + "sp-npos-elections", + "sp-runtime", +] + [[package]] name = "frame-executive" version = "4.0.0-dev" @@ -2065,7 +2359,7 @@ dependencies = [ "hex-literal", "pallet-balances", "pallet-transaction-payment", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-inherents", @@ -2078,12 +2372,12 @@ dependencies = [ [[package]] name = "frame-metadata" -version = "14.2.0" +version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ed5e5c346de62ca5c184b4325a6600d1eaca210666e4606fe4e449574978d0" +checksum = "df6bb8542ef006ef0de09a5c4420787d79823c0ed7924225822362fd2bf2ff2d" dependencies = [ "cfg-if 1.0.0", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "serde", ] @@ -2098,17 +2392,18 @@ dependencies = [ "frame-support-procedural", "frame-system", "impl-trait-for-tuples", - "log 0.4.14", + "log 0.4.16", "once_cell", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "parity-util-mem", - "paste 1.0.6", - "pretty_assertions 0.6.1", + "paste 1.0.7", + "pretty_assertions", "scale-info", "serde", "smallvec 1.8.0", "sp-arithmetic", "sp-core", + "sp-core-hashing-proc-macro", "sp-inherents", "sp-io", "sp-runtime", @@ -2116,6 +2411,7 @@ dependencies = [ "sp-state-machine", "sp-std", "sp-tracing", + "tt-call", ] [[package]] @@ -2156,8 +2452,8 @@ dependencies = [ "frame-support", "frame-support-test-pallet", "frame-system", - "parity-scale-codec", - "pretty_assertions 0.6.1", + "parity-scale-codec 3.1.2", + "pretty_assertions", "rustversion", "scale-info", "serde", @@ -2171,13 +2467,26 @@ dependencies = [ "trybuild", ] +[[package]] +name = "frame-support-test-compile-pass" +version = "4.0.0-dev" +dependencies = [ + "frame-support", + "frame-system", + "parity-scale-codec 3.1.2", + "scale-info", + "sp-core", + "sp-runtime", + "sp-version", +] + [[package]] name = "frame-support-test-pallet" version = "4.0.0-dev" dependencies = [ "frame-support", "frame-system", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", ] @@ -2187,8 +2496,8 @@ version = "4.0.0-dev" dependencies = [ "criterion", "frame-support", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "scale-info", "serde", "sp-core", @@ -2207,7 +2516,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -2219,7 +2528,7 @@ dependencies = [ name = "frame-system-rpc-runtime-api" version = "4.0.0-dev" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sp-api", ] @@ -2289,6 +2598,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.1.31" @@ -2377,8 +2692,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a1387e07917c711fb4ee4f48ea0adb04a3c9739e53ef85bf43ae1edc2937a8b" dependencies = [ "futures-io", - "rustls", - "webpki", + "rustls 0.19.1", + "webpki 0.21.4", ] [[package]] @@ -2393,12 +2708,6 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" -[[package]] -name = "futures-timer" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1de7508b218029b0f01662ed8f61b1c964b3ae99d6f25462d0f55a595109df6" - [[package]] name = "futures-timer" version = "3.0.2" @@ -2435,7 +2744,7 @@ dependencies = [ [[package]] name = "generate-bags" -version = "3.0.0" +version = "4.0.0-dev" dependencies = [ "chrono", "frame-election-provider-support", @@ -2445,7 +2754,6 @@ dependencies = [ "num-format", "pallet-staking", "sp-io", - "structopt", ] [[package]] @@ -2482,15 +2790,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if 1.0.0", - "js-sys", "libc", "wasi 0.10.0+wasi-snapshot-preview1", - "wasm-bindgen", ] [[package]] @@ -2505,9 +2811,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.22.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" +checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" dependencies = [ "fallible-iterator", "indexmap", @@ -2516,21 +2822,15 @@ dependencies = [ [[package]] name = "gimli" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" +checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" dependencies = [ "fallible-iterator", "indexmap", "stable_deref_trait", ] -[[package]] -name = "gimli" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" - [[package]] name = "git2" version = "0.13.25" @@ -2540,7 +2840,7 @@ dependencies = [ "bitflags", "libc", "libgit2-sys", - "log 0.4.14", + "log 0.4.16", "url 2.2.2", ] @@ -2559,7 +2859,7 @@ dependencies = [ "aho-corasick", "bstr", "fnv", - "log 0.4.14", + "log 0.4.16", "regex", ] @@ -2575,6 +2875,36 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "group" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5ac374b108929de78460075f3dc439fa66df9d8fc77e8f12caa5165fcf0c89" +dependencies = [ + "ff", + "rand_core 0.6.3", + "subtle", +] + +[[package]] +name = "h2" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +dependencies = [ + "bytes 1.1.0", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util 0.7.1", + "tracing", +] + [[package]] name = "half" version = "1.8.2" @@ -2583,11 +2913,11 @@ checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" [[package]] name = "handlebars" -version = "3.5.5" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4498fc115fa7d34de968184e473529abb40eeb6be8bc5f7faba3d08c316cb3e3" +checksum = "99d6a30320f094710245150395bc763ad23128d6a1ebbad7594dc4164b62c56b" dependencies = [ - "log 0.4.14", + "log 0.4.16", "pest", "pest_derive", "quick-error 2.0.1", @@ -2625,6 +2955,15 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c21d40587b92fa6a6c6e3c1bdbf87d75511db5672f9c93175574b3a00df1758" +dependencies = [ + "ahash", +] + [[package]] name = "heck" version = "0.3.3" @@ -2667,16 +3006,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" -[[package]] -name = "hmac" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" -dependencies = [ - "crypto-mac 0.7.0", - "digest 0.8.1", -] - [[package]] name = "hmac" version = "0.8.1" @@ -2699,24 +3028,13 @@ dependencies = [ [[package]] name = "hmac-drbg" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e570451493f10f6581b48cdd530413b63ea9e780f544bfd3bdcaa0d89d1a7b" +checksum = "17ea0a1394df5b6574da6e0c1ade9e78868c9fb0a4e5ef4428e32da4676b85b1" dependencies = [ - "digest 0.8.1", - "generic-array 0.12.4", - "hmac 0.7.1", -] - -[[package]] -name = "hmac-drbg" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ea0a1394df5b6574da6e0c1ade9e78868c9fb0a4e5ef4428e32da4676b85b1" -dependencies = [ - "digest 0.9.0", - "generic-array 0.14.5", - "hmac 0.8.1", + "digest 0.9.0", + "generic-array 0.14.5", + "hmac 0.8.1", ] [[package]] @@ -2802,14 +3120,15 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.17" +version = "0.14.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043f0e083e9901b6cc658a77d1eb86f4fc650bbb977a4337dd63192826aa85dd" +checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" dependencies = [ "bytes 1.1.0", "futures-channel", "futures-core", "futures-util", + "h2", "http", "http-body", "httparse", @@ -2831,13 +3150,13 @@ checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" dependencies = [ "ct-logs", "futures-util", - "hyper 0.14.17", - "log 0.4.14", - "rustls", - "rustls-native-certs", + "hyper 0.14.18", + "log 0.4.16", + "rustls 0.19.1", + "rustls-native-certs 0.5.0", "tokio", - "tokio-rustls", - "webpki", + "tokio-rustls 0.22.0", + "webpki 0.21.4", ] [[package]] @@ -2847,7 +3166,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.1.0", - "hyper 0.14.17", + "hyper 0.14.18", "native-tls", "tokio", "tokio-native-tls", @@ -2892,6 +3211,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "if-addrs" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc0fa01ffc752e9dbc72818cdb072cd028b86be5e09dd04c5a643704fe101a9" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "if-addrs-sys" version = "0.3.2" @@ -2911,20 +3240,20 @@ dependencies = [ "async-io", "futures 0.3.21", "futures-lite", - "if-addrs", + "if-addrs 0.6.7", "ipnet", "libc", - "log 0.4.14", + "log 0.4.16", "winapi 0.3.9", ] [[package]] name = "impl-codec" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "161ebdfec3c8e3b52bf61c4f3550a1eea4f9579d10dc1b936f3171ebdcd6c443" +checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", ] [[package]] @@ -2949,12 +3278,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg 1.1.0", - "hashbrown", + "hashbrown 0.11.2", "serde", ] @@ -2977,13 +3306,12 @@ dependencies = [ ] [[package]] -name = "intervalier" -version = "0.4.0" +name = "io-lifetimes" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64fa110ec7b8f493f416eed552740d10e7030ad5f63b2308f82c9608ec2df275" +checksum = "f6ef6787e7f0faedc040f95716bdd0e62bcfcf4ba93da053b62dea2691c13864" dependencies = [ - "futures 0.3.21", - "futures-timer 2.0.2", + "winapi 0.3.9", ] [[package]] @@ -3008,15 +3336,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" dependencies = [ "socket2 0.3.19", - "widestring", + "widestring 0.4.3", "winapi 0.3.9", - "winreg", + "winreg 0.6.2", +] + +[[package]] +name = "ipconfig" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723519edce41262b05d4143ceb95050e4c614f483e78e9fd9e39a8275a84ad98" +dependencies = [ + "socket2 0.4.4", + "widestring 0.5.1", + "winapi 0.3.9", + "winreg 0.7.0", ] [[package]] name = "ipfs" version = "0.2.1" -source = "git+https://github.com/rs-ipfs/rust-ipfs#3eff4e15342d430870bfa4fad7c2604c865266b2" +source = "git+https://github.com/rs-ipfs/rust-ipfs?branch=master#275b7615b8373b368d74e075323ea9cd9b08a802" dependencies = [ "anyhow", "async-stream", @@ -3031,39 +3371,39 @@ dependencies = [ "hash_hasher", "ipfs-bitswap", "ipfs-unixfs", - "libp2p", + "libp2p 0.43.0", "multibase 0.9.1", "multihash 0.11.4", "once_cell", - "prost", - "prost-build", + "prost 0.9.0", + "prost-build 0.8.0", "serde", "serde_json", "sled", "thiserror", "tokio", "tokio-stream", - "tokio-util", + "tokio-util 0.6.9", "tracing", "tracing-futures", - "trust-dns-resolver", + "trust-dns-resolver 0.20.4", "void", ] [[package]] name = "ipfs-bitswap" version = "0.1.0" -source = "git+https://github.com/rs-ipfs/rust-ipfs#3eff4e15342d430870bfa4fad7c2604c865266b2" +source = "git+https://github.com/rs-ipfs/rust-ipfs?branch=master#275b7615b8373b368d74e075323ea9cd9b08a802" dependencies = [ "cid 0.5.1", "fnv", "futures 0.3.21", "hash_hasher", - "libp2p-core", - "libp2p-swarm", + "libp2p-core 0.32.1", + "libp2p-swarm 0.34.0", "multihash 0.11.4", - "prost", - "prost-build", + "prost 0.9.0", + "prost-build 0.8.0", "thiserror", "tokio", "tracing", @@ -3073,7 +3413,7 @@ dependencies = [ [[package]] name = "ipfs-unixfs" version = "0.2.0" -source = "git+https://github.com/rs-ipfs/rust-ipfs#3eff4e15342d430870bfa4fad7c2604c865266b2" +source = "git+https://github.com/rs-ipfs/rust-ipfs?branch=master#275b7615b8373b368d74e075323ea9cd9b08a802" dependencies = [ "cid 0.5.1", "either", @@ -3121,9 +3461,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" dependencies = [ "wasm-bindgen", ] @@ -3136,11 +3476,11 @@ checksum = "d2b99d4207e2a04fb4581746903c2bb7eb376f88de9c699d0f3e10feeac0cd3a" dependencies = [ "derive_more", "futures 0.3.21", - "hyper 0.14.17", + "hyper 0.14.18", "hyper-tls", "jsonrpc-core", "jsonrpc-pubsub", - "log 0.4.14", + "log 0.4.16", "serde", "serde_json", "tokio", @@ -3157,7 +3497,7 @@ dependencies = [ "futures 0.3.21", "futures-executor", "futures-util", - "log 0.4.14", + "log 0.4.16", "serde", "serde_derive", "serde_json", @@ -3192,10 +3532,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1dea6e07251d9ce6a552abfb5d7ad6bc290a4596c8dcc3d795fae2bbdc1f3ff" dependencies = [ "futures 0.3.21", - "hyper 0.14.17", + "hyper 0.14.18", "jsonrpc-core", "jsonrpc-server-utils", - "log 0.4.14", + "log 0.4.16", "net2", "parking_lot 0.11.2", "unicase 2.6.0", @@ -3210,7 +3550,7 @@ dependencies = [ "futures 0.3.21", "jsonrpc-core", "jsonrpc-server-utils", - "log 0.4.14", + "log 0.4.16", "parity-tokio-ipc", "parking_lot 0.11.2", "tower-service", @@ -3225,7 +3565,7 @@ dependencies = [ "futures 0.3.21", "jsonrpc-core", "lazy_static", - "log 0.4.14", + "log 0.4.16", "parking_lot 0.11.2", "rand 0.7.3", "serde", @@ -3242,10 +3582,10 @@ dependencies = [ "globset", "jsonrpc-core", "lazy_static", - "log 0.4.14", + "log 0.4.16", "tokio", "tokio-stream", - "tokio-util", + "tokio-util 0.6.9", "unicase 2.6.0", ] @@ -3258,20 +3598,74 @@ dependencies = [ "futures 0.3.21", "jsonrpc-core", "jsonrpc-server-utils", - "log 0.4.14", + "log 0.4.16", "parity-ws", "parking_lot 0.11.2", "slab", ] +[[package]] +name = "jsonrpsee" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91dc760c341fa81173f9a434931aaf32baad5552b0230cc6c93e8fb7eaad4c19" +dependencies = [ + "jsonrpsee-core", + "jsonrpsee-proc-macros", + "jsonrpsee-types", + "jsonrpsee-ws-client", +] + +[[package]] +name = "jsonrpsee-client-transport" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765f7a36d5087f74e3b3b47805c2188fef8eb54afcb587b078d9f8ebfe9c7220" +dependencies = [ + "futures 0.3.21", + "http", + "jsonrpsee-core", + "jsonrpsee-types", + "pin-project 1.0.10", + "rustls-native-certs 0.6.1", + "soketto 0.7.1", + "thiserror", + "tokio", + "tokio-rustls 0.23.3", + "tokio-util 0.7.1", + "tracing", + "webpki-roots 0.22.3", +] + +[[package]] +name = "jsonrpsee-core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82ef77ecd20c2254d54f5da8c0738eacca61e6b6511268a8f2753e3148c6c706" +dependencies = [ + "anyhow", + "arrayvec 0.7.2", + "async-trait", + "beef", + "futures-channel", + "futures-util", + "hyper 0.14.18", + "jsonrpsee-types", + "rustc-hash", + "serde", + "serde_json", + "soketto 0.7.1", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "jsonrpsee-proc-macros" -version = "0.3.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8edb341d35279b59c79d7fe9e060a51aec29d45af99cc7c72ea7caa350fa71a4" +checksum = "b7291c72805bc7d413b457e50d8ef3e87aa554da65ecbbc278abb7dfc283e7f0" dependencies = [ - "Inflector", - "bae", "proc-macro-crate 1.1.3", "proc-macro2", "quote", @@ -3280,44 +3674,39 @@ dependencies = [ [[package]] name = "jsonrpsee-types" -version = "0.3.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc738fd55b676ada3271ef7c383a14a0867a2a88b0fa941311bf5fc0a29d498" +checksum = "38b6aa52f322cbf20c762407629b8300f39bcc0cf0619840d9252a2f65fd2dd9" dependencies = [ - "async-trait", + "anyhow", "beef", - "futures-channel", - "futures-util", - "hyper 0.14.17", - "log 0.4.14", "serde", "serde_json", - "soketto 0.6.0", "thiserror", + "tracing", ] [[package]] name = "jsonrpsee-ws-client" -version = "0.3.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9841352dbecf4c2ed5dc71698df9f1660262ae4e0b610e968602529bdbcf7b30" +checksum = "dd66d18bab78d956df24dd0d2e41e4c00afbb818fda94a98264bdd12ce8506ac" dependencies = [ - "async-trait", - "fnv", - "futures 0.3.21", + "jsonrpsee-client-transport", + "jsonrpsee-core", "jsonrpsee-types", - "log 0.4.14", - "pin-project 1.0.10", - "rustls", - "rustls-native-certs", - "serde", - "serde_json", - "soketto 0.6.0", - "thiserror", - "tokio", - "tokio-rustls", - "tokio-util", - "url 2.2.2", +] + +[[package]] +name = "k256" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19c3a5e0a0b8450278feda242592512e09f61c72e018b8cd5c859482802daf2d" +dependencies = [ + "cfg-if 1.0.0", + "ecdsa", + "elliptic-curve", + "sec1", ] [[package]] @@ -3353,14 +3742,14 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" dependencies = [ - "log 0.4.14", + "log 0.4.16", ] [[package]] name = "kvdb" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a3f58dc069ec0e205a27f5b45920722a46faed802a0541538241af6228f512" +checksum = "a301d8ecb7989d4a6e2c57a49baca77d353bdbf879909debe3f375fe25d61f86" dependencies = [ "parity-util-mem", "smallvec 1.8.0", @@ -3368,28 +3757,28 @@ dependencies = [ [[package]] name = "kvdb-memorydb" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3b6b85fc643f5acd0bffb2cc8a6d150209379267af0d41db72170021841f9f5" +checksum = "ece7e668abd21387aeb6628130a6f4c802787f014fa46bc83221448322250357" dependencies = [ "kvdb", "parity-util-mem", - "parking_lot 0.11.2", + "parking_lot 0.12.0", ] [[package]] name = "kvdb-rocksdb" -version = "0.14.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1b6ea8f2536f504b645ad78419c8246550e19d2c3419a167080ce08edee35a" +checksum = "ca7fbdfd71cd663dceb0faf3367a99f8cf724514933e9867cec4995b6027cbc1" dependencies = [ "fs-swap", "kvdb", - "log 0.4.14", + "log 0.4.16", "num_cpus", "owning_ref", "parity-util-mem", - "parking_lot 0.11.2", + "parking_lot 0.12.0", "regex", "rocksdb", "smallvec 1.8.0", @@ -3421,9 +3810,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.119" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" +checksum = "ec647867e2bf0772e28c8bcde4f0d19a9216916e890543b5a03ed8ef27b8f259" [[package]] name = "libgit2-sys" @@ -3447,16 +3836,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "libloading" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883" -dependencies = [ - "cfg-if 1.0.0", - "winapi 0.3.9", -] - [[package]] name = "libloading" version = "0.7.3" @@ -3475,48 +3854,83 @@ checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db" [[package]] name = "libp2p" -version = "0.39.1" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9004c06878ef8f3b4b4067e69a140d87ed20bf777287f82223e49713b36ee433" +checksum = "3bec54343492ba5940a6c555e512c6721139835d28c59bc22febece72dfd0d9d" dependencies = [ "atomic", "bytes 1.1.0", "futures 0.3.21", "lazy_static", - "libp2p-core", + "libp2p-core 0.30.2", "libp2p-deflate", - "libp2p-dns", - "libp2p-floodsub", + "libp2p-dns 0.30.0", + "libp2p-floodsub 0.31.0", "libp2p-gossipsub", - "libp2p-identify", - "libp2p-kad", + "libp2p-identify 0.31.0", + "libp2p-kad 0.32.0", "libp2p-mdns", - "libp2p-mplex", - "libp2p-noise", - "libp2p-ping", + "libp2p-metrics 0.1.0", + "libp2p-mplex 0.30.0", + "libp2p-noise 0.33.0", + "libp2p-ping 0.31.0", "libp2p-plaintext", "libp2p-pnet", "libp2p-relay", + "libp2p-rendezvous", "libp2p-request-response", - "libp2p-swarm", - "libp2p-swarm-derive", - "libp2p-tcp", + "libp2p-swarm 0.31.0", + "libp2p-swarm-derive 0.25.0", + "libp2p-tcp 0.30.0", "libp2p-uds", "libp2p-wasm-ext", "libp2p-websocket", - "libp2p-yamux", - "multiaddr", + "libp2p-yamux 0.34.0", + "multiaddr 0.13.0", "parking_lot 0.11.2", "pin-project 1.0.10", "smallvec 1.8.0", "wasm-timer", ] +[[package]] +name = "libp2p" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e8570e25fa03d4385405dbeaf540ba00e3ee50942f03d84e1a8928a029f35f9" +dependencies = [ + "atomic", + "bytes 1.1.0", + "futures 0.3.21", + "futures-timer", + "getrandom 0.2.6", + "instant", + "lazy_static", + "libp2p-core 0.32.1", + "libp2p-dns 0.32.1", + "libp2p-floodsub 0.34.0", + "libp2p-identify 0.34.0", + "libp2p-kad 0.35.0", + "libp2p-metrics 0.4.0", + "libp2p-mplex 0.32.0", + "libp2p-noise 0.35.0", + "libp2p-ping 0.34.0", + "libp2p-swarm 0.34.0", + "libp2p-swarm-derive 0.27.1", + "libp2p-tcp 0.32.0", + "libp2p-yamux 0.36.0", + "multiaddr 0.14.0", + "parking_lot 0.12.0", + "pin-project 1.0.10", + "rand 0.7.3", + "smallvec 1.8.0", +] + [[package]] name = "libp2p-core" -version = "0.29.0" +version = "0.30.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af9b4abdeaa420593a297c8592f63fad4234f4b88dc9343b8fd8e736c35faa59" +checksum = "86aad7d54df283db817becded03e611137698a6509d4237a96881976a162340c" dependencies = [ "asn1_der", "bs58", @@ -3524,18 +3938,19 @@ dependencies = [ "either", "fnv", "futures 0.3.21", - "futures-timer 3.0.2", + "futures-timer", + "instant", "lazy_static", - "libsecp256k1 0.5.0", - "log 0.4.14", - "multiaddr", + "libsecp256k1", + "log 0.4.16", + "multiaddr 0.13.0", "multihash 0.14.0", - "multistream-select", + "multistream-select 0.10.4", "parking_lot 0.11.2", "pin-project 1.0.10", - "prost", - "prost-build", - "rand 0.7.3", + "prost 0.9.0", + "prost-build 0.9.0", + "rand 0.8.5", "ring", "rw-stream-sink", "sha2 0.9.9", @@ -3546,54 +3961,119 @@ dependencies = [ "zeroize", ] +[[package]] +name = "libp2p-core" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b02602099fb75cb2d16f9ea860a320d6eb82ce41e95ab680912c454805cd5" +dependencies = [ + "asn1_der", + "bs58", + "ed25519-dalek", + "either", + "fnv", + "futures 0.3.21", + "futures-timer", + "instant", + "lazy_static", + "log 0.4.16", + "multiaddr 0.14.0", + "multihash 0.16.2", + "multistream-select 0.11.0", + "parking_lot 0.12.0", + "pin-project 1.0.10", + "prost 0.9.0", + "prost-build 0.9.0", + "rand 0.8.5", + "ring", + "rw-stream-sink", + "sha2 0.10.2", + "smallvec 1.8.0", + "thiserror", + "unsigned-varint 0.7.1", + "void", + "zeroize", +] + [[package]] name = "libp2p-deflate" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66097fccc0b7f8579f90a03ea76ba6196332ea049fd07fd969490a06819dcdc8" +checksum = "51a800adb195f33de63f4b17b63fe64cfc23bf2c6a0d3d0d5321328664e65197" dependencies = [ "flate2", "futures 0.3.21", - "libp2p-core", + "libp2p-core 0.30.2", ] [[package]] name = "libp2p-dns" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ff08b3196b85a17f202d80589e93b1660a574af67275706657fdc762e42c32" +checksum = "bb8f89d15cb6e3c5bc22afff7513b11bab7856f2872d3cfba86f7f63a06bc498" dependencies = [ "async-std-resolver", "futures 0.3.21", - "libp2p-core", - "log 0.4.14", + "libp2p-core 0.30.2", + "log 0.4.16", "smallvec 1.8.0", - "trust-dns-resolver", + "trust-dns-resolver 0.20.4", +] + +[[package]] +name = "libp2p-dns" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "066e33e854e10b5c93fc650458bf2179c7e0d143db260b0963e44a94859817f1" +dependencies = [ + "futures 0.3.21", + "libp2p-core 0.32.1", + "log 0.4.16", + "smallvec 1.8.0", + "trust-dns-resolver 0.21.2", ] [[package]] name = "libp2p-floodsub" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "404eca8720967179dac7a5b4275eb91f904a53859c69ca8d018560ad6beb214f" +checksum = "aab3d7210901ea51b7bae2b581aa34521797af8c4ec738c980bda4a06434067f" dependencies = [ "cuckoofilter", "fnv", "futures 0.3.21", - "libp2p-core", - "libp2p-swarm", - "log 0.4.14", - "prost", - "prost-build", + "libp2p-core 0.30.2", + "libp2p-swarm 0.31.0", + "log 0.4.16", + "prost 0.9.0", + "prost-build 0.9.0", + "rand 0.7.3", + "smallvec 1.8.0", +] + +[[package]] +name = "libp2p-floodsub" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0b7d6c3fa2ead77a5bbeff580bd7507efcc9d7fa9d0caf873795b097d385c0" +dependencies = [ + "cuckoofilter", + "fnv", + "futures 0.3.21", + "libp2p-core 0.32.1", + "libp2p-swarm 0.34.0", + "log 0.4.16", + "prost 0.9.0", + "prost-build 0.9.0", "rand 0.7.3", "smallvec 1.8.0", ] [[package]] name = "libp2p-gossipsub" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1cc48709bcbc3a3321f08a73560b4bbb4166a7d56f6fdb615bc775f4f91058e" +checksum = "dfeead619eb5dac46e65acc78c535a60aaec803d1428cca6407c3a4fc74d698d" dependencies = [ "asynchronous-codec 0.6.0", "base64 0.13.0", @@ -3602,11 +4082,11 @@ dependencies = [ "fnv", "futures 0.3.21", "hex_fmt", - "libp2p-core", - "libp2p-swarm", - "log 0.4.14", - "prost", - "prost-build", + "libp2p-core 0.30.2", + "libp2p-swarm 0.31.0", + "log 0.4.16", + "prost 0.9.0", + "prost-build 0.9.0", "rand 0.7.3", "regex", "sha2 0.9.9", @@ -3617,25 +4097,43 @@ dependencies = [ [[package]] name = "libp2p-identify" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7b61f6cf07664fb97016c318c4d4512b3dd4cc07238607f3f0163245f99008e" +checksum = "cca1275574183f288ff8b72d535d5ffa5ea9292ef7829af8b47dcb197c7b0dcd" dependencies = [ "futures 0.3.21", - "libp2p-core", - "libp2p-swarm", - "log 0.4.14", - "prost", - "prost-build", + "libp2p-core 0.30.2", + "libp2p-swarm 0.31.0", + "log 0.4.16", + "lru 0.6.6", + "prost 0.9.0", + "prost-build 0.9.0", "smallvec 1.8.0", "wasm-timer", ] +[[package]] +name = "libp2p-identify" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f219b4d4660fe3a04bf5fe6b5970902b7c1918e25b2536be8c70efc480f88f8" +dependencies = [ + "futures 0.3.21", + "futures-timer", + "libp2p-core 0.32.1", + "libp2p-swarm 0.34.0", + "log 0.4.16", + "lru 0.7.5", + "prost 0.9.0", + "prost-build 0.9.0", + "smallvec 1.8.0", +] + [[package]] name = "libp2p-kad" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50ed78489c87924235665a0ab345b298ee34dff0f7ad62c0ba6608b2144fb75e" +checksum = "a2297dc0ca285f3a09d1368bde02449e539b46f94d32d53233f53f6625bcd3ba" dependencies = [ "arrayvec 0.5.2", "asynchronous-codec 0.6.0", @@ -3643,11 +4141,11 @@ dependencies = [ "either", "fnv", "futures 0.3.21", - "libp2p-core", - "libp2p-swarm", - "log 0.4.14", - "prost", - "prost-build", + "libp2p-core 0.30.2", + "libp2p-swarm 0.31.0", + "log 0.4.16", + "prost 0.9.0", + "prost-build 0.9.0", "rand 0.7.3", "sha2 0.9.9", "smallvec 1.8.0", @@ -3657,11 +4155,39 @@ dependencies = [ "wasm-timer", ] +[[package]] +name = "libp2p-kad" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aead5ee2322a7b825c7633065370909c8383046f955cda5b56797e6904db7a72" +dependencies = [ + "arrayvec 0.5.2", + "asynchronous-codec 0.6.0", + "bytes 1.1.0", + "either", + "fnv", + "futures 0.3.21", + "futures-timer", + "instant", + "libp2p-core 0.32.1", + "libp2p-swarm 0.34.0", + "log 0.4.16", + "prost 0.9.0", + "prost-build 0.9.0", + "rand 0.7.3", + "sha2 0.10.2", + "smallvec 1.8.0", + "thiserror", + "uint", + "unsigned-varint 0.7.1", + "void", +] + [[package]] name = "libp2p-mdns" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a29e6cbc2a24b8471b6567e580a0e8e7b70a6d0f0ea2be0844d1e842d7d4fa33" +checksum = "14c864b64bdc8a84ff3910a0df88e6535f256191a450870f1e7e10cbf8e64d45" dependencies = [ "async-io", "data-encoding", @@ -3669,26 +4195,54 @@ dependencies = [ "futures 0.3.21", "if-watch", "lazy_static", - "libp2p-core", - "libp2p-swarm", - "log 0.4.14", + "libp2p-core 0.30.2", + "libp2p-swarm 0.31.0", + "log 0.4.16", "rand 0.8.5", "smallvec 1.8.0", "socket2 0.4.4", "void", ] +[[package]] +name = "libp2p-metrics" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4af432fcdd2f8ba4579b846489f8f0812cfd738ced2c0af39df9b1c48bbb6ab2" +dependencies = [ + "libp2p-core 0.30.2", + "libp2p-identify 0.31.0", + "libp2p-kad 0.32.0", + "libp2p-ping 0.31.0", + "libp2p-swarm 0.31.0", + "open-metrics-client", +] + +[[package]] +name = "libp2p-metrics" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29e4e5e4c5aa567fe1ee3133afe088dc2d2fd104e20c5c2c5c2649f75129677" +dependencies = [ + "libp2p-core 0.32.1", + "libp2p-identify 0.34.0", + "libp2p-kad 0.35.0", + "libp2p-ping 0.34.0", + "libp2p-swarm 0.34.0", + "prometheus-client", +] + [[package]] name = "libp2p-mplex" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d9ea526c68df4425f580024e67a9d3ffd49f2c33de5154b1f5019816f7a99" +checksum = "7f2cd64ef597f40e14bfce0497f50ecb63dd6d201c61796daeb4227078834fbf" dependencies = [ "asynchronous-codec 0.6.0", "bytes 1.1.0", "futures 0.3.21", - "libp2p-core", - "log 0.4.14", + "libp2p-core 0.30.2", + "log 0.4.16", "nohash-hasher", "parking_lot 0.11.2", "rand 0.7.3", @@ -3697,22 +4251,62 @@ dependencies = [ ] [[package]] -name = "libp2p-noise" +name = "libp2p-mplex" version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1db7212f342b6ba7c981cc40e31f76e9e56cb48e65fa4c142ecaca5839523e" +checksum = "442eb0c9fff0bf22a34f015724b4143ce01877e079ed0963c722d94c07c72160" +dependencies = [ + "asynchronous-codec 0.6.0", + "bytes 1.1.0", + "futures 0.3.21", + "libp2p-core 0.32.1", + "log 0.4.16", + "nohash-hasher", + "parking_lot 0.12.0", + "rand 0.7.3", + "smallvec 1.8.0", + "unsigned-varint 0.7.1", +] + +[[package]] +name = "libp2p-noise" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8772c7a99088221bb7ca9c5c0574bf55046a7ab4c319f3619b275f28c8fb87a" dependencies = [ "bytes 1.1.0", "curve25519-dalek 3.2.0", "futures 0.3.21", "lazy_static", - "libp2p-core", - "log 0.4.14", - "prost", - "prost-build", + "libp2p-core 0.30.2", + "log 0.4.16", + "prost 0.9.0", + "prost-build 0.9.0", "rand 0.8.5", "sha2 0.9.9", - "snow", + "snow 0.8.0", + "static_assertions", + "x25519-dalek", + "zeroize", +] + +[[package]] +name = "libp2p-noise" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd7e0c94051cda67123be68cf6b65211ba3dde7277be9068412de3e7ffd63ef" +dependencies = [ + "bytes 1.1.0", + "curve25519-dalek 3.2.0", + "futures 0.3.21", + "lazy_static", + "libp2p-core 0.32.1", + "log 0.4.16", + "prost 0.9.0", + "prost-build 0.9.0", + "rand 0.8.5", + "sha2 0.10.2", + "snow 0.9.0", "static_assertions", "x25519-dalek", "zeroize", @@ -3720,66 +4314,82 @@ dependencies = [ [[package]] name = "libp2p-ping" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2482cfd9eb0b7a0baaf3e7b329dc4f2785181a161b1a47b7192f8d758f54a439" +checksum = "80ef7b0ec5cf06530d9eb6cf59ae49d46a2c45663bde31c25a12f682664adbcf" dependencies = [ "futures 0.3.21", - "libp2p-core", - "libp2p-swarm", - "log 0.4.14", + "libp2p-core 0.30.2", + "libp2p-swarm 0.31.0", + "log 0.4.16", "rand 0.7.3", "void", "wasm-timer", ] +[[package]] +name = "libp2p-ping" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab44a12d372d6abdd326c468c1d5b002be06fbd923c5a799d6a9d3b36646ca3" +dependencies = [ + "futures 0.3.21", + "futures-timer", + "instant", + "libp2p-core 0.32.1", + "libp2p-swarm 0.34.0", + "log 0.4.16", + "rand 0.7.3", + "void", +] + [[package]] name = "libp2p-plaintext" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13b4783e5423870b9a5c199f65a7a3bc66d86ab56b2b9beebf3c338d889cf8e4" +checksum = "5fba1a6ff33e4a274c89a3b1d78b9f34f32af13265cc5c46c16938262d4e945a" dependencies = [ "asynchronous-codec 0.6.0", "bytes 1.1.0", "futures 0.3.21", - "libp2p-core", - "log 0.4.14", - "prost", - "prost-build", + "libp2p-core 0.30.2", + "log 0.4.16", + "prost 0.9.0", + "prost-build 0.9.0", "unsigned-varint 0.7.1", "void", ] [[package]] name = "libp2p-pnet" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cb4dd4b917e5b40ddefe49b96b07adcd8d342e0317011d175b7b2bb1dcc974" +checksum = "0f1a458bbda880107b5b36fcb9b5a1ef0c329685da0e203ed692a8ebe64cc92c" dependencies = [ "futures 0.3.21", - "log 0.4.14", + "log 0.4.16", "pin-project 1.0.10", "rand 0.7.3", "salsa20", - "sha3", + "sha3 0.9.1", ] [[package]] name = "libp2p-relay" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0133f6cfd81cdc16e716de2982e012c62e6b9d4f12e41967b3ee361051c622aa" +checksum = "2852b61c90fa8ce3c8fcc2aba76e6cefc20d648f9df29157d6b3a916278ef3e3" dependencies = [ "asynchronous-codec 0.6.0", "bytes 1.1.0", "futures 0.3.21", - "futures-timer 3.0.2", - "libp2p-core", - "libp2p-swarm", - "log 0.4.14", + "futures-timer", + "libp2p-core 0.30.2", + "libp2p-swarm 0.31.0", + "log 0.4.16", "pin-project 1.0.10", - "prost", - "prost-build", + "prost 0.9.0", + "prost-build 0.9.0", "rand 0.7.3", "smallvec 1.8.0", "unsigned-varint 0.7.1", @@ -3787,20 +4397,41 @@ dependencies = [ "wasm-timer", ] +[[package]] +name = "libp2p-rendezvous" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14a6d2b9e7677eff61dc3d2854876aaf3976d84a01ef6664b610c77a0c9407c5" +dependencies = [ + "asynchronous-codec 0.6.0", + "bimap", + "futures 0.3.21", + "libp2p-core 0.30.2", + "libp2p-swarm 0.31.0", + "log 0.4.16", + "prost 0.9.0", + "prost-build 0.9.0", + "rand 0.8.5", + "sha2 0.9.9", + "thiserror", + "unsigned-varint 0.7.1", + "void", + "wasm-timer", +] + [[package]] name = "libp2p-request-response" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06cdae44b6821466123af93cbcdec7c9e6ba9534a8af9cdc296446d39416d241" +checksum = "a877a4ced6d46bf84677e1974e8cf61fb434af73b2e96fb48d6cb6223a4634d8" dependencies = [ "async-trait", "bytes 1.1.0", "futures 0.3.21", - "libp2p-core", - "libp2p-swarm", - "log 0.4.14", - "lru", - "minicbor", + "libp2p-core 0.30.2", + "libp2p-swarm 0.31.0", + "log 0.4.16", + "lru 0.7.5", "rand 0.7.3", "smallvec 1.8.0", "unsigned-varint 0.7.1", @@ -3809,25 +4440,55 @@ dependencies = [ [[package]] name = "libp2p-swarm" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7083861341e1555467863b4cd802bea1e8c4787c0f7b5110097d0f1f3248f9a9" +checksum = "3f5184a508f223bc100a12665517773fb8730e9f36fc09eefb670bf01b107ae9" dependencies = [ "either", "futures 0.3.21", - "libp2p-core", - "log 0.4.14", + "libp2p-core 0.30.2", + "log 0.4.16", "rand 0.7.3", "smallvec 1.8.0", "void", "wasm-timer", ] +[[package]] +name = "libp2p-swarm" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53ab2d4eb8ef2966b10fdf859245cdd231026df76d3c6ed2cf9e418a8f688ec9" +dependencies = [ + "either", + "fnv", + "futures 0.3.21", + "futures-timer", + "instant", + "libp2p-core 0.32.1", + "log 0.4.16", + "pin-project 1.0.10", + "rand 0.7.3", + "smallvec 1.8.0", + "thiserror", + "void", +] + [[package]] name = "libp2p-swarm-derive" -version = "0.24.0" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "072c290f727d39bdc4e9d6d1c847978693d25a673bd757813681e33e5f6c00c2" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "libp2p-swarm-derive" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab8cb308d4fc854869f5abb54fdab0833d2cf670d407c745849dc47e6e08d79c" +checksum = "daf2fe8c80b43561355f4d51875273b5b6dfbac37952e8f64b1270769305c9d7" dependencies = [ "quote", "syn", @@ -3835,44 +4496,59 @@ dependencies = [ [[package]] name = "libp2p-tcp" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79edd26b6b4bb5feee210dcda562dca186940dfecb0024b979c3f50824b3bf28" +checksum = "7399c5b6361ef525d41c11fcf51635724f832baf5819b30d3d873eabb4fbae4b" dependencies = [ "async-io", "futures 0.3.21", - "futures-timer 3.0.2", - "if-addrs", + "futures-timer", "if-watch", "ipnet", "libc", - "libp2p-core", - "log 0.4.14", + "libp2p-core 0.30.2", + "log 0.4.16", + "socket2 0.4.4", +] + +[[package]] +name = "libp2p-tcp" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "193447aa729c85aac2376828df76d171c1a589c9e6b58fcc7f9d9a020734122c" +dependencies = [ + "futures 0.3.21", + "futures-timer", + "if-addrs 0.7.0", + "ipnet", + "libc", + "libp2p-core 0.32.1", + "log 0.4.16", "socket2 0.4.4", "tokio", ] [[package]] name = "libp2p-uds" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280e793440dd4e9f273d714f4497325c72cddb0fe85a49f9a03c88f41dd20182" +checksum = "b8b7563e46218165dfd60f64b96f7ce84590d75f53ecbdc74a7dd01450dc5973" dependencies = [ "async-std", "futures 0.3.21", - "libp2p-core", - "log 0.4.14", + "libp2p-core 0.30.2", + "log 0.4.16", ] [[package]] name = "libp2p-wasm-ext" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f553b7140fad3d7a76f50497b0ea591e26737d9607428a75509fc191e4d1b1f6" +checksum = "1008a302b73c5020251f9708c653f5ed08368e530e247cc9cd2f109ff30042cf" dependencies = [ "futures 0.3.21", "js-sys", - "libp2p-core", + "libp2p-core 0.30.2", "parity-send-wrapper", "wasm-bindgen", "wasm-bindgen-futures", @@ -3880,99 +4556,61 @@ dependencies = [ [[package]] name = "libp2p-websocket" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf99dcbf5063e9d59087f61b1e85c686ceab2f5abedb472d32288065c0e5e27" +checksum = "22e12df82d1ed64969371a9e65ea92b91064658604cc2576c2757f18ead9a1cf" dependencies = [ "either", "futures 0.3.21", "futures-rustls", - "libp2p-core", - "log 0.4.14", + "libp2p-core 0.30.2", + "log 0.4.16", "quicksink", "rw-stream-sink", - "soketto 0.4.2", + "soketto 0.7.1", "url 2.2.2", - "webpki-roots", + "webpki-roots 0.21.1", ] [[package]] name = "libp2p-yamux" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214cc0dd9c37cbed27f0bb1eba0c41bbafdb93a8be5e9d6ae1e6b4b42cd044bf" +checksum = "4e7362abb8867d7187e7e93df17f460d554c997fc5c8ac57dc1259057f6889af" dependencies = [ "futures 0.3.21", - "libp2p-core", + "libp2p-core 0.30.2", "parking_lot 0.11.2", "thiserror", - "yamux", + "yamux 0.9.0", ] [[package]] -name = "librocksdb-sys" -version = "6.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c309a9d2470844aceb9a4a098cf5286154d20596868b75a6b36357d2bb9ca25d" -dependencies = [ - "bindgen", - "cc", - "glob", - "libc", -] - -[[package]] -name = "libsecp256k1" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc1e2c808481a63dc6da2074752fdd4336a3c8fcc68b83db6f1fd5224ae7962" -dependencies = [ - "arrayref", - "crunchy", - "digest 0.8.1", - "hmac-drbg 0.2.0", - "rand 0.7.3", - "sha2 0.8.2", - "subtle 2.4.1", - "typenum", -] - -[[package]] -name = "libsecp256k1" -version = "0.5.0" +name = "libp2p-yamux" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd1137239ab33b41aa9637a88a28249e5e70c40a42ccc92db7f12cc356c1fcd7" +checksum = "be902ebd89193cd020e89e89107726a38cfc0d16d18f613f4a37d046e92c7517" dependencies = [ - "arrayref", - "base64 0.12.3", - "digest 0.9.0", - "hmac-drbg 0.3.0", - "libsecp256k1-core 0.2.2", - "libsecp256k1-gen-ecmult 0.2.1", - "libsecp256k1-gen-genmult 0.2.1", - "rand 0.7.3", - "serde", - "sha2 0.9.9", - "typenum", + "futures 0.3.21", + "libp2p-core 0.32.1", + "parking_lot 0.12.0", + "thiserror", + "yamux 0.10.1", ] [[package]] -name = "libsecp256k1" -version = "0.6.0" +name = "librocksdb-sys" +version = "0.6.1+6.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d220bc1feda2ac231cb78c3d26f27676b8cf82c96971f7aeef3d0cf2797c73" +checksum = "81bc587013734dadb7cf23468e531aa120788b87243648be42e2d3a072186291" dependencies = [ - "arrayref", - "base64 0.12.3", - "digest 0.9.0", - "hmac-drbg 0.3.0", - "libsecp256k1-core 0.2.2", - "libsecp256k1-gen-ecmult 0.2.1", - "libsecp256k1-gen-genmult 0.2.1", - "rand 0.7.3", - "serde", - "sha2 0.9.9", - "typenum", + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "tikv-jemalloc-sys", ] [[package]] @@ -3984,23 +4622,14 @@ dependencies = [ "arrayref", "base64 0.13.0", "digest 0.9.0", - "libsecp256k1-core 0.3.0", - "libsecp256k1-gen-ecmult 0.3.0", - "libsecp256k1-gen-genmult 0.3.0", + "hmac-drbg", + "libsecp256k1-core", + "libsecp256k1-gen-ecmult", + "libsecp256k1-gen-genmult", "rand 0.8.5", "serde", "sha2 0.9.9", -] - -[[package]] -name = "libsecp256k1-core" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f6ab710cec28cef759c5f18671a27dae2a5f952cdaaee1d8e2908cb2478a80" -dependencies = [ - "crunchy", - "digest 0.9.0", - "subtle 2.4.1", + "typenum", ] [[package]] @@ -4011,16 +4640,7 @@ checksum = "5be9b9bb642d8522a44d533eab56c16c738301965504753b03ad1de3425d5451" dependencies = [ "crunchy", "digest 0.9.0", - "subtle 2.4.1", -] - -[[package]] -name = "libsecp256k1-gen-ecmult" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccab96b584d38fac86a83f07e659f0deafd0253dc096dab5a36d53efe653c5c3" -dependencies = [ - "libsecp256k1-core 0.2.2", + "subtle", ] [[package]] @@ -4029,16 +4649,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3038c808c55c87e8a172643a7d87187fc6c4174468159cb3090659d55bcb4809" dependencies = [ - "libsecp256k1-core 0.3.0", -] - -[[package]] -name = "libsecp256k1-gen-genmult" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67abfe149395e3aa1c48a2beb32b068e2334402df8181f818d3aee2b304c4f5d" -dependencies = [ - "libsecp256k1-core 0.2.2", + "libsecp256k1-core", ] [[package]] @@ -4047,7 +4658,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3db8d6ba2cec9eacc40e6e8ccc98931840301f1006e95647ceb2dd5c3aa06f7c" dependencies = [ - "libsecp256k1-core 0.3.0", + "libsecp256k1-core", ] [[package]] @@ -4087,6 +4698,12 @@ dependencies = [ "statrs", ] +[[package]] +name = "linux-raw-sys" +version = "0.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a261afc61b7a5e323933b402ca6a1765183687c614789b1e4db7762ed4230bca" + [[package]] name = "lite-json" version = "0.1.3" @@ -4129,26 +4746,56 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" dependencies = [ - "log 0.4.14", + "log 0.4.16", ] [[package]] name = "log" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" dependencies = [ "cfg-if 1.0.0", "value-bag", ] +[[package]] +name = "loupe" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6a72dfa44fe15b5e76b94307eeb2ff995a8c5b283b55008940c02e0c5b634d" +dependencies = [ + "indexmap", + "loupe-derive", + "rustversion", +] + +[[package]] +name = "loupe-derive" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fbfc88337168279f2e9ae06e157cfed4efd3316e14dc96ed074d4f2e6c5952" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "lru" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ea2d928b485416e8908cff2d97d621db22b27f7b3b6729e438bcf42c671ba91" +dependencies = [ + "hashbrown 0.11.2", +] + [[package]] name = "lru" -version = "0.6.6" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ea2d928b485416e8908cff2d97d621db22b27f7b3b6729e438bcf42c671ba91" +checksum = "32613e41de4c47ab04970c348ca7ae7382cf116625755af070b008a15516a889" dependencies = [ - "hashbrown", + "hashbrown 0.11.2", ] [[package]] @@ -4276,12 +4923,12 @@ dependencies = [ [[package]] name = "memory-db" -version = "0.27.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de006e09d04fc301a5f7e817b75aa49801c4479a8af753764416b085337ddcc5" +checksum = "6566c70c1016f525ced45d7b7f97730a2bafb037c788211d0c186ef5b2189f0a" dependencies = [ "hash-db", - "hashbrown", + "hashbrown 0.12.0", "parity-util-mem", ] @@ -4312,26 +4959,6 @@ dependencies = [ "log 0.3.9", ] -[[package]] -name = "minicbor" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51aa5bb0ca22415daca596a227b507f880ad1b2318a87fa9325312a5d285ca0d" -dependencies = [ - "minicbor-derive", -] - -[[package]] -name = "minicbor-derive" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54999f917cd092b13904737e26631aa2b2b88d625db68e4bab461dcd8006c788" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -4360,7 +4987,7 @@ dependencies = [ "iovec", "kernel32-sys", "libc", - "log 0.4.14", + "log 0.4.16", "miow 0.2.2", "net2", "slab", @@ -4369,12 +4996,12 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba42135c6a5917b9db9cd7b293e5409e1c6b041e6f9825e92e55a894c63b6f8" +checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" dependencies = [ "libc", - "log 0.4.14", + "log 0.4.16", "miow 0.3.7", "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", @@ -4388,7 +5015,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52403fe290012ce777c4626790c8951324a2b9e3316b3143779c72b029742f19" dependencies = [ "lazycell", - "log 0.4.14", + "log 0.4.16", "mio 0.6.23", "slab", ] @@ -4438,6 +5065,24 @@ dependencies = [ "url 2.2.2", ] +[[package]] +name = "multiaddr" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c580bfdd8803cce319b047d239559a22f809094aaea4ac13902a1fdcfcd4261" +dependencies = [ + "arrayref", + "bs58", + "byteorder", + "data-encoding", + "multihash 0.16.2", + "percent-encoding 2.1.0", + "serde", + "static_assertions", + "unsigned-varint 0.7.1", + "url 2.2.2", +] + [[package]] name = "multibase" version = "0.8.0" @@ -4471,7 +5116,7 @@ dependencies = [ "digest 0.9.0", "sha-1 0.9.8", "sha2 0.9.9", - "sha3", + "sha3 0.9.1", "unsigned-varint 0.5.1", ] @@ -4486,9 +5131,9 @@ dependencies = [ "blake3", "digest 0.9.0", "generic-array 0.14.5", - "multihash-derive", + "multihash-derive 0.7.2", "sha2 0.9.9", - "sha3", + "sha3 0.9.1", "unsigned-varint 0.5.1", ] @@ -4500,11 +5145,24 @@ checksum = "752a61cd890ff691b4411423d23816d5866dd5621e4d1c5687a53b94b5a979d8" dependencies = [ "digest 0.9.0", "generic-array 0.14.5", - "multihash-derive", + "multihash-derive 0.7.2", "sha2 0.9.9", "unsigned-varint 0.7.1", ] +[[package]] +name = "multihash" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3db354f401db558759dfc1e568d010a5d4146f4d3f637be1275ec4a3cf09689" +dependencies = [ + "core2", + "digest 0.10.3", + "multihash-derive 0.8.0", + "sha2 0.10.2", + "unsigned-varint 0.7.1", +] + [[package]] name = "multihash-derive" version = "0.7.2" @@ -4519,6 +5177,20 @@ dependencies = [ "synstructure", ] +[[package]] +name = "multihash-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc076939022111618a5026d3be019fd8b366e76314538ff9a1b59ffbcbf98bcd" +dependencies = [ + "proc-macro-crate 1.1.3", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "multimap" version = "0.8.3" @@ -4533,7 +5205,21 @@ checksum = "56a336acba8bc87c8876f6425407dbbe6c417bf478b22015f8fb0994ef3bc0ab" dependencies = [ "bytes 1.1.0", "futures 0.3.21", - "log 0.4.14", + "log 0.4.16", + "pin-project 1.0.10", + "smallvec 1.8.0", + "unsigned-varint 0.7.1", +] + +[[package]] +name = "multistream-select" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "363a84be6453a70e63513660f4894ef815daf88e3356bffcda9ca27d810ce83b" +dependencies = [ + "bytes 1.1.0", + "futures 0.3.21", + "log 0.4.16", "pin-project 1.0.10", "smallvec 1.8.0", "unsigned-varint 0.7.1", @@ -4570,22 +5256,22 @@ dependencies = [ [[package]] name = "names" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a8690bf09abf659851e58cd666c3d37ac6af07c2bd7a9e332cfba471715775" +checksum = "e7d66043b25d4a6cccb23619d10c19c25304b355a7dccd4a8e11423dd2382146" dependencies = [ "rand 0.8.5", ] [[package]] name = "native-tls" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" dependencies = [ "lazy_static", "libc", - "log 0.4.14", + "log 0.4.16", "openssl", "openssl-probe", "openssl-sys", @@ -4608,20 +5294,22 @@ dependencies = [ [[package]] name = "nix" -version = "0.19.1" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ccba0cfe4fdf15982d1674c69b1fd80bad427d293849982668dfe454bd61f2" +checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" dependencies = [ "bitflags", "cc", "cfg-if 1.0.0", "libc", + "memoffset", ] [[package]] name = "node-bench" version = "0.9.0-dev" dependencies = [ + "clap 3.1.8", "derive_more", "fs_extra", "futures 0.3.21", @@ -4630,7 +5318,7 @@ dependencies = [ "kvdb", "kvdb-rocksdb", "lazy_static", - "log 0.4.14", + "log 0.4.16", "node-primitives", "node-runtime", "node-testing", @@ -4651,7 +5339,6 @@ dependencies = [ "sp-timestamp", "sp-tracing", "sp-trie", - "structopt", "tempfile", ] @@ -4661,28 +5348,35 @@ version = "3.0.0-dev" dependencies = [ "assert_cmd", "async-std", + "clap 3.1.8", + "clap_complete", + "criterion", "frame-benchmarking-cli", "frame-system", + "frame-system-rpc-runtime-api", "futures 0.3.21", "hex-literal", "ipfs", - "jsonrpsee-ws-client", - "log 0.4.14", + "log 0.4.16", "nix", "node-executor", "node-inspect", "node-primitives", "node-rpc", "node-runtime", + "pallet-asset-tx-payment", + "pallet-balances", "pallet-im-online", + "pallet-timestamp", "pallet-transaction-payment", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "platforms", - "rand 0.7.3", + "rand 0.8.5", "regex", "remote-externalities", "sc-authority-discovery", "sc-basic-authorship", + "sc-block-builder", "sc-chain-spec", "sc-cli", "sc-client-api", @@ -4706,8 +5400,10 @@ dependencies = [ "serde", "serde_json", "soketto 0.4.2", + "sp-api", "sp-authority-discovery", "sp-authorship", + "sp-blockchain", "sp-consensus", "sp-consensus-babe", "sp-core", @@ -4747,16 +5443,18 @@ dependencies = [ "pallet-im-online", "pallet-timestamp", "pallet-treasury", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sc-executor", "scale-info", "sp-application-crypto", "sp-consensus-babe", "sp-core", "sp-externalities", + "sp-keyring", "sp-keystore", "sp-runtime", "sp-state-machine", + "sp-tracing", "sp-trie", "wat", ] @@ -4765,8 +5463,8 @@ dependencies = [ name = "node-inspect" version = "0.9.0-dev" dependencies = [ - "derive_more", - "parity-scale-codec", + "clap 3.1.8", + "parity-scale-codec 3.1.2", "sc-cli", "sc-client-api", "sc-executor", @@ -4774,7 +5472,7 @@ dependencies = [ "sp-blockchain", "sp-core", "sp-runtime", - "structopt", + "thiserror", ] [[package]] @@ -4782,7 +5480,7 @@ name = "node-primitives" version = "2.0.0" dependencies = [ "frame-system", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-application-crypto", "sp-core", @@ -4818,17 +5516,7 @@ dependencies = [ "sp-keystore", "sp-runtime", "substrate-frame-rpc-system", -] - -[[package]] -name = "node-rpc-client" -version = "2.0.0" -dependencies = [ - "futures 0.3.21", - "jsonrpc-core-client", - "node-primitives", - "sc-rpc", - "sp-tracing", + "substrate-state-trie-migration-rpc", ] [[package]] @@ -4844,8 +5532,9 @@ dependencies = [ "frame-system-rpc-runtime-api", "frame-try-runtime", "hex-literal", - "log 0.4.14", + "log 0.4.16", "node-primitives", + "pallet-asset-tx-payment", "pallet-assets", "pallet-authority-discovery", "pallet-authorship", @@ -4853,10 +5542,12 @@ dependencies = [ "pallet-bags-list", "pallet-balances", "pallet-bounties", + "pallet-child-bounties", "pallet-collective", "pallet-contracts", "pallet-contracts-primitives", "pallet-contracts-rpc-runtime-api", + "pallet-conviction-voting", "pallet-democracy", "pallet-election-provider-multi-phase", "pallet-elections-phragmen", @@ -4871,15 +5562,18 @@ dependencies = [ "pallet-multisig", "pallet-offences", "pallet-offences-benchmarking", + "pallet-preimage", "pallet-proxy", "pallet-randomness-collective-flip", "pallet-recovery", + "pallet-referenda", "pallet-scheduler", "pallet-session", "pallet-session-benchmarking", "pallet-society", "pallet-staking", "pallet-staking-reward-curve", + "pallet-state-trie-migration", "pallet-sudo", "pallet-timestamp", "pallet-tips", @@ -4890,7 +5584,8 @@ dependencies = [ "pallet-uniques", "pallet-utility", "pallet-vesting", - "parity-scale-codec", + "pallet-whitelist", + "parity-scale-codec 3.1.2", "scale-info", "sp-api", "sp-authority-discovery", @@ -4899,10 +5594,9 @@ dependencies = [ "sp-core", "sp-inherents", "sp-io", - "sp-keyring", - "sp-npos-elections", "sp-offchain", "sp-runtime", + "sp-sandbox", "sp-session", "sp-staking", "sp-std", @@ -4916,23 +5610,26 @@ dependencies = [ name = "node-runtime-generate-bags" version = "3.0.0" dependencies = [ + "clap 3.1.8", "generate-bags", "node-runtime", - "structopt", ] [[package]] name = "node-template" -version = "3.0.0" +version = "4.0.0-dev" dependencies = [ + "clap 3.1.8", "frame-benchmarking", "frame-benchmarking-cli", + "frame-system", "jsonrpc-core", "node-primitives", "node-template-runtime", "pallet-contracts-rpc", "pallet-im-online", "pallet-iris-rpc", + "pallet-transaction-payment", "pallet-transaction-payment-rpc", "sc-basic-authorship", "sc-cli", @@ -4955,17 +5652,19 @@ dependencies = [ "sp-consensus-aura", "sp-core", "sp-finality-grandpa", + "sp-inherents", + "sp-keyring", "sp-keystore", "sp-runtime", "sp-timestamp", - "structopt", "substrate-build-script-utils", "substrate-frame-rpc-system", + "try-runtime-cli", ] [[package]] name = "node-template-runtime" -version = "3.0.0" +version = "4.0.0-dev" dependencies = [ "frame-benchmarking", "frame-executive", @@ -4973,8 +5672,9 @@ dependencies = [ "frame-system", "frame-system-benchmarking", "frame-system-rpc-runtime-api", + "frame-try-runtime", "hex-literal", - "log 0.4.14", + "log 0.4.16", "node-primitives", "pallet-assets", "pallet-aura", @@ -4994,7 +5694,7 @@ dependencies = [ "pallet-timestamp", "pallet-transaction-payment", "pallet-transaction-payment-rpc-runtime-api", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-api", "sp-block-builder", @@ -5017,12 +5717,13 @@ dependencies = [ "frame-system", "fs_extra", "futures 0.3.21", - "log 0.4.14", + "log 0.4.16", "node-executor", "node-primitives", "node-runtime", + "pallet-asset-tx-payment", "pallet-transaction-payment", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sc-block-builder", "sc-client-api", "sc-client-db", @@ -5057,13 +5758,12 @@ checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" [[package]] name = "nom" -version = "7.1.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" dependencies = [ "memchr", "minimal-lexical", - "version_check 0.9.4", ] [[package]] @@ -5160,34 +5860,27 @@ dependencies = [ [[package]] name = "object" -version = "0.22.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397" +checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" dependencies = [ "crc32fast", "indexmap", + "memchr", ] [[package]] name = "object" -version = "0.26.2" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39f37e50073ccad23b6d09bcb5b263f4e76d3bb6038e4a3c08e52162ffa8abc2" +checksum = "40bec70ba014595f99f7aa110b84331ffe1ee9aece7fe6f387cc7e3ecda4d456" dependencies = [ "crc32fast", + "hashbrown 0.11.2", "indexmap", "memchr", ] -[[package]] -name = "object" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.10.0" @@ -5212,6 +5905,29 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "open-metrics-client" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7337d80c23c2d8b1349563981bc4fb531220733743ba8115454a67b181173f0d" +dependencies = [ + "dtoa 0.4.8", + "itoa 0.4.8", + "open-metrics-client-derive-text-encode", + "owning_ref", +] + +[[package]] +name = "open-metrics-client-derive-text-encode" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15c83b586f00268c619c1cb3340ec1a6f59dd9ba1d9833a273a68e6d5cd8ffc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl" version = "0.10.38" @@ -5245,6 +5961,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "os_str_bytes" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" +dependencies = [ + "memchr", +] + [[package]] name = "output_vt100" version = "0.1.3" @@ -5263,6 +5988,28 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "pallet-asset-tx-payment" +version = "4.0.0-dev" +dependencies = [ + "frame-support", + "frame-system", + "pallet-assets", + "pallet-authorship", + "pallet-balances", + "pallet-transaction-payment", + "parity-scale-codec 3.1.2", + "scale-info", + "serde", + "serde_json", + "smallvec 1.8.0", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", + "sp-storage", +] + [[package]] name = "pallet-assets" version = "4.0.0-dev" @@ -5271,7 +6018,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5286,7 +6033,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5301,7 +6048,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-timestamp", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-application-crypto", "sp-consensus-aura", @@ -5318,7 +6065,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-session", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-application-crypto", "sp-authority-discovery", @@ -5335,7 +6082,7 @@ dependencies = [ "frame-support", "frame-system", "impl-trait-for-tuples", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-authorship", "sp-core", @@ -5352,7 +6099,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-authorship", "pallet-balances", "pallet-offences", @@ -5360,7 +6107,7 @@ dependencies = [ "pallet-staking", "pallet-staking-reward-curve", "pallet-timestamp", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-application-crypto", "sp-consensus-babe", @@ -5381,9 +6128,9 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5392,6 +6139,35 @@ dependencies = [ "sp-tracing", ] +[[package]] +name = "pallet-bags-list-fuzzer" +version = "4.0.0-dev" +dependencies = [ + "frame-election-provider-support", + "honggfuzz", + "pallet-bags-list", + "rand 0.8.5", +] + +[[package]] +name = "pallet-bags-list-remote-tests" +version = "4.0.0-dev" +dependencies = [ + "frame-election-provider-support", + "frame-support", + "frame-system", + "log 0.4.16", + "pallet-bags-list", + "pallet-staking", + "remote-externalities", + "sp-core", + "sp-runtime", + "sp-std", + "sp-storage", + "sp-tracing", + "tokio", +] + [[package]] name = "pallet-balances" version = "4.0.0-dev" @@ -5399,9 +6175,9 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-transaction-payment", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5417,7 +6193,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-session", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "serde", "sp-core", @@ -5437,13 +6213,13 @@ dependencies = [ "frame-system", "hex", "hex-literal", - "libsecp256k1 0.7.0", - "log 0.4.14", + "k256", + "log 0.4.16", "pallet-beefy", "pallet-mmr", "pallet-mmr-primitives", "pallet-session", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "serde", "sp-core", @@ -5460,10 +6236,29 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", + "pallet-balances", + "pallet-treasury", + "parity-scale-codec 3.1.2", + "scale-info", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "pallet-child-bounties" +version = "4.0.0-dev" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "log 0.4.16", "pallet-balances", + "pallet-bounties", "pallet-treasury", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5478,8 +6273,8 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5493,44 +6288,46 @@ version = "4.0.0-dev" dependencies = [ "assert_matches", "bitflags", + "env_logger 0.9.0", "frame-benchmarking", "frame-support", "frame-system", "hex-literal", - "libsecp256k1 0.3.5", - "log 0.4.14", + "log 0.4.16", "pallet-balances", "pallet-contracts-primitives", "pallet-contracts-proc-macro", "pallet-randomness-collective-flip", "pallet-timestamp", "pallet-utility", - "parity-scale-codec", - "pretty_assertions 0.7.2", - "pwasm-utils", - "rand 0.7.3", - "rand_pcg 0.2.1", + "parity-scale-codec 3.1.2", + "pretty_assertions", + "rand 0.8.5", + "rand_pcg 0.3.1", "scale-info", "serde", "smallvec 1.8.0", "sp-core", "sp-io", + "sp-keystore", "sp-runtime", "sp-sandbox", "sp-std", + "wasm-instrument", "wasmi-validation", "wat", ] [[package]] name = "pallet-contracts-primitives" -version = "4.0.0-dev" +version = "6.0.0" dependencies = [ "bitflags", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "serde", "sp-core", + "sp-rpc", "sp-runtime", "sp-std", ] @@ -5553,7 +6350,7 @@ dependencies = [ "jsonrpc-derive", "pallet-contracts-primitives", "pallet-contracts-rpc-runtime-api", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "serde", "serde_json", "sp-api", @@ -5568,7 +6365,7 @@ name = "pallet-contracts-rpc-runtime-api" version = "4.0.0-dev" dependencies = [ "pallet-contracts-primitives", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-api", "sp-runtime", @@ -5576,15 +6373,16 @@ dependencies = [ ] [[package]] -name = "pallet-democracy" +name = "pallet-conviction-voting" version = "4.0.0-dev" dependencies = [ + "assert_matches", "frame-benchmarking", "frame-support", "frame-system", "pallet-balances", "pallet-scheduler", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "serde", "sp-core", @@ -5594,44 +6392,46 @@ dependencies = [ ] [[package]] -name = "pallet-election-provider-multi-phase" +name = "pallet-democracy" version = "4.0.0-dev" dependencies = [ "frame-benchmarking", - "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", "pallet-balances", - "parity-scale-codec", - "parking_lot 0.11.2", - "rand 0.7.3", + "pallet-scheduler", + "parity-scale-codec 3.1.2", "scale-info", - "sp-arithmetic", + "serde", "sp-core", "sp-io", - "sp-npos-elections", "sp-runtime", "sp-std", - "sp-tracing", - "static_assertions", - "strum 0.21.0", - "strum_macros 0.21.1", ] [[package]] -name = "pallet-elections" +name = "pallet-election-provider-multi-phase" version = "4.0.0-dev" dependencies = [ + "frame-benchmarking", + "frame-election-provider-support", "frame-support", "frame-system", + "log 0.4.16", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", + "rand 0.7.3", "scale-info", + "sp-arithmetic", "sp-core", "sp-io", + "sp-npos-elections", "sp-runtime", "sp-std", + "sp-tracing", + "static_assertions", + "strum", ] [[package]] @@ -5641,9 +6441,9 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5654,15 +6454,15 @@ dependencies = [ ] [[package]] -name = "pallet-example" +name = "pallet-example-basic" version = "4.0.0-dev" dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5677,8 +6477,8 @@ dependencies = [ "frame-support", "frame-system", "lite-json", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5693,7 +6493,7 @@ version = "3.0.0-dev" dependencies = [ "frame-support", "frame-system", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5710,7 +6510,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-arithmetic", "sp-core", @@ -5728,7 +6528,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-authorship", "pallet-balances", "pallet-offences", @@ -5736,7 +6536,7 @@ dependencies = [ "pallet-staking", "pallet-staking-reward-curve", "pallet-timestamp", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-application-crypto", "sp-core", @@ -5758,7 +6558,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5773,10 +6573,10 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-authorship", "pallet-session", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-application-crypto", "sp-core", @@ -5794,7 +6594,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5810,10 +6610,10 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-assets", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5829,9 +6629,9 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5848,7 +6648,7 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "pallet-iris-rpc-runtime-api", - "parity-scale-codec", + "parity-scale-codec 2.3.1", "sp-api", "sp-blockchain", "sp-core", @@ -5862,7 +6662,7 @@ name = "pallet-iris-rpc-runtime-api" version = "4.0.0-dev" dependencies = [ "pallet-iris-session", - "parity-scale-codec", + "parity-scale-codec 2.3.1", "sp-api", "sp-core", "sp-runtime", @@ -5876,13 +6676,13 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-assets", "pallet-balances", "pallet-im-online", "pallet-iris-assets", "pallet-session", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5901,7 +6701,7 @@ dependencies = [ "frame-support-test", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5916,8 +6716,8 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5936,7 +6736,7 @@ dependencies = [ "frame-system", "hex-literal", "pallet-mmr-primitives", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -5951,8 +6751,8 @@ dependencies = [ "frame-support", "frame-system", "hex-literal", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "serde", "sp-api", "sp-core", @@ -5968,7 +6768,7 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "pallet-mmr-primitives", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "serde", "serde_json", "sp-api", @@ -5985,7 +6785,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -6000,7 +6800,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -6014,8 +6814,8 @@ version = "4.0.0-dev" dependencies = [ "frame-support", "frame-system", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -6029,9 +6829,9 @@ version = "4.0.0-dev" dependencies = [ "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "serde", "sp-core", @@ -6058,12 +6858,28 @@ dependencies = [ "pallet-staking", "pallet-staking-reward-curve", "pallet-timestamp", - "parity-scale-codec", + "parity-scale-codec 3.1.2", + "scale-info", + "sp-core", + "sp-io", + "sp-runtime", + "sp-staking", + "sp-std", +] + +[[package]] +name = "pallet-preimage" +version = "4.0.0-dev" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "pallet-balances", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", "sp-runtime", - "sp-staking", "sp-std", ] @@ -6076,7 +6892,7 @@ dependencies = [ "frame-system", "pallet-balances", "pallet-utility", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -6090,7 +6906,7 @@ version = "4.0.0-dev" dependencies = [ "frame-support", "frame-system", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "safe-mix", "scale-info", "sp-core", @@ -6106,8 +6922,28 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", + "scale-info", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "pallet-referenda" +version = "4.0.0-dev" +dependencies = [ + "assert_matches", + "frame-benchmarking", + "frame-support", + "frame-system", + "pallet-balances", + "pallet-preimage", + "pallet-scheduler", + "parity-scale-codec 3.1.2", "scale-info", + "serde", "sp-core", "sp-io", "sp-runtime", @@ -6121,8 +6957,9 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "pallet-preimage", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -6138,7 +6975,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -6153,9 +6990,9 @@ dependencies = [ "frame-support", "frame-system", "impl-trait-for-tuples", - "log 0.4.14", + "log 0.4.16", "pallet-timestamp", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -6179,7 +7016,7 @@ dependencies = [ "pallet-staking", "pallet-staking-reward-curve", "pallet-timestamp", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "rand 0.7.3", "scale-info", "sp-core", @@ -6197,7 +7034,7 @@ dependencies = [ "frame-support-test", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "rand_chacha 0.2.2", "scale-info", "sp-core", @@ -6214,14 +7051,14 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-authorship", "pallet-bags-list", "pallet-balances", "pallet-session", "pallet-staking-reward-curve", "pallet-timestamp", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "rand_chacha 0.2.2", "scale-info", "serde", @@ -6251,17 +7088,42 @@ dependencies = [ name = "pallet-staking-reward-fn" version = "4.0.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.16", "sp-arithmetic", ] +[[package]] +name = "pallet-state-trie-migration" +version = "4.0.0-dev" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "log 0.4.16", + "pallet-balances", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", + "remote-externalities", + "scale-info", + "serde", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", + "sp-tracing", + "substrate-state-trie-migration-rpc", + "thousands", + "tokio", + "zstd", +] + [[package]] name = "pallet-sudo" version = "4.0.0-dev" dependencies = [ "frame-support", "frame-system", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -6276,8 +7138,8 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-inherents", @@ -6294,10 +7156,10 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", "pallet-balances", "pallet-treasury", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "serde", "sp-core", @@ -6314,7 +7176,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "serde", "serde_json", @@ -6333,7 +7195,7 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "pallet-transaction-payment-rpc-runtime-api", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sp-api", "sp-blockchain", "sp-core", @@ -6346,7 +7208,7 @@ name = "pallet-transaction-payment-rpc-runtime-api" version = "4.0.0-dev" dependencies = [ "pallet-transaction-payment", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sp-api", "sp-runtime", ] @@ -6360,7 +7222,7 @@ dependencies = [ "frame-system", "hex-literal", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "serde", "sp-core", @@ -6380,7 +7242,7 @@ dependencies = [ "frame-system", "impl-trait-for-tuples", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "serde", "sp-core", @@ -6396,8 +7258,9 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", + "log 0.4.16", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -6413,7 +7276,7 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-io", @@ -6428,10 +7291,28 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "log 0.4.14", + "log 0.4.16", + "pallet-balances", + "parity-scale-codec 3.1.2", + "scale-info", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + +[[package]] +name = "pallet-whitelist" +version = "4.0.0-dev" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", "pallet-balances", - "parity-scale-codec", + "pallet-preimage", + "parity-scale-codec 3.1.2", "scale-info", + "sp-api", "sp-core", "sp-io", "sp-runtime", @@ -6440,16 +7321,16 @@ dependencies = [ [[package]] name = "parity-db" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "865edee5b792f537356d9e55cbc138e7f4718dc881a7ea45a18b37bf61c21e3d" +checksum = "b3e7f385d61562f5834282b90aa50b41f38a35cf64d5209b8b05487b50553dbe" dependencies = [ "blake2-rfc", "crc32fast", "fs2", "hex", "libc", - "log 0.4.14", + "log 0.4.16", "lz4", "memmap2 0.2.3", "parking_lot 0.11.2", @@ -6464,10 +7345,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "373b1a4c1338d9cd3d1fa53b3a11bdab5ab6bd80a20f7f7becd76953ae2be909" dependencies = [ "arrayvec 0.7.2", - "bitvec", + "bitvec 0.20.4", + "byte-slice-cast", + "impl-trait-for-tuples", + "parity-scale-codec-derive 2.3.1", + "serde", +] + +[[package]] +name = "parity-scale-codec" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b44461635bbb1a0300f100a841e571e7d919c81c73075ef5d152ffdb521066" +dependencies = [ + "arrayvec 0.7.2", + "bitvec 1.0.0", "byte-slice-cast", "impl-trait-for-tuples", - "parity-scale-codec-derive", + "parity-scale-codec-derive 3.1.2", "serde", ] @@ -6483,6 +7378,18 @@ dependencies = [ "syn", ] +[[package]] +name = "parity-scale-codec-derive" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c45ed1f39709f5a89338fab50e59816b2e8815f5bb58276e7ddf9afd495f73f8" +dependencies = [ + "proc-macro-crate 1.1.3", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "parity-send-wrapper" version = "0.1.0" @@ -6497,7 +7404,7 @@ checksum = "9981e32fb75e004cc148f5fb70342f393830e0a4aa62e3cc93b50976218d42b6" dependencies = [ "futures 0.3.21", "libc", - "log 0.4.14", + "log 0.4.16", "rand 0.7.3", "tokio", "winapi 0.3.9", @@ -6505,15 +7412,15 @@ dependencies = [ [[package]] name = "parity-util-mem" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f4cb4e169446179cbc6b8b6320cc9fca49bd2e94e8db25f25f200a8ea774770" +checksum = "c32561d248d352148124f036cac253a644685a21dc9fea383eb4907d7bd35a8f" dependencies = [ "cfg-if 1.0.0", - "hashbrown", + "hashbrown 0.12.0", "impl-trait-for-tuples", "parity-util-mem-derive", - "parking_lot 0.11.2", + "parking_lot 0.12.0", "primitive-types", "smallvec 1.8.0", "winapi 0.3.9", @@ -6554,7 +7461,7 @@ dependencies = [ "byteorder", "bytes 0.4.12", "httparse", - "log 0.4.14", + "log 0.4.16", "mio 0.6.23", "mio-extras", "rand 0.7.3", @@ -6580,16 +7487,6 @@ dependencies = [ "rustc_version 0.2.3", ] -[[package]] -name = "parking_lot" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" -dependencies = [ - "lock_api 0.3.4", - "parking_lot_core 0.7.2", -] - [[package]] name = "parking_lot" version = "0.11.2" @@ -6608,7 +7505,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ "lock_api 0.4.6", - "parking_lot_core 0.9.1", + "parking_lot_core 0.9.2", ] [[package]] @@ -6626,20 +7523,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "parking_lot_core" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" -dependencies = [ - "cfg-if 0.1.10", - "cloudabi", - "libc", - "redox_syscall 0.1.57", - "smallvec 1.8.0", - "winapi 0.3.9", -] - [[package]] name = "parking_lot_core" version = "0.8.5" @@ -6649,20 +7532,20 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.2.11", + "redox_syscall 0.2.13", "smallvec 1.8.0", "winapi 0.3.9", ] [[package]] name = "parking_lot_core" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.11", + "redox_syscall 0.2.13", "smallvec 1.8.0", "windows-sys", ] @@ -6679,9 +7562,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" [[package]] name = "paste-impl" @@ -6777,7 +7660,17 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" dependencies = [ - "fixedbitset", + "fixedbitset 0.2.0", + "indexmap", +] + +[[package]] +name = "petgraph" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f" +dependencies = [ + "fixedbitset 0.4.1", "indexmap", ] @@ -6839,17 +7732,28 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cabda3fb821068a9a4fab19a683eac3af12edf0f34b94a8be53c4972b8149d0" +dependencies = [ + "der", + "spki", + "zeroize", +] + [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "platforms" -version = "1.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989d43012e2ca1c4a02507c67282691a0a3207f9dc67cec596b43fe925b3d325" +checksum = "e8d0eef3571242013a0d5dc84861c3ae4a652e56e12adf8bdc26ff5f8cb34c94" [[package]] name = "plotters" @@ -6887,7 +7791,7 @@ checksum = "685404d509889fade3e86fe3a5803bca2ec09b0c0778d5ada6ec8bf7a8de5259" dependencies = [ "cfg-if 1.0.0", "libc", - "log 0.4.14", + "log 0.4.16", "wepoll-ffi", "winapi 0.3.9", ] @@ -6898,7 +7802,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "048aeb476be11a4b6ca432ca569e375810de9294ae78f4774e78ea98a9246ede" dependencies = [ - "cpufeatures 0.2.1", + "cpufeatures 0.2.2", "opaque-debug 0.3.0", "universal-hash", ] @@ -6910,7 +7814,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1" dependencies = [ "cfg-if 1.0.0", - "cpufeatures 0.2.1", + "cpufeatures 0.2.2", "opaque-debug 0.3.0", "universal-hash", ] @@ -6950,23 +7854,11 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427" -dependencies = [ - "ansi_term 0.11.0", - "ctor", - "difference", - "output_vt100", -] - -[[package]] -name = "pretty_assertions" -version = "0.7.2" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cab0e7c02cf376875e9335e0ba1da535775beb5450d21e1dffca068818ed98b" +checksum = "c89f989ac94207d048d92db058e4f6ec7342b0971fc58d1271ca148b799b3563" dependencies = [ - "ansi_term 0.12.1", + "ansi_term", "ctor", "diff", "output_vt100", @@ -6974,9 +7866,9 @@ dependencies = [ [[package]] name = "primitive-types" -version = "0.10.1" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05e4722c697a58a99d5d06a08c30821d7c082a4632198de1eaa5a6c22ef42373" +checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" dependencies = [ "fixed-hash", "impl-codec", @@ -7036,27 +7928,50 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" dependencies = [ "unicode-xid", ] [[package]] name = "prometheus" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8425533e7122f0c3cc7a37e6244b16ad3a2cc32ae7ac6276e2a75da0d9c200d" +checksum = "b7f64969ffd5dd8f39bd57a68ac53c163a095ed9d0fb707146da1b27025a3504" dependencies = [ "cfg-if 1.0.0", "fnv", "lazy_static", + "memchr", "parking_lot 0.11.2", - "regex", "thiserror", ] +[[package]] +name = "prometheus-client" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a896938cc6018c64f279888b8c7559d3725210d5db9a3a1ee6bc7188d51d34" +dependencies = [ + "dtoa 1.0.2", + "itoa 1.0.1", + "owning_ref", + "prometheus-client-derive-text-encode", +] + +[[package]] +name = "prometheus-client-derive-text-encode" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8e12d01b9d66ad9eb4529c57666b6263fc1993cb30261d83ead658fdd932652" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "prost" version = "0.8.0" @@ -7064,7 +7979,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020" dependencies = [ "bytes 1.1.0", - "prost-derive", + "prost-derive 0.8.0", +] + +[[package]] +name = "prost" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" +dependencies = [ + "bytes 1.1.0", + "prost-derive 0.9.0", ] [[package]] @@ -7076,11 +8001,31 @@ dependencies = [ "bytes 1.1.0", "heck 0.3.3", "itertools", - "log 0.4.14", + "log 0.4.16", + "multimap", + "petgraph 0.5.1", + "prost 0.8.0", + "prost-types 0.8.0", + "tempfile", + "which", +] + +[[package]] +name = "prost-build" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" +dependencies = [ + "bytes 1.1.0", + "heck 0.3.3", + "itertools", + "lazy_static", + "log 0.4.16", "multimap", - "petgraph", - "prost", - "prost-types", + "petgraph 0.6.0", + "prost 0.9.0", + "prost-types 0.9.0", + "regex", "tempfile", "which", ] @@ -7098,6 +8043,19 @@ dependencies = [ "syn", ] +[[package]] +name = "prost-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "prost-types" version = "0.8.0" @@ -7105,27 +8063,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" dependencies = [ "bytes 1.1.0", - "prost", + "prost 0.8.0", +] + +[[package]] +name = "prost-types" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" +dependencies = [ + "bytes 1.1.0", + "prost 0.9.0", ] [[package]] name = "psm" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eca0fa5dd7c4c96e184cec588f0b1db1ee3165e678db21c09793105acb17e6f" +checksum = "871372391786ccec00d3c5d3d6608905b3d4db263639cfe075d3b60a736d115a" dependencies = [ "cc", ] [[package]] -name = "pwasm-utils" -version = "0.18.2" +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "880b3384fb00b8f6ecccd5d358b93bd2201900ae3daad213791d1864f6441f5c" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" dependencies = [ - "byteorder", - "log 0.4.14", - "parity-wasm 0.42.2", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -7156,7 +8133,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ "env_logger 0.8.4", - "log 0.4.14", + "log 0.4.16", "rand 0.8.5", ] @@ -7173,9 +8150,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145" +checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58" dependencies = [ "proc-macro2", ] @@ -7186,6 +8163,12 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.6.5" @@ -7290,7 +8273,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom 0.2.5", + "getrandom 0.2.6", ] [[package]] @@ -7374,6 +8357,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core 0.6.3", +] + [[package]] name = "rand_xorshift" version = "0.1.1" @@ -7409,7 +8401,7 @@ checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" dependencies = [ "crossbeam-channel", "crossbeam-deque", - "crossbeam-utils 0.8.7", + "crossbeam-utils 0.8.8", "lazy_static", "num_cpus", ] @@ -7431,21 +8423,22 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.11" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.5", - "redox_syscall 0.2.11", + "getrandom 0.2.6", + "redox_syscall 0.2.13", + "thiserror", ] [[package]] @@ -7474,9 +8467,19 @@ version = "0.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "571f7f397d61c4755285cd37853fe8e03271c243424a907415909379659381c5" dependencies = [ - "log 0.4.14", + "log 0.4.16", + "rustc-hash", + "smallvec 1.8.0", +] + +[[package]] +name = "regalloc" +version = "0.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d808cff91dfca7b239d40b972ba628add94892b1d9e19a842aedc5cfae8ab1a" +dependencies = [ + "log 0.4.16", "rustc-hash", - "serde", "smallvec 1.8.0", ] @@ -7518,16 +8521,28 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "region" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76e189c2369884dce920945e2ddf79b3dff49e071a167dd1817fa9c4c00d512e" +dependencies = [ + "bitflags", + "libc", + "mach", + "winapi 0.3.9", +] + [[package]] name = "remote-externalities" version = "0.10.0-dev" dependencies = [ "env_logger 0.9.0", - "jsonrpsee-proc-macros", - "jsonrpsee-ws-client", - "log 0.4.14", + "frame-support", + "jsonrpsee", + "log 0.4.16", "pallet-elections-phragmen", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "serde", "serde_json", "sp-core", @@ -7546,6 +8561,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rend" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79af64b4b6362ffba04eef3a4e10829718a4896dac19daa741851c86781edf95" +dependencies = [ + "bytecheck", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -7577,11 +8601,36 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rkyv" +version = "0.7.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f08c8062c1fe1253064043b8fc07bfea1b9702b71b4a86c11ea3588183b12e1" +dependencies = [ + "bytecheck", + "hashbrown 0.12.0", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e289706df51226e84814bf6ba1a9e1013112ae29bc7a9878f73fce360520c403" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "rocksdb" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a62eca5cacf2c8261128631bed9f045598d40bfbe4b29f5163f0f802f8f44a7" +checksum = "620f4129485ff1a7128d184bc687470c21c7951b64779ebc9cfdad3dcd920290" dependencies = [ "libc", "librocksdb-sys", @@ -7639,7 +8688,21 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.6", + "semver 1.0.7", +] + +[[package]] +name = "rustix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2dcfc2778a90e38f56a708bfc90572422e11d6c7ee233d053d1f782cf9df6d2" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "winapi 0.3.9", ] [[package]] @@ -7649,10 +8712,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" dependencies = [ "base64 0.13.0", - "log 0.4.14", + "log 0.4.16", + "ring", + "sct 0.6.1", + "webpki 0.21.4", +] + +[[package]] +name = "rustls" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fbfeb8d0ddb84706bc597a5574ab8912817c52a397f819e5b614e2265206921" +dependencies = [ + "log 0.4.16", "ring", - "sct", - "webpki", + "sct 0.7.0", + "webpki 0.22.0", ] [[package]] @@ -7662,11 +8737,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" dependencies = [ "openssl-probe", - "rustls", + "rustls 0.19.1", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca9ebdfa27d3fc180e42879037b5338ab1c040c06affd00d8338598e7800943" +dependencies = [ + "openssl-probe", + "rustls-pemfile", "schannel", "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" +dependencies = [ + "base64 0.13.0", +] + [[package]] name = "rustversion" version = "1.0.6" @@ -7707,9 +8803,9 @@ checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" [[package]] name = "salsa20" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecbd2eb639fd7cab5804a0837fe373cc2172d15437e804c054a9fb885cb923b0" +checksum = "0c0fbb5f676da676c260ba276a8f43a8dc67cf02d1438423aeb1c677a7212686" dependencies = [ "cipher", ] @@ -7725,9 +8821,9 @@ dependencies = [ [[package]] name = "sc-allocator" -version = "4.0.0-dev" +version = "4.1.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.16", "sp-core", "sp-wasm-interface", "thiserror", @@ -7738,15 +8834,14 @@ name = "sc-authority-discovery" version = "0.10.0-dev" dependencies = [ "async-trait", - "derive_more", "futures 0.3.21", - "futures-timer 3.0.2", + "futures-timer", "ip_network", - "libp2p", - "log 0.4.14", - "parity-scale-codec", - "prost", - "prost-build", + "libp2p 0.40.0", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "prost 0.9.0", + "prost-build 0.9.0", "quickcheck", "rand 0.7.3", "sc-client-api", @@ -7760,6 +8855,7 @@ dependencies = [ "sp-tracing", "substrate-prometheus-endpoint", "substrate-test-runtime-client", + "thiserror", ] [[package]] @@ -7767,10 +8863,10 @@ name = "sc-basic-authorship" version = "0.10.0-dev" dependencies = [ "futures 0.3.21", - "futures-timer 3.0.2", - "log 0.4.14", - "parity-scale-codec", - "parking_lot 0.11.2", + "futures-timer", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "sc-block-builder", "sc-client-api", "sc-proposer-metrics", @@ -7791,7 +8887,7 @@ dependencies = [ name = "sc-block-builder" version = "0.10.0-dev" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sc-client-api", "sp-api", "sp-block-builder", @@ -7808,7 +8904,8 @@ name = "sc-chain-spec" version = "4.0.0-dev" dependencies = [ "impl-trait-for-tuples", - "parity-scale-codec", + "memmap2 0.5.3", + "parity-scale-codec 3.1.2", "sc-chain-spec-derive", "sc-network", "sc-telemetry", @@ -7833,13 +8930,14 @@ name = "sc-cli" version = "0.10.0-dev" dependencies = [ "chrono", + "clap 3.1.8", "fdlimit", "futures 0.3.21", "hex", - "libp2p", - "log 0.4.14", + "libp2p 0.40.0", + "log 0.4.16", "names", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "rand 0.7.3", "regex", "rpassword", @@ -7859,7 +8957,6 @@ dependencies = [ "sp-panic-handler", "sp-runtime", "sp-version", - "structopt", "tempfile", "thiserror", "tiny-bip39", @@ -7873,9 +8970,9 @@ dependencies = [ "fnv", "futures 0.3.21", "hash-db", - "log 0.4.14", - "parity-scale-codec", - "parking_lot 0.11.2", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "sc-executor", "sc-transaction-pool-api", "sc-utils", @@ -7905,10 +9002,10 @@ dependencies = [ "kvdb-memorydb", "kvdb-rocksdb", "linked-hash-map", - "log 0.4.14", + "log 0.4.16", "parity-db", - "parity-scale-codec", - "parking_lot 0.11.2", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "quickcheck", "sc-client-api", "sc-state-db", @@ -7930,10 +9027,10 @@ version = "0.10.0-dev" dependencies = [ "async-trait", "futures 0.3.21", - "futures-timer 3.0.2", - "libp2p", - "log 0.4.14", - "parking_lot 0.11.2", + "futures-timer", + "libp2p 0.40.0", + "log 0.4.16", + "parking_lot 0.12.0", "sc-client-api", "sc-utils", "serde", @@ -7953,12 +9050,10 @@ name = "sc-consensus-aura" version = "0.10.0-dev" dependencies = [ "async-trait", - "derive_more", "futures 0.3.21", - "getrandom 0.2.5", - "log 0.4.14", - "parity-scale-codec", - "parking_lot 0.11.2", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "sc-block-builder", "sc-client-api", "sc-consensus", @@ -7984,6 +9079,7 @@ dependencies = [ "substrate-prometheus-endpoint", "substrate-test-runtime-client", "tempfile", + "thiserror", ] [[package]] @@ -7991,16 +9087,15 @@ name = "sc-consensus-babe" version = "0.10.0-dev" dependencies = [ "async-trait", - "derive_more", "fork-tree", "futures 0.3.21", - "log 0.4.14", + "log 0.4.16", "merlin", "num-bigint", "num-rational 0.2.4", "num-traits", - "parity-scale-codec", - "parking_lot 0.11.2", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "rand 0.7.3", "rand_chacha 0.2.2", "retain_mut", @@ -8034,13 +9129,13 @@ dependencies = [ "substrate-prometheus-endpoint", "substrate-test-runtime-client", "tempfile", + "thiserror", ] [[package]] name = "sc-consensus-babe-rpc" version = "0.10.0-dev" dependencies = [ - "derive_more", "futures 0.3.21", "jsonrpc-core", "jsonrpc-core-client", @@ -8063,6 +9158,7 @@ dependencies = [ "sp-runtime", "substrate-test-runtime-client", "tempfile", + "thiserror", ] [[package]] @@ -8070,7 +9166,7 @@ name = "sc-consensus-epochs" version = "0.10.0-dev" dependencies = [ "fork-tree", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sc-client-api", "sc-consensus", "sp-blockchain", @@ -8083,16 +9179,16 @@ version = "0.10.0-dev" dependencies = [ "assert_matches", "async-trait", - "derive_more", "futures 0.3.21", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "sc-basic-authorship", "sc-client-api", "sc-consensus", + "sc-consensus-aura", "sc-consensus-babe", "sc-consensus-epochs", "sc-transaction-pool", @@ -8101,6 +9197,7 @@ dependencies = [ "sp-api", "sp-blockchain", "sp-consensus", + "sp-consensus-aura", "sp-consensus-babe", "sp-consensus-slots", "sp-core", @@ -8111,6 +9208,7 @@ dependencies = [ "substrate-prometheus-endpoint", "substrate-test-runtime-client", "substrate-test-runtime-transaction-pool", + "thiserror", "tokio", ] @@ -8119,12 +9217,11 @@ name = "sc-consensus-pow" version = "0.10.0-dev" dependencies = [ "async-trait", - "derive_more", "futures 0.3.21", - "futures-timer 3.0.2", - "log 0.4.14", - "parity-scale-codec", - "parking_lot 0.11.2", + "futures-timer", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "sc-client-api", "sc-consensus", "sp-api", @@ -8136,6 +9233,7 @@ dependencies = [ "sp-inherents", "sp-runtime", "substrate-prometheus-endpoint", + "thiserror", ] [[package]] @@ -8144,13 +9242,12 @@ version = "0.10.0-dev" dependencies = [ "async-trait", "futures 0.3.21", - "futures-timer 3.0.2", - "log 0.4.14", - "parity-scale-codec", + "futures-timer", + "log 0.4.16", + "parity-scale-codec 3.1.2", "sc-client-api", "sc-consensus", "sc-telemetry", - "sp-api", "sp-arithmetic", "sp-blockchain", "sp-consensus", @@ -8178,13 +9275,14 @@ dependencies = [ name = "sc-executor" version = "0.10.0-dev" dependencies = [ + "criterion", + "env_logger 0.9.0", "hex-literal", "lazy_static", - "libsecp256k1 0.6.0", - "log 0.4.14", - "parity-scale-codec", - "parking_lot 0.11.2", - "paste 1.0.6", + "lru 0.7.5", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", + "paste 1.0.7", "regex", "sc-executor-common", "sc-executor-wasmi", @@ -8193,6 +9291,7 @@ dependencies = [ "sc-tracing", "sp-api", "sp-core", + "sp-core-hashing-proc-macro", "sp-externalities", "sp-io", "sp-maybe-compressed-blob", @@ -8215,18 +9314,16 @@ dependencies = [ name = "sc-executor-common" version = "0.10.0-dev" dependencies = [ - "derive_more", "environmental", - "parity-scale-codec", - "pwasm-utils", + "parity-scale-codec 3.1.2", "sc-allocator", "sp-core", "sp-maybe-compressed-blob", "sp-serializer", "sp-wasm-interface", "thiserror", + "wasm-instrument", "wasmer", - "wasmer-compiler-singlepass", "wasmi", ] @@ -8234,8 +9331,8 @@ dependencies = [ name = "sc-executor-wasmi" version = "0.10.0-dev" dependencies = [ - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "sc-allocator", "sc-executor-common", "scoped-tls", @@ -8251,8 +9348,8 @@ version = "0.10.0-dev" dependencies = [ "cfg-if 1.0.0", "libc", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "parity-wasm 0.42.2", "sc-allocator", "sc-executor-common", @@ -8269,19 +9366,21 @@ dependencies = [ name = "sc-finality-grandpa" version = "0.10.0-dev" dependencies = [ + "ahash", "assert_matches", "async-trait", - "derive_more", "dyn-clone", "finality-grandpa", "fork-tree", "futures 0.3.21", - "futures-timer 3.0.2", - "log 0.4.14", - "parity-scale-codec", - "parking_lot 0.11.2", + "futures-timer", + "hex", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "rand 0.8.5", "sc-block-builder", + "sc-chain-spec", "sc-client-api", "sc-consensus", "sc-keystore", @@ -8290,6 +9389,7 @@ dependencies = [ "sc-network-test", "sc-telemetry", "sc-utils", + "serde", "serde_json", "sp-api", "sp-application-crypto", @@ -8305,6 +9405,7 @@ dependencies = [ "substrate-prometheus-endpoint", "substrate-test-runtime-client", "tempfile", + "thiserror", "tokio", ] @@ -8312,15 +9413,14 @@ dependencies = [ name = "sc-finality-grandpa-rpc" version = "0.10.0-dev" dependencies = [ - "derive_more", "finality-grandpa", "futures 0.3.21", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-pubsub", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "sc-block-builder", "sc-client-api", "sc-finality-grandpa", @@ -8333,16 +9433,17 @@ dependencies = [ "sp-keyring", "sp-runtime", "substrate-test-runtime-client", + "thiserror", ] [[package]] name = "sc-informant" version = "0.10.0-dev" dependencies = [ - "ansi_term 0.12.1", + "ansi_term", "futures 0.3.21", - "futures-timer 3.0.2", - "log 0.4.14", + "futures-timer", + "log 0.4.16", "parity-util-mem", "sc-client-api", "sc-network", @@ -8356,31 +9457,14 @@ name = "sc-keystore" version = "4.0.0-dev" dependencies = [ "async-trait", - "derive_more", "hex", - "parking_lot 0.11.2", + "parking_lot 0.12.0", "serde_json", "sp-application-crypto", "sp-core", "sp-keystore", "tempfile", -] - -[[package]] -name = "sc-light" -version = "4.0.0-dev" -dependencies = [ - "hash-db", - "parity-scale-codec", - "parking_lot 0.11.2", - "sc-client-api", - "sc-executor", - "sp-api", - "sp-blockchain", - "sp-core", - "sp-externalities", - "sp-runtime", - "sp-state-machine", + "thiserror", ] [[package]] @@ -8394,24 +9478,23 @@ dependencies = [ "bitflags", "bytes 1.1.0", "cid 0.6.1", - "derive_more", "either", "fnv", "fork-tree", "futures 0.3.21", - "futures-timer 3.0.2", + "futures-timer", "hex", "ip_network", - "libp2p", + "libp2p 0.40.0", "linked-hash-map", "linked_hash_set", - "log 0.4.14", - "lru", - "parity-scale-codec", - "parking_lot 0.11.2", + "log 0.4.16", + "lru 0.7.5", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "pin-project 1.0.10", - "prost", - "prost-build", + "prost 0.9.0", + "prost-build 0.9.0", "quickcheck", "rand 0.7.3", "sc-block-builder", @@ -8444,12 +9527,13 @@ dependencies = [ name = "sc-network-gossip" version = "0.10.0-dev" dependencies = [ + "ahash", "async-std", "futures 0.3.21", - "futures-timer 3.0.2", - "libp2p", - "log 0.4.14", - "lru", + "futures-timer", + "libp2p 0.40.0", + "log 0.4.16", + "lru 0.7.5", "quickcheck", "sc-network", "sp-runtime", @@ -8465,10 +9549,10 @@ dependencies = [ "async-std", "async-trait", "futures 0.3.21", - "futures-timer 3.0.2", - "libp2p", - "log 0.4.14", - "parking_lot 0.11.2", + "futures-timer", + "libp2p 0.40.0", + "log 0.4.16", + "parking_lot 0.12.0", "rand 0.7.3", "sc-block-builder", "sc-client-api", @@ -8493,16 +9577,17 @@ dependencies = [ "cid 0.5.1", "fnv", "futures 0.3.21", - "futures-timer 3.0.2", + "futures-timer", "hex", - "hyper 0.14.17", + "hyper 0.14.18", "hyper-rustls", "ipfs", "lazy_static", - "log 0.4.14", + "log 0.4.16", "num_cpus", - "parity-scale-codec", - "parking_lot 0.11.2", + "once_cell", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "rand 0.7.3", "sc-block-builder", "sc-client-api", @@ -8520,6 +9605,7 @@ dependencies = [ "substrate-test-runtime-client", "threadpool", "tokio", + "tracing", ] [[package]] @@ -8527,8 +9613,8 @@ name = "sc-peerset" version = "4.0.0-dev" dependencies = [ "futures 0.3.21", - "libp2p", - "log 0.4.14", + "libp2p 0.40.0", + "log 0.4.16", "rand 0.7.3", "sc-utils", "serde_json", @@ -8537,9 +9623,9 @@ dependencies = [ [[package]] name = "sc-proposer-metrics" -version = "0.9.0" +version = "0.10.0-dev" dependencies = [ - "log 0.4.14", + "log 0.4.16", "substrate-prometheus-endpoint", ] @@ -8553,9 +9639,9 @@ dependencies = [ "jsonrpc-core", "jsonrpc-pubsub", "lazy_static", - "log 0.4.14", - "parity-scale-codec", - "parking_lot 0.11.2", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "sc-block-builder", "sc-chain-spec", "sc-client-api", @@ -8589,11 +9675,12 @@ dependencies = [ "jsonrpc-core-client", "jsonrpc-derive", "jsonrpc-pubsub", - "log 0.4.14", - "parity-scale-codec", - "parking_lot 0.11.2", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "sc-chain-spec", "sc-transaction-pool-api", + "scale-info", "serde", "serde_json", "sp-core", @@ -8614,7 +9701,7 @@ dependencies = [ "jsonrpc-ipc-server", "jsonrpc-pubsub", "jsonrpc-ws-server", - "log 0.4.14", + "log 0.4.16", "serde_json", "substrate-prometheus-endpoint", "tokio", @@ -8624,6 +9711,7 @@ dependencies = [ name = "sc-runtime-test" version = "2.0.0" dependencies = [ + "paste 1.0.7", "sp-core", "sp-io", "sp-runtime", @@ -8642,14 +9730,14 @@ dependencies = [ "directories", "exit-future", "futures 0.3.21", - "futures-timer 3.0.2", + "futures-timer", "hash-db", "jsonrpc-core", "jsonrpc-pubsub", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "parity-util-mem", - "parking_lot 0.11.2", + "parking_lot 0.12.0", "pin-project 1.0.10", "rand 0.7.3", "sc-block-builder", @@ -8660,7 +9748,6 @@ dependencies = [ "sc-executor", "sc-informant", "sc-keystore", - "sc-light", "sc-network", "sc-offchain", "sc-rpc", @@ -8706,16 +9793,16 @@ version = "2.0.0" dependencies = [ "fdlimit", "futures 0.3.21", + "hex", "hex-literal", - "log 0.4.14", - "parity-scale-codec", - "parking_lot 0.11.2", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "sc-block-builder", "sc-client-api", "sc-client-db", "sc-consensus", "sc-executor", - "sc-light", "sc-network", "sc-service", "sc-transaction-pool-api", @@ -8740,11 +9827,11 @@ dependencies = [ name = "sc-state-db" version = "0.10.0-dev" dependencies = [ - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "parity-util-mem", "parity-util-mem-derive", - "parking_lot 0.11.2", + "parking_lot 0.12.0", "sc-client-api", "sp-core", ] @@ -8756,13 +9843,12 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sc-chain-spec", "sc-client-api", "sc-consensus-babe", "sc-consensus-epochs", "sc-finality-grandpa", - "sc-rpc-api", "serde", "serde_json", "sp-blockchain", @@ -8776,9 +9862,9 @@ version = "4.0.0-dev" dependencies = [ "chrono", "futures 0.3.21", - "libp2p", - "log 0.4.14", - "parking_lot 0.11.2", + "libp2p 0.40.0", + "log 0.4.16", + "parking_lot 0.12.0", "pin-project 1.0.10", "rand 0.7.3", "serde", @@ -8791,12 +9877,15 @@ dependencies = [ name = "sc-tracing" version = "4.0.0-dev" dependencies = [ - "ansi_term 0.12.1", + "ansi_term", "atty", + "chrono", + "criterion", "lazy_static", - "log 0.4.14", + "libc", + "log 0.4.16", "once_cell", - "parking_lot 0.11.2", + "parking_lot 0.12.0", "regex", "rustc-hash", "sc-client-api", @@ -8832,13 +9921,13 @@ dependencies = [ "assert_matches", "criterion", "futures 0.3.21", + "futures-timer", "hex", - "intervalier", "linked-hash-map", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "parity-util-mem", - "parking_lot 0.11.2", + "parking_lot 0.12.0", "retain_mut", "sc-block-builder", "sc-client-api", @@ -8863,9 +9952,8 @@ dependencies = [ name = "sc-transaction-pool-api" version = "4.0.0-dev" dependencies = [ - "derive_more", "futures 0.3.21", - "log 0.4.14", + "log 0.4.16", "serde", "sp-blockchain", "sp-runtime", @@ -8877,30 +9965,33 @@ name = "sc-utils" version = "4.0.0-dev" dependencies = [ "futures 0.3.21", - "futures-timer 3.0.2", + "futures-timer", "lazy_static", + "log 0.4.16", + "parking_lot 0.12.0", "prometheus", + "tokio-test", ] [[package]] name = "scale-info" -version = "1.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55b744399c25532d63a0d2789b109df8d46fc93752d46b0782991a931a782f" +checksum = "0563970d79bcbf3c537ce3ad36d859b30d36fc5b190efd227f1f7a84d7cf0d42" dependencies = [ - "bitvec", + "bitvec 1.0.0", "cfg-if 1.0.0", "derive_more", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info-derive", "serde", ] [[package]] name = "scale-info-derive" -version = "1.0.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baeb2780690380592f86205aa4ee49815feb2acad8c2f59e6dd207148c3f1fcd" +checksum = "b7805950c36512db9e3251c970bb7ac425f326716941862205d612ab3b5e46e2" dependencies = [ "proc-macro-crate 1.1.3", "proc-macro2", @@ -8932,7 +10023,7 @@ dependencies = [ "rand 0.7.3", "rand_core 0.5.1", "sha2 0.8.2", - "subtle 2.4.1", + "subtle", "zeroize", ] @@ -8959,10 +10050,57 @@ dependencies = [ ] [[package]] -name = "secrecy" +name = "sct" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0673d6a6449f5e7d12a1caf424fd9363e2af3a4953023ed455e3c4beef4597c0" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + +[[package]] +name = "sec1" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08da66b8b0965a5555b6bd6639e68ccba85e1e2506f5fbb089e93f8a04e1a2d1" +dependencies = [ + "der", + "generic-array 0.14.5", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "secp256k1" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c42e6f1735c5f00f51e43e28d6634141f2bcad10931b2609ddd74a86d751260" +dependencies = [ + "secp256k1-sys", +] + +[[package]] +name = "secp256k1-sys" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957da2573cde917463ece3570eab4a0b3f19de6f1646cde62e6fd3868f566036" +dependencies = [ + "cc", +] + +[[package]] +name = "secrecy" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" dependencies = [ "zeroize", ] @@ -9015,14 +10153,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" dependencies = [ "semver-parser 0.10.2", - "serde", ] [[package]] name = "semver" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" +checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" +dependencies = [ + "serde", +] [[package]] name = "semver-parser" @@ -9089,6 +10229,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_nanos" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e44969a61f5d316be20a42ff97816efb3b407a924d06824c3d8a49fa8450de0e" +dependencies = [ + "serde", +] + [[package]] name = "sha-1" version = "0.8.2" @@ -9109,7 +10258,7 @@ checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", - "cpufeatures 0.2.1", + "cpufeatures 0.2.2", "digest 0.9.0", "opaque-debug 0.3.0", ] @@ -9149,11 +10298,22 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", - "cpufeatures 0.2.1", + "cpufeatures 0.2.2", "digest 0.9.0", "opaque-debug 0.3.0", ] +[[package]] +name = "sha2" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures 0.2.2", + "digest 0.10.3", +] + [[package]] name = "sha3" version = "0.9.1" @@ -9166,6 +10326,16 @@ dependencies = [ "opaque-debug 0.3.0", ] +[[package]] +name = "sha3" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "881bf8156c87b6301fc5ca6b27f11eeb2761224c7081e69b409d5a1951a70c86" +dependencies = [ + "digest 0.10.3", + "keccak", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -9181,16 +10351,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" -[[package]] -name = "signal-hook" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "647c97df271007dcea485bb74ffdb57f2e683f1306c854f468a0c244badabf2d" -dependencies = [ - "libc", - "signal-hook-registry", -] - [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -9202,9 +10362,12 @@ dependencies = [ [[package]] name = "signature" -version = "1.5.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f054c6c1a6e95179d6f23ed974060dcefb2d9388bb7256900badad682c499de4" +checksum = "02658e48d89f2bec991f9a78e69cfa4c316f8d6a6c4ec12fae1aeb263d486788" +dependencies = [ + "rand_core 0.6.3", +] [[package]] name = "simba" @@ -9215,14 +10378,14 @@ dependencies = [ "approx", "num-complex", "num-traits", - "paste 1.0.6", + "paste 1.0.7", ] [[package]] name = "slab" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "sled" @@ -9232,23 +10395,14 @@ checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" dependencies = [ "crc32fast", "crossbeam-epoch", - "crossbeam-utils 0.8.7", + "crossbeam-utils 0.8.8", "fs2", "fxhash", "libc", - "log 0.4.14", + "log 0.4.16", "parking_lot 0.11.2", ] -[[package]] -name = "slog" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" -dependencies = [ - "erased-serde", -] - [[package]] name = "smallvec" version = "0.6.14" @@ -9277,17 +10431,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6142f7c25e94f6fd25a32c3348ec230df9109b463f59c8c7acc4bd34936babb7" dependencies = [ "aes-gcm", - "blake2", - "chacha20poly1305", + "blake2 0.9.2", + "chacha20poly1305 0.8.0", "rand 0.8.5", "rand_core 0.6.3", "ring", "rustc_version 0.3.3", "sha2 0.9.9", - "subtle 2.4.1", + "subtle", "x25519-dalek", ] +[[package]] +name = "snow" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774d05a3edae07ce6d68ea6984f3c05e9bba8927e3dd591e3b479e5b03213d0d" +dependencies = [ + "aes-gcm", + "blake2 0.10.4", + "chacha20poly1305 0.9.0", + "curve25519-dalek 4.0.0-pre.1", + "rand_core 0.6.3", + "ring", + "rustc_version 0.4.0", + "sha2 0.10.2", + "subtle", +] + [[package]] name = "socket2" version = "0.3.19" @@ -9317,25 +10488,25 @@ checksum = "b5c71ed3d54db0a699f4948e1bb3e45b450fa31fe602621dee6680361d569c88" dependencies = [ "base64 0.12.3", "bytes 0.5.6", - "flate2", "futures 0.3.21", "httparse", - "log 0.4.14", + "log 0.4.16", "rand 0.7.3", "sha-1 0.9.8", ] [[package]] name = "soketto" -version = "0.6.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a74e48087dbeed4833785c2f3352b59140095dc192dce966a3bfc155020a439f" +checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" dependencies = [ "base64 0.13.0", "bytes 1.1.0", + "flate2", "futures 0.3.21", "httparse", - "log 0.4.14", + "log 0.4.16", "rand 0.8.5", "sha-1 0.9.8", ] @@ -9345,8 +10516,8 @@ name = "sp-api" version = "4.0.0-dev" dependencies = [ "hash-db", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "sp-api-proc-macro", "sp-core", "sp-runtime", @@ -9361,7 +10532,7 @@ dependencies = [ name = "sp-api-proc-macro" version = "4.0.0-dev" dependencies = [ - "blake2-rfc", + "blake2 0.10.4", "proc-macro-crate 1.1.3", "proc-macro2", "quote", @@ -9374,8 +10545,8 @@ version = "2.0.1" dependencies = [ "criterion", "futures 0.3.21", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "rustversion", "sc-block-builder", "sp-api", @@ -9391,9 +10562,9 @@ dependencies = [ [[package]] name = "sp-application-crypto" -version = "4.0.0-dev" +version = "6.0.0" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "serde", "sp-core", @@ -9415,12 +10586,12 @@ dependencies = [ [[package]] name = "sp-arithmetic" -version = "4.0.0-dev" +version = "5.0.0" dependencies = [ "criterion", "integer-sqrt", "num-traits", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "primitive-types", "rand 0.7.3", "scale-info", @@ -9444,7 +10615,7 @@ dependencies = [ name = "sp-authority-discovery" version = "4.0.0-dev" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-api", "sp-application-crypto", @@ -9457,7 +10628,7 @@ name = "sp-authorship" version = "4.0.0-dev" dependencies = [ "async-trait", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sp-inherents", "sp-runtime", "sp-std", @@ -9467,7 +10638,7 @@ dependencies = [ name = "sp-block-builder" version = "4.0.0-dev" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sp-api", "sp-inherents", "sp-runtime", @@ -9479,10 +10650,10 @@ name = "sp-blockchain" version = "4.0.0-dev" dependencies = [ "futures 0.3.21", - "log 0.4.14", - "lru", - "parity-scale-codec", - "parking_lot 0.11.2", + "log 0.4.16", + "lru 0.7.5", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "sp-api", "sp-consensus", "sp-database", @@ -9497,9 +10668,9 @@ version = "0.10.0-dev" dependencies = [ "async-trait", "futures 0.3.21", - "futures-timer 3.0.2", - "log 0.4.14", - "parity-scale-codec", + "futures-timer", + "log 0.4.16", + "parity-scale-codec 3.1.2", "sp-core", "sp-inherents", "sp-runtime", @@ -9515,7 +10686,7 @@ name = "sp-consensus-aura" version = "0.10.0-dev" dependencies = [ "async-trait", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-api", "sp-application-crypto", @@ -9533,7 +10704,7 @@ version = "0.10.0-dev" dependencies = [ "async-trait", "merlin", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "serde", "sp-api", @@ -9553,7 +10724,7 @@ dependencies = [ name = "sp-consensus-pow" version = "0.10.0-dev" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sp-api", "sp-core", "sp-runtime", @@ -9564,17 +10735,20 @@ dependencies = [ name = "sp-consensus-slots" version = "0.10.0-dev" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", + "serde", "sp-arithmetic", "sp-runtime", + "sp-std", + "sp-timestamp", ] [[package]] name = "sp-consensus-vrf" version = "0.10.0-dev" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "schnorrkel", "sp-core", "sp-runtime", @@ -9583,9 +10757,10 @@ dependencies = [ [[package]] name = "sp-core" -version = "4.0.0-dev" +version = "6.0.0" dependencies = [ "base58", + "bitflags", "blake2-rfc", "byteorder", "criterion", @@ -9598,48 +10773,72 @@ dependencies = [ "hex-literal", "impl-serde", "lazy_static", - "libsecp256k1 0.6.0", - "log 0.4.14", + "libsecp256k1", + "log 0.4.16", "merlin", "num-traits", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "parity-util-mem", - "parking_lot 0.11.2", + "parking_lot 0.12.0", "primitive-types", "rand 0.7.3", "regex", "scale-info", "schnorrkel", + "secp256k1", "secrecy", "serde", "serde_json", - "sha2 0.9.9", + "sp-core-hashing", + "sp-core-hashing-proc-macro", "sp-debug-derive", "sp-externalities", "sp-runtime-interface", "sp-serializer", "sp-std", "sp-storage", + "ss58-registry", "substrate-bip39", "thiserror", "tiny-bip39", - "tiny-keccak", - "twox-hash", "wasmi", "zeroize", ] +[[package]] +name = "sp-core-hashing" +version = "4.0.0" +dependencies = [ + "blake2 0.10.4", + "byteorder", + "digest 0.10.3", + "sha2 0.10.2", + "sha3 0.10.1", + "sp-std", + "twox-hash", +] + +[[package]] +name = "sp-core-hashing-proc-macro" +version = "5.0.0" +dependencies = [ + "proc-macro2", + "quote", + "sp-core-hashing", + "syn", +] + [[package]] name = "sp-database" version = "4.0.0-dev" dependencies = [ "kvdb", - "parking_lot 0.11.2", + "parking_lot 0.12.0", ] [[package]] name = "sp-debug-derive" -version = "3.0.0" +version = "4.0.0" dependencies = [ "proc-macro2", "quote", @@ -9648,10 +10847,10 @@ dependencies = [ [[package]] name = "sp-externalities" -version = "0.10.0-dev" +version = "0.12.0" dependencies = [ "environmental", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sp-std", "sp-storage", ] @@ -9661,8 +10860,8 @@ name = "sp-finality-grandpa" version = "4.0.0-dev" dependencies = [ "finality-grandpa", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "scale-info", "serde", "sp-api", @@ -9680,7 +10879,7 @@ dependencies = [ "async-trait", "futures 0.3.21", "impl-trait-for-tuples", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sp-core", "sp-runtime", "sp-std", @@ -9689,14 +10888,15 @@ dependencies = [ [[package]] name = "sp-io" -version = "4.0.0-dev" +version = "6.0.0" dependencies = [ "futures 0.3.21", "hash-db", - "libsecp256k1 0.6.0", - "log 0.4.14", - "parity-scale-codec", - "parking_lot 0.11.2", + "libsecp256k1", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", + "secp256k1", "sp-core", "sp-externalities", "sp-keystore", @@ -9712,36 +10912,37 @@ dependencies = [ [[package]] name = "sp-keyring" -version = "4.0.0-dev" +version = "6.0.0" dependencies = [ "lazy_static", "sp-core", "sp-runtime", - "strum 0.20.0", + "strum", ] [[package]] name = "sp-keystore" -version = "0.10.0-dev" +version = "0.12.0" dependencies = [ "async-trait", - "derive_more", "futures 0.3.21", "merlin", - "parity-scale-codec", - "parking_lot 0.11.2", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "rand 0.7.3", "rand_chacha 0.2.2", "schnorrkel", "serde", "sp-core", "sp-externalities", + "thiserror", ] [[package]] name = "sp-maybe-compressed-blob" -version = "4.0.0-dev" +version = "4.1.0-dev" dependencies = [ + "thiserror", "zstd", ] @@ -9749,13 +10950,12 @@ dependencies = [ name = "sp-npos-elections" version = "4.0.0-dev" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "rand 0.7.3", "scale-info", "serde", "sp-arithmetic", "sp-core", - "sp-npos-elections-solution-type", "sp-runtime", "sp-std", "substrate-test-utils", @@ -9765,28 +10965,13 @@ dependencies = [ name = "sp-npos-elections-fuzzer" version = "2.0.0-alpha.5" dependencies = [ + "clap 3.1.8", "honggfuzz", - "parity-scale-codec", - "rand 0.7.3", + "parity-scale-codec 3.1.2", + "rand 0.8.5", "scale-info", "sp-npos-elections", "sp-runtime", - "structopt", -] - -[[package]] -name = "sp-npos-elections-solution-type" -version = "4.0.0-dev" -dependencies = [ - "parity-scale-codec", - "proc-macro-crate 1.1.3", - "proc-macro2", - "quote", - "scale-info", - "sp-arithmetic", - "sp-npos-elections", - "syn", - "trybuild", ] [[package]] @@ -9800,14 +10985,16 @@ dependencies = [ [[package]] name = "sp-panic-handler" -version = "3.0.0" +version = "4.0.0" dependencies = [ "backtrace", + "lazy_static", + "regex", ] [[package]] name = "sp-rpc" -version = "4.0.0-dev" +version = "6.0.0" dependencies = [ "rustc-hash", "serde", @@ -9817,15 +11004,15 @@ dependencies = [ [[package]] name = "sp-runtime" -version = "4.0.0-dev" +version = "6.0.0" dependencies = [ "either", "hash256-std-hasher", "impl-trait-for-tuples", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "parity-util-mem", - "paste 1.0.6", + "paste 1.0.7", "rand 0.7.3", "scale-info", "serde", @@ -9839,14 +11026,15 @@ dependencies = [ "sp-std", "sp-tracing", "substrate-test-runtime-client", + "zstd", ] [[package]] name = "sp-runtime-interface" -version = "4.0.0-dev" +version = "6.0.0" dependencies = [ "impl-trait-for-tuples", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "primitive-types", "rustversion", "sp-core", @@ -9865,7 +11053,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" -version = "4.0.0-dev" +version = "5.0.0" dependencies = [ "Inflector", "proc-macro-crate 1.1.3", @@ -9917,8 +11105,8 @@ name = "sp-sandbox" version = "0.10.0-dev" dependencies = [ "assert_matches", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "sp-core", "sp-io", "sp-std", @@ -9929,7 +11117,7 @@ dependencies = [ [[package]] name = "sp-serializer" -version = "3.0.0" +version = "4.0.0-dev" dependencies = [ "serde", "serde_json", @@ -9939,7 +11127,7 @@ dependencies = [ name = "sp-session" version = "4.0.0-dev" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-api", "sp-core", @@ -9952,7 +11140,7 @@ dependencies = [ name = "sp-staking" version = "4.0.0-dev" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-runtime", "sp-std", @@ -9960,15 +11148,15 @@ dependencies = [ [[package]] name = "sp-state-machine" -version = "0.10.0-dev" +version = "0.12.0" dependencies = [ "hash-db", "hex-literal", - "log 0.4.14", + "log 0.4.16", "num-traits", - "parity-scale-codec", - "parking_lot 0.11.2", - "pretty_assertions 0.6.1", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", + "pretty_assertions", "rand 0.7.3", "smallvec 1.8.0", "sp-core", @@ -9979,20 +11167,19 @@ dependencies = [ "sp-trie", "thiserror", "tracing", - "trie-db", "trie-root", ] [[package]] name = "sp-std" -version = "4.0.0-dev" +version = "4.0.0" [[package]] name = "sp-storage" -version = "4.0.0-dev" +version = "6.0.0" dependencies = [ "impl-serde", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "ref-cast", "serde", "sp-debug-derive", @@ -10003,8 +11190,8 @@ dependencies = [ name = "sp-tasks" version = "4.0.0-dev" dependencies = [ - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "sp-core", "sp-externalities", "sp-io", @@ -10016,7 +11203,7 @@ dependencies = [ name = "sp-test-primitives" version = "2.0.0" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "parity-util-mem", "serde", "sp-application-crypto", @@ -10029,9 +11216,9 @@ name = "sp-timestamp" version = "4.0.0-dev" dependencies = [ "async-trait", - "futures-timer 3.0.2", - "log 0.4.14", - "parity-scale-codec", + "futures-timer", + "log 0.4.16", + "parity-scale-codec 3.1.2", "sp-api", "sp-inherents", "sp-runtime", @@ -10041,15 +11228,9 @@ dependencies = [ [[package]] name = "sp-tracing" -version = "4.0.0-dev" +version = "5.0.0" dependencies = [ - "erased-serde", - "log 0.4.14", - "parity-scale-codec", - "parking_lot 0.10.2", - "serde", - "serde_json", - "slog", + "parity-scale-codec 3.1.2", "sp-std", "tracing", "tracing-core", @@ -10069,8 +11250,8 @@ name = "sp-transaction-storage-proof" version = "4.0.0-dev" dependencies = [ "async-trait", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-inherents", @@ -10081,17 +11262,18 @@ dependencies = [ [[package]] name = "sp-trie" -version = "4.0.0-dev" +version = "6.0.0" dependencies = [ "criterion", "hash-db", "hex-literal", "memory-db", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "scale-info", "sp-core", "sp-runtime", "sp-std", + "thiserror", "trie-bench", "trie-db", "trie-root", @@ -10100,13 +11282,14 @@ dependencies = [ [[package]] name = "sp-version" -version = "4.0.0-dev" +version = "5.0.0" dependencies = [ "impl-serde", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "parity-wasm 0.42.2", "scale-info", "serde", + "sp-core-hashing-proc-macro", "sp-runtime", "sp-std", "sp-version-proc-macro", @@ -10117,7 +11300,7 @@ dependencies = [ name = "sp-version-proc-macro" version = "4.0.0-dev" dependencies = [ - "parity-scale-codec", + "parity-scale-codec 3.1.2", "proc-macro2", "quote", "sp-version", @@ -10126,12 +11309,14 @@ dependencies = [ [[package]] name = "sp-wasm-interface" -version = "4.0.0-dev" +version = "6.0.0" dependencies = [ "impl-trait-for-tuples", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "sp-std", "wasmi", + "wasmtime", ] [[package]] @@ -10140,6 +11325,31 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spki" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d01ac02a6ccf3e07db148d2be087da624fea0221a16152ed01f0496a6b0a27" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "ss58-registry" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b84a70894df7a73666e0694f44b41a9571625e9546fb58a0818a565d2c7e084" +dependencies = [ + "Inflector", + "num-format", + "proc-macro2", + "quote", + "serde", + "serde_json", + "unicode-xid", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -10183,7 +11393,7 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" dependencies = [ - "clap", + "clap 2.34.0", "lazy_static", "structopt-derive", ] @@ -10203,43 +11413,23 @@ dependencies = [ [[package]] name = "strum" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7318c509b5ba57f18533982607f24070a55d353e90d4cae30c467cdb2ad5ac5c" -dependencies = [ - "strum_macros 0.20.1", -] - -[[package]] -name = "strum" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf86bbcfd1fa9670b7a129f64fc0c9fcbbfe4f1bc4210e9e98fe71ffc12cde2" -dependencies = [ - "strum_macros 0.21.1", -] - -[[package]] -name = "strum_macros" -version = "0.20.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee8bc6b87a5112aeeab1f4a9f7ab634fe6cbefc4850006df31267f4cfb9e3149" +checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" dependencies = [ - "heck 0.3.3", - "proc-macro2", - "quote", - "syn", + "strum_macros", ] [[package]] name = "strum_macros" -version = "0.21.1" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d06aaeeee809dbc59eb4556183dd927df67db1540de5be8d3ec0b6636358a5ec" +checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" dependencies = [ "heck 0.3.3", "proc-macro2", "quote", + "rustversion", "syn", ] @@ -10247,8 +11437,8 @@ dependencies = [ name = "subkey" version = "2.0.1" dependencies = [ + "clap 3.1.8", "sc-cli", - "structopt", ] [[package]] @@ -10275,12 +11465,12 @@ dependencies = [ name = "substrate-frame-cli" version = "4.0.0-dev" dependencies = [ + "clap 3.1.8", "frame-support", "frame-system", "sc-cli", "sp-core", "sp-runtime", - "structopt", ] [[package]] @@ -10291,7 +11481,7 @@ dependencies = [ "frame-system", "futures 0.3.21", "jsonrpc-client-transports", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sc-rpc-api", "scale-info", "serde", @@ -10308,8 +11498,8 @@ dependencies = [ "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", - "log 0.4.14", - "parity-scale-codec", + "log 0.4.16", + "parity-scale-codec 3.1.2", "sc-client-api", "sc-rpc-api", "sc-transaction-pool", @@ -10325,17 +11515,39 @@ dependencies = [ [[package]] name = "substrate-prometheus-endpoint" -version = "0.9.0" +version = "0.10.0-dev" dependencies = [ - "async-std", - "derive_more", "futures-util", - "hyper 0.14.17", - "log 0.4.14", + "hyper 0.14.18", + "log 0.4.16", "prometheus", + "thiserror", "tokio", ] +[[package]] +name = "substrate-state-trie-migration-rpc" +version = "4.0.0-dev" +dependencies = [ + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "log 0.4.16", + "parity-scale-codec 3.1.2", + "sc-client-api", + "sc-rpc-api", + "scale-info", + "serde", + "serde_json", + "sp-core", + "sp-io", + "sp-runtime", + "sp-state-machine", + "sp-std", + "sp-trie", + "trie-db", +] + [[package]] name = "substrate-test-client" version = "2.0.1" @@ -10343,12 +11555,11 @@ dependencies = [ "async-trait", "futures 0.3.21", "hex", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sc-client-api", "sc-client-db", "sc-consensus", "sc-executor", - "sc-light", "sc-offchain", "sc-service", "serde", @@ -10366,16 +11577,17 @@ dependencies = [ name = "substrate-test-runtime" version = "2.0.0" dependencies = [ + "beefy-primitives", "cfg-if 1.0.0", "frame-support", "frame-system", "frame-system-rpc-runtime-api", "futures 0.3.21", - "log 0.4.14", + "log 0.4.16", "memory-db", "pallet-babe", "pallet-timestamp", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "parity-util-mem", "sc-block-builder", "sc-executor", @@ -10413,11 +11625,10 @@ name = "substrate-test-runtime-client" version = "2.0.0" dependencies = [ "futures 0.3.21", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "sc-block-builder", "sc-client-api", "sc-consensus", - "sc-light", "sp-api", "sp-blockchain", "sp-consensus", @@ -10431,15 +11642,15 @@ dependencies = [ name = "substrate-test-runtime-transaction-pool" version = "2.0.0" dependencies = [ - "derive_more", "futures 0.3.21", - "parity-scale-codec", - "parking_lot 0.11.2", + "parity-scale-codec 3.1.2", + "parking_lot 0.12.0", "sc-transaction-pool", "sc-transaction-pool-api", "sp-blockchain", "sp-runtime", "substrate-test-runtime-client", + "thiserror", ] [[package]] @@ -10476,22 +11687,17 @@ dependencies = [ name = "substrate-wasm-builder" version = "5.0.0-dev" dependencies = [ - "ansi_term 0.12.1", + "ansi_term", "build-helper", "cargo_metadata", "sp-maybe-compressed-blob", + "strum", "tempfile", "toml", "walkdir", "wasm-gc-api", ] -[[package]] -name = "subtle" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" - [[package]] name = "subtle" version = "2.4.1" @@ -10500,9 +11706,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.86" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" +checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" dependencies = [ "proc-macro2", "quote", @@ -10527,12 +11733,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" -[[package]] -name = "target-lexicon" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422045212ea98508ae3d28025bc5aaa2bd4a9cdaecd442a08da2ee620ee9ea95" - [[package]] name = "target-lexicon" version = "0.12.3" @@ -10548,7 +11748,7 @@ dependencies = [ "cfg-if 1.0.0", "fastrand", "libc", - "redox_syscall 0.2.11", + "redox_syscall 0.2.13", "remove_dir_all", "winapi 0.3.9", ] @@ -10568,72 +11768,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" -[[package]] -name = "test-runner" -version = "0.9.0" -dependencies = [ - "frame-system", - "futures 0.3.21", - "jsonrpc-core", - "log 0.4.14", - "num-traits", - "sc-basic-authorship", - "sc-cli", - "sc-client-api", - "sc-consensus", - "sc-consensus-babe", - "sc-consensus-manual-seal", - "sc-executor", - "sc-finality-grandpa", - "sc-informant", - "sc-network", - "sc-rpc", - "sc-rpc-server", - "sc-service", - "sc-transaction-pool", - "sc-transaction-pool-api", - "sp-api", - "sp-block-builder", - "sp-blockchain", - "sp-consensus", - "sp-consensus-babe", - "sp-core", - "sp-externalities", - "sp-finality-grandpa", - "sp-inherents", - "sp-keyring", - "sp-offchain", - "sp-runtime", - "sp-runtime-interface", - "sp-session", - "sp-state-machine", - "sp-transaction-pool", - "sp-wasm-interface", - "tokio", -] - -[[package]] -name = "test-runner-example" -version = "0.1.0" -dependencies = [ - "frame-benchmarking", - "frame-system", - "node-cli", - "node-primitives", - "node-runtime", - "pallet-transaction-payment", - "sc-consensus", - "sc-consensus-babe", - "sc-consensus-manual-seal", - "sc-executor", - "sc-finality-grandpa", - "sc-service", - "sp-consensus-babe", - "sp-keyring", - "sp-runtime", - "test-runner", -] - [[package]] name = "textwrap" version = "0.11.0" @@ -10643,6 +11777,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "textwrap" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" + [[package]] name = "thiserror" version = "1.0.30" @@ -10663,6 +11803,12 @@ dependencies = [ "syn", ] +[[package]] +name = "thousands" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" + [[package]] name = "thread_local" version = "1.1.4" @@ -10681,6 +11827,17 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.4.3+5.2.1-patched.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1792ccb507d955b46af42c123ea8863668fae24d03721e40cad6a41773dbb49" +dependencies = [ + "cc", + "fs_extra", + "libc", +] + [[package]] name = "time" version = "0.1.44" @@ -10754,7 +11911,7 @@ dependencies = [ "bytes 1.1.0", "libc", "memchr", - "mio 0.8.1", + "mio 0.8.2", "num_cpus", "once_cell", "parking_lot 0.12.0", @@ -10794,7 +11951,7 @@ checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" dependencies = [ "bytes 0.4.12", "futures 0.1.31", - "log 0.4.14", + "log 0.4.16", ] [[package]] @@ -10827,7 +11984,7 @@ dependencies = [ "crossbeam-utils 0.7.2", "futures 0.1.31", "lazy_static", - "log 0.4.14", + "log 0.4.16", "mio 0.6.23", "num_cpus", "parking_lot 0.9.0", @@ -10843,9 +12000,20 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ - "rustls", + "rustls 0.19.1", + "tokio", + "webpki 0.21.4", +] + +[[package]] +name = "tokio-rustls" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4151fda0cf2798550ad0b34bcfc9b9dcc2a9d2471c895c68f3a8818e54f2389e" +dependencies = [ + "rustls 0.20.4", "tokio", - "webpki", + "webpki 0.22.0", ] [[package]] @@ -10883,6 +12051,19 @@ dependencies = [ "tokio-reactor", ] +[[package]] +name = "tokio-test" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3" +dependencies = [ + "async-stream", + "bytes 1.1.0", + "futures-core", + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-tls" version = "0.2.1" @@ -10899,14 +12080,28 @@ name = "tokio-util" version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +dependencies = [ + "bytes 1.1.0", + "futures-core", + "futures-sink", + "log 0.4.16", + "pin-project-lite 0.2.8", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" dependencies = [ "bytes 1.1.0", "futures-core", "futures-io", "futures-sink", - "log 0.4.14", "pin-project-lite 0.2.8", "tokio", + "tracing", ] [[package]] @@ -10931,7 +12126,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" dependencies = [ "cfg-if 1.0.0", - "log 0.4.14", + "log 0.4.16", "pin-project-lite 0.2.8", "tracing-attributes", "tracing-core", @@ -10950,9 +12145,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c" +checksum = "90442985ee2f57c9e1b548ee72ae842f4a9a20e3f417cc38dbc5dc684d9bb4ee" dependencies = [ "lazy_static", "valuable", @@ -10977,7 +12172,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" dependencies = [ "lazy_static", - "log 0.4.14", + "log 0.4.16", "tracing-core", ] @@ -10997,10 +12192,11 @@ version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" dependencies = [ - "ansi_term 0.12.1", + "ansi_term", "chrono", "lazy_static", "matchers", + "parking_lot 0.11.2", "regex", "serde", "serde_json", @@ -11021,15 +12217,15 @@ checksum = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" [[package]] name = "trie-bench" -version = "0.28.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4edd9bdf0c2e08fd77c0fb2608179cac7ebed997ae18f58d47a2d96425ff51f0" +checksum = "57ecec5d10427b35e9ae374b059dccc0801d02d832617c04c78afc7a8c5c4a34" dependencies = [ "criterion", "hash-db", "keccak-hasher", "memory-db", - "parity-scale-codec", + "parity-scale-codec 3.1.2", "trie-db", "trie-root", "trie-standardmap", @@ -11037,22 +12233,22 @@ dependencies = [ [[package]] name = "trie-db" -version = "0.22.6" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eac131e334e81b6b3be07399482042838adcd7957aa0010231d0813e39e02fa" +checksum = "d32d034c0d3db64b43c31de38e945f15b40cd4ca6d2dcfc26d4798ce8de4ab83" dependencies = [ "hash-db", - "hashbrown", - "log 0.4.14", + "hashbrown 0.12.0", + "log 0.4.16", "rustc-hex", "smallvec 1.8.0", ] [[package]] name = "trie-root" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "652931506d2c1244d7217a70b99f56718a7b4161b37f04e7cd868072a99f68cd" +checksum = "9a36c5ca3911ed3c9a5416ee6c679042064b93fc637ded67e25f92e68d783891" dependencies = [ "hash-db", ] @@ -11076,14 +12272,39 @@ dependencies = [ "async-trait", "cfg-if 1.0.0", "data-encoding", - "enum-as-inner", + "enum-as-inner 0.3.4", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.2.3", + "ipnet", + "lazy_static", + "log 0.4.16", + "rand 0.8.5", + "smallvec 1.8.0", + "thiserror", + "tinyvec", + "tokio", + "url 2.2.2", +] + +[[package]] +name = "trust-dns-proto" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c31f240f59877c3d4bb3b3ea0ec5a6a0cff07323580ff8c7a605cd7d08b255d" +dependencies = [ + "async-trait", + "cfg-if 1.0.0", + "data-encoding", + "enum-as-inner 0.4.0", "futures-channel", "futures-io", "futures-util", "idna 0.2.3", "ipnet", "lazy_static", - "log 0.4.14", + "log 0.4.16", "rand 0.8.5", "smallvec 1.8.0", "thiserror", @@ -11100,16 +12321,36 @@ checksum = "ecae383baad9995efaa34ce8e57d12c3f305e545887472a492b838f4b5cfb77a" dependencies = [ "cfg-if 1.0.0", "futures-util", - "ipconfig", + "ipconfig 0.2.2", "lazy_static", - "log 0.4.14", + "log 0.4.16", "lru-cache", "parking_lot 0.11.2", "resolv-conf", "smallvec 1.8.0", "thiserror", "tokio", - "trust-dns-proto", + "trust-dns-proto 0.20.4", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ba72c2ea84515690c9fcef4c6c660bb9df3036ed1051686de84605b74fd558" +dependencies = [ + "cfg-if 1.0.0", + "futures-util", + "ipconfig 0.3.0", + "lazy_static", + "log 0.4.16", + "lru-cache", + "parking_lot 0.12.0", + "resolv-conf", + "smallvec 1.8.0", + "thiserror", + "tokio", + "trust-dns-proto 0.21.2", ] [[package]] @@ -11122,9 +12363,10 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" name = "try-runtime-cli" version = "0.10.0-dev" dependencies = [ - "jsonrpsee-ws-client", - "log 0.4.14", - "parity-scale-codec", + "clap 3.1.8", + "jsonrpsee", + "log 0.4.16", + "parity-scale-codec 3.1.2", "remote-externalities", "sc-chain-spec", "sc-cli", @@ -11138,24 +12380,31 @@ dependencies = [ "sp-runtime", "sp-state-machine", "sp-version", - "structopt", + "zstd", ] [[package]] name = "trybuild" -version = "1.0.56" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d60539445867cdd9680b2bfe2d0428f1814b7d5c9652f09d8d3eae9d19308db" +checksum = "606ab3fe0065741fdbb51f64bcb6ba76f13fad49f1723030041826c631782764" dependencies = [ "dissimilar", "glob", "once_cell", "serde", + "serde_derive", "serde_json", "termcolor", "toml", ] +[[package]] +name = "tt-call" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e66dcbec4290c69dd03c57e76c2469ea5c7ce109c6dd4351c13055cf71ea055" + [[package]] name = "twox-hash" version = "1.6.2" @@ -11163,6 +12412,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee73e6e4924fe940354b8d4d98cad5231175d615cd855b758adc658c0aac6a0" dependencies = [ "cfg-if 1.0.0", + "digest 0.10.3", "rand 0.8.5", "static_assertions", ] @@ -11255,7 +12505,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" dependencies = [ "generic-array 0.14.5", - "subtle 2.4.1", + "subtle", ] [[package]] @@ -11401,7 +12651,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ - "log 0.4.14", + "log 0.4.16", "try-lock", ] @@ -11425,9 +12675,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -11435,13 +12685,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" dependencies = [ "bumpalo", "lazy_static", - "log 0.4.14", + "log 0.4.16", "proc-macro2", "quote", "syn", @@ -11450,9 +12700,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" +checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -11462,9 +12712,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -11472,9 +12722,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" dependencies = [ "proc-macro2", "quote", @@ -11485,9 +12735,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" [[package]] name = "wasm-gc-api" @@ -11495,11 +12745,20 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0c32691b6c7e6c14e7f8fd55361a9088b507aa49620fcd06c09b3a1082186b9" dependencies = [ - "log 0.4.14", + "log 0.4.16", "parity-wasm 0.32.0", "rustc-demangle", ] +[[package]] +name = "wasm-instrument" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962e5b0401bbb6c887f54e69b8c496ea36f704df65db73e81fd5ff8dc3e63a9f" +dependencies = [ + "parity-wasm 0.42.2", +] + [[package]] name = "wasm-timer" version = "0.2.5" @@ -11517,21 +12776,25 @@ dependencies = [ [[package]] name = "wasmer" -version = "1.0.2" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a70cfae554988d904d64ca17ab0e7cd652ee5c8a0807094819c1ea93eb9d6866" +checksum = "f727a39e7161f7438ddb8eafe571b67c576a8c2fb459f666d9053b5bba4afdea" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "indexmap", + "js-sys", + "loupe", "more-asserts", - "target-lexicon 0.11.2", + "target-lexicon", "thiserror", + "wasm-bindgen", "wasmer-compiler", "wasmer-compiler-cranelift", + "wasmer-compiler-singlepass", "wasmer-derive", "wasmer-engine", - "wasmer-engine-jit", - "wasmer-engine-native", + "wasmer-engine-dylib", + "wasmer-engine-universal", "wasmer-types", "wasmer-vm", "wat", @@ -11540,34 +12803,38 @@ dependencies = [ [[package]] name = "wasmer-compiler" -version = "1.0.2" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7732a9cab472bd921d5a0c422f45b3d03f62fa2c40a89e0770cef6d47e383e" +checksum = "4e9951599222eb12bd13d4d91bcded0a880e4c22c2dfdabdf5dc7e5e803b7bf3" dependencies = [ "enumset", + "loupe", + "rkyv", "serde", "serde_bytes", "smallvec 1.8.0", - "target-lexicon 0.11.2", + "target-lexicon", "thiserror", "wasmer-types", "wasmer-vm", - "wasmparser 0.65.0", + "wasmparser 0.78.2", ] [[package]] name = "wasmer-compiler-cranelift" -version = "1.0.2" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb9395f094e1d81534f4c5e330ed4cdb424e8df870d29ad585620284f5fddb" +checksum = "44c83273bce44e668f3a2b9ccb7f1193db918b1d6806f64acc5ff71f6ece5f20" dependencies = [ - "cranelift-codegen 0.68.0", - "cranelift-frontend 0.68.0", - "gimli 0.22.0", + "cranelift-codegen 0.76.0", + "cranelift-entity 0.76.0", + "cranelift-frontend 0.76.0", + "gimli 0.25.0", + "loupe", "more-asserts", "rayon", - "serde", "smallvec 1.8.0", + "target-lexicon", "tracing", "wasmer-compiler", "wasmer-types", @@ -11576,17 +12843,17 @@ dependencies = [ [[package]] name = "wasmer-compiler-singlepass" -version = "1.0.2" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "426ae6ef0f606ca815510f3e2ef6f520e217514bfb7a664defe180b9a9e75d07" +checksum = "5432e993840cdb8e6875ddc8c9eea64e7a129579b4706bd91b8eb474d9c4a860" dependencies = [ "byteorder", "dynasm", "dynasmrt", "lazy_static", + "loupe", "more-asserts", "rayon", - "serde", "smallvec 1.8.0", "wasmer-compiler", "wasmer-types", @@ -11595,9 +12862,9 @@ dependencies = [ [[package]] name = "wasmer-derive" -version = "1.0.2" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8b86dcd2c3efdb8390728a2b56f762db07789aaa5aa872a9dc776ba3a7912ed" +checksum = "458dbd9718a837e6dbc52003aef84487d79eedef5fa28c7d28b6784be98ac08e" dependencies = [ "proc-macro-error", "proc-macro2", @@ -11607,19 +12874,20 @@ dependencies = [ [[package]] name = "wasmer-engine" -version = "1.0.2" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efe4667d6bd888f26ae8062a63a9379fa697415b4b4e380f33832e8418fd71b5" +checksum = "6ed603a6d037ebbb14014d7f739ae996a78455a4b86c41cfa4e81c590a1253b9" dependencies = [ "backtrace", - "bincode", + "enumset", "lazy_static", - "memmap2 0.2.3", + "loupe", + "memmap2 0.5.3", "more-asserts", "rustc-demangle", "serde", "serde_bytes", - "target-lexicon 0.11.2", + "target-lexicon", "thiserror", "wasmer-compiler", "wasmer-types", @@ -11627,51 +12895,57 @@ dependencies = [ ] [[package]] -name = "wasmer-engine-jit" -version = "1.0.2" +name = "wasmer-engine-dylib" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26770be802888011b4a3072f2a282fc2faa68aa48c71b3db6252a3937a85f3da" +checksum = "ccd7fdc60e252a795c849b3f78a81a134783051407e7e279c10b7019139ef8dc" dependencies = [ - "bincode", - "cfg-if 0.1.10", - "region", + "cfg-if 1.0.0", + "enum-iterator", + "enumset", + "leb128", + "libloading 0.7.3", + "loupe", + "object 0.28.3", + "rkyv", "serde", - "serde_bytes", + "tempfile", + "tracing", "wasmer-compiler", "wasmer-engine", + "wasmer-object", "wasmer-types", "wasmer-vm", - "winapi 0.3.9", + "which", ] [[package]] -name = "wasmer-engine-native" -version = "1.0.2" +name = "wasmer-engine-universal" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb4083a6c69f2cd4b000b82a80717f37c6cc2e536aee3a8ffe9af3edc276a8b" +checksum = "dcff0cd2c01a8de6009fd863b14ea883132a468a24f2d2ee59dc34453d3a31b5" dependencies = [ - "bincode", - "cfg-if 0.1.10", + "cfg-if 1.0.0", + "enum-iterator", + "enumset", "leb128", - "libloading 0.6.7", - "serde", - "tempfile", - "tracing", + "loupe", + "region 3.0.0", + "rkyv", "wasmer-compiler", "wasmer-engine", - "wasmer-object", "wasmer-types", "wasmer-vm", - "which", + "winapi 0.3.9", ] [[package]] name = "wasmer-object" -version = "1.0.2" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf8e0c12b82ff81ebecd30d7e118be5fec871d6de885a90eeb105df0a769a7b" +checksum = "24ce18ac2877050e59580d27ee1a88f3192d7a31e77fbba0852abc7888d6e0b5" dependencies = [ - "object 0.22.0", + "object 0.28.3", "thiserror", "wasmer-compiler", "wasmer-types", @@ -11679,29 +12953,34 @@ dependencies = [ [[package]] name = "wasmer-types" -version = "1.0.2" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f4ac28c2951cd792c18332f03da523ed06b170f5cf6bb5b1bdd7e36c2a8218" +checksum = "659fa3dd6c76f62630deff4ac8c7657b07f0b1e4d7e0f8243a552b9d9b448e24" dependencies = [ - "cranelift-entity 0.68.0", + "indexmap", + "loupe", + "rkyv", "serde", "thiserror", ] [[package]] name = "wasmer-vm" -version = "1.0.2" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7635ba0b6d2fd325f588d69a950ad9fa04dddbf6ad08b6b2a183146319bf6ae" +checksum = "afdc46158517c2769f9938bc222a7d41b3bb330824196279d8aa2d667cd40641" dependencies = [ "backtrace", "cc", - "cfg-if 0.1.10", + "cfg-if 1.0.0", + "enum-iterator", "indexmap", "libc", + "loupe", "memoffset", "more-asserts", - "region", + "region 3.0.0", + "rkyv", "serde", "thiserror", "wasmer-types", @@ -11736,21 +13015,21 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.65.0" +version = "0.78.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc2fe6350834b4e528ba0901e7aa405d78b89dc1fa3145359eb4de0e323fcf" +checksum = "52144d4c78e5cf8b055ceab8e5fa22814ce4315d6002ad32cfd914f37c12fd65" [[package]] name = "wasmparser" -version = "0.79.0" +version = "0.81.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b5894be15a559c85779254700e1d35f02f843b5a69152e5c82c626d9fd66c0e" +checksum = "98930446519f63d00a836efdc22f67766ceae8dbcc1571379f2bcabc6b2b9abc" [[package]] name = "wasmtime" -version = "0.29.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bbb8a082a8ef50f7eeb8b82dda9709ef1e68963ea3c94e45581644dd4041835" +checksum = "4c9c724da92e39a85d2231d4c2a942c8be295211441dbca581c6c3f3f45a9f00" dependencies = [ "anyhow", "backtrace", @@ -11760,37 +13039,37 @@ dependencies = [ "indexmap", "lazy_static", "libc", - "log 0.4.14", - "paste 1.0.6", + "log 0.4.16", + "object 0.27.1", + "paste 1.0.7", "psm", - "region", + "rayon", + "region 2.2.0", "rustc-demangle", "serde", - "smallvec 1.8.0", - "target-lexicon 0.12.3", - "wasmparser 0.79.0", + "target-lexicon", + "wasmparser 0.81.0", "wasmtime-cache", + "wasmtime-cranelift", "wasmtime-environ", "wasmtime-jit", - "wasmtime-profiling", "wasmtime-runtime", "winapi 0.3.9", ] [[package]] name = "wasmtime-cache" -version = "0.29.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d73391579ca7f24573138ef768b73b2aed5f9d542385c64979b65d60d0912399" +checksum = "da4439d99100298344567c0eb6916ad5864e99e54760b8177c427e529077fb30" dependencies = [ "anyhow", "base64 0.13.0", "bincode", "directories-next", - "errno", "file-per-thread-logger", - "libc", - "log 0.4.14", + "log 0.4.16", + "rustix", "serde", "sha2 0.9.9", "toml", @@ -11800,122 +13079,73 @@ dependencies = [ [[package]] name = "wasmtime-cranelift" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81c6f5ae9205382345c7cd7454932a906186836999a2161c385e38a15f52e1fe" -dependencies = [ - "cranelift-codegen 0.76.0", - "cranelift-entity 0.76.0", - "cranelift-frontend 0.76.0", - "cranelift-wasm", - "target-lexicon 0.12.3", - "wasmparser 0.79.0", - "wasmtime-environ", -] - -[[package]] -name = "wasmtime-debug" -version = "0.29.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c69e08f55e12f15f50b1b533bc3626723e7224254a065de6576934c86258c9e8" +checksum = "1762765dd69245f00e5d9783b695039e449a7be0f9c5383e4c78465dd6131aeb" dependencies = [ "anyhow", - "gimli 0.25.0", + "cranelift-codegen 0.80.1", + "cranelift-entity 0.80.1", + "cranelift-frontend 0.80.1", + "cranelift-native", + "cranelift-wasm", + "gimli 0.26.1", + "log 0.4.16", "more-asserts", - "object 0.26.2", - "target-lexicon 0.12.3", + "object 0.27.1", + "target-lexicon", "thiserror", - "wasmparser 0.79.0", + "wasmparser 0.81.0", "wasmtime-environ", ] [[package]] name = "wasmtime-environ" -version = "0.29.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005d93174040af37fb8625f891cd9827afdad314261f7ec4ee61ec497d6e9d3c" +checksum = "c4468301d95ec71710bb6261382efe27d1296447711645e3dbabaea6e4de3504" dependencies = [ - "cfg-if 1.0.0", - "cranelift-codegen 0.76.0", - "cranelift-entity 0.76.0", - "cranelift-wasm", - "gimli 0.25.0", + "anyhow", + "cranelift-entity 0.80.1", + "gimli 0.26.1", "indexmap", - "log 0.4.14", + "log 0.4.16", "more-asserts", + "object 0.27.1", "serde", + "target-lexicon", "thiserror", - "wasmparser 0.79.0", + "wasmparser 0.81.0", + "wasmtime-types", ] [[package]] name = "wasmtime-jit" -version = "0.29.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0bf1dfb213a35d8f21aefae40e597fe72778a907011ffdff7affb029a02af9a" +checksum = "ab0ae6e581ff014b470ec35847ea3c0b4c3ace89a55df5a04c802a11f4574e7d" dependencies = [ - "addr2line 0.16.0", + "addr2line", "anyhow", + "bincode", "cfg-if 1.0.0", - "cranelift-codegen 0.76.0", - "cranelift-entity 0.76.0", - "cranelift-frontend 0.76.0", - "cranelift-native", - "cranelift-wasm", - "gimli 0.25.0", - "log 0.4.14", - "more-asserts", - "object 0.26.2", - "rayon", - "region", + "gimli 0.26.1", + "object 0.27.1", + "region 2.2.0", + "rustix", "serde", - "target-lexicon 0.12.3", + "target-lexicon", "thiserror", - "wasmparser 0.79.0", - "wasmtime-cranelift", - "wasmtime-debug", "wasmtime-environ", - "wasmtime-obj", - "wasmtime-profiling", "wasmtime-runtime", "winapi 0.3.9", ] -[[package]] -name = "wasmtime-obj" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231491878e710c68015228c9f9fc5955fe5c96dbf1485c15f7bed55b622c83c" -dependencies = [ - "anyhow", - "more-asserts", - "object 0.26.2", - "target-lexicon 0.12.3", - "wasmtime-debug", - "wasmtime-environ", -] - -[[package]] -name = "wasmtime-profiling" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21486cfb5255c2069666c1f116f9e949d4e35c9a494f11112fa407879e42198d" -dependencies = [ - "anyhow", - "cfg-if 1.0.0", - "lazy_static", - "libc", - "serde", - "target-lexicon 0.12.3", - "wasmtime-environ", - "wasmtime-runtime", -] - [[package]] name = "wasmtime-runtime" -version = "0.29.0" +version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7ddfdf32e0a20d81f48be9dacd31612bc61de5a174d1356fef806d300f507de" +checksum = "6d9c28877ae37a367cda7b52b8887589816152e95dde9b7c80cc686f52761961" dependencies = [ "anyhow", "backtrace", @@ -11924,22 +13154,35 @@ dependencies = [ "indexmap", "lazy_static", "libc", - "log 0.4.14", + "log 0.4.16", "mach", "memoffset", "more-asserts", "rand 0.8.5", - "region", + "region 2.2.0", + "rustix", "thiserror", "wasmtime-environ", "winapi 0.3.9", ] +[[package]] +name = "wasmtime-types" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395726e8f5dd8c57cb0db445627b842343f7e29ed7489467fdf7953ed9d3cd4f" +dependencies = [ + "cranelift-entity 0.80.1", + "serde", + "thiserror", + "wasmparser 0.81.0", +] + [[package]] name = "wast" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9bbbd53432b267421186feee3e52436531fa69a7cfee9403f5204352df3dd05" +checksum = "9bb4f48a8b083dbc50e291e430afb8f524092bb00428957bcc63f49f856c64ac" dependencies = [ "leb128", "memchr", @@ -11948,18 +13191,18 @@ dependencies = [ [[package]] name = "wat" -version = "1.0.41" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab98ed25494f97c69f28758617f27c3e92e5336040b5c3a14634f2dd3fe61830" +checksum = "0401b6395ce0db91629a75b29597ccb66ea29950af9fc859f1bb3a736609c76e" dependencies = [ "wast", ] [[package]] name = "web-sys" -version = "0.3.56" +version = "0.3.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" dependencies = [ "js-sys", "wasm-bindgen", @@ -11975,13 +13218,32 @@ dependencies = [ "untrusted", ] +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "webpki-roots" version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" dependencies = [ - "webpki", + "webpki 0.21.4", +] + +[[package]] +name = "webpki-roots" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d8de8415c823c8abd270ad483c6feeac771fad964890779f9a8cb24fbbc1bf" +dependencies = [ + "webpki 0.22.0", ] [[package]] @@ -12036,9 +13298,9 @@ dependencies = [ [[package]] name = "which" -version = "4.2.4" +version = "4.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a5a7e487e921cf220206864a94a89b6c6905bfc19f1057fa26a4cb360e5c1d2" +checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" dependencies = [ "either", "lazy_static", @@ -12051,6 +13313,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" +[[package]] +name = "widestring" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" + [[package]] name = "winapi" version = "0.2.8" @@ -12096,9 +13364,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -12109,33 +13377,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" +checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" [[package]] name = "windows_i686_gnu" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" +checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" [[package]] name = "windows_i686_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" [[package]] name = "windows_x86_64_gnu" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" [[package]] name = "windows_x86_64_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" [[package]] name = "winreg" @@ -12146,6 +13414,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "ws2_32-sys" version = "0.2.1" @@ -12162,6 +13439,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" +[[package]] +name = "wyz" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b31594f29d27036c383b53b59ed3476874d518f0efb151b27a4c275141390e" +dependencies = [ + "tap", +] + [[package]] name = "x25519-dalek" version = "1.1.1" @@ -12180,18 +13466,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d9028f208dd5e63c614be69f115c1b53cacc1111437d4c765185856666c107" dependencies = [ "futures 0.3.21", - "log 0.4.14", + "log 0.4.16", "nohash-hasher", "parking_lot 0.11.2", "rand 0.8.5", "static_assertions", ] +[[package]] +name = "yamux" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0608f53c1dc0bad505d03a34bbd49fbf2ad7b51eb036123e896365532745a1" +dependencies = [ + "futures 0.3.21", + "log 0.4.16", + "nohash-hasher", + "parking_lot 0.12.0", + "rand 0.8.5", + "static_assertions", +] + [[package]] name = "zeroize" -version = "1.5.3" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50344758e2f40e3a1fcfc8f6f91aa57b5f8ebd8d27919fe6451f15aaaf9ee608" +checksum = "d68d9dcec5f9b43a30d38c49f91dfedfaac384cb8f085faca366c26207dd1619" dependencies = [ "zeroize_derive", ] diff --git a/Cargo.toml b/Cargo.toml index 3f428bcccc7a..09f22c8a5a54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,11 +7,9 @@ members = [ "bin/node-template/runtime", "bin/node/bench", "bin/node/cli", - "bin/node/test-runner-example", "bin/node/executor", "bin/node/primitives", "bin/node/rpc", - "bin/node/rpc-client", "bin/node/runtime", "bin/node/testing", "bin/utils/chain-spec-builder", @@ -43,7 +41,6 @@ members = [ "client/finality-grandpa", "client/informant", "client/keystore", - "client/light", "client/network", "client/network-gossip", "client/network/test", @@ -70,24 +67,30 @@ members = [ "frame/authority-discovery", "frame/authorship", "frame/babe", + "frame/bags-list", + "frame/bags-list/fuzzer", + "frame/bags-list/remote-tests", "frame/balances", "frame/beefy", "frame/beefy-mmr", "frame/beefy-mmr/primitives", "frame/benchmarking", "frame/bounties", + "frame/child-bounties", "frame/collective", "frame/contracts", "frame/contracts/rpc", "frame/contracts/rpc/runtime-api", + "frame/conviction-voting", "frame/democracy", "frame/try-runtime", - "frame/elections", "frame/election-provider-multi-phase", "frame/election-provider-support", - "frame/example", - "frame/example-offchain-worker", - "frame/example-parallel", + "frame/election-provider-support/solution-type", + "frame/election-provider-support/solution-type/fuzzer", + "frame/examples/basic", + "frame/examples/offchain-worker", + "frame/examples/parallel", "frame/executive", "frame/gilt", "frame/grandpa", @@ -103,9 +106,11 @@ members = [ "frame/nicks", "frame/node-authorization", "frame/offences", + "frame/preimage", "frame/proxy", "frame/randomness-collective-flip", "frame/recovery", + "frame/referenda", "frame/scheduler", "frame/scored-pool", "frame/session", @@ -114,17 +119,20 @@ members = [ "frame/staking", "frame/staking/reward-curve", "frame/staking/reward-fn", + "frame/state-trie-migration", "frame/sudo", "frame/support", "frame/support/procedural", "frame/support/procedural/tools", "frame/support/procedural/tools/derive", "frame/support/test", + "frame/support/test/compile_pass", "frame/system", "frame/system/benchmarking", "frame/system/rpc/runtime-api", "frame/timestamp", "frame/transaction-payment", + "frame/transaction-payment/asset-tx-payment", "frame/transaction-payment/rpc", "frame/transaction-payment/rpc/runtime-api", "frame/transaction-storage", @@ -133,7 +141,7 @@ members = [ "frame/uniques", "frame/utility", "frame/vesting", - "frame/bags-list", + "frame/whitelist", "primitives/api", "primitives/api/proc-macro", "primitives/api/test", @@ -152,6 +160,8 @@ members = [ "primitives/consensus/pow", "primitives/consensus/vrf", "primitives/core", + "primitives/core/hashing", + "primitives/core/hashing/proc-macro", "primitives/database", "primitives/debug-derive", "primitives/externalities", @@ -162,7 +172,6 @@ members = [ "primitives/keystore", "primitives/maybe-compressed-blob", "primitives/npos-elections", - "primitives/npos-elections/solution-type", "primitives/npos-elections/fuzzer", "primitives/offchain", "primitives/panic-handler", @@ -195,7 +204,6 @@ members = [ "test-utils/runtime", "test-utils/runtime/client", "test-utils/runtime/transaction-pool", - "test-utils/test-runner", "test-utils/test-crate", "utils/build-script-utils", "utils/fork-tree", @@ -203,6 +211,7 @@ members = [ "utils/frame/remote-externalities", "utils/frame/frame-utilities-cli", "utils/frame/try-runtime/cli", + "utils/frame/rpc/state-trie-migration-rpc", "utils/frame/rpc/support", "utils/frame/rpc/system", "utils/frame/generate-bags", @@ -245,6 +254,7 @@ hash-db = { opt-level = 3 } hmac = { opt-level = 3 } httparse = { opt-level = 3 } integer-sqrt = { opt-level = 3 } +k256 = { opt-level = 3 } keccak = { opt-level = 3 } libm = { opt-level = 3 } librocksdb-sys = { opt-level = 3 } @@ -259,6 +269,7 @@ percent-encoding = { opt-level = 3 } primitive-types = { opt-level = 3 } ring = { opt-level = 3 } rustls = { opt-level = 3 } +secp256k1 = { opt-level = 3 } sha2 = { opt-level = 3 } sha3 = { opt-level = 3 } smallvec = { opt-level = 3 } @@ -269,6 +280,17 @@ wasmi = { opt-level = 3 } x25519-dalek = { opt-level = 3 } yamux = { opt-level = 3 } zeroize = { opt-level = 3 } + [profile.release] # Substrate runtime requires unwinding. -panic = "unwind" \ No newline at end of file +panic = "unwind" + +[profile.production] +inherits = "release" + +# Sacrifice compile speed for execution speed by using optimization flags: + +# https://doc.rust-lang.org/rustc/linker-plugin-lto.html +lto = "fat" +# https://doc.rust-lang.org/rustc/codegen-options/index.html#codegen-units +codegen-units = 1 diff --git a/HEADER-APACHE2 b/HEADER-APACHE2 index f364f4bdf845..58baa53894ca 100644 --- a/HEADER-APACHE2 +++ b/HEADER-APACHE2 @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/HEADER-GPL3 b/HEADER-GPL3 index 0dd7e4f76028..9412b5a70bb3 100644 --- a/HEADER-GPL3 +++ b/HEADER-GPL3 @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/Process.json b/Process.json deleted file mode 100644 index 540bd644311c..000000000000 --- a/Process.json +++ /dev/null @@ -1,29 +0,0 @@ -[{ - "project_name": "Networking", - "owner": "tomaka", - "matrix_room_id": "!vUADSGcyXmxhKLeDsW:matrix.parity.io" -}, -{ "project_name": "Client", - "owner": "gnunicorn", - "matrix_room_id": "!aenJixaHcSKbJOWxYk:matrix.parity.io" -}, -{ - "project_name": "Runtime", - "owner": "gavofyork", - "matrix_room_id": "!yBKstWVBkwzUkPslsp:matrix.parity.io" -}, -{ - "project_name": "Consensus", - "owner": "andresilva", - "matrix_room_id": "!XdNWDTfVNFVixljKZU:matrix.parity.io" -}, -{ - "project_name": "Smart Contracts", - "owner": "pepyakin", - "matrix_room_id": "!yBKstWVBkwzUkPslsp:matrix.parity.io" -}, -{ - "project_name": "Benchmarking and Weights", - "owner": "shawntabrizi", - "matrix_room_id": "!pZPWqCRLVtORZTEsEf:matrix.parity.io" -}] diff --git a/README.md b/README.md index 6288540548a0..b716794428a0 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ Substrate is a next-generation framework for blockchain innovation 🚀. ## Trying it out -Simply go to [substrate.dev](https://substrate.dev) and follow the -[installation](https://substrate.dev/docs/en/knowledgebase/getting-started/) instructions. You can -also try out one of the [tutorials](https://substrate.dev/en/tutorials). +Simply go to [docs.substrate.io](https://docs.substrate.io) and follow the +[installation](https://docs.substrate.io/v3/getting-started/overview) instructions. You can +also try out one of the [tutorials](https://docs.substrate.io/tutorials/). ## Contributions & Code of Conduct @@ -28,3 +28,4 @@ The security policy and procedures can be found in [`docs/SECURITY.md`](docs/SEC The reason for the split-licensing is to ensure that for the vast majority of teams using Substrate to create feature-chains, then all changes can be made entirely in Apache2-licensed code, allowing teams full freedom over what and how they release and giving licensing clarity to commercial teams. In the interests of the community, we require any deeper improvements made to Substrate's core logic (e.g. Substrate's internal consensus, crypto or database code) to be contributed back so everyone can benefit. + diff --git a/bin/node-template/README.md b/bin/node-template/README.md index cd977fac8449..52b117b8a782 100644 --- a/bin/node-template/README.md +++ b/bin/node-template/README.md @@ -1,21 +1,32 @@ # Substrate Node Template +[![Try on playground](https://img.shields.io/badge/Playground-Node_Template-brightgreen?logo=Parity%20Substrate)](https://docs.substrate.io/playground/) [![Matrix](https://img.shields.io/matrix/substrate-technical:matrix.org)](https://matrix.to/#/#substrate-technical:matrix.org) + A fresh FRAME-based [Substrate](https://www.substrate.io/) node, ready for hacking :rocket: ## Getting Started -Follow these steps to get started with the Node Template :hammer_and_wrench: +Follow the steps below to get started with the Node Template, or get it up and running right from +your browser in just a few clicks using +the [Substrate Playground](https://docs.substrate.io/playground/) :hammer_and_wrench: + +### Using Nix + +Install [nix](https://nixos.org/) and optionally [direnv](https://github.com/direnv/direnv) and +[lorri](https://github.com/target/lorri) for a fully plug and play experience for setting up the +development environment. To get all the correct dependencies activate direnv `direnv allow` and +lorri `lorri shell`. ### Rust Setup -First, complete the [basic Rust setup instructions](./doc/rust-setup.md). +First, complete the [basic Rust setup instructions](./docs/rust-setup.md). ### Run Use Rust's native `cargo` command to build and launch the template node: ```sh -cargo run --release -- --dev --tmp +cargo run --release -- --dev ``` ### Build @@ -44,7 +55,7 @@ node. ### Single-Node Development Chain -This command will start the single-node development chain with persistent state: +This command will start the single-node development chain with non-persistent state: ```bash ./target/release/node-template --dev @@ -62,10 +73,48 @@ Start the development chain with detailed logging: RUST_BACKTRACE=1 ./target/release/node-template -ldebug --dev ``` +> Development chain means that the state of our chain will be in a tmp folder while the nodes are +> running. Also, **alice** account will be authority and sudo account as declared in the +> [genesis state](https://github.com/substrate-developer-hub/substrate-node-template/blob/main/node/src/chain_spec.rs#L49). +> At the same time the following accounts will be pre-funded: +> - Alice +> - Bob +> - Alice//stash +> - Bob//stash + +In case of being interested in maintaining the chain' state between runs a base path must be added +so the db can be stored in the provided folder instead of a temporal one. We could use this folder +to store different chain databases, as a different folder will be created per different chain that +is ran. The following commands shows how to use a newly created folder as our db base path. + +```bash +// Create a folder to use as the db base path +$ mkdir my-chain-state + +// Use of that folder to store the chain state +$ ./target/release/node-template --dev --base-path ./my-chain-state/ + +// Check the folder structure created inside the base path after running the chain +$ ls ./my-chain-state +chains +$ ls ./my-chain-state/chains/ +dev +$ ls ./my-chain-state/chains/dev +db keystore network +``` + + +### Connect with Polkadot-JS Apps Front-end + +Once the node template is running locally, you can connect it with **Polkadot-JS Apps** front-end +to interact with your chain. [Click +here](https://polkadot.js.org/apps/#/explorer?rpc=ws://localhost:9944) connecting the Apps to your +local node template. + ### Multi-Node Local Testnet -If you want to see the multi-node consensus algorithm in action, refer to -[our Start a Private Network tutorial](https://substrate.dev/docs/en/tutorials/start-a-private-network/). +If you want to see the multi-node consensus algorithm in action, refer to our +[Start a Private Network tutorial](https://docs.substrate.io/tutorials/v3/private-network). ## Template Structure @@ -77,34 +126,34 @@ directories. A blockchain node is an application that allows users to participate in a blockchain network. Substrate-based blockchain nodes expose a number of capabilities: -- Networking: Substrate nodes use the [`libp2p`](https://libp2p.io/) networking stack to allow the - nodes in the network to communicate with one another. -- Consensus: Blockchains must have a way to come to - [consensus](https://substrate.dev/docs/en/knowledgebase/advanced/consensus) on the state of the - network. Substrate makes it possible to supply custom consensus engines and also ships with - several consensus mechanisms that have been built on top of - [Web3 Foundation research](https://research.web3.foundation/en/latest/polkadot/NPoS/index.html). -- RPC Server: A remote procedure call (RPC) server is used to interact with Substrate nodes. +- Networking: Substrate nodes use the [`libp2p`](https://libp2p.io/) networking stack to allow the + nodes in the network to communicate with one another. +- Consensus: Blockchains must have a way to come to + [consensus](https://docs.substrate.io/v3/advanced/consensus) on the state of the + network. Substrate makes it possible to supply custom consensus engines and also ships with + several consensus mechanisms that have been built on top of + [Web3 Foundation research](https://research.web3.foundation/en/latest/polkadot/NPoS/index.html). +- RPC Server: A remote procedure call (RPC) server is used to interact with Substrate nodes. There are several files in the `node` directory - take special note of the following: -- [`chain_spec.rs`](./node/src/chain_spec.rs): A - [chain specification](https://substrate.dev/docs/en/knowledgebase/integrate/chain-spec) is a - source code file that defines a Substrate chain's initial (genesis) state. Chain specifications - are useful for development and testing, and critical when architecting the launch of a - production chain. Take note of the `development_config` and `testnet_genesis` functions, which - are used to define the genesis state for the local development chain configuration. These - functions identify some - [well-known accounts](https://substrate.dev/docs/en/knowledgebase/integrate/subkey#well-known-keys) - and use them to configure the blockchain's initial state. -- [`service.rs`](./node/src/service.rs): This file defines the node implementation. Take note of - the libraries that this file imports and the names of the functions it invokes. In particular, - there are references to consensus-related topics, such as the - [longest chain rule](https://substrate.dev/docs/en/knowledgebase/advanced/consensus#longest-chain-rule), - the [Aura](https://substrate.dev/docs/en/knowledgebase/advanced/consensus#aura) block authoring - mechanism and the - [GRANDPA](https://substrate.dev/docs/en/knowledgebase/advanced/consensus#grandpa) finality - gadget. +- [`chain_spec.rs`](./node/src/chain_spec.rs): A + [chain specification](https://docs.substrate.io/v3/runtime/chain-specs) is a + source code file that defines a Substrate chain's initial (genesis) state. Chain specifications + are useful for development and testing, and critical when architecting the launch of a + production chain. Take note of the `development_config` and `testnet_genesis` functions, which + are used to define the genesis state for the local development chain configuration. These + functions identify some + [well-known accounts](https://docs.substrate.io/v3/tools/subkey#well-known-keys) + and use them to configure the blockchain's initial state. +- [`service.rs`](./node/src/service.rs): This file defines the node implementation. Take note of + the libraries that this file imports and the names of the functions it invokes. In particular, + there are references to consensus-related topics, such as the + [longest chain rule](https://docs.substrate.io/v3/advanced/consensus#longest-chain-rule), + the [Aura](https://docs.substrate.io/v3/advanced/consensus#aura) block authoring + mechanism and the + [GRANDPA](https://docs.substrate.io/v3/advanced/consensus#grandpa) finality + gadget. After the node has been [built](#build), refer to the embedded documentation to learn more about the capabilities and configuration parameters that it exposes: @@ -116,27 +165,27 @@ capabilities and configuration parameters that it exposes: ### Runtime In Substrate, the terms -"[runtime](https://substrate.dev/docs/en/knowledgebase/getting-started/glossary#runtime)" and -"[state transition function](https://substrate.dev/docs/en/knowledgebase/getting-started/glossary#stf-state-transition-function)" +"[runtime](https://docs.substrate.io/v3/getting-started/glossary#runtime)" and +"[state transition function](https://docs.substrate.io/v3/getting-started/glossary#state-transition-function-stf)" are analogous - they refer to the core logic of the blockchain that is responsible for validating blocks and executing the state changes they define. The Substrate project in this repository uses -the [FRAME](https://substrate.dev/docs/en/knowledgebase/runtime/frame) framework to construct a +the [FRAME](https://docs.substrate.io/v3/runtime/frame) framework to construct a blockchain runtime. FRAME allows runtime developers to declare domain-specific logic in modules called "pallets". At the heart of FRAME is a helpful -[macro language](https://substrate.dev/docs/en/knowledgebase/runtime/macros) that makes it easy to +[macro language](https://docs.substrate.io/v3/runtime/macros) that makes it easy to create pallets and flexibly compose them to create blockchains that can address [a variety of needs](https://www.substrate.io/substrate-users/). Review the [FRAME runtime implementation](./runtime/src/lib.rs) included in this template and note the following: -- This file configures several pallets to include in the runtime. Each pallet configuration is - defined by a code block that begins with `impl $PALLET_NAME::Config for Runtime`. -- The pallets are composed into a single runtime by way of the - [`construct_runtime!`](https://crates.parity.io/frame_support/macro.construct_runtime.html) - macro, which is part of the core - [FRAME Support](https://substrate.dev/docs/en/knowledgebase/runtime/frame#support-library) - library. +- This file configures several pallets to include in the runtime. Each pallet configuration is + defined by a code block that begins with `impl $PALLET_NAME::Config for Runtime`. +- The pallets are composed into a single runtime by way of the + [`construct_runtime!`](https://crates.parity.io/frame_support/macro.construct_runtime.html) + macro, which is part of the core + [FRAME Support](https://docs.substrate.io/v3/runtime/frame#support-crate) + library. ### Pallets @@ -146,17 +195,17 @@ template pallet that is [defined in the `pallets`](./pallets/template/src/lib.rs A FRAME pallet is compromised of a number of blockchain primitives: -- Storage: FRAME defines a rich set of powerful - [storage abstractions](https://substrate.dev/docs/en/knowledgebase/runtime/storage) that makes - it easy to use Substrate's efficient key-value database to manage the evolving state of a - blockchain. -- Dispatchables: FRAME pallets define special types of functions that can be invoked (dispatched) - from outside of the runtime in order to update its state. -- Events: Substrate uses [events](https://substrate.dev/docs/en/knowledgebase/runtime/events) to - notify users of important changes in the runtime. -- Errors: When a dispatchable fails, it returns an error. -- Config: The `Config` configuration interface is used to define the types and parameters upon - which a FRAME pallet depends. +- Storage: FRAME defines a rich set of powerful + [storage abstractions](https://docs.substrate.io/v3/runtime/storage) that makes + it easy to use Substrate's efficient key-value database to manage the evolving state of a + blockchain. +- Dispatchables: FRAME pallets define special types of functions that can be invoked (dispatched) + from outside of the runtime in order to update its state. +- Events: Substrate uses [events and errors](https://docs.substrate.io/v3/runtime/events-and-errors) + to notify users of important changes in the runtime. +- Errors: When a dispatchable fails, it returns an error. +- Config: The `Config` configuration interface is used to define the types and parameters upon + which a FRAME pallet depends. ### Run in Docker @@ -170,7 +219,8 @@ Then run the following command to start a single node development chain. ``` This command will firstly compile your code, and then start a local development network. You can -also replace the default command (`cargo build --release && ./target/release/node-template --dev --ws-external`) +also replace the default command +(`cargo build --release && ./target/release/node-template --dev --ws-external`) by appending your own. A few useful ones are as follow. ```bash diff --git a/bin/node-template/docs/rust-setup.md b/bin/node-template/docs/rust-setup.md index 34f6e43e7f0d..ea133ca847af 100644 --- a/bin/node-template/docs/rust-setup.md +++ b/bin/node-template/docs/rust-setup.md @@ -2,31 +2,21 @@ title: Installation --- -This page will guide you through the steps needed to prepare a computer for development with the -Substrate Node Template. Since Substrate is built with -[the Rust programming language](https://www.rust-lang.org/), the first thing you will need to do is -prepare the computer for Rust development - these steps will vary based on the computer's operating -system. Once Rust is configured, you will use its toolchains to interact with Rust projects; the -commands for Rust's toolchains will be the same for all supported, Unix-based operating systems. +This guide is for reference only, please check the latest information on getting starting with Substrate +[here](https://docs.substrate.io/v3/getting-started/installation/). -## Unix-Based Operating Systems +This page will guide you through the **2 steps** needed to prepare a computer for **Substrate** development. +Since Substrate is built with [the Rust programming language](https://www.rust-lang.org/), the first +thing you will need to do is prepare the computer for Rust development - these steps will vary based +on the computer's operating system. Once Rust is configured, you will use its toolchains to interact +with Rust projects; the commands for Rust's toolchains will be the same for all supported, +Unix-based operating systems. -Substrate development is easiest on Unix-based operating systems like macOS or Linux. The examples -in the Substrate [Tutorials](https://substrate.dev/tutorials) and [Recipes](https://substrate.dev/recipes/) -use Unix-style terminals to demonstrate how to interact with Substrate from the command line. - -### macOS +## Build dependencies -Open the Terminal application and execute the following commands: - -```bash -# Install Homebrew if necessary https://brew.sh/ -/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" - -# Make sure Homebrew is up-to-date, install openssl and cmake -brew update -brew install openssl cmake -``` +Substrate development is easiest on Unix-based operating systems like macOS or Linux. The examples +in the [Substrate Docs](https://docs.substrate.io) use Unix-style terminals to demonstrate how to +interact with Substrate from the command line. ### Ubuntu/Debian @@ -35,7 +25,7 @@ Use a terminal shell to execute the following commands: ```bash sudo apt update # May prompt for location information -sudo apt install -y cmake pkg-config libssl-dev git build-essential clang libclang-dev curl +sudo apt install -y git clang curl libssl-dev llvm libudev-dev ``` ### Arch Linux @@ -43,39 +33,193 @@ sudo apt install -y cmake pkg-config libssl-dev git build-essential clang libcla Run these commands from a terminal: ```bash -pacman -Syu --needed --noconfirm cmake gcc openssl-1.0 pkgconf git clang -export OPENSSL_LIB_DIR="/usr/lib/openssl-1.0" -export OPENSSL_INCLUDE_DIR="/usr/include/openssl-1.0" +pacman -Syu --needed --noconfirm curl git clang ``` -### Fedora/RHEL/CentOS +### Fedora -Use a terminal to run the following commands: +Run these commands from a terminal: ```bash -# Update sudo dnf update -# Install packages -sudo dnf install cmake pkgconfig rocksdb rocksdb-devel llvm git libcurl libcurl-devel curl-devel clang +sudo dnf install clang curl git openssl-devel +``` + +### OpenSUSE + +Run these commands from a terminal: + +```bash +sudo zypper install clang curl git openssl-devel llvm-devel libudev-devel +``` + +### macOS + +> **Apple M1 ARM** +> If you have an Apple M1 ARM system on a chip, make sure that you have Apple Rosetta 2 +> installed through `softwareupdate --install-rosetta`. This is only needed to run the +> `protoc` tool during the build. The build itself and the target binaries would remain native. + +Open the Terminal application and execute the following commands: + +```bash +# Install Homebrew if necessary https://brew.sh/ +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" + +# Make sure Homebrew is up-to-date, install openssl +brew update +brew install openssl ``` -## Rust Developer Environment +### Windows + +**_PLEASE NOTE:_** Native development of Substrate is _not_ very well supported! It is _highly_ +recommend to use [Windows Subsystem Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10) +(WSL) and follow the instructions for [Ubuntu/Debian](#ubuntudebian). +Please refer to the separate +[guide for native Windows development](https://docs.substrate.io/v3/getting-started/windows-users/). -This project uses [`rustup`](https://rustup.rs/) to help manage the Rust toolchain. First install -and configure `rustup`: +## Rust developer environment + +This guide uses installer and the `rustup` tool to manage the Rust toolchain. +First install and configure `rustup`: ```bash # Install -curl https://sh.rustup.rs -sSf | sh +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh # Configure source ~/.cargo/env ``` -Finally, configure the Rust toolchain: +Configure the Rust toolchain to default to the latest stable version, add nightly and the nightly wasm target: ```bash rustup default stable +rustup update +rustup update nightly +rustup target add wasm32-unknown-unknown --toolchain nightly +``` + +## Test your set-up + +Now the best way to ensure that you have successfully prepared a computer for Substrate +development is to follow the steps in [our first Substrate tutorial](https://docs.substrate.io/tutorials/v3/create-your-first-substrate-chain/). + +## Troubleshooting Substrate builds + +Sometimes you can't get the Substrate node template +to compile out of the box. Here are some tips to help you work through that. + +### Rust configuration check + +To see what Rust toolchain you are presently using, run: + +```bash +rustup show +``` + +This will show something like this (Ubuntu example) output: + +```text +Default host: x86_64-unknown-linux-gnu +rustup home: /home/user/.rustup + +installed toolchains +-------------------- + +stable-x86_64-unknown-linux-gnu (default) +nightly-2020-10-06-x86_64-unknown-linux-gnu +nightly-x86_64-unknown-linux-gnu + +installed targets for active toolchain +-------------------------------------- + +wasm32-unknown-unknown +x86_64-unknown-linux-gnu + +active toolchain +---------------- + +stable-x86_64-unknown-linux-gnu (default) +rustc 1.50.0 (cb75ad5db 2021-02-10) +``` + +As you can see above, the default toolchain is stable, and the +`nightly-x86_64-unknown-linux-gnu` toolchain as well as its `wasm32-unknown-unknown` target is installed. +You also see that `nightly-2020-10-06-x86_64-unknown-linux-gnu` is installed, but is not used unless explicitly defined as illustrated in the [specify your nightly version](#specifying-nightly-version) +section. + +### WebAssembly compilation + +Substrate uses [WebAssembly](https://webassembly.org) (Wasm) to produce portable blockchain +runtimes. You will need to configure your Rust compiler to use +[`nightly` builds](https://doc.rust-lang.org/book/appendix-07-nightly-rust.html) to allow you to +compile Substrate runtime code to the Wasm target. + +> There are upstream issues in Rust that need to be resolved before all of Substrate can use the stable Rust toolchain. +> [This is our tracking issue](https://github.com/paritytech/substrate/issues/1252) if you're curious as to why and how this will be resolved. + +#### Latest nightly for Substrate `master` + +Developers who are building Substrate _itself_ should always use the latest bug-free versions of +Rust stable and nightly. This is because the Substrate codebase follows the tip of Rust nightly, +which means that changes in Substrate often depend on upstream changes in the Rust nightly compiler. +To ensure your Rust compiler is always up to date, you should run: + +```bash +rustup update rustup update nightly -rustup update stable rustup target add wasm32-unknown-unknown --toolchain nightly ``` + +> NOTE: It may be necessary to occasionally rerun `rustup update` if a change in the upstream Substrate +> codebase depends on a new feature of the Rust compiler. When you do this, both your nightly +> and stable toolchains will be pulled to the most recent release, and for nightly, it is +> generally _not_ expected to compile WASM without error (although it very often does). +> Be sure to [specify your nightly version](#specifying-nightly-version) if you get WASM build errors +> from `rustup` and [downgrade nightly as needed](#downgrading-rust-nightly). + +#### Rust nightly toolchain + +If you want to guarantee that your build works on your computer as you update Rust and other +dependencies, you should use a specific Rust nightly version that is known to be +compatible with the version of Substrate they are using; this version will vary from project to +project and different projects may use different mechanisms to communicate this version to +developers. For instance, the Polkadot client specifies this information in its +[release notes](https://github.com/paritytech/polkadot/releases). + +```bash +# Specify the specific nightly toolchain in the date below: +rustup install nightly- +``` + +#### Wasm toolchain + +Now, configure the nightly version to work with the Wasm compilation target: + +```bash +rustup target add wasm32-unknown-unknown --toolchain nightly- +``` + +### Specifying nightly version + +Use the `WASM_BUILD_TOOLCHAIN` environment variable to specify the Rust nightly version a Substrate +project should use for Wasm compilation: + +```bash +WASM_BUILD_TOOLCHAIN=nightly- cargo build --release +``` + +> Note that this only builds _the runtime_ with the specified nightly. The rest of project will be +> compiled with **your default toolchain**, i.e. the latest installed stable toolchain. + +### Downgrading Rust nightly + +If your computer is configured to use the latest Rust nightly and you would like to downgrade to a +specific nightly version, follow these steps: + +```bash +rustup uninstall nightly +rustup install nightly- +rustup target add wasm32-unknown-unknown --toolchain nightly- +``` diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index 9493ab3c338d..d81279259fa7 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -1,14 +1,14 @@ [package] name = "node-template" -version = "3.0.0" -authors = ["Substrate DevHub "] +version = "4.0.0-dev" description = "A fresh FRAME-based Substrate node, ready for hacking." -edition = "2018" +authors = ["Substrate DevHub "] +homepage = "https://substrate.io/" +edition = "2021" license = "Unlicense" -build = "build.rs" -homepage = "https://substrate.dev" -repository = "https://github.com/substrate-developer-hub/substrate-node-template/" publish = false +repository = "https://github.com/substrate-developer-hub/substrate-node-template/" +build = "build.rs" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] @@ -17,10 +17,10 @@ targets = ["x86_64-unknown-linux-gnu"] name = "node-template" [dependencies] -structopt = "0.3.8" +clap = { version = "3.1.6", features = ["derive"] } sc-cli = { version = "0.10.0-dev", path = "../../../client/cli", features = ["wasmtime"] } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } sc-executor = { version = "0.10.0-dev", path = "../../../client/executor", features = ["wasmtime"] } sc-service = { version = "0.10.0-dev", path = "../../../client/service", features = ["wasmtime"] } sc-telemetry = { version = "4.0.0-dev", path = "../../../client/telemetry" } @@ -34,9 +34,13 @@ sc-consensus = { version = "0.10.0-dev", path = "../../../client/consensus/commo sc-finality-grandpa = { version = "0.10.0-dev", path = "../../../client/finality-grandpa" } sp-finality-grandpa = { version = "4.0.0-dev", path = "../../../primitives/finality-grandpa" } sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } sp-timestamp = { version = "4.0.0-dev", path = "../../../primitives/timestamp" } -sp-keystore = { version = "0.10.0-dev", path = "../../../primitives/keystore" } +sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } +sp-keyring = { version = "6.0.0", path = "../../../primitives/keyring" } +frame-system = { version = "4.0.0-dev", path = "../../../frame/system" } +pallet-transaction-payment = { version = "4.0.0-dev", default-features = false, path = "../../../frame/transaction-payment" } +sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" } pallet-im-online = { version = "4.0.0-dev", default-features = false, path = "../../../frame/im-online" } # These dependencies are used for the node template's RPCs @@ -57,13 +61,18 @@ pallet-contracts-rpc = { version = "4.0.0-dev", path = "../../../frame/contracts frame-benchmarking = { version = "4.0.0-dev", path = "../../../frame/benchmarking" } frame-benchmarking-cli = { version = "4.0.0-dev", path = "../../../utils/frame/benchmarking-cli" } -node-template-runtime = { version = "3.0.0", path = "../runtime" } +# Local Dependencies +node-template-runtime = { version = "4.0.0-dev", path = "../runtime" } + +# CLI-specific dependencies +try-runtime-cli = { version = "0.10.0-dev", optional = true, path = "../../../utils/frame/try-runtime/cli" } [build-dependencies] substrate-build-script-utils = { version = "3.0.0", path = "../../../utils/build-script-utils" } [features] default = [] -runtime-benchmarks = [ - "node-template-runtime/runtime-benchmarks", -] +runtime-benchmarks = ["node-template-runtime/runtime-benchmarks"] +# Enable features that allow the runtime to be tried and debugged. Name might be subject to change +# in the near future. +try-runtime = ["node-template-runtime/try-runtime", "try-runtime-cli"] diff --git a/bin/node-template/node/src/chain_spec.rs b/bin/node-template/node/src/chain_spec.rs index 8140e6e48a61..77d361dd4651 100644 --- a/bin/node-template/node/src/chain_spec.rs +++ b/bin/node-template/node/src/chain_spec.rs @@ -83,6 +83,7 @@ pub fn development_config() -> Result { None, // Protocol ID None, + None, // Properties None, // Extensions @@ -132,6 +133,7 @@ pub fn local_testnet_config() -> Result { None, // Properties None, + None, // Extensions None, )) @@ -149,7 +151,6 @@ fn testnet_genesis( system: SystemConfig { // Add Wasm runtime to storage. code: wasm_binary.to_vec(), - changes_trie_config: Default::default(), }, balances: BalancesConfig { // Configure endowed accounts with initial balance of 1 << 60. @@ -172,7 +173,8 @@ fn testnet_genesis( im_online: ImOnlineConfig { keys: vec![] }, sudo: SudoConfig { // Assign network admin rights. - key: root_key, + key: Some(root_key), }, + // transaction_payment: Default::default(), } } diff --git a/bin/node-template/node/src/cli.rs b/bin/node-template/node/src/cli.rs index 8ed1d35ba5f9..710c7f3f9e14 100644 --- a/bin/node-template/node/src/cli.rs +++ b/bin/node-template/node/src/cli.rs @@ -1,19 +1,20 @@ use sc_cli::RunCmd; -use structopt::StructOpt; -#[derive(Debug, StructOpt)] +#[derive(Debug, clap::Parser)] pub struct Cli { - #[structopt(subcommand)] + #[clap(subcommand)] pub subcommand: Option, - #[structopt(flatten)] + #[clap(flatten)] pub run: RunCmd, } -#[derive(Debug, StructOpt)] +#[derive(Debug, clap::Subcommand)] pub enum Subcommand { /// Key management cli utilities + #[clap(subcommand)] Key(sc_cli::KeySubcommand), + /// Build a chain specification. BuildSpec(sc_cli::BuildSpecCmd), @@ -35,7 +36,15 @@ pub enum Subcommand { /// Revert the chain to a previous state. Revert(sc_cli::RevertCmd), - /// The custom benchmark subcommand benchmarking runtime pallets. - #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] + /// Sub-commands concerned with benchmarking. + #[clap(subcommand)] Benchmark(frame_benchmarking_cli::BenchmarkCmd), + + /// Try some command against runtime state. + #[cfg(feature = "try-runtime")] + TryRuntime(try_runtime_cli::TryRuntimeCmd), + + /// Try some command against runtime state. Note: `try-runtime` feature must be enabled. + #[cfg(not(feature = "try-runtime"))] + TryRuntime, } diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index e948c3f53b71..ede969b3572c 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -1,11 +1,14 @@ use crate::{ chain_spec, cli::{Cli, Subcommand}, + command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder}, service, }; +use frame_benchmarking_cli::BenchmarkCmd; use node_template_runtime::Block; -use sc_cli::{ChainSpec, Role, RuntimeVersion, SubstrateCli}; +use sc_cli::{ChainSpec, RuntimeVersion, SubstrateCli}; use sc_service::PartialComponents; +use std::sync::Arc; impl SubstrateCli for Cli { fn impl_name() -> String { @@ -95,27 +98,69 @@ pub fn run() -> sc_cli::Result<()> { runner.async_run(|config| { let PartialComponents { client, task_manager, backend, .. } = service::new_partial(&config)?; - Ok((cmd.run(client, backend), task_manager)) + let aux_revert = Box::new(move |client, _, blocks| { + sc_finality_grandpa::revert(client, blocks)?; + Ok(()) + }); + Ok((cmd.run(client, backend, Some(aux_revert)), task_manager)) }) }, - Some(Subcommand::Benchmark(cmd)) => - if cfg!(feature = "runtime-benchmarks") { - let runner = cli.create_runner(cmd)?; + Some(Subcommand::Benchmark(cmd)) => { + let runner = cli.create_runner(cmd)?; + + runner.sync_run(|config| { + let PartialComponents { client, backend, .. } = service::new_partial(&config)?; + + // This switch needs to be in the client, since the client decides + // which sub-commands it wants to support. + match cmd { + BenchmarkCmd::Pallet(cmd) => { + if !cfg!(feature = "runtime-benchmarks") { + return Err( + "Runtime benchmarking wasn't enabled when building the node. \ + You can enable it with `--features runtime-benchmarks`." + .into(), + ) + } + + cmd.run::(config) + }, + BenchmarkCmd::Block(cmd) => cmd.run(client), + BenchmarkCmd::Storage(cmd) => { + let db = backend.expose_db(); + let storage = backend.expose_storage(); - runner.sync_run(|config| cmd.run::(config)) - } else { - Err("Benchmarking wasn't enabled when building the node. You can enable it with \ - `--features runtime-benchmarks`." - .into()) - }, + cmd.run(config, client, db, storage) + }, + BenchmarkCmd::Overhead(cmd) => { + let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone()); + + cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) + }, + } + }) + }, + #[cfg(feature = "try-runtime")] + Some(Subcommand::TryRuntime(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + // we don't need any of the components of new_partial, just a runtime, or a task + // manager to do `async_run`. + let registry = config.prometheus_config.as_ref().map(|cfg| &cfg.registry); + let task_manager = + sc_service::TaskManager::new(config.tokio_handle.clone(), registry) + .map_err(|e| sc_cli::Error::Service(sc_service::Error::Prometheus(e)))?; + Ok((cmd.run::(config), task_manager)) + }) + }, + #[cfg(not(feature = "try-runtime"))] + Some(Subcommand::TryRuntime) => Err("TryRuntime wasn't enabled when building the node. \ + You can enable it with `--features try-runtime`." + .into()), None => { let runner = cli.create_runner(&cli.run)?; runner.run_node_until_exit(|config| async move { - match config.role { - Role::Light => service::new_light(config), - _ => service::new_full(config), - } - .map_err(sc_cli::Error::Service) + service::new_full(config).map_err(sc_cli::Error::Service) }) }, } diff --git a/bin/node-template/node/src/command_helper.rs b/bin/node-template/node/src/command_helper.rs new file mode 100644 index 000000000000..287e81b1e96b --- /dev/null +++ b/bin/node-template/node/src/command_helper.rs @@ -0,0 +1,131 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Contains code to setup the command invocations in [`super::command`] which would +//! otherwise bloat that module. + +use crate::service::FullClient; + +use node_template_runtime as runtime; +use runtime::SystemCall; +use sc_cli::Result; +use sc_client_api::BlockBackend; +use sp_core::{Encode, Pair}; +use sp_inherents::{InherentData, InherentDataProvider}; +use sp_keyring::Sr25519Keyring; +use sp_runtime::{OpaqueExtrinsic, SaturatedConversion}; + +use std::{sync::Arc, time::Duration}; + +/// Generates extrinsics for the `benchmark overhead` command. +/// +/// Note: Should only be used for benchmarking. +pub struct BenchmarkExtrinsicBuilder { + client: Arc, +} + +impl BenchmarkExtrinsicBuilder { + /// Creates a new [`Self`] from the given client. + pub fn new(client: Arc) -> Self { + Self { client } + } +} + +impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder { + fn remark(&self, nonce: u32) -> std::result::Result { + let acc = Sr25519Keyring::Bob.pair(); + let extrinsic: OpaqueExtrinsic = create_benchmark_extrinsic( + self.client.as_ref(), + acc, + SystemCall::remark { remark: vec![] }.into(), + nonce, + ) + .into(); + + Ok(extrinsic) + } +} + +/// Create a transaction using the given `call`. +/// +/// Note: Should only be used for benchmarking. +pub fn create_benchmark_extrinsic( + client: &FullClient, + sender: sp_core::sr25519::Pair, + call: runtime::Call, + nonce: u32, +) -> runtime::UncheckedExtrinsic { + let genesis_hash = client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"); + let best_hash = client.chain_info().best_hash; + let best_block = client.chain_info().best_number; + + let period = runtime::BlockHashCount::get() + .checked_next_power_of_two() + .map(|c| c / 2) + .unwrap_or(2) as u64; + let extra: runtime::SignedExtra = ( + frame_system::CheckNonZeroSender::::new(), + frame_system::CheckSpecVersion::::new(), + frame_system::CheckTxVersion::::new(), + frame_system::CheckGenesis::::new(), + frame_system::CheckEra::::from(sp_runtime::generic::Era::mortal( + period, + best_block.saturated_into(), + )), + frame_system::CheckNonce::::from(nonce), + frame_system::CheckWeight::::new(), + pallet_transaction_payment::ChargeTransactionPayment::::from(0), + ); + + let raw_payload = runtime::SignedPayload::from_raw( + call.clone(), + extra.clone(), + ( + (), + runtime::VERSION.spec_version, + runtime::VERSION.transaction_version, + genesis_hash, + best_hash, + (), + (), + (), + ), + ); + let signature = raw_payload.using_encoded(|e| sender.sign(e)); + + runtime::UncheckedExtrinsic::new_signed( + call.clone(), + sp_runtime::AccountId32::from(sender.public()).into(), + runtime::Signature::Sr25519(signature.clone()), + extra.clone(), + ) +} + +/// Generates inherent data for the `benchmark overhead` command. +/// +/// Note: Should only be used for benchmarking. +pub fn inherent_benchmark_data() -> Result { + let mut inherent_data = InherentData::new(); + let d = Duration::from_millis(0); + let timestamp = sp_timestamp::InherentDataProvider::new(d.into()); + + timestamp + .provide_inherent_data(&mut inherent_data) + .map_err(|e| format!("creating inherent data: {:?}", e))?; + Ok(inherent_data) +} diff --git a/bin/node-template/node/src/main.rs b/bin/node-template/node/src/main.rs index 4449d28b9fa4..0f2fbd5a909c 100644 --- a/bin/node-template/node/src/main.rs +++ b/bin/node-template/node/src/main.rs @@ -6,6 +6,7 @@ mod chain_spec; mod service; mod cli; mod command; +mod command_helper; mod rpc; fn main() -> sc_cli::Result<()> { diff --git a/bin/node-template/node/src/rpc.rs b/bin/node-template/node/src/rpc.rs index 6d650bd934f4..d69ac1226a84 100644 --- a/bin/node-template/node/src/rpc.rs +++ b/bin/node-template/node/src/rpc.rs @@ -50,8 +50,10 @@ where io.extend_with(TransactionPaymentApi::to_delegate(TransactionPayment::new(client.clone()))); + // IRIS rpc io.extend_with(IrisApi::to_delegate(Iris::new(client.clone()))); + // Contracts RPC io.extend_with(ContractsApi::to_delegate(Contracts::new(client.clone()))); io diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 55db2721da8d..c6a9ca0ca89d 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -1,14 +1,13 @@ //! Service and ServiceFactory implementation. Specialized wrapper over substrate service. use node_template_runtime::{self, opaque::Block, RuntimeApi}; -use sc_client_api::{ExecutorProvider, RemoteBackend}; +use sc_client_api::{BlockBackend, ExecutorProvider}; use sc_consensus_aura::{ImportQueueParams, SlotProportion, StartAuraParams}; pub use sc_executor::NativeElseWasmExecutor; use sc_finality_grandpa::SharedVoterState; use sc_keystore::LocalKeystore; use sc_service::{error::Error as ServiceError, Configuration, TaskManager}; use sc_telemetry::{Telemetry, TelemetryWorker}; -use sp_consensus::SlotData; use sp_consensus_aura::sr25519::AuthorityPair as AuraPair; use std::{sync::Arc, time::Duration}; @@ -16,7 +15,12 @@ use std::{sync::Arc, time::Duration}; pub struct ExecutorDispatch; impl sc_executor::NativeExecutionDispatch for ExecutorDispatch { + /// Only enable the benchmarking host functions when we actually want to benchmark. + #[cfg(feature = "runtime-benchmarks")] type ExtendHostFunctions = frame_benchmarking::benchmarking::HostFunctions; + /// Otherwise we only use the default Substrate host functions. + #[cfg(not(feature = "runtime-benchmarks"))] + type ExtendHostFunctions = (); fn dispatch(method: &str, data: &[u8]) -> Option> { node_template_runtime::api::dispatch(method, data) @@ -27,7 +31,7 @@ impl sc_executor::NativeExecutionDispatch for ExecutorDispatch { } } -type FullClient = +pub(crate) type FullClient = sc_service::TFullClient>; type FullBackend = sc_service::TFullBackend; type FullSelectChain = sc_consensus::LongestChain; @@ -55,7 +59,7 @@ pub fn new_partial( ServiceError, > { if config.keystore_remote.is_some() { - return Err(ServiceError::Other(format!("Remote Keystores are not supported."))) + return Err(ServiceError::Other("Remote Keystores are not supported.".into())) } let telemetry = config @@ -73,6 +77,7 @@ pub fn new_partial( config.wasm_method, config.default_heap_pages, config.max_runtime_instances, + config.runtime_cache_size, ); let (client, backend, keystore_container, task_manager) = @@ -84,7 +89,7 @@ pub fn new_partial( let client = Arc::new(client); let telemetry = telemetry.map(|(worker, telemetry)| { - task_manager.spawn_handle().spawn("telemetry", worker.run()); + task_manager.spawn_handle().spawn("telemetry", None, worker.run()); telemetry }); @@ -105,7 +110,7 @@ pub fn new_partial( telemetry.as_ref().map(|x| x.handle()), )?; - let slot_duration = sc_consensus_aura::slot_duration(&*client)?.slot_duration(); + let slot_duration = sc_consensus_aura::slot_duration(&*client)?; let import_queue = sc_consensus_aura::import_queue::(ImportQueueParams { @@ -116,7 +121,7 @@ pub fn new_partial( let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); let slot = - sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_duration( + sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_slot_duration( *timestamp, slot_duration, ); @@ -174,11 +179,19 @@ pub fn new_full(mut config: Configuration) -> Result ))), }; } + let grandpa_protocol_name = sc_finality_grandpa::protocol_standard_name( + &client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"), + &config.chain_spec, + ); - config.network.extra_sets.push(sc_finality_grandpa::grandpa_peers_set_config()); + config + .network + .extra_sets + .push(sc_finality_grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone())); let warp_sync = Arc::new(sc_finality_grandpa::warp_proof::NetworkProvider::new( backend.clone(), grandpa_link.shared_authority_set().clone(), + Vec::default(), )); let (network, system_rpc_tx, network_starter) = @@ -188,10 +201,10 @@ pub fn new_full(mut config: Configuration) -> Result transaction_pool: transaction_pool.clone(), spawn_handle: task_manager.spawn_handle(), import_queue, - on_demand: None, block_announce_validator_builder: None, warp_sync: Some(warp_sync), })?; + if config.offchain_worker.enabled { sc_service::build_offchain_workers( &config, @@ -228,8 +241,6 @@ pub fn new_full(mut config: Configuration) -> Result task_manager: &mut task_manager, transaction_pool: transaction_pool.clone(), rpc_extensions_builder, - on_demand: None, - remote_blockchain: None, backend, system_rpc_tx, config, @@ -249,7 +260,6 @@ pub fn new_full(mut config: Configuration) -> Result sp_consensus::CanAuthorWithNativeVersion::new(client.executor().clone()); let slot_duration = sc_consensus_aura::slot_duration(&*client)?; - let raw_slot_duration = slot_duration.slot_duration(); let aura = sc_consensus_aura::start_aura::( StartAuraParams { @@ -262,9 +272,9 @@ pub fn new_full(mut config: Configuration) -> Result let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); let slot = - sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_duration( + sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_slot_duration( *timestamp, - raw_slot_duration, + slot_duration, ); Ok((timestamp, slot)) @@ -283,7 +293,9 @@ pub fn new_full(mut config: Configuration) -> Result // the AURA authoring task is considered essential, i.e. if it // fails we take down the service with it. - task_manager.spawn_essential_handle().spawn_blocking("aura", aura); + task_manager + .spawn_essential_handle() + .spawn_blocking("aura", Some("block-authoring"), aura); } // if the node isn't actively participating in consensus then it doesn't @@ -300,6 +312,7 @@ pub fn new_full(mut config: Configuration) -> Result keystore, local_role: role, telemetry: telemetry.as_ref().map(|x| x.handle()), + protocol_name: grandpa_protocol_name, }; if enable_grandpa { @@ -323,6 +336,7 @@ pub fn new_full(mut config: Configuration) -> Result // if it fails we take down the service with it. task_manager.spawn_essential_handle().spawn_blocking( "grandpa-voter", + None, sc_finality_grandpa::run_grandpa_voter(grandpa_config)?, ); } @@ -330,144 +344,3 @@ pub fn new_full(mut config: Configuration) -> Result network_starter.start_network(); Ok(task_manager) } - -/// Builds a new service for a light client. -pub fn new_light(mut config: Configuration) -> Result { - let telemetry = config - .telemetry_endpoints - .clone() - .filter(|x| !x.is_empty()) - .map(|endpoints| -> Result<_, sc_telemetry::Error> { - let worker = TelemetryWorker::new(16)?; - let telemetry = worker.handle().new_telemetry(endpoints); - Ok((worker, telemetry)) - }) - .transpose()?; - - let executor = NativeElseWasmExecutor::::new( - config.wasm_method, - config.default_heap_pages, - config.max_runtime_instances, - ); - - let (client, backend, keystore_container, mut task_manager, on_demand) = - sc_service::new_light_parts::( - &config, - telemetry.as_ref().map(|(_, telemetry)| telemetry.handle()), - executor, - )?; - - let mut telemetry = telemetry.map(|(worker, telemetry)| { - task_manager.spawn_handle().spawn("telemetry", worker.run()); - telemetry - }); - - config.network.extra_sets.push(sc_finality_grandpa::grandpa_peers_set_config()); - - let select_chain = sc_consensus::LongestChain::new(backend.clone()); - - let transaction_pool = Arc::new(sc_transaction_pool::BasicPool::new_light( - config.transaction_pool.clone(), - config.prometheus_registry(), - task_manager.spawn_essential_handle(), - client.clone(), - on_demand.clone(), - )); - - let (grandpa_block_import, grandpa_link) = sc_finality_grandpa::block_import( - client.clone(), - &(client.clone() as Arc<_>), - select_chain.clone(), - telemetry.as_ref().map(|x| x.handle()), - )?; - - let slot_duration = sc_consensus_aura::slot_duration(&*client)?.slot_duration(); - - let import_queue = - sc_consensus_aura::import_queue::(ImportQueueParams { - block_import: grandpa_block_import.clone(), - justification_import: Some(Box::new(grandpa_block_import.clone())), - client: client.clone(), - create_inherent_data_providers: move |_, ()| async move { - let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); - - let slot = - sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_duration( - *timestamp, - slot_duration, - ); - - Ok((timestamp, slot)) - }, - spawner: &task_manager.spawn_essential_handle(), - can_author_with: sp_consensus::NeverCanAuthor, - registry: config.prometheus_registry(), - check_for_equivocation: Default::default(), - telemetry: telemetry.as_ref().map(|x| x.handle()), - })?; - - let warp_sync = Arc::new(sc_finality_grandpa::warp_proof::NetworkProvider::new( - backend.clone(), - grandpa_link.shared_authority_set().clone(), - )); - - let (network, system_rpc_tx, network_starter) = - sc_service::build_network(sc_service::BuildNetworkParams { - config: &config, - client: client.clone(), - transaction_pool: transaction_pool.clone(), - spawn_handle: task_manager.spawn_handle(), - import_queue, - on_demand: Some(on_demand.clone()), - block_announce_validator_builder: None, - warp_sync: Some(warp_sync), - })?; - - if config.offchain_worker.enabled { - sc_service::build_offchain_workers( - &config, - task_manager.spawn_handle(), - client.clone(), - network.clone(), - task_manager.ipfs_rt.as_ref().unwrap().clone(), - ); - } - - let enable_grandpa = !config.disable_grandpa; - if enable_grandpa { - let name = config.network.node_name.clone(); - - let config = sc_finality_grandpa::Config { - gossip_duration: std::time::Duration::from_millis(333), - justification_period: 512, - name: Some(name), - observer_enabled: false, - keystore: None, - local_role: config.role.clone(), - telemetry: telemetry.as_ref().map(|x| x.handle()), - }; - - task_manager.spawn_handle().spawn_blocking( - "grandpa-observer", - sc_finality_grandpa::run_grandpa_observer(config, grandpa_link, network.clone())?, - ); - } - - sc_service::spawn_tasks(sc_service::SpawnTasksParams { - remote_blockchain: Some(backend.remote_blockchain()), - transaction_pool, - task_manager: &mut task_manager, - on_demand: Some(on_demand), - rpc_extensions_builder: Box::new(|_, _| Ok(())), - config, - client, - keystore: keystore_container.sync_keystore(), - backend, - network, - system_rpc_tx, - telemetry: telemetry.as_mut(), - })?; - - network_starter.start_network(); - Ok(task_manager) -} diff --git a/bin/node-template/pallets/iris-assets/Cargo.toml b/bin/node-template/pallets/iris-assets/Cargo.toml index ae017ba59de4..4af4bcfc9ba3 100644 --- a/bin/node-template/pallets/iris-assets/Cargo.toml +++ b/bin/node-template/pallets/iris-assets/Cargo.toml @@ -14,16 +14,16 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0", default-features = false, features = ["derive"] } -sp-io = { default-features = false, version = "4.0.0-dev", path = "../../../../primitives/io" } +sp-io = { default-features = false, version = "6.0.0", path = "../../../../primitives/io" } sp-std = { default-features = false, version = "4.0.0-dev", path = "../../../../primitives/std" } -sp-core = { default-features = false, version = "4.0.0-dev", path = "../../../../primitives/core" } -sp-runtime = { default-features = false, version = "4.0.0-dev", path = "../../../../primitives/runtime" } -sp-keystore = { version = "0.10.0-dev", path = "../../../../primitives/keystore", optional = true } +sp-core = { default-features = false, version = "6.0.0", path = "../../../../primitives/core" } +sp-runtime = { default-features = false, version = "6.0.0", path = "../../../../primitives/runtime" } +sp-keystore = { version = "0.12.0", path = "../../../../primitives/keystore", optional = true } frame-support = { default-features = false, version = "4.0.0-dev", path = "../../../../frame/support" } frame-system = { default-features = false, version = "4.0.0-dev", path = "../../../../frame/system" } diff --git a/bin/node-template/pallets/iris-assets/rpc/Cargo.toml b/bin/node-template/pallets/iris-assets/rpc/Cargo.toml index e35392debab3..8de7295feb7c 100644 --- a/bin/node-template/pallets/iris-assets/rpc/Cargo.toml +++ b/bin/node-template/pallets/iris-assets/rpc/Cargo.toml @@ -21,8 +21,8 @@ jsonrpc-derive = "18.0.0" sp-api = { version = "4.0.0-dev", path = "../../../../../primitives/api" } sp-std = { default-features = false, version = "4.0.0-dev", path = "../../../../../primitives/std" } sp-blockchain = { version = "4.0.0-dev", path = "../../../../../primitives/blockchain" } -sp-core = { version = "4.0.0-dev", path = "../../../../../primitives/core" } -sp-rpc = { version = "4.0.0-dev", path = "../../../../../primitives/rpc" } -sp-runtime = { version = "4.0.0-dev", path = "../../../../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../../../../primitives/core" } +sp-rpc = { version = "6.0.0", path = "../../../../../primitives/rpc" } +sp-runtime = { version = "6.0.0", path = "../../../../../primitives/runtime" } pallet-iris-rpc-runtime-api = { version = "4.0.0-dev", path = "./runtime-api" } diff --git a/bin/node-template/pallets/iris-assets/rpc/runtime-api/Cargo.toml b/bin/node-template/pallets/iris-assets/rpc/runtime-api/Cargo.toml index f5c23fc281be..0780158384c9 100644 --- a/bin/node-template/pallets/iris-assets/rpc/runtime-api/Cargo.toml +++ b/bin/node-template/pallets/iris-assets/rpc/runtime-api/Cargo.toml @@ -15,9 +15,9 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../../../../primitives/api" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../../../../../primitives/core" } +sp-core = { version = "6.0.0", default-features = false, path = "../../../../../../primitives/core" } sp-std = { default-features = false, version = "4.0.0-dev", path = "../../../../../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../../../../primitives/runtime" } pallet-iris-session = { version = "1.0.0", default-features = false, path = "../../../iris-session" } [features] diff --git a/bin/node-template/pallets/iris-assets/src/lib.rs b/bin/node-template/pallets/iris-assets/src/lib.rs index 0a285ef1f8c1..980122448d9a 100644 --- a/bin/node-template/pallets/iris-assets/src/lib.rs +++ b/bin/node-template/pallets/iris-assets/src/lib.rs @@ -36,7 +36,7 @@ use sp_runtime::{ traits::{StaticLookup, Verify, IdentifyAccount}, }; use sp_std::{ - vec::Vec, + // vec::Vec, prelude::*, }; use scale_info::prelude::string::String; @@ -73,7 +73,11 @@ pub mod pallet { pallet_prelude::*, }; use sp_core::offchain::OpaqueMultiaddr; - use sp_std::{str, vec::Vec}; + use sp_std::{ + str, + prelude::*, + // vec::Vec + }; #[pallet::config] /// the module configuration trait @@ -88,6 +92,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); /// A queue of data to publish or obtain on IPFS. @@ -348,7 +353,7 @@ pub mod pallet { // verify asset access // in the future this is where the composable access rules will be executed // for now we just check if they account has a positive balance of assets - let true_asset_balance = >::account(asset_id.clone(), who.clone()).balance; + let true_asset_balance = >::account(asset_id.clone(), who.clone()).unwrap().balance; let zero_balance: T::Balance = 0u32.into(); ensure!(true_asset_balance != zero_balance, Error::::InsufficientBalance); // submit command to dataqueue diff --git a/bin/node-template/pallets/iris-ledger/Cargo.toml b/bin/node-template/pallets/iris-ledger/Cargo.toml index 1eac5cb2b6e9..0dcc8cceccab 100644 --- a/bin/node-template/pallets/iris-ledger/Cargo.toml +++ b/bin/node-template/pallets/iris-ledger/Cargo.toml @@ -14,16 +14,16 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } -sp-io = { default-features = false, version = "4.0.0-dev", path = "../../../../primitives/io" } -sp-std = { default-features = false, version = "4.0.0-dev", path = "../../../../primitives/std" } -sp-core = { default-features = false, version = "4.0.0-dev", path = "../../../../primitives/core" } -sp-runtime = { default-features = false, version = "4.0.0-dev", path = "../../../../primitives/runtime" } -sp-keystore = { version = "0.10.0-dev", path = "../../../../primitives/keystore", optional = true } +sp-io = { default-features = false, version = "6.0.0", path = "../../../../primitives/io" } +sp-std = { default-features = false, version = "4.0.0", path = "../../../../primitives/std" } +sp-core = { default-features = false, version = "6.0.0", path = "../../../../primitives/core" } +sp-runtime = { default-features = false, version = "6.0.0", path = "../../../../primitives/runtime" } +sp-keystore = { version = "0.12.0", path = "../../../../primitives/keystore", optional = true } frame-support = { default-features = false, version = "4.0.0-dev", path = "../../../../frame/support" } frame-system = { default-features = false, version = "4.0.0-dev", path = "../../../../frame/system" } @@ -33,7 +33,7 @@ log = { version = "0.4.14", default-features = false } pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/balances" } [features] -default = ['std'] +default = ["std"] std = [ 'codec/std', 'scale-info/std', @@ -48,5 +48,5 @@ std = [ 'pallet-balances/std', ] -runtime-benchmarks = ["frame-benchmarking"] +runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"] try-runtime = ["frame-support/try-runtime"] diff --git a/bin/node-template/pallets/iris-ledger/src/lib.rs b/bin/node-template/pallets/iris-ledger/src/lib.rs index 66ddead93ecc..a1643c483e9d 100644 --- a/bin/node-template/pallets/iris-ledger/src/lib.rs +++ b/bin/node-template/pallets/iris-ledger/src/lib.rs @@ -104,10 +104,10 @@ pub mod pallet { Unlocked(T::AccountId), } - #[pallet::error] - pub enum Error { + // #[pallet::error] + // pub enum Error { - } + // } #[pallet::call] impl Pallet { diff --git a/bin/node-template/pallets/iris-session/Cargo.toml b/bin/node-template/pallets/iris-session/Cargo.toml index 53d0aae68025..cd85893cd2c3 100644 --- a/bin/node-template/pallets/iris-session/Cargo.toml +++ b/bin/node-template/pallets/iris-session/Cargo.toml @@ -14,17 +14,17 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0", default-features = false, features = ["derive"] } -sp-io = { default-features = false, version = "4.0.0-dev", path = "../../../../primitives/io" } +sp-io = { default-features = false, version = "6.0.0", path = "../../../../primitives/io" } sp-std = { default-features = false, version = "4.0.0-dev", path = "../../../../primitives/std" } -sp-core = { default-features = false, version = "4.0.0-dev", path = "../../../../primitives/core" } -sp-runtime = { default-features = false, version = "4.0.0-dev", path = "../../../../primitives/runtime" } +sp-core = { default-features = false, version = "6.0.0", path = "../../../../primitives/core" } +sp-runtime = { default-features = false, version = "6.0.0", path = "../../../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../../../primitives/staking" } -sp-keystore = { version = "0.10.0-dev", path = "../../../../primitives/keystore", optional = true } +sp-keystore = { version = "0.12.0", path = "../../../../primitives/keystore", optional = true } frame-support = { default-features = false, version = "4.0.0-dev", path = "../../../../frame/support" } frame-system = { default-features = false, version = "4.0.0-dev", path = "../../../../frame/system" } diff --git a/bin/node-template/pallets/iris-session/src/benchmarking.rs b/bin/node-template/pallets/iris-session/src/benchmarking.rs index 2117c048cfbd..d496a9fc89b1 100644 --- a/bin/node-template/pallets/iris-session/src/benchmarking.rs +++ b/bin/node-template/pallets/iris-session/src/benchmarking.rs @@ -4,7 +4,7 @@ use super::*; #[allow(unused)] use crate::Pallet as Template; -use frame_benchmarking::{benchmarks, impl_benchmark_test_suite, whitelisted_caller}; +use frame_benchmarking::{benchmarks, whitelisted_caller}; use frame_system::RawOrigin; benchmarks! { @@ -15,6 +15,6 @@ benchmarks! { verify { assert_eq!(Something::::get(), Some(s)); } -} -impl_benchmark_test_suite!(Template, crate::mock::new_test_ext(), crate::mock::Test); + impl_benchmark_test_suite!(Template, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/bin/node-template/pallets/iris-session/src/lib.rs b/bin/node-template/pallets/iris-session/src/lib.rs index d6bece9622b6..75b0a3cd3044 100644 --- a/bin/node-template/pallets/iris-session/src/lib.rs +++ b/bin/node-template/pallets/iris-session/src/lib.rs @@ -72,16 +72,23 @@ use sp_runtime::{ use pallet_iris_assets::{ DataCommand, }; +use sp_std::convert::TryFrom; +use sp_std::convert::TryInto; pub const LOG_TARGET: &'static str = "runtime::iris-session"; // TODO: should a new KeyTypeId be defined? e.g. b"iris" pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"aura"); pub mod crypto { - use crate::KEY_TYPE; + // use crate::KEY_TYPE; + use sp_core::crypto::KeyTypeId; use sp_core::sr25519::Signature as Sr25519Signature; use sp_runtime::app_crypto::{app_crypto, sr25519}; use sp_runtime::{traits::Verify, MultiSignature, MultiSigner}; + use sp_std::convert::TryInto; + use sp_std::convert::TryFrom; + + pub const KEY_TYPE: KeyTypeId = KeyTypeId(*b"aura"); app_crypto!(sr25519, KEY_TYPE); @@ -139,6 +146,7 @@ pub mod pallet { CreateSignedTransaction, } }; + use frame_support::pallet_prelude::*; /// Configure the pallet by specifying the parameters and types on which it /// depends. @@ -166,6 +174,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); /// map the ipfs public key to a list of multiaddresses @@ -229,7 +238,7 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn eras_reward_points)] pub type ErasRewardPoints = StorageDoubleMap< - _, Blake2_128Concat, EraIndex, Blake2_128Concat, T::AssetId, EraRewardPoints, ValueQuery, + _, Twox64Concat, EraIndex, Twox64Concat, T::AssetId, EraRewardPoints, >; /// @@ -487,16 +496,17 @@ pub mod pallet { )?; // award point to all validators if let Some(active_era) = ActiveEra::::get() { - >::mutate(active_era.clone(), id, |era_rewards| { - // reward all validators - for v in >::get() { - SessionParticipation::::mutate(active_era.clone(), |participants| { - participants.push(v.clone()); - }); - *era_rewards.individual.entry(v.clone()).or_default() += 1; - era_rewards.total += 1; - } - }); + // WIP: TODO + // >::mutate(active_era.clone(), id, |era_rewards| { + // // reward all validators + // for v in >::get() { + // SessionParticipation::::mutate(active_era.clone(), |participants| { + // participants.push(v.clone()); + // }); + // *era_rewards.unwrap().individual.entry(v.clone()).or_default() += 1; + // era_rewards.unwrap().total += 1; + // } + // }); } else { // error } @@ -550,10 +560,11 @@ pub mod pallet { SessionParticipation::::mutate(active_era.clone(), |p| { p.push(pinner.clone()); }); - >::mutate(active_era, asset_id, |era_rewards| { - *era_rewards.individual.entry(pinner.clone()).or_default() += 1; - era_rewards.total += 1; - }); + // WIP: TODO + // >::mutate(active_era, asset_id, |era_rewards| { + // *era_rewards.unwrap().individual.entry(pinner.clone()).or_default() += 1; + // era_rewards.unwrap().total += 1; + // }); } Ok(()) } @@ -570,18 +581,19 @@ pub mod pallet { asset_id: T::AssetId, ) -> DispatchResult { // ensure_signed(origin)?; - if let Some(active_era) = ActiveEra::::get() { - >::mutate(active_era.clone(), asset_id.clone(), |era_rewards| { - // reward all active storage providers - for k in StorageProviders::::get(asset_id.clone()).into_iter() { - SessionParticipation::::mutate(active_era.clone(), |p| { - p.push(k.clone()); - }); - *era_rewards.individual.entry(k.clone()).or_default() += 1; - era_rewards.total += 1; - } - }); - } + // WIP: TODO + // if let Some(active_era) = ActiveEra::::get() { + // >::mutate(active_era.clone(), asset_id.clone(), |era_rewards| { + // // reward all active storage providers + // for k in StorageProviders::::get(asset_id.clone()).into_iter() { + // SessionParticipation::::mutate(active_era.clone(), |p| { + // p.push(k.clone()); + // }); + // *era_rewards.unwrap().individual.entry(k.clone()).or_default() += 1; + // era_rewards.unwrap().total += 1; + // } + // }); + // } Ok(()) } } diff --git a/bin/node-template/pallets/template/src/lib.rs b/bin/node-template/pallets/template/src/lib.rs new file mode 100644 index 000000000000..f5ce8c5a0f7f --- /dev/null +++ b/bin/node-template/pallets/template/src/lib.rs @@ -0,0 +1,102 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +/// Edit this file to define custom logic or remove it if it is not needed. +/// Learn more about FRAME and the core library of Substrate FRAME pallets: +/// +pub use pallet::*; + +#[cfg(test)] +mod mock; + +#[cfg(test)] +mod tests; + +#[cfg(feature = "runtime-benchmarks")] +mod benchmarking; + +#[frame_support::pallet] +pub mod pallet { + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + /// Configure the pallet by specifying the parameters and types on which it depends. + #[pallet::config] + pub trait Config: frame_system::Config { + /// Because this pallet emits events, it depends on the runtime's definition of an event. + type Event: From> + IsType<::Event>; + } + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(_); + + // The pallet's runtime storage items. + // https://docs.substrate.io/v3/runtime/storage + #[pallet::storage] + #[pallet::getter(fn something)] + // Learn more about declaring storage items: + // https://docs.substrate.io/v3/runtime/storage#declaring-storage-items + pub type Something = StorageValue<_, u32>; + + // Pallets use events to inform users when important changes are made. + // https://docs.substrate.io/v3/runtime/events-and-errors + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// Event documentation should end with an array that provides descriptive names for event + /// parameters. [something, who] + SomethingStored(u32, T::AccountId), + } + + // Errors inform users that something went wrong. + #[pallet::error] + pub enum Error { + /// Error names should be descriptive. + NoneValue, + /// Errors should have helpful documentation associated with them. + StorageOverflow, + } + + // Dispatchable functions allows users to interact with the pallet and invoke state changes. + // These functions materialize as "extrinsics", which are often compared to transactions. + // Dispatchable functions must be annotated with a weight and must return a DispatchResult. + #[pallet::call] + impl Pallet { + /// An example dispatchable that takes a singles value as a parameter, writes the value to + /// storage and emits an event. This function must be dispatched by a signed extrinsic. + #[pallet::weight(10_000 + T::DbWeight::get().writes(1))] + pub fn do_something(origin: OriginFor, something: u32) -> DispatchResult { + // Check that the extrinsic was signed and get the signer. + // This function will return an error if the extrinsic is not signed. + // https://docs.substrate.io/v3/runtime/origins + let who = ensure_signed(origin)?; + + // Update storage. + >::put(something); + + // Emit an event. + Self::deposit_event(Event::SomethingStored(something, who)); + // Return a successful DispatchResultWithPostInfo + Ok(()) + } + + /// An example dispatchable that may throw a custom error. + #[pallet::weight(10_000 + T::DbWeight::get().reads_writes(1,1))] + pub fn cause_error(origin: OriginFor) -> DispatchResult { + let _who = ensure_signed(origin)?; + + // Read a value from storage. + match >::get() { + // Return an error if the value has not been set. + None => Err(Error::::NoneValue)?, + Some(old) => { + // Increment the value read from storage; will error in the event of overflow. + let new = old.checked_add(1).ok_or(Error::::StorageOverflow)?; + // Update the value in storage with the incremented result. + >::put(new); + Ok(()) + }, + } + } + } +} diff --git a/bin/node-template/pallets/template/src/mock.rs b/bin/node-template/pallets/template/src/mock.rs new file mode 100644 index 000000000000..8721fe6c7885 --- /dev/null +++ b/bin/node-template/pallets/template/src/mock.rs @@ -0,0 +1,59 @@ +use crate as pallet_template; +use frame_support::traits::{ConstU16, ConstU64}; +use frame_system as system; +use sp_core::H256; +use sp_runtime::{ + testing::Header, + traits::{BlakeTwo256, IdentityLookup}, +}; + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; + +// Configure a mock runtime to test the pallet. +frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + TemplateModule: pallet_template::{Pallet, Call, Storage, Event}, + } +); + +impl system::Config for Test { + type BaseCallFilter = frame_support::traits::Everything; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Origin = Origin; + type Call = Call; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = ConstU64<250>; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = ConstU16<42>; + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; +} + +impl pallet_template::Config for Test { + type Event = Event; +} + +// Build genesis storage according to the mock runtime. +pub fn new_test_ext() -> sp_io::TestExternalities { + system::GenesisConfig::default().build_storage::().unwrap().into() +} diff --git a/bin/node-template/runtime/Cargo.toml b/bin/node-template/runtime/Cargo.toml index 0a7f9c7e3343..2915b5d0064e 100644 --- a/bin/node-template/runtime/Cargo.toml +++ b/bin/node-template/runtime/Cargo.toml @@ -1,19 +1,20 @@ [package] name = "node-template-runtime" -version = "3.0.0" +version = "4.0.0-dev" +description = "A fresh FRAME-based Substrate node, ready for hacking." authors = ["Substrate DevHub "] -edition = "2018" +homepage = "https://substrate.io/" +edition = "2021" license = "Unlicense" -homepage = "https://substrate.dev" -repository = "https://github.com/substrate-developer-hub/substrate-node-template/" publish = false +repository = "https://github.com/substrate-developer-hub/substrate-node-template/" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } pallet-aura = { version = "4.0.0-dev", default-features = false, path = "../../../frame/aura" } pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../../frame/balances" } @@ -22,6 +23,8 @@ pallet-grandpa = { version = "4.0.0-dev", default-features = false, path = "../. # TODO: REMOVE THIS pallet-randomness-collective-flip = { version = "4.0.0-dev", default-features = false, path = "../../../frame/randomness-collective-flip" } pallet-sudo = { version = "4.0.0-dev", default-features = false, path = "../../../frame/sudo" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system" } +frame-try-runtime = { version = "0.10.0-dev", default-features = false, path = "../../../frame/try-runtime", optional = true } pallet-im-online = { version = "4.0.0-dev", default-features = false, path = "../../../frame/im-online" } pallet-assets = { version = "4.0.0-dev", default-features = false, path = "../../../frame/assets" } pallet-session = { version = "4.0.0-dev", default-features = false, features = ["historical"], path = "../../../frame/session" } @@ -31,23 +34,23 @@ pallet-iris-ledger = { version = "1.0.0", default-features = false, path = "../p pallet-timestamp = { version = "4.0.0-dev", default-features = false, path = "../../../frame/timestamp" } pallet-transaction-payment = { version = "4.0.0-dev", default-features = false, path = "../../../frame/transaction-payment" } pallet-contracts = { version = "4.0.0-dev", default-features = false, path = "../../../frame/contracts" } -pallet-contracts-primitives = { version = "4.0.0-dev", default-features = false, path = "../../../frame/contracts/common/" } +pallet-contracts-primitives = { version = "6.0.0", default-features = false, path = "../../../frame/contracts/common/" } # primitives node-primitives = { version = "2.0.0", default-features = false, path = "../../node/primitives" } -frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system" } +# frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system" } frame-executive = { version = "4.0.0-dev", default-features = false, path = "../../../frame/executive" } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/api" } -sp-block-builder = { path = "../../../primitives/block-builder", default-features = false, version = "4.0.0-dev"} +sp-block-builder = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/block-builder"} sp-consensus-aura = { version = "0.10.0-dev", default-features = false, path = "../../../primitives/consensus/aura" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/core" } -sp-inherents = { path = "../../../primitives/inherents", default-features = false, version = "4.0.0-dev"} +sp-core = { version = "6.0.0", default-features = false, path = "../../../primitives/core" } +sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/inherents"} sp-offchain = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/offchain" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } sp-session = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/session" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/std" } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } sp-transaction-pool = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/transaction-pool" } -sp-version = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/version" } +sp-version = { version = "5.0.0", default-features = false, path = "../../../primitives/version" } # Used for the node template's RPCs frame-system-rpc-runtime-api = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } @@ -58,7 +61,7 @@ pallet-contracts-rpc-runtime-api = { version = "4.0.0-dev", default-features = f frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../../../frame/benchmarking", optional = true } frame-system-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system/benchmarking", optional = true } -hex-literal = { version = "0.3.1", optional = true } +hex-literal = { version = "0.3.4", optional = true } log = { version = "0.4.14", default-features = false } @@ -66,7 +69,7 @@ log = { version = "0.4.14", default-features = false } substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } [features] -default = ["std"] +default = ["std", "contracts-unstable-interface"] std = [ "codec/std", "scale-info/std", @@ -105,7 +108,7 @@ std = [ "pallet-contracts-rpc-runtime-api/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking", "frame-system/runtime-benchmarks", @@ -119,8 +122,17 @@ runtime-benchmarks = [ "pallet-contracts/runtime-benchmarks", ] try-runtime = [ + "frame-executive/try-runtime", + "frame-try-runtime", + "frame-system/try-runtime", + "pallet-balances/try-runtime", + "pallet-grandpa/try-runtime", + "pallet-randomness-collective-flip/try-runtime", + "pallet-sudo/try-runtime", + "pallet-timestamp/try-runtime", + "pallet-transaction-payment/try-runtime", "pallet-contracts/try-runtime", ] # Make contract callable functions marked as __unstable__ available. Do not enable # on live chains as those are subject to change. -contracts-unstable-interface = ["pallet-contracts/unstable-interface"] \ No newline at end of file +contracts-unstable-interface = ["pallet-contracts/unstable-interface"] diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index 5380b5e0b270..5a094a341357 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -37,17 +37,20 @@ use frame_system::{ }; use pallet_im_online::sr25519::AuthorityId as ImOnlineId; use pallet_contracts::weights::WeightInfo; +use frame_support::traits::Nothing; // A few exports that help ease life for downstream crates. pub use frame_support::{ - debug, construct_runtime, parameter_types, - traits::{KeyOwnerProofSystem, Randomness, StorageInfo, Nothing}, + construct_runtime, parameter_types, + traits::{ConstU128, ConstU32, ConstU8, KeyOwnerProofSystem, Randomness, StorageInfo}, + debug, weights::{ constants::{BlockExecutionWeight, ExtrinsicBaseWeight, RocksDbWeight, WEIGHT_PER_SECOND}, IdentityFee, Weight, DispatchClass, }, StorageValue, }; +pub use frame_system::Call as SystemCall; pub use pallet_balances::Call as BalancesCall; pub use pallet_timestamp::Call as TimestampCall; pub use pallet_assets::Call as AssetsCall; @@ -81,10 +84,28 @@ pub type Index = u32; /// A hash of some data used by the chain. pub type Hash = sp_core::H256; +// Prints debug output of the `contracts` pallet to stdout if the node is +// started with `-lruntime::contracts=debug`. +const CONTRACTS_DEBUG_OUTPUT: bool = true; + +// Unit = the base number of indivisible units for balances +const UNIT: Balance = 1_000_000_000_000; +const MILLIUNIT: Balance = 1_000_000_000; +const EXISTENTIAL_DEPOSIT: Balance = MILLIUNIT; + +const fn deposit(items: u32, bytes: u32) -> Balance { + (items as Balance * UNIT + (bytes as Balance) * (5 * MILLIUNIT / 100)) / 10 +} + +// TODO: needed? pub const MILLICENTS: Balance = 1_000_000_000; pub const CENTS: Balance = 1_000 * MILLICENTS; // assume this is worth about a cent. pub const DOLLARS: Balance = 100 * CENTS; +// const fn deposit(items: u32, bytes: u32) -> Balance { +// (items as Balance * UNIT + (bytes as Balance) * (5 * MILLIUNIT / 100)) / 10 +// } + /// Opaque types. These are used by the CLI to instantiate machinery that don't need to know /// the specifics of the runtime. They can then be made to be agnostic over specific formats /// of data like extrinsics, allowing for them to continue syncing the network through upgrades @@ -112,12 +133,8 @@ pub mod opaque { use node_primitives::Balance; -pub const fn deposit(items: u32, bytes: u32) -> Balance { - items as Balance * 15 * CENTS + (bytes as Balance) * 6 * CENTS -} - // To learn more about runtime versioning and what each of the following value means: -// https://substrate.dev/docs/en/knowledgebase/runtime/upgrades#runtime-versioning +// https://docs.substrate.io/v3/runtime/upgrades#runtime-versioning #[sp_version::runtime_version] pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("node-template"), @@ -132,6 +149,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { impl_version: 1, apis: RUNTIME_API_VERSIONS, transaction_version: 1, + state_version: 1, }; /// This determines the average expected block time that we are targeting. @@ -245,16 +263,14 @@ impl frame_system::Config for Runtime { type SS58Prefix = SS58Prefix; /// The set code logic, just the default since we're not a parachain. type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } impl pallet_randomness_collective_flip::Config for Runtime {} parameter_types! { - pub ContractDeposit: Balance = deposit( - 1, - >::contract_info_size(), - ); - pub const MaxValueSize: u32 = 16 * 1024; + pub const DepositPerItem: Balance = deposit(1, 0); + pub const DepositPerByte: Balance = deposit(0, 1); // The lazy deletion runs inside on_initialize. pub DeletionWeightLimit: Weight = AVERAGE_ON_INITIALIZE_RATIO * RuntimeBlockWeights::get().max_block; @@ -264,7 +280,18 @@ parameter_types! { ::WeightInfo::on_initialize_per_queue_item(1) - ::WeightInfo::on_initialize_per_queue_item(0) )) / 5) as u32; - pub Schedule: pallet_contracts::Schedule = Default::default(); + pub Schedule: pallet_contracts::Schedule = { + let mut schedule = pallet_contracts::Schedule::::default(); + // We decided to **temporarily* increase the default allowed contract size here + // (the default is `128 * 1024`). + // + // Our reasoning is that a number of people ran into `CodeTooLarge` when trying + // to deploy their contracts. We are currently introducing a number of optimizations + // into ink! which should bring the contract sizes lower. In the meantime we don't + // want to pose additional friction on developers. + schedule.limits.code_len = 256 * 1024; + schedule + }; } impl pallet_contracts::Config for Runtime { @@ -279,15 +306,17 @@ impl pallet_contracts::Config for Runtime { /// and make sure they are stable. Dispatchables exposed to contracts are not allowed to /// change because that would break already deployed contracts. The `Call` structure itself /// is not allowed to change the indices of existing pallets, too. - type CallFilter = Nothing; - type ContractDeposit = ContractDeposit; - type CallStack = [pallet_contracts::Frame; 31]; + type CallFilter = frame_support::traits::Nothing; + type DepositPerItem = DepositPerItem; + type DepositPerByte = DepositPerByte; type WeightPrice = pallet_transaction_payment::Pallet; type WeightInfo = pallet_contracts::weights::SubstrateWeight; type ChainExtension = IrisExtension; type DeletionQueueDepth = DeletionQueueDepth; type DeletionWeightLimit = DeletionWeightLimit; type Schedule = Schedule; + type CallStack = [pallet_contracts::Frame; 31]; + type AddressGenerator = pallet_contracts::DefaultAddressGenerator; } parameter_types! { @@ -319,7 +348,7 @@ impl pallet_session::Config for Runtime { type SessionHandler = ::KeyTypeIdProviders; type Keys = opaque::SessionKeys; type WeightInfo = pallet_session::weights::SubstrateWeight; - type DisabledValidatorsThreshold = (); + // type DisabledValidatorsThreshold = (); type Event = Event; } @@ -350,7 +379,7 @@ parameter_types! { impl pallet_aura::Config for Runtime { type AuthorityId = AuraId; type DisabledValidators = (); - type MaxAuthorities = MaxAuthorities; + type MaxAuthorities = ConstU32<32>; } impl pallet_grandpa::Config for Runtime { @@ -370,7 +399,7 @@ impl pallet_grandpa::Config for Runtime { type HandleEquivocation = (); type WeightInfo = (); - type MaxAuthorities = MaxAuthorities; + type MaxAuthorities = ConstU32<32>; } parameter_types! { @@ -385,13 +414,8 @@ impl pallet_timestamp::Config for Runtime { type WeightInfo = (); } -parameter_types! { - pub const ExistentialDeposit: u128 = 500; - pub const MaxLocks: u32 = 50; -} - impl pallet_balances::Config for Runtime { - type MaxLocks = MaxLocks; + type MaxLocks = ConstU32<50>; type MaxReserves = (); type ReserveIdentifier = [u8; 8]; /// The type for recording an account's balance. @@ -399,7 +423,7 @@ impl pallet_balances::Config for Runtime { /// The ubiquitous event type. type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU128<500>; type AccountStore = System; type WeightInfo = pallet_balances::weights::SubstrateWeight; } @@ -412,13 +436,16 @@ parameter_types! { impl pallet_transaction_payment::Config for Runtime { type OnChargeTransaction = CurrencyAdapter; - type TransactionByteFee = TransactionByteFee; + type OperationalFeeMultiplier = ConstU8<5>; type WeightToFee = IdentityFee; + type LengthToFee = IdentityFee; type FeeMultiplierUpdate = (); } parameter_types! { pub const AssetDeposit: Balance = 100 * DOLLARS; + // TODO: What is the proper amount for this? + pub const AssetAccountDeposit: Balance = 1 * DOLLARS; pub const ApprovalDeposit: Balance = 1 * DOLLARS; pub const StringLimit: u32 = 50; pub const MetadataDepositBase: Balance = 10 * DOLLARS; @@ -437,6 +464,7 @@ impl pallet_assets::Config for Runtime { type Currency = Balances; type ForceOrigin = EnsureRoot; type AssetDeposit = AssetDeposit; + type AssetAccountDeposit = AssetAccountDeposit; type MetadataDepositBase = MetadataDepositBase; type MetadataDepositPerByte = MetadataDepositPerByte; type ApprovalDeposit = ApprovalDeposit; @@ -483,6 +511,7 @@ where .saturating_sub(1); let tip = 0; let extra: SignedExtra = ( + frame_system::CheckNonZeroSender::::new(), frame_system::CheckSpecVersion::::new(), frame_system::CheckTxVersion::::new(), frame_system::CheckGenesis::::new(), @@ -552,6 +581,7 @@ pub type Header = generic::Header; pub type Block = generic::Block; /// The SignedExtension to the basic transaction logic. pub type SignedExtra = ( + frame_system::CheckNonZeroSender, frame_system::CheckSpecVersion, frame_system::CheckTxVersion, frame_system::CheckGenesis, @@ -562,15 +592,32 @@ pub type SignedExtra = ( ); /// Unchecked extrinsic type as expected by this runtime. pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; +/// The payload being signed in transactions. +// pub type SignedPayload = generic::SignedPayload; /// Executive: handles dispatch to the various modules. pub type Executive = frame_executive::Executive< Runtime, Block, frame_system::ChainContext, Runtime, - AllPallets, + AllPalletsWithSystem, >; +#[cfg(feature = "runtime-benchmarks")] +#[macro_use] +extern crate frame_benchmarking; + +#[cfg(feature = "runtime-benchmarks")] +mod benches { + define_benchmarks!( + [frame_benchmarking, BaselineBench::] + [frame_system, SystemBench::] + [pallet_balances, Balances] + [pallet_timestamp, Timestamp] + [pallet_template, TemplateModule] + ); +} + impl_runtime_apis! { impl sp_api::Core for Runtime { fn version() -> RuntimeVersion { @@ -688,9 +735,7 @@ impl_runtime_apis! { } - impl pallet_contracts_rpc_runtime_api::ContractsApi< - Block, AccountId, Balance, BlockNumber, Hash, - > + impl pallet_contracts_rpc_runtime_api::ContractsApi for Runtime { fn call( @@ -698,21 +743,32 @@ impl_runtime_apis! { dest: AccountId, value: Balance, gas_limit: u64, + storage_deposit_limit: Option, input_data: Vec, - ) -> pallet_contracts_primitives::ContractExecResult { - Contracts::bare_call(origin, dest, value, gas_limit, input_data, true) + ) -> pallet_contracts_primitives::ContractExecResult { + Contracts::bare_call(origin, dest, value, gas_limit, storage_deposit_limit, input_data, CONTRACTS_DEBUG_OUTPUT) } fn instantiate( origin: AccountId, - endowment: Balance, + value: Balance, gas_limit: u64, + storage_deposit_limit: Option, code: pallet_contracts_primitives::Code, data: Vec, salt: Vec, - ) -> pallet_contracts_primitives::ContractInstantiateResult + ) -> pallet_contracts_primitives::ContractInstantiateResult + { + Contracts::bare_instantiate(origin, value, gas_limit, storage_deposit_limit, code, data, salt, CONTRACTS_DEBUG_OUTPUT) + } + + fn upload_code( + origin: AccountId, + code: Vec, + storage_deposit_limit: Option, + ) -> pallet_contracts_primitives::CodeUploadResult { - Contracts::bare_instantiate(origin, endowment, gas_limit, code, data, salt, true) + Contracts::bare_upload_code(origin, code, storage_deposit_limit) } fn get_storage( @@ -755,15 +811,16 @@ impl_runtime_apis! { Vec, Vec, ) { - use frame_benchmarking::{list_benchmark, Benchmarking, BenchmarkList}; + use frame_benchmarking::{baseline, Benchmarking, BenchmarkList}; use frame_support::traits::StorageInfoTrait; use frame_system_benchmarking::Pallet as SystemBench; + use baseline::Pallet as BaselineBench; let mut list = Vec::::new(); list_benchmark!(list, extra, pallet_assets, Assets); list_benchmark!(list, extra, frame_system, SystemBench::); - list_benchmark!(list, extra, pallet_contracts, Contracts); + // list_benchmark!(list, extra, pallet_contracts, Contracts); list_benchmark!(list, extra, pallet_balances, Balances); list_benchmark!(list, extra, pallet_timestamp, Timestamp); list_benchmark!(list, extra, pallet_iris, Iris); @@ -776,10 +833,13 @@ impl_runtime_apis! { fn dispatch_benchmark( config: frame_benchmarking::BenchmarkConfig ) -> Result, sp_runtime::RuntimeString> { - use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark, TrackedStorageKey}; + use frame_benchmarking::{baseline, Benchmarking, BenchmarkBatch, TrackedStorageKey}; use frame_system_benchmarking::Pallet as SystemBench; + use baseline::Pallet as BaselineBench; + impl frame_system_benchmarking::Config for Runtime {} + impl baseline::Config for Runtime {} let whitelist: Vec = vec![ // Block Number @@ -796,10 +856,11 @@ impl_runtime_apis! { let mut batches = Vec::::new(); let params = (&config, &whitelist); + add_benchmarks!(params, batches); add_benchmark!(params, batches, pallet_assets, Assets); add_benchmark!(params, batches, frame_system, SystemBench::); - add_benchmark!(params, batches, pallet_contracts, Contracts); + // add_benchmark!(params, batches, pallet_contracts, Contracts); add_benchmark!(params, batches, pallet_balances, Balances); add_benchmark!(params, batches, pallet_timestamp, Timestamp); add_benchmark!(params, batches, pallet_iris_assets, Iris); @@ -808,6 +869,21 @@ impl_runtime_apis! { Ok(batches) } } + + #[cfg(feature = "try-runtime")] + impl frame_try_runtime::TryRuntime for Runtime { + fn on_runtime_upgrade() -> (Weight, Weight) { + // NOTE: intentional unwrap: we don't want to propagate the error backwards, and want to + // have a backtrace here. If any of the pre/post migration checks fail, we shall stop + // right here and right now. + let weight = Executive::try_runtime_upgrade().unwrap(); + (weight, BlockWeights::get().max_block) + } + + fn execute_block_no_check(block: Block) -> Weight { + Executive::execute_block_no_check(block) + } + } } /** diff --git a/bin/node-template/shell.nix b/bin/node-template/shell.nix new file mode 100644 index 000000000000..c08005c1630e --- /dev/null +++ b/bin/node-template/shell.nix @@ -0,0 +1,35 @@ +let + mozillaOverlay = + import (builtins.fetchGit { + url = "https://github.com/mozilla/nixpkgs-mozilla.git"; + rev = "57c8084c7ef41366993909c20491e359bbb90f54"; + }); + pinned = builtins.fetchGit { + # Descriptive name to make the store path easier to identify + url = "https://github.com/nixos/nixpkgs/"; + # Commit hash for nixos-unstable as of 2020-04-26 + # `git ls-remote https://github.com/nixos/nixpkgs nixos-unstable` + ref = "refs/heads/nixos-unstable"; + rev = "1fe6ed37fd9beb92afe90671c0c2a662a03463dd"; + }; + nixpkgs = import pinned { overlays = [ mozillaOverlay ]; }; + toolchain = with nixpkgs; (rustChannelOf { date = "2021-09-14"; channel = "nightly"; }); + rust-wasm = toolchain.rust.override { + targets = [ "wasm32-unknown-unknown" ]; + }; +in +with nixpkgs; pkgs.mkShell { + buildInputs = [ + clang + pkg-config + rust-wasm + ] ++ stdenv.lib.optionals stdenv.isDarwin [ + darwin.apple_sdk.frameworks.Security + ]; + + LIBCLANG_PATH = "${llvmPackages.libclang}/lib"; + PROTOC = "${protobuf}/bin/protoc"; + RUST_SRC_PATH = "${toolchain.rust-src}/lib/rustlib/src/rust/library/"; + ROCKSDB_LIB_DIR = "${rocksdb}/lib"; + +} diff --git a/bin/node/bench/Cargo.toml b/bin/node/bench/Cargo.toml index b19a71966fb8..45a959ac16b1 100644 --- a/bin/node/bench/Cargo.toml +++ b/bin/node/bench/Cargo.toml @@ -3,42 +3,40 @@ name = "node-bench" version = "0.9.0-dev" authors = ["Parity Technologies "] description = "Substrate node integration benchmarks." -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +clap = { version = "3.1.6", features = ["derive"] } log = "0.4.8" node-primitives = { version = "2.0.0", path = "../primitives" } node-testing = { version = "3.0.0-dev", path = "../testing" } node-runtime = { version = "3.0.0-dev", path = "../runtime" } sc-client-api = { version = "4.0.0-dev", path = "../../../client/api/" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -sp-state-machine = { version = "0.10.0-dev", path = "../../../primitives/state-machine" } -serde = "1.0.126" -serde_json = "1.0.68" -structopt = "0.3" -derive_more = "0.99.2" -kvdb = "0.10.0" -kvdb-rocksdb = "0.14.0" -sp-trie = { version = "4.0.0-dev", path = "../../../primitives/trie" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } +sp-state-machine = { version = "0.12.0", path = "../../../primitives/state-machine" } +serde = "1.0.136" +serde_json = "1.0.79" +derive_more = "0.99.16" +kvdb = "0.11.0" +kvdb-rocksdb = "0.15.1" +sp-trie = { version = "6.0.0", path = "../../../primitives/trie" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } sc-basic-authorship = { version = "0.10.0-dev", path = "../../../client/basic-authorship" } sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } sp-timestamp = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/timestamp" } -sp-tracing = { version = "4.0.0-dev", path = "../../../primitives/tracing" } +sp-tracing = { version = "5.0.0", path = "../../../primitives/tracing" } hash-db = "0.15.2" tempfile = "3.1.0" fs_extra = "1" hex = "0.4.0" rand = { version = "0.7.2", features = ["small_rng"] } lazy_static = "1.4.0" -parity-util-mem = { version = "0.10.0", default-features = false, features = [ - "primitive-types", -] } +parity-util-mem = { version = "0.11.0", default-features = false, features = ["primitive-types"] } parity-db = { version = "0.3" } sc-transaction-pool = { version = "4.0.0-dev", path = "../../../client/transaction-pool" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../../client/transaction-pool/api" } -futures = { version = "0.3.4", features = ["thread-pool"] } +futures = { version = "0.3.21", features = ["thread-pool"] } diff --git a/bin/node/bench/src/common.rs b/bin/node/bench/src/common.rs index d04d79e9907a..839e26f9f6d1 100644 --- a/bin/node/bench/src/common.rs +++ b/bin/node/bench/src/common.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/bench/src/construct.rs b/bin/node/bench/src/construct.rs index 1532e02bd3ef..50b9db9f019d 100644 --- a/bin/node/bench/src/construct.rs +++ b/bin/node/bench/src/construct.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -30,8 +30,8 @@ use std::{borrow::Cow, collections::HashMap, pin::Pin, sync::Arc}; use node_primitives::Block; use node_testing::bench::{BenchDb, BlockType, DatabaseType, KeyTypes, Profile}; use sc_transaction_pool_api::{ - ImportNotificationStream, PoolFuture, PoolStatus, TransactionFor, TransactionSource, - TransactionStatusStreamFor, TxHash, + ImportNotificationStream, PoolFuture, PoolStatus, ReadyTransactions, TransactionFor, + TransactionSource, TransactionStatusStreamFor, TxHash, }; use sp_consensus::{Environment, Proposer}; use sp_inherents::InherentDataProvider; @@ -216,6 +216,19 @@ impl sc_transaction_pool_api::InPoolTransaction for PoolTransaction { #[derive(Clone, Debug)] pub struct Transactions(Vec>); +pub struct TransactionsIterator(std::vec::IntoIter>); + +impl Iterator for TransactionsIterator { + type Item = Arc; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +impl ReadyTransactions for TransactionsIterator { + fn report_invalid(&mut self, _tx: &Self::Item) {} +} impl sc_transaction_pool_api::TransactionPool for Transactions { type Block = Block; @@ -257,16 +270,17 @@ impl sc_transaction_pool_api::TransactionPool for Transactions { _at: NumberFor, ) -> Pin< Box< - dyn Future> + Send>> - + Send, + dyn Future< + Output = Box> + Send>, + > + Send, >, > { - let iter: Box> + Send> = - Box::new(self.0.clone().into_iter()); + let iter: Box> + Send> = + Box::new(TransactionsIterator(self.0.clone().into_iter())); Box::pin(futures::future::ready(iter)) } - fn ready(&self) -> Box> + Send> { + fn ready(&self) -> Box> + Send> { unimplemented!() } diff --git a/bin/node/bench/src/core.rs b/bin/node/bench/src/core.rs index 56c0f3526a4d..b6ad3ecd8006 100644 --- a/bin/node/bench/src/core.rs +++ b/bin/node/bench/src/core.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index e3aa1192b5d1..2b26ed9089a5 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,7 +20,7 @@ use std::{collections::HashMap, sync::Arc}; use kvdb::KeyValueDB; use node_primitives::Hash; -use sp_trie::{trie_types::TrieDBMut, TrieMut}; +use sp_trie::{trie_types::TrieDBMutV1, TrieMut}; use crate::simple_trie::SimpleTrie; @@ -43,8 +43,7 @@ pub fn generate_trie( ); let mut trie = SimpleTrie { db, overlay: &mut overlay }; { - let mut trie_db = TrieDBMut::new(&mut trie, &mut root); - + let mut trie_db = TrieDBMutV1::::new(&mut trie, &mut root); for (key, value) in key_values { trie_db.insert(&key, &value).expect("trie insertion failed"); } diff --git a/bin/node/bench/src/import.rs b/bin/node/bench/src/import.rs index 5bbf1ddf3b73..faba85468b1f 100644 --- a/bin/node/bench/src/import.rs +++ b/bin/node/bench/src/import.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -135,18 +135,20 @@ impl core::Benchmark for ImportBenchmark { .inspect_state(|| { match self.block_type { BlockType::RandomTransfersKeepAlive => { - // should be 5 per signed extrinsic + 1 per unsigned + // should be 7 per signed extrinsic + 1 per unsigned // we have 1 unsigned and the rest are signed in the block - // those 5 events per signed are: - // - new account (RawEvent::NewAccount) as we always transfer fund to - // non-existant account - // - endowed (RawEvent::Endowed) for this new account - // - successful transfer (RawEvent::Transfer) for this transfer operation - // - deposit event for charging transaction fee + // those 7 events per signed are: + // - withdraw (Balances::Withdraw) for charging the transaction fee + // - new account (System::NewAccount) as we always transfer fund to + // non-existent account + // - endowed (Balances::Endowed) for this new account + // - successful transfer (Event::Transfer) for this transfer operation + // - 2x deposit (Balances::Deposit and Treasury::Deposit) for depositing + // the transaction fee into the treasury // - extrinsic success assert_eq!( node_runtime::System::events().len(), - (self.block.extrinsics.len() - 1) * 5 + 1, + (self.block.extrinsics.len() - 1) * 7 + 1, ); }, BlockType::Noop => { diff --git a/bin/node/bench/src/main.rs b/bin/node/bench/src/main.rs index 4b006b387d0e..0e50447d464f 100644 --- a/bin/node/bench/src/main.rs +++ b/bin/node/bench/src/main.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -28,7 +28,7 @@ mod tempdb; mod trie; mod txpool; -use structopt::StructOpt; +use clap::Parser; use node_testing::bench::{BlockType, DatabaseType as BenchDataBaseType, KeyTypes, Profile}; @@ -42,19 +42,19 @@ use crate::{ txpool::PoolBenchmarkDescription, }; -#[derive(Debug, StructOpt)] -#[structopt(name = "node-bench", about = "Node integration benchmarks")] +#[derive(Debug, Parser)] +#[clap(name = "node-bench", about = "Node integration benchmarks")] struct Opt { /// Show list of all available benchmarks. /// /// Will output ("name", "path"). Benchmarks can then be filtered by path. - #[structopt(short, long)] + #[clap(short, long)] list: bool, /// Machine readable json output. /// /// This also suppresses all regular output (except to stderr) - #[structopt(short, long)] + #[clap(short, long)] json: bool, /// Filter benchmarks. @@ -63,7 +63,7 @@ struct Opt { filter: Option, /// Number of transactions for block import with `custom` size. - #[structopt(long)] + #[clap(long)] transactions: Option, /// Mode @@ -72,12 +72,12 @@ struct Opt { /// /// "profile" mode adds pauses between measurable runs, /// so that actual interval can be selected in the profiler of choice. - #[structopt(short, long, default_value = "regular")] + #[clap(short, long, default_value = "regular")] mode: BenchmarkMode, } fn main() { - let opt = Opt::from_args(); + let opt = Opt::parse(); if !opt.json { sp_tracing::try_init_simple(); diff --git a/bin/node/bench/src/simple_trie.rs b/bin/node/bench/src/simple_trie.rs index 651772c71575..c59389570e53 100644 --- a/bin/node/bench/src/simple_trie.rs +++ b/bin/node/bench/src/simple_trie.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/bench/src/state_sizes.rs b/bin/node/bench/src/state_sizes.rs index 27112ed42d45..f97645423edc 100644 --- a/bin/node/bench/src/state_sizes.rs +++ b/bin/node/bench/src/state_sizes.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/bench/src/tempdb.rs b/bin/node/bench/src/tempdb.rs index 518c0dd96127..1e5f3dfbce14 100644 --- a/bin/node/bench/src/tempdb.rs +++ b/bin/node/bench/src/tempdb.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index a17e386ca879..1b4534cbd0f7 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,7 +23,7 @@ use kvdb::KeyValueDB; use lazy_static::lazy_static; use rand::Rng; use sp_state_machine::Backend as _; -use sp_trie::{trie_types::TrieDBMut, TrieMut as _}; +use sp_trie::{trie_types::TrieDBMutV1, TrieMut as _}; use std::{borrow::Cow, collections::HashMap, sync::Arc}; use node_primitives::Hash; @@ -286,8 +286,8 @@ impl core::Benchmark for TrieWriteBenchmark { let mut overlay = HashMap::new(); let mut trie = SimpleTrie { db: kvdb.clone(), overlay: &mut overlay }; - let mut trie_db_mut = - TrieDBMut::from_existing(&mut trie, &mut new_root).expect("Failed to create TrieDBMut"); + let mut trie_db_mut = TrieDBMutV1::from_existing(&mut trie, &mut new_root) + .expect("Failed to create TrieDBMut"); for (warmup_key, warmup_value) in self.warmup_keys.iter() { let value = trie_db_mut diff --git a/bin/node/bench/src/txpool.rs b/bin/node/bench/src/txpool.rs index b0db73453485..322dc352e897 100644 --- a/bin/node/bench/src/txpool.rs +++ b/bin/node/bench/src/txpool.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 45e5b6311655..5545e8efa82a 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -4,10 +4,10 @@ version = "3.0.0-dev" authors = ["Parity Technologies "] description = "Generic Substrate node implementation in Rust." build = "build.rs" -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" default-run = "substrate" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.wasm-pack.profile.release] @@ -34,26 +34,28 @@ crate-type = ["cdylib", "rlib"] [dependencies] # third-party dependencies -codec = { package = "parity-scale-codec", version = "2.0.0" } -serde = { version = "1.0.126", features = ["derive"] } -futures = "0.3.16" -hex-literal = "0.3.1" +clap = { version = "3.1.6", features = ["derive"], optional = true } +codec = { package = "parity-scale-codec", version = "3.0.0" } +serde = { version = "1.0.136", features = ["derive"] } +futures = "0.3.21" +hex-literal = "0.3.4" log = "0.4.8" -rand = "0.7.2" +rand = "0.8" structopt = { version = "0.3.8", optional = true } -ipfs = { git = "https://github.com/rs-ipfs/rust-ipfs" } +ipfs = { git = "https://github.com/rs-ipfs/rust-ipfs", branch = "master" } # primitives sp-authority-discovery = { version = "4.0.0-dev", path = "../../../primitives/authority-discovery" } sp-consensus-babe = { version = "0.10.0-dev", path = "../../../primitives/consensus/babe" } grandpa-primitives = { version = "4.0.0-dev", package = "sp-finality-grandpa", path = "../../../primitives/finality-grandpa" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } sp-timestamp = { version = "4.0.0-dev", path = "../../../primitives/timestamp" } sp-authorship = { version = "4.0.0-dev", path = "../../../primitives/authorship" } sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } -sp-keyring = { version = "4.0.0-dev", path = "../../../primitives/keyring" } -sp-keystore = { version = "0.10.0-dev", path = "../../../primitives/keystore" } +sp-keyring = { version = "6.0.0", path = "../../../primitives/keyring" } +sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" } sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } sp-transaction-pool = { version = "4.0.0-dev", path = "../../../primitives/transaction-pool" } sp-transaction-storage-proof = { version = "4.0.0-dev", path = "../../../primitives/transaction-storage-proof" } @@ -79,7 +81,9 @@ sc-sync-state-rpc = { version = "0.10.0-dev", path = "../../../client/sync-state # frame dependencies frame-system = { version = "4.0.0-dev", path = "../../../frame/system" } +frame-system-rpc-runtime-api = { version = "4.0.0-dev", path = "../../../frame/system/rpc/runtime-api" } pallet-transaction-payment = { version = "4.0.0-dev", path = "../../../frame/transaction-payment" } +pallet-asset-tx-payment = { version = "4.0.0-dev", path = "../../../frame/transaction-payment/asset-tx-payment/" } pallet-im-online = { version = "4.0.0-dev", default-features = false, path = "../../../frame/im-online" } # node-specific dependencies @@ -94,17 +98,13 @@ frame-benchmarking-cli = { version = "4.0.0-dev", optional = true, path = "../.. node-inspect = { version = "0.9.0-dev", optional = true, path = "../inspect" } try-runtime-cli = { version = "0.10.0-dev", optional = true, path = "../../../utils/frame/try-runtime/cli" } -[target.'cfg(target_arch="x86_64")'.dependencies] -node-executor = { version = "3.0.0-dev", path = "../executor", features = [ - "wasmtime", -] } -sc-cli = { version = "0.10.0-dev", optional = true, path = "../../../client/cli", features = [ - "wasmtime", -] } +[target.'cfg(any(target_arch="x86_64", target_arch="aarch64"))'.dependencies] +node-executor = { version = "3.0.0-dev", path = "../executor", features = ["wasmtime"] } +sc-cli = { version = "0.10.0-dev", optional = true, path = "../../../client/cli", features = ["wasmtime"] } sc-service = { version = "0.10.0-dev", default-features = false, path = "../../../client/service", features = [ "wasmtime", ] } -sp-trie = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/trie", features = [ +sp-trie = { version = "6.0.0", default-features = false, path = "../../../primitives/trie", features = [ "memory-tracker", ] } @@ -115,29 +115,34 @@ sc-consensus = { version = "0.10.0-dev", path = "../../../client/consensus/commo sc-consensus-babe = { version = "0.10.0-dev", path = "../../../client/consensus/babe" } sc-consensus-epochs = { version = "0.10.0-dev", path = "../../../client/consensus/epochs" } sc-service-test = { version = "2.0.0", path = "../../../client/service/test" } -sp-tracing = { version = "4.0.0-dev", path = "../../../primitives/tracing" } -futures = "0.3.16" +sc-block-builder = { version = "0.10.0-dev", path = "../../../client/block-builder" } +sp-tracing = { version = "5.0.0", path = "../../../primitives/tracing" } +sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } +futures = "0.3.21" tempfile = "3.1.0" -assert_cmd = "1.0" -nix = "0.19" +assert_cmd = "2.0.2" +nix = "0.23" serde_json = "1.0" -regex = "1" -platforms = "1.1" +regex = "1.5.5" +platforms = "2.0" async-std = { version = "1.10.0", features = ["attributes"] } soketto = "0.4.2" -jsonrpsee-ws-client = { version = "0.3.0", default-features = false, features = ["tokio1"] } -tokio = { version = "1.10", features = ["macros", "time"] } +criterion = { version = "0.3.5", features = ["async_tokio"] } +tokio = { version = "1.17.0", features = ["macros", "time", "parking_lot"] } wait-timeout = "0.2" remote-externalities = { path = "../../../utils/frame/remote-externalities" } +pallet-timestamp = { version = "4.0.0-dev", path = "../../../frame/timestamp" } [build-dependencies] -structopt = { version = "0.3.8", optional = true } +clap = { version = "3.1.6", optional = true } +clap_complete = { version = "3.0", optional = true } node-inspect = { version = "0.9.0-dev", optional = true, path = "../inspect" } frame-benchmarking-cli = { version = "4.0.0-dev", optional = true, path = "../../../utils/frame/benchmarking-cli" } substrate-build-script-utils = { version = "3.0.0", optional = true, path = "../../../utils/build-script-utils" } substrate-frame-cli = { version = "4.0.0-dev", optional = true, path = "../../../utils/frame/frame-utilities-cli" } try-runtime-cli = { version = "0.10.0-dev", optional = true, path = "../../../utils/frame/try-runtime/cli" } sc-cli = { version = "0.10.0-dev", path = "../../../client/cli", optional = true } +pallet-balances = { version = "4.0.0-dev", path = "../../../frame/balances" } [features] default = ["cli"] @@ -148,14 +153,20 @@ cli = [ "frame-benchmarking-cli", "substrate-frame-cli", "sc-service/db", - "structopt", + "clap", + "clap_complete", "substrate-build-script-utils", "try-runtime-cli", ] -runtime-benchmarks = [ - "node-runtime/runtime-benchmarks", - "frame-benchmarking-cli", -] +runtime-benchmarks = ["node-runtime/runtime-benchmarks", "frame-benchmarking-cli"] # Enable features that allow the runtime to be tried and debugged. Name might be subject to change # in the near future. try-runtime = ["node-runtime/try-runtime", "try-runtime-cli"] + +[[bench]] +name = "transaction_pool" +harness = false + +[[bench]] +name = "block_production" +harness = false diff --git a/bin/node/cli/benches/block_production.rs b/bin/node/cli/benches/block_production.rs new file mode 100644 index 000000000000..77b51fa28dd1 --- /dev/null +++ b/bin/node/cli/benches/block_production.rs @@ -0,0 +1,237 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; + +use node_cli::service::{create_extrinsic, FullClient}; +use node_runtime::{constants::currency::*, BalancesCall}; +use sc_block_builder::{BlockBuilderProvider, BuiltBlock, RecordProof}; +use sc_client_api::execution_extensions::ExecutionStrategies; +use sc_consensus::{ + block_import::{BlockImportParams, ForkChoiceStrategy}, + BlockImport, StateAction, +}; +use sc_service::{ + config::{ + DatabaseSource, KeepBlocks, KeystoreConfig, NetworkConfiguration, OffchainWorkerConfig, + PruningMode, WasmExecutionMethod, + }, + BasePath, Configuration, Role, +}; +use sp_blockchain::{ApplyExtrinsicFailed::Validity, Error::ApplyExtrinsicFailed}; +use sp_consensus::BlockOrigin; +use sp_keyring::Sr25519Keyring; +use sp_runtime::{ + generic::BlockId, + transaction_validity::{InvalidTransaction, TransactionValidityError}, + AccountId32, MultiAddress, OpaqueExtrinsic, +}; +use tokio::runtime::Handle; + +fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase { + let base_path = BasePath::new_temp_dir() + .expect("getting the base path of a temporary path doesn't fail; qed"); + let root = base_path.path().to_path_buf(); + + let network_config = NetworkConfiguration::new( + Sr25519Keyring::Alice.to_seed(), + "network/test/0.1", + Default::default(), + None, + ); + + let spec = Box::new(node_cli::chain_spec::development_config()); + + // NOTE: We enforce the use of the WASM runtime to benchmark block production using WASM. + let execution_strategy = sc_client_api::ExecutionStrategy::AlwaysWasm; + + let config = Configuration { + impl_name: "BenchmarkImpl".into(), + impl_version: "1.0".into(), + // We don't use the authority role since that would start producing blocks + // in the background which would mess with our benchmark. + role: Role::Full, + tokio_handle, + transaction_pool: Default::default(), + network: network_config, + keystore: KeystoreConfig::InMemory, + keystore_remote: Default::default(), + database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 }, + state_cache_size: 67108864, + state_cache_child_ratio: None, + state_pruning: PruningMode::ArchiveAll, + keep_blocks: KeepBlocks::All, + chain_spec: spec, + wasm_method: WasmExecutionMethod::Compiled, + execution_strategies: ExecutionStrategies { + syncing: execution_strategy, + importing: execution_strategy, + block_construction: execution_strategy, + offchain_worker: execution_strategy, + other: execution_strategy, + }, + rpc_http: None, + rpc_ws: None, + rpc_ipc: None, + rpc_ws_max_connections: None, + rpc_cors: None, + rpc_methods: Default::default(), + rpc_max_payload: None, + ws_max_out_buffer_capacity: None, + prometheus_config: None, + telemetry_endpoints: None, + default_heap_pages: None, + offchain_worker: OffchainWorkerConfig { enabled: true, indexing_enabled: false }, + force_authoring: false, + disable_grandpa: false, + dev_key_seed: Some(Sr25519Keyring::Alice.to_seed()), + tracing_targets: None, + tracing_receiver: Default::default(), + max_runtime_instances: 8, + runtime_cache_size: 2, + announce_block: true, + base_path: Some(base_path), + informant_output_format: Default::default(), + wasm_runtime_overrides: None, + }; + + node_cli::service::new_full_base(config, |_, _| ()).expect("creating a full node doesn't fail") +} + +fn extrinsic_set_time(now: u64) -> OpaqueExtrinsic { + node_runtime::UncheckedExtrinsic { + signature: None, + function: node_runtime::Call::Timestamp(pallet_timestamp::Call::set { now }), + } + .into() +} + +fn import_block( + mut client: &FullClient, + built: BuiltBlock< + node_primitives::Block, + >::StateBackend, + >, +) { + let mut params = BlockImportParams::new(BlockOrigin::File, built.block.header); + params.state_action = + StateAction::ApplyChanges(sc_consensus::StorageChanges::Changes(built.storage_changes)); + params.fork_choice = Some(ForkChoiceStrategy::LongestChain); + futures::executor::block_on(client.import_block(params, Default::default())) + .expect("importing a block doesn't fail"); +} + +fn prepare_benchmark(client: &FullClient) -> (usize, Vec) { + const MINIMUM_PERIOD_FOR_BLOCKS: u64 = 1500; + + let mut max_transfer_count = 0; + let mut extrinsics = Vec::new(); + let mut block_builder = client.new_block(Default::default()).unwrap(); + + // Every block needs one timestamp extrinsic. + let extrinsic_set_time = extrinsic_set_time(1 + MINIMUM_PERIOD_FOR_BLOCKS); + block_builder.push(extrinsic_set_time.clone()).unwrap(); + extrinsics.push(extrinsic_set_time); + + // Creating those is surprisingly costly, so let's only do it once and later just `clone` them. + let src = Sr25519Keyring::Alice.pair(); + let dst: MultiAddress = Sr25519Keyring::Bob.to_account_id().into(); + + // Add as many tranfer extrinsics as possible into a single block. + for nonce in 0.. { + let extrinsic: OpaqueExtrinsic = create_extrinsic( + client, + src.clone(), + BalancesCall::transfer { dest: dst.clone(), value: 1 * DOLLARS }, + Some(nonce), + ) + .into(); + + match block_builder.push(extrinsic.clone()) { + Ok(_) => {}, + Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid( + InvalidTransaction::ExhaustsResources, + )))) => break, + Err(error) => panic!("{}", error), + } + + extrinsics.push(extrinsic); + max_transfer_count += 1; + } + + (max_transfer_count, extrinsics) +} + +fn block_production(c: &mut Criterion) { + sp_tracing::try_init_simple(); + + let runtime = tokio::runtime::Runtime::new().expect("creating tokio runtime doesn't fail; qed"); + let tokio_handle = runtime.handle().clone(); + + let node = new_node(tokio_handle.clone()); + let client = &*node.client; + + // Buliding the very first block is around ~30x slower than any subsequent one, + // so let's make sure it's built and imported before we benchmark anything. + let mut block_builder = client.new_block(Default::default()).unwrap(); + block_builder.push(extrinsic_set_time(1)).unwrap(); + import_block(client, block_builder.build().unwrap()); + + let (max_transfer_count, extrinsics) = prepare_benchmark(&client); + log::info!("Maximum transfer count: {}", max_transfer_count); + + let mut group = c.benchmark_group("Block production"); + + group.sample_size(10); + group.throughput(Throughput::Elements(max_transfer_count as u64)); + + let block_id = BlockId::Hash(client.chain_info().best_hash); + + group.bench_function(format!("{} transfers (no proof)", max_transfer_count), |b| { + b.iter_batched( + || extrinsics.clone(), + |extrinsics| { + let mut block_builder = + client.new_block_at(&block_id, Default::default(), RecordProof::No).unwrap(); + for extrinsic in extrinsics { + block_builder.push(extrinsic).unwrap(); + } + block_builder.build().unwrap() + }, + BatchSize::SmallInput, + ) + }); + + group.bench_function(format!("{} transfers (with proof)", max_transfer_count), |b| { + b.iter_batched( + || extrinsics.clone(), + |extrinsics| { + let mut block_builder = + client.new_block_at(&block_id, Default::default(), RecordProof::Yes).unwrap(); + for extrinsic in extrinsics { + block_builder.push(extrinsic).unwrap(); + } + block_builder.build().unwrap() + }, + BatchSize::SmallInput, + ) + }); +} + +criterion_group!(benches, block_production); +criterion_main!(benches); diff --git a/bin/node/cli/benches/transaction_pool.rs b/bin/node/cli/benches/transaction_pool.rs new file mode 100644 index 000000000000..e89527f2333a --- /dev/null +++ b/bin/node/cli/benches/transaction_pool.rs @@ -0,0 +1,274 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; +use futures::{future, StreamExt}; +use node_cli::service::{create_extrinsic, fetch_nonce, FullClient, TransactionPool}; +use node_primitives::AccountId; +use node_runtime::{constants::currency::*, BalancesCall, SudoCall}; +use sc_client_api::execution_extensions::ExecutionStrategies; +use sc_service::{ + config::{ + DatabaseSource, KeepBlocks, KeystoreConfig, NetworkConfiguration, OffchainWorkerConfig, + PruningMode, TransactionPoolOptions, WasmExecutionMethod, + }, + BasePath, Configuration, Role, +}; +use sc_transaction_pool::PoolLimit; +use sc_transaction_pool_api::{TransactionPool as _, TransactionSource, TransactionStatus}; +use sp_core::{crypto::Pair, sr25519}; +use sp_keyring::Sr25519Keyring; +use sp_runtime::{generic::BlockId, OpaqueExtrinsic}; +use tokio::runtime::Handle; + +fn new_node(tokio_handle: Handle) -> node_cli::service::NewFullBase { + let base_path = BasePath::new_temp_dir().expect("Creates base path"); + let root = base_path.path().to_path_buf(); + + let network_config = NetworkConfiguration::new( + Sr25519Keyring::Alice.to_seed(), + "network/test/0.1", + Default::default(), + None, + ); + + let spec = Box::new(node_cli::chain_spec::development_config()); + + let config = Configuration { + impl_name: "BenchmarkImpl".into(), + impl_version: "1.0".into(), + role: Role::Authority, + tokio_handle, + transaction_pool: TransactionPoolOptions { + ready: PoolLimit { count: 100_000, total_bytes: 100 * 1024 * 1024 }, + future: PoolLimit { count: 100_000, total_bytes: 100 * 1024 * 1024 }, + reject_future_transactions: false, + }, + network: network_config, + keystore: KeystoreConfig::InMemory, + keystore_remote: Default::default(), + database: DatabaseSource::RocksDb { path: root.join("db"), cache_size: 128 }, + state_cache_size: 67108864, + state_cache_child_ratio: None, + state_pruning: PruningMode::ArchiveAll, + keep_blocks: KeepBlocks::All, + chain_spec: spec, + wasm_method: WasmExecutionMethod::Interpreted, + // NOTE: we enforce the use of the native runtime to make the errors more debuggable + execution_strategies: ExecutionStrategies { + syncing: sc_client_api::ExecutionStrategy::NativeWhenPossible, + importing: sc_client_api::ExecutionStrategy::NativeWhenPossible, + block_construction: sc_client_api::ExecutionStrategy::NativeWhenPossible, + offchain_worker: sc_client_api::ExecutionStrategy::NativeWhenPossible, + other: sc_client_api::ExecutionStrategy::NativeWhenPossible, + }, + rpc_http: None, + rpc_ws: None, + rpc_ipc: None, + rpc_ws_max_connections: None, + rpc_cors: None, + rpc_methods: Default::default(), + rpc_max_payload: None, + ws_max_out_buffer_capacity: None, + prometheus_config: None, + telemetry_endpoints: None, + default_heap_pages: None, + offchain_worker: OffchainWorkerConfig { enabled: true, indexing_enabled: false }, + force_authoring: false, + disable_grandpa: false, + dev_key_seed: Some(Sr25519Keyring::Alice.to_seed()), + tracing_targets: None, + tracing_receiver: Default::default(), + max_runtime_instances: 8, + runtime_cache_size: 2, + announce_block: true, + base_path: Some(base_path), + informant_output_format: Default::default(), + wasm_runtime_overrides: None, + }; + + node_cli::service::new_full_base(config, |_, _| ()).expect("Creates node") +} + +fn create_accounts(num: usize) -> Vec { + (0..num) + .map(|i| { + Pair::from_string(&format!("{}/{}", Sr25519Keyring::Alice.to_seed(), i), None) + .expect("Creates account pair") + }) + .collect() +} + +/// Create the extrinsics that will initialize the accounts from the sudo account (Alice). +/// +/// `start_nonce` is the current nonce of Alice. +fn create_account_extrinsics( + client: &FullClient, + accounts: &[sr25519::Pair], +) -> Vec { + let start_nonce = fetch_nonce(client, Sr25519Keyring::Alice.pair()); + + accounts + .iter() + .enumerate() + .map(|(i, a)| { + vec![ + // Reset the nonce by removing any funds + create_extrinsic( + client, + Sr25519Keyring::Alice.pair(), + SudoCall::sudo { + call: Box::new( + BalancesCall::set_balance { + who: AccountId::from(a.public()).into(), + new_free: 0, + new_reserved: 0, + } + .into(), + ), + }, + Some(start_nonce + (i as u32) * 2), + ), + // Give back funds + create_extrinsic( + client, + Sr25519Keyring::Alice.pair(), + SudoCall::sudo { + call: Box::new( + BalancesCall::set_balance { + who: AccountId::from(a.public()).into(), + new_free: 1_000_000 * DOLLARS, + new_reserved: 0, + } + .into(), + ), + }, + Some(start_nonce + (i as u32) * 2 + 1), + ), + ] + }) + .flatten() + .map(OpaqueExtrinsic::from) + .collect() +} + +fn create_benchmark_extrinsics( + client: &FullClient, + accounts: &[sr25519::Pair], + extrinsics_per_account: usize, +) -> Vec { + accounts + .iter() + .map(|account| { + (0..extrinsics_per_account).map(move |nonce| { + create_extrinsic( + client, + account.clone(), + BalancesCall::transfer { + dest: Sr25519Keyring::Bob.to_account_id().into(), + value: 1 * DOLLARS, + }, + Some(nonce as u32), + ) + }) + }) + .flatten() + .map(OpaqueExtrinsic::from) + .collect() +} + +async fn submit_tx_and_wait_for_inclusion( + tx_pool: &TransactionPool, + tx: OpaqueExtrinsic, + client: &FullClient, + wait_for_finalized: bool, +) { + let best_hash = client.chain_info().best_hash; + + let mut watch = tx_pool + .submit_and_watch(&BlockId::Hash(best_hash), TransactionSource::External, tx.clone()) + .await + .expect("Submits tx to pool") + .fuse(); + + loop { + match watch.select_next_some().await { + TransactionStatus::Finalized(_) => break, + TransactionStatus::InBlock(_) if !wait_for_finalized => break, + _ => {}, + } + } +} + +fn transaction_pool_benchmarks(c: &mut Criterion) { + sp_tracing::try_init_simple(); + + let runtime = tokio::runtime::Runtime::new().expect("Creates tokio runtime"); + let tokio_handle = runtime.handle().clone(); + + let node = new_node(tokio_handle.clone()); + + let account_num = 10; + let extrinsics_per_account = 2000; + let accounts = create_accounts(account_num); + + let mut group = c.benchmark_group("Transaction pool"); + + group.sample_size(10); + group.throughput(Throughput::Elements(account_num as u64 * extrinsics_per_account as u64)); + + let mut counter = 1; + group.bench_function( + format!("{} transfers from {} accounts", account_num * extrinsics_per_account, account_num), + move |b| { + b.iter_batched( + || { + let prepare_extrinsics = create_account_extrinsics(&*node.client, &accounts); + + runtime.block_on(future::join_all(prepare_extrinsics.into_iter().map(|tx| { + submit_tx_and_wait_for_inclusion( + &node.transaction_pool, + tx, + &*node.client, + true, + ) + }))); + + create_benchmark_extrinsics(&*node.client, &accounts, extrinsics_per_account) + }, + |extrinsics| { + runtime.block_on(future::join_all(extrinsics.into_iter().map(|tx| { + submit_tx_and_wait_for_inclusion( + &node.transaction_pool, + tx, + &*node.client, + false, + ) + }))); + + println!("Finished {}", counter); + counter += 1; + }, + BatchSize::SmallInput, + ) + }, + ); +} + +criterion_group!(benches, transaction_pool_benchmarks); +criterion_main!(benches); diff --git a/bin/node/cli/bin/main.rs b/bin/node/cli/bin/main.rs index cf32a7cf2886..3ae295754561 100644 --- a/bin/node/cli/bin/main.rs +++ b/bin/node/cli/bin/main.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/cli/build.rs b/bin/node/cli/build.rs index 90aec2222c9e..6a3d13dda6a0 100644 --- a/bin/node/cli/build.rs +++ b/bin/node/cli/build.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -25,7 +25,8 @@ fn main() { mod cli { include!("src/cli.rs"); - use sc_cli::structopt::clap::Shell; + use clap::{ArgEnum, CommandFactory}; + use clap_complete::{generate_to, Shell}; use std::{env, fs, path::Path}; use substrate_build_script_utils::{generate_cargo_keys, rerun_if_git_head_changed}; @@ -37,9 +38,8 @@ mod cli { } /// Build shell completion scripts for all known shells - /// Full list in https://github.com/kbknapp/clap-rs/blob/e9d0562a1dc5dfe731ed7c767e6cee0af08f0cf9/src/app/parser.rs#L123 fn build_shell_completion() { - for shell in &[Shell::Bash, Shell::Fish, Shell::Zsh, Shell::Elvish, Shell::PowerShell] { + for shell in Shell::value_variants() { build_completion(shell); } } @@ -61,6 +61,6 @@ mod cli { fs::create_dir(&path).ok(); - Cli::clap().gen_completions("substrate-node", *shell, &path); + let _ = generate_to(*shell, &mut Cli::command(), "substrate-node", &path); } } diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index 5e727afa304b..11516f964903 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,8 +23,8 @@ use hex_literal::hex; use node_runtime::{ constants::currency::*, wasm_binary_unwrap, AuthorityDiscoveryConfig, BabeConfig, BalancesConfig, Block, CouncilConfig, DemocracyConfig, ElectionsConfig, GrandpaConfig, - ImOnlineConfig, IndicesConfig, SessionConfig, SessionKeys, SocietyConfig, StakerStatus, - StakingConfig, SudoConfig, SystemConfig, TechnicalCommitteeConfig, MAX_NOMINATIONS, + ImOnlineConfig, IndicesConfig, MaxNominations, SessionConfig, SessionKeys, SocietyConfig, + StakerStatus, StakingConfig, SudoConfig, SystemConfig, TechnicalCommitteeConfig, }; use pallet_im_online::sr25519::AuthorityId as ImOnlineId; use sc_chain_spec::ChainSpecExtension; @@ -196,6 +196,7 @@ pub fn staging_testnet_config() -> ChainSpec { ), None, None, + None, Default::default(), ) } @@ -277,7 +278,7 @@ pub fn testnet_genesis( .map(|x| (x.0.clone(), x.1.clone(), STASH, StakerStatus::Validator)) .chain(initial_nominators.iter().map(|x| { use rand::{seq::SliceRandom, Rng}; - let limit = (MAX_NOMINATIONS as usize).min(initial_authorities.len()); + let limit = (MaxNominations::get() as usize).min(initial_authorities.len()); let count = rng.gen::() % limit; let nominations = initial_authorities .as_slice() @@ -295,10 +296,7 @@ pub fn testnet_genesis( const STASH: Balance = ENDOWMENT / 1000; GenesisConfig { - system: SystemConfig { - code: wasm_binary_unwrap().to_vec(), - changes_trie_config: Default::default(), - }, + system: SystemConfig { code: wasm_binary_unwrap().to_vec() }, balances: BalancesConfig { balances: endowed_accounts.iter().cloned().map(|x| (x, ENDOWMENT)).collect(), }, @@ -341,7 +339,7 @@ pub fn testnet_genesis( .collect(), phantom: Default::default(), }, - sudo: SudoConfig { key: root_key }, + sudo: SudoConfig { key: Some(root_key) }, babe: BabeConfig { authorities: vec![], epoch_config: Some(node_runtime::BABE_GENESIS_EPOCH_CONFIG), @@ -361,8 +359,10 @@ pub fn testnet_genesis( max_members: 999, }, vesting: Default::default(), + assets: Default::default(), gilt: Default::default(), transaction_storage: Default::default(), + transaction_payment: Default::default(), } } @@ -386,6 +386,7 @@ pub fn development_config() -> ChainSpec { None, None, None, + None, Default::default(), ) } @@ -410,6 +411,7 @@ pub fn local_testnet_config() -> ChainSpec { None, None, None, + None, Default::default(), ) } @@ -417,7 +419,7 @@ pub fn local_testnet_config() -> ChainSpec { #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::service::{new_full_base, new_light_base, NewFullBase}; + use crate::service::{new_full_base, NewFullBase}; use sc_service_test; use sp_runtime::BuildStorage; @@ -441,6 +443,7 @@ pub(crate) mod tests { None, None, None, + None, Default::default(), ) } @@ -456,6 +459,7 @@ pub(crate) mod tests { None, None, None, + None, Default::default(), ) } @@ -465,28 +469,16 @@ pub(crate) mod tests { fn test_connectivity() { sp_tracing::try_init_simple(); - sc_service_test::connectivity( - integration_test_config_with_two_authorities(), - |config| { - let NewFullBase { task_manager, client, network, transaction_pool, .. } = - new_full_base(config, |_, _| ())?; - Ok(sc_service_test::TestNetComponents::new( - task_manager, - client, - network, - transaction_pool, - )) - }, - |config| { - let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; - Ok(sc_service_test::TestNetComponents::new( - keep_alive, - client, - network, - transaction_pool, - )) - }, - ); + sc_service_test::connectivity(integration_test_config_with_two_authorities(), |config| { + let NewFullBase { task_manager, client, network, transaction_pool, .. } = + new_full_base(config, |_, _| ())?; + Ok(sc_service_test::TestNetComponents::new( + task_manager, + client, + network, + transaction_pool, + )) + }); } #[test] diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 850581748fde..7430cc46f4cc 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,35 +16,31 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use sc_cli::{KeySubcommand, RunCmd, SignCmd, VanityCmd, VerifyCmd}; -use structopt::StructOpt; - /// An overarching CLI command definition. -#[derive(Debug, StructOpt)] +#[derive(Debug, clap::Parser)] pub struct Cli { /// Possible subcommand with parameters. - #[structopt(subcommand)] + #[clap(subcommand)] pub subcommand: Option, + #[allow(missing_docs)] - #[structopt(flatten)] - pub run: RunCmd, + #[clap(flatten)] + pub run: sc_cli::RunCmd, } /// Possible subcommands of the main binary. -#[derive(Debug, StructOpt)] +#[derive(Debug, clap::Subcommand)] pub enum Subcommand { - /// Key management cli utilities - Key(KeySubcommand), - /// The custom inspect subcommmand for decoding blocks and extrinsics. - #[structopt( + #[clap( name = "inspect", about = "Decode given block or extrinsic using current native runtime." )] Inspect(node_inspect::cli::InspectCmd), - /// The custom benchmark subcommmand benchmarking runtime pallets. - #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] + /// Sub-commands concerned with benchmarking. + /// The pallet benchmarking moved to the `pallet` sub-command. + #[clap(subcommand)] Benchmark(frame_benchmarking_cli::BenchmarkCmd), /// Try some command against runtime state. @@ -55,14 +51,18 @@ pub enum Subcommand { #[cfg(not(feature = "try-runtime"))] TryRuntime, + /// Key management cli utilities + #[clap(subcommand)] + Key(sc_cli::KeySubcommand), + /// Verify a signature for a message, provided on STDIN, with a given (public or secret) key. - Verify(VerifyCmd), + Verify(sc_cli::VerifyCmd), /// Generate a seed that provides a vanity address. - Vanity(VanityCmd), + Vanity(sc_cli::VanityCmd), /// Sign a message, with a given (secret) key. - Sign(SignCmd), + Sign(sc_cli::SignCmd), /// Build a chain specification. BuildSpec(sc_cli::BuildSpecCmd), diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 17375094f2a1..c752b1c30ae2 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,12 +16,21 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::{chain_spec, service, service::new_partial, Cli, Subcommand}; +use super::command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder}; +use crate::{ + chain_spec, service, + service::{new_partial, FullClient}, + Cli, Subcommand, +}; +use frame_benchmarking_cli::*; use node_executor::ExecutorDispatch; -use node_runtime::{Block, RuntimeApi}; -use sc_cli::{ChainSpec, Result, Role, RuntimeVersion, SubstrateCli}; +use node_primitives::Block; +use node_runtime::RuntimeApi; +use sc_cli::{ChainSpec, Result, RuntimeVersion, SubstrateCli}; use sc_service::PartialComponents; +use std::sync::Arc; + impl SubstrateCli for Cli { fn impl_name() -> String { "Substrate Node".into() @@ -77,11 +86,7 @@ pub fn run() -> Result<()> { None => { let runner = cli.create_runner(&cli.run)?; runner.run_node_until_exit(|config| async move { - match config.role { - Role::Light => service::new_light(config), - _ => service::new_full(config), - } - .map_err(sc_cli::Error::Service) + service::new_full(config).map_err(sc_cli::Error::Service) }) }, Some(Subcommand::Inspect(cmd)) => { @@ -89,16 +94,41 @@ pub fn run() -> Result<()> { runner.sync_run(|config| cmd.run::(config)) }, - Some(Subcommand::Benchmark(cmd)) => - if cfg!(feature = "runtime-benchmarks") { - let runner = cli.create_runner(cmd)?; - - runner.sync_run(|config| cmd.run::(config)) - } else { - Err("Benchmarking wasn't enabled when building the node. \ - You can enable it with `--features runtime-benchmarks`." - .into()) - }, + Some(Subcommand::Benchmark(cmd)) => { + let runner = cli.create_runner(cmd)?; + + runner.sync_run(|config| { + let PartialComponents { client, backend, .. } = new_partial(&config)?; + + // This switch needs to be in the client, since the client decides + // which sub-commands it wants to support. + match cmd { + BenchmarkCmd::Pallet(cmd) => { + if !cfg!(feature = "runtime-benchmarks") { + return Err( + "Runtime benchmarking wasn't enabled when building the node. \ + You can enable it with `--features runtime-benchmarks`." + .into(), + ) + } + + cmd.run::(config) + }, + BenchmarkCmd::Block(cmd) => cmd.run(client), + BenchmarkCmd::Storage(cmd) => { + let db = backend.expose_db(); + let storage = backend.expose_storage(); + + cmd.run(config, client, db, storage) + }, + BenchmarkCmd::Overhead(cmd) => { + let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone()); + + cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) + }, + } + }) + }, Some(Subcommand::Key(cmd)) => cmd.run(&cli), Some(Subcommand::Sign(cmd)) => cmd.run(), Some(Subcommand::Verify(cmd)) => cmd.run(), @@ -145,7 +175,12 @@ pub fn run() -> Result<()> { let runner = cli.create_runner(cmd)?; runner.async_run(|config| { let PartialComponents { client, task_manager, backend, .. } = new_partial(&config)?; - Ok((cmd.run(client, backend), task_manager)) + let aux_revert = Box::new(move |client: Arc, backend, blocks| { + sc_consensus_babe::revert(client.clone(), backend, blocks)?; + grandpa::revert(client, blocks)?; + Ok(()) + }); + Ok((cmd.run(client, backend, Some(aux_revert)), task_manager)) }) }, #[cfg(feature = "try-runtime")] diff --git a/bin/node/cli/src/command_helper.rs b/bin/node/cli/src/command_helper.rs new file mode 100644 index 000000000000..84d85ee367ca --- /dev/null +++ b/bin/node/cli/src/command_helper.rs @@ -0,0 +1,69 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Contains code to setup the command invocations in [`super::command`] which would +//! otherwise bloat that module. + +use crate::service::{create_extrinsic, FullClient}; + +use node_runtime::SystemCall; +use sc_cli::Result; +use sp_inherents::{InherentData, InherentDataProvider}; +use sp_keyring::Sr25519Keyring; +use sp_runtime::OpaqueExtrinsic; + +use std::{sync::Arc, time::Duration}; + +/// Generates extrinsics for the `benchmark overhead` command. +pub struct BenchmarkExtrinsicBuilder { + client: Arc, +} + +impl BenchmarkExtrinsicBuilder { + /// Creates a new [`Self`] from the given client. + pub fn new(client: Arc) -> Self { + Self { client } + } +} + +impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder { + fn remark(&self, nonce: u32) -> std::result::Result { + let acc = Sr25519Keyring::Bob.pair(); + let extrinsic: OpaqueExtrinsic = create_extrinsic( + self.client.as_ref(), + acc, + SystemCall::remark { remark: vec![] }, + Some(nonce), + ) + .into(); + + Ok(extrinsic) + } +} + +/// Generates inherent data for the `benchmark overhead` command. +pub fn inherent_benchmark_data() -> Result { + let mut inherent_data = InherentData::new(); + let d = Duration::from_millis(0); + let timestamp = sp_timestamp::InherentDataProvider::new(d.into()); + + timestamp + .provide_inherent_data(&mut inherent_data) + .map_err(|e| format!("creating inherent data: {:?}", e))?; + Ok(inherent_data) +} diff --git a/bin/node/cli/src/lib.rs b/bin/node/cli/src/lib.rs index 1a4c1b0eab8d..06c0bcccbc29 100644 --- a/bin/node/cli/src/lib.rs +++ b/bin/node/cli/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -33,11 +33,13 @@ pub mod chain_spec; #[macro_use] -mod service; +pub mod service; #[cfg(feature = "cli")] mod cli; #[cfg(feature = "cli")] mod command; +#[cfg(feature = "cli")] +mod command_helper; #[cfg(feature = "cli")] pub use cli::*; diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index a68358a9cfdd..28c5ea346f2a 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,28 +20,107 @@ //! Service implementation. Specialized wrapper over substrate service. +use codec::Encode; +use frame_system_rpc_runtime_api::AccountNonceApi; use futures::prelude::*; use node_executor::ExecutorDispatch; use node_primitives::Block; use node_runtime::RuntimeApi; -use sc_client_api::{ExecutorProvider, RemoteBackend}; +use sc_client_api::{BlockBackend, ExecutorProvider}; use sc_consensus_babe::{self, SlotProportion}; use sc_executor::NativeElseWasmExecutor; use sc_network::{Event, NetworkService}; use sc_service::{config::Configuration, error::Error as ServiceError, RpcHandlers, TaskManager}; use sc_telemetry::{Telemetry, TelemetryWorker}; -use sp_runtime::traits::Block as BlockT; +use sp_api::ProvideRuntimeApi; +use sp_core::crypto::Pair; +use sp_runtime::{generic, traits::Block as BlockT, SaturatedConversion}; use std::sync::Arc; -type FullClient = +/// The full client type definition. +pub type FullClient = sc_service::TFullClient>; type FullBackend = sc_service::TFullBackend; type FullSelectChain = sc_consensus::LongestChain; type FullGrandpaBlockImport = grandpa::GrandpaBlockImport; -type LightClient = - sc_service::TLightClient>; +/// The transaction pool type defintion. +pub type TransactionPool = sc_transaction_pool::FullPool; + +/// Fetch the nonce of the given `account` from the chain state. +/// +/// Note: Should only be used for tests. +pub fn fetch_nonce(client: &FullClient, account: sp_core::sr25519::Pair) -> u32 { + let best_hash = client.chain_info().best_hash; + client + .runtime_api() + .account_nonce(&generic::BlockId::Hash(best_hash), account.public().into()) + .expect("Fetching account nonce works; qed") +} + +/// Create a transaction using the given `call`. +/// +/// The transaction will be signed by `sender`. If `nonce` is `None` it will be fetched from the +/// state of the best block. +/// +/// Note: Should only be used for tests. +pub fn create_extrinsic( + client: &FullClient, + sender: sp_core::sr25519::Pair, + function: impl Into, + nonce: Option, +) -> node_runtime::UncheckedExtrinsic { + let function = function.into(); + let genesis_hash = client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"); + let best_hash = client.chain_info().best_hash; + let best_block = client.chain_info().best_number; + let nonce = nonce.unwrap_or_else(|| fetch_nonce(client, sender.clone())); + + let period = node_runtime::BlockHashCount::get() + .checked_next_power_of_two() + .map(|c| c / 2) + .unwrap_or(2) as u64; + let tip = 0; + let extra: node_runtime::SignedExtra = ( + frame_system::CheckNonZeroSender::::new(), + frame_system::CheckSpecVersion::::new(), + frame_system::CheckTxVersion::::new(), + frame_system::CheckGenesis::::new(), + frame_system::CheckEra::::from(generic::Era::mortal( + period, + best_block.saturated_into(), + )), + frame_system::CheckNonce::::from(nonce), + frame_system::CheckWeight::::new(), + pallet_asset_tx_payment::ChargeAssetTxPayment::::from(tip, None), + ); + + let raw_payload = node_runtime::SignedPayload::from_raw( + function.clone(), + extra.clone(), + ( + (), + node_runtime::VERSION.spec_version, + node_runtime::VERSION.transaction_version, + genesis_hash, + best_hash, + (), + (), + (), + ), + ); + let signature = raw_payload.using_encoded(|e| sender.sign(e)); + + node_runtime::UncheckedExtrinsic::new_signed( + function.clone(), + sp_runtime::AccountId32::from(sender.public()).into(), + node_runtime::Signature::Sr25519(signature.clone()), + extra.clone(), + ) +} + +/// Creates a new partial node. pub fn new_partial( config: &Configuration, ) -> Result< @@ -82,6 +161,7 @@ pub fn new_partial( config.wasm_method, config.default_heap_pages, config.max_runtime_instances, + config.runtime_cache_size, ); let (client, backend, keystore_container, task_manager) = @@ -93,7 +173,7 @@ pub fn new_partial( let client = Arc::new(client); let telemetry = telemetry.map(|(worker, telemetry)| { - task_manager.spawn_handle().spawn("telemetry", worker.run()); + task_manager.spawn_handle().spawn("telemetry", None, worker.run()); telemetry }); @@ -116,7 +196,7 @@ pub fn new_partial( let justification_import = grandpa_block_import.clone(); let (block_import, babe_link) = sc_consensus_babe::block_import( - sc_consensus_babe::Config::get_or_compute(&*client)?, + sc_consensus_babe::Config::get(&*client)?, grandpa_block_import, client.clone(), )?; @@ -132,7 +212,7 @@ pub fn new_partial( let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); let slot = - sp_consensus_babe::inherents::InherentDataProvider::from_timestamp_and_duration( + sp_consensus_babe::inherents::InherentDataProvider::from_timestamp_and_slot_duration( *timestamp, slot_duration, ); @@ -172,6 +252,7 @@ pub fn new_partial( let keystore = keystore_container.sync_keystore(); let chain_spec = config.chain_spec.cloned_box(); + let rpc_backend = backend.clone(); let rpc_extensions_builder = move |deny_unsafe, subscription_executor| { let deps = node_rpc::FullDeps { client: client.clone(), @@ -193,7 +274,7 @@ pub fn new_partial( }, }; - node_rpc::create_full(deps).map_err(Into::into) + node_rpc::create_full(deps, rpc_backend.clone()).map_err(Into::into) }; (rpc_extensions_builder, rpc_setup) @@ -211,11 +292,18 @@ pub fn new_partial( }) } +/// Result of [`new_full_base`]. pub struct NewFullBase { + /// The task manager of the node. pub task_manager: TaskManager, + /// The client instance of the node. pub client: Arc, + /// The networking service of the node. pub network: Arc::Hash>>, - pub transaction_pool: Arc>, + /// The transaction pool of the node. + pub transaction_pool: Arc, + /// The rpc handlers of the node. + pub rpc_handlers: RpcHandlers, } /// Creates a full service from the configuration. @@ -239,11 +327,19 @@ pub fn new_full_base( let shared_voter_state = rpc_setup; let auth_disc_publish_non_global_ips = config.network.allow_non_globals_in_dht; + let grandpa_protocol_name = grandpa::protocol_standard_name( + &client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"), + &config.chain_spec, + ); - config.network.extra_sets.push(grandpa::grandpa_peers_set_config()); + config + .network + .extra_sets + .push(grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone())); let warp_sync = Arc::new(grandpa::warp_proof::NetworkProvider::new( backend.clone(), import_setup.1.shared_authority_set().clone(), + Vec::default(), )); let (network, system_rpc_tx, network_starter) = @@ -253,7 +349,6 @@ pub fn new_full_base( transaction_pool: transaction_pool.clone(), spawn_handle: task_manager.spawn_handle(), import_queue, - on_demand: None, block_announce_validator_builder: None, warp_sync: Some(warp_sync), })?; @@ -276,7 +371,7 @@ pub fn new_full_base( let enable_grandpa = !config.disable_grandpa; let prometheus_registry = config.prometheus_registry().cloned(); - let _rpc_handlers = sc_service::spawn_tasks(sc_service::SpawnTasksParams { + let rpc_handlers = sc_service::spawn_tasks(sc_service::SpawnTasksParams { config, backend, client: client.clone(), @@ -285,8 +380,6 @@ pub fn new_full_base( rpc_extensions_builder: Box::new(rpc_extensions_builder), transaction_pool: transaction_pool.clone(), task_manager: &mut task_manager, - on_demand: None, - remote_blockchain: None, system_rpc_tx, telemetry: telemetry.as_mut(), })?; @@ -328,7 +421,7 @@ pub fn new_full_base( let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); let slot = - sp_consensus_babe::inherents::InherentDataProvider::from_timestamp_and_duration( + sp_consensus_babe::inherents::InherentDataProvider::from_timestamp_and_slot_duration( *timestamp, slot_duration, ); @@ -352,7 +445,11 @@ pub fn new_full_base( }; let babe = sc_consensus_babe::start_babe(babe_config)?; - task_manager.spawn_essential_handle().spawn_blocking("babe-proposer", babe); + task_manager.spawn_essential_handle().spawn_blocking( + "babe-proposer", + Some("block-authoring"), + babe, + ); } // Spawn authority discovery module. @@ -379,9 +476,11 @@ pub fn new_full_base( prometheus_registry.clone(), ); - task_manager - .spawn_handle() - .spawn("authority-discovery-worker", authority_discovery_worker.run()); + task_manager.spawn_handle().spawn( + "authority-discovery-worker", + Some("networking"), + authority_discovery_worker.run(), + ); } // if the node isn't actively participating in consensus then it doesn't @@ -398,6 +497,7 @@ pub fn new_full_base( keystore, local_role: role, telemetry: telemetry.as_ref().map(|x| x.handle()), + protocol_name: grandpa_protocol_name, }; if enable_grandpa { @@ -419,13 +519,15 @@ pub fn new_full_base( // the GRANDPA voter task is considered infallible, i.e. // if it fails we take down the service with it. - task_manager - .spawn_essential_handle() - .spawn_blocking("grandpa-voter", grandpa::run_grandpa_voter(grandpa_config)?); + task_manager.spawn_essential_handle().spawn_blocking( + "grandpa-voter", + None, + grandpa::run_grandpa_voter(grandpa_config)?, + ); } network_starter.start_network(); - Ok(NewFullBase { task_manager, client, network, transaction_pool }) + Ok(NewFullBase { task_manager, client, network, transaction_pool, rpc_handlers }) } /// Builds a new service for a full client. @@ -433,185 +535,9 @@ pub fn new_full(config: Configuration) -> Result { new_full_base(config, |_, _| ()).map(|NewFullBase { task_manager, .. }| task_manager) } -pub fn new_light_base( - mut config: Configuration, -) -> Result< - ( - TaskManager, - RpcHandlers, - Arc, - Arc::Hash>>, - Arc< - sc_transaction_pool::LightPool>, - >, - ), - ServiceError, -> { - let telemetry = config - .telemetry_endpoints - .clone() - .filter(|x| !x.is_empty()) - .map(|endpoints| -> Result<_, sc_telemetry::Error> { - let worker = TelemetryWorker::new(16)?; - let telemetry = worker.handle().new_telemetry(endpoints); - Ok((worker, telemetry)) - }) - .transpose()?; - - let executor = NativeElseWasmExecutor::::new( - config.wasm_method, - config.default_heap_pages, - config.max_runtime_instances, - ); - - let (client, backend, keystore_container, mut task_manager, on_demand) = - sc_service::new_light_parts::( - &config, - telemetry.as_ref().map(|(_, telemetry)| telemetry.handle()), - executor, - )?; - - let mut telemetry = telemetry.map(|(worker, telemetry)| { - task_manager.spawn_handle().spawn("telemetry", worker.run()); - telemetry - }); - - config.network.extra_sets.push(grandpa::grandpa_peers_set_config()); - - let select_chain = sc_consensus::LongestChain::new(backend.clone()); - - let transaction_pool = Arc::new(sc_transaction_pool::BasicPool::new_light( - config.transaction_pool.clone(), - config.prometheus_registry(), - task_manager.spawn_essential_handle(), - client.clone(), - on_demand.clone(), - )); - - let (grandpa_block_import, grandpa_link) = grandpa::block_import( - client.clone(), - &(client.clone() as Arc<_>), - select_chain.clone(), - telemetry.as_ref().map(|x| x.handle()), - )?; - let justification_import = grandpa_block_import.clone(); - - let (babe_block_import, babe_link) = sc_consensus_babe::block_import( - sc_consensus_babe::Config::get_or_compute(&*client)?, - grandpa_block_import, - client.clone(), - )?; - - let slot_duration = babe_link.config().slot_duration(); - let import_queue = sc_consensus_babe::import_queue( - babe_link, - babe_block_import, - Some(Box::new(justification_import)), - client.clone(), - select_chain, - move |_, ()| async move { - let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); - - let slot = - sp_consensus_babe::inherents::InherentDataProvider::from_timestamp_and_duration( - *timestamp, - slot_duration, - ); - - let uncles = - sp_authorship::InherentDataProvider::<::Header>::check_inherents(); - - Ok((timestamp, slot, uncles)) - }, - &task_manager.spawn_essential_handle(), - config.prometheus_registry(), - sp_consensus::NeverCanAuthor, - telemetry.as_ref().map(|x| x.handle()), - )?; - - let warp_sync = Arc::new(grandpa::warp_proof::NetworkProvider::new( - backend.clone(), - grandpa_link.shared_authority_set().clone(), - )); - - let (network, system_rpc_tx, network_starter) = - sc_service::build_network(sc_service::BuildNetworkParams { - config: &config, - client: client.clone(), - transaction_pool: transaction_pool.clone(), - spawn_handle: task_manager.spawn_handle(), - import_queue, - on_demand: Some(on_demand.clone()), - block_announce_validator_builder: None, - warp_sync: Some(warp_sync), - })?; - - let enable_grandpa = !config.disable_grandpa; - if enable_grandpa { - let name = config.network.node_name.clone(); - - let config = grandpa::Config { - gossip_duration: std::time::Duration::from_millis(333), - justification_period: 512, - name: Some(name), - observer_enabled: false, - keystore: None, - local_role: config.role.clone(), - telemetry: telemetry.as_ref().map(|x| x.handle()), - }; - - task_manager.spawn_handle().spawn_blocking( - "grandpa-observer", - grandpa::run_grandpa_observer(config, grandpa_link, network.clone())?, - ); - } - - if config.offchain_worker.enabled { - sc_service::build_offchain_workers( - &config, - task_manager.spawn_handle(), - client.clone(), - network.clone(), - task_manager.ipfs_rt.as_ref().unwrap().clone(), - ); - } - - let light_deps = node_rpc::LightDeps { - remote_blockchain: backend.remote_blockchain(), - fetcher: on_demand.clone(), - client: client.clone(), - pool: transaction_pool.clone(), - }; - - let rpc_extensions = node_rpc::create_light(light_deps); - - let rpc_handlers = sc_service::spawn_tasks(sc_service::SpawnTasksParams { - on_demand: Some(on_demand), - remote_blockchain: Some(backend.remote_blockchain()), - rpc_extensions_builder: Box::new(sc_service::NoopRpcExtensionBuilder(rpc_extensions)), - client: client.clone(), - transaction_pool: transaction_pool.clone(), - keystore: keystore_container.sync_keystore(), - config, - backend, - system_rpc_tx, - network: network.clone(), - task_manager: &mut task_manager, - telemetry: telemetry.as_mut(), - })?; - - network_starter.start_network(); - Ok((task_manager, rpc_handlers, client, network, transaction_pool)) -} - -/// Builds a new service for a light client. -pub fn new_light(config: Configuration) -> Result { - new_light_base(config).map(|(task_manager, _, _, _, _)| task_manager) -} - #[cfg(test)] mod tests { - use crate::service::{new_full_base, new_light_base, NewFullBase}; + use crate::service::{new_full_base, NewFullBase}; use codec::Encode; use node_primitives::{Block, DigestItem, Signature}; use node_runtime::{ @@ -626,7 +552,7 @@ mod tests { use sc_service_test::TestNetNode; use sc_transaction_pool_api::{ChainEvent, MaintainedTransactionPool}; use sp_consensus::{BlockOrigin, Environment, Proposer}; - use sp_core::{crypto::Pair as CryptoPair, Public, H256}; + use sp_core::{crypto::Pair as CryptoPair, Public}; use sp_inherents::InherentDataProvider; use sp_keyring::AccountKeyring; use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; @@ -637,7 +563,7 @@ mod tests { RuntimeAppPublic, }; use sp_timestamp; - use std::{borrow::Cow, convert::TryInto, sync::Arc}; + use std::{borrow::Cow, sync::Arc}; type AccountPublic = ::Signer; @@ -687,15 +613,6 @@ mod tests { ); Ok((node, setup_handles.unwrap())) }, - |config| { - let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; - Ok(sc_service_test::TestNetComponents::new( - keep_alive, - client, - network, - transaction_pool, - )) - }, |service, &mut (ref mut block_import, ref babe_link)| { let parent_id = BlockId::number(service.client().chain_info().best_number); let parent_header = service.client().header(&parent_id).unwrap().unwrap(); @@ -714,7 +631,7 @@ mod tests { None, ); - let mut digest = Digest::::default(); + let mut digest = Digest::default(); // even though there's only one authority some slots might be empty, // so we must keep trying the next slots until we can claim one. @@ -735,7 +652,10 @@ mod tests { .epoch_changes() .shared_data() .epoch_data(&epoch_descriptor, |slot| { - sc_consensus_babe::Epoch::genesis(&babe_link.config(), slot) + sc_consensus_babe::Epoch::genesis( + babe_link.config().genesis_config(), + slot, + ) }) .unwrap(); @@ -815,26 +735,28 @@ mod tests { let function = Call::Balances(BalancesCall::transfer { dest: to.into(), value: amount }); + let check_non_zero_sender = frame_system::CheckNonZeroSender::new(); let check_spec_version = frame_system::CheckSpecVersion::new(); let check_tx_version = frame_system::CheckTxVersion::new(); let check_genesis = frame_system::CheckGenesis::new(); let check_era = frame_system::CheckEra::from(Era::Immortal); let check_nonce = frame_system::CheckNonce::from(index); let check_weight = frame_system::CheckWeight::new(); - let payment = pallet_transaction_payment::ChargeTransactionPayment::from(0); + let tx_payment = pallet_asset_tx_payment::ChargeAssetTxPayment::from(0, None); let extra = ( + check_non_zero_sender, check_spec_version, check_tx_version, check_genesis, check_era, check_nonce, check_weight, - payment, + tx_payment, ); let raw_payload = SignedPayload::from_raw( function, extra, - (spec_version, transaction_version, genesis_hash, genesis_hash, (), (), ()), + ((), spec_version, transaction_version, genesis_hash, genesis_hash, (), (), ()), ); let signature = raw_payload.using_encoded(|payload| signer.sign(payload)); let (function, extra, _) = raw_payload.deconstruct(); @@ -862,15 +784,6 @@ mod tests { transaction_pool, )) }, - |config| { - let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; - Ok(sc_service_test::TestNetComponents::new( - keep_alive, - client, - network, - transaction_pool, - )) - }, vec!["//Alice".into(), "//Bob".into()], ) } diff --git a/bin/node/cli/tests/benchmark_block_works.rs b/bin/node/cli/tests/benchmark_block_works.rs new file mode 100644 index 000000000000..37a4db25f363 --- /dev/null +++ b/bin/node/cli/tests/benchmark_block_works.rs @@ -0,0 +1,48 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// Unix only since it uses signals from [`common::run_node_for_a_while`]. +#![cfg(unix)] + +use assert_cmd::cargo::cargo_bin; +use std::process::Command; +use tempfile::tempdir; + +pub mod common; + +/// `benchmark block` works for the dev runtime using the wasm executor. +#[tokio::test] +async fn benchmark_block_works() { + let base_dir = tempdir().expect("could not create a temp dir"); + + common::run_node_for_a_while(base_dir.path(), &["--dev"]).await; + + // Invoke `benchmark block` with all options to make sure that they are valid. + let status = Command::new(cargo_bin("substrate")) + .args(["benchmark", "block", "--dev"]) + .arg("-d") + .arg(base_dir.path()) + .args(["--pruning", "archive"]) + .args(["--from", "1", "--to", "1"]) + .args(["--repeat", "1"]) + .args(["--execution", "wasm", "--wasm-execution", "compiled"]) + .status() + .unwrap(); + + assert!(status.success()) +} diff --git a/bin/node/cli/tests/benchmark_overhead_works.rs b/bin/node/cli/tests/benchmark_overhead_works.rs new file mode 100644 index 000000000000..44dcebfbc0c3 --- /dev/null +++ b/bin/node/cli/tests/benchmark_overhead_works.rs @@ -0,0 +1,46 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use assert_cmd::cargo::cargo_bin; +use std::process::Command; +use tempfile::tempdir; + +/// Tests that the `benchmark overhead` command works for the substrate dev runtime. +#[test] +fn benchmark_overhead_works() { + let tmp_dir = tempdir().expect("could not create a temp dir"); + let base_path = tmp_dir.path(); + + // Only put 10 extrinsics into the block otherwise it takes forever to build it + // especially for a non-release build. + let status = Command::new(cargo_bin("substrate")) + .args(&["benchmark", "overhead", "--dev", "-d"]) + .arg(base_path) + .arg("--weight-path") + .arg(base_path) + .args(["--warmup", "10", "--repeat", "10"]) + .args(["--add", "100", "--mul", "1.2", "--metric", "p75"]) + .args(["--max-ext-per-block", "10"]) + .status() + .unwrap(); + assert!(status.success()); + + // Weight files have been created. + assert!(base_path.join("block_weights.rs").exists()); + assert!(base_path.join("extrinsic_weights.rs").exists()); +} diff --git a/bin/node/cli/tests/benchmark_storage_works.rs b/bin/node/cli/tests/benchmark_storage_works.rs new file mode 100644 index 000000000000..30f860e48459 --- /dev/null +++ b/bin/node/cli/tests/benchmark_storage_works.rs @@ -0,0 +1,52 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use assert_cmd::cargo::cargo_bin; +use std::{ + path::Path, + process::{Command, ExitStatus}, +}; +use tempfile::tempdir; + +/// Tests that the `benchmark storage` command works for the dev runtime. +#[test] +fn benchmark_storage_works() { + let tmp_dir = tempdir().expect("could not create a temp dir"); + let base_path = tmp_dir.path(); + + // Benchmarking the storage works and creates the correct weight file. + assert!(benchmark_storage("rocksdb", base_path).success()); + assert!(base_path.join("rocksdb_weights.rs").exists()); + + assert!(benchmark_storage("paritydb", base_path).success()); + assert!(base_path.join("paritydb_weights.rs").exists()); +} + +fn benchmark_storage(db: &str, base_path: &Path) -> ExitStatus { + Command::new(cargo_bin("substrate")) + .args(&["benchmark", "storage", "--dev"]) + .arg("--db") + .arg(db) + .arg("--weight-path") + .arg(base_path) + .args(["--state-version", "1"]) + .args(["--warmups", "0"]) + .args(["--add", "100", "--mul", "1.2", "--metric", "p75"]) + .status() + .unwrap() +} diff --git a/bin/node/cli/tests/build_spec_works.rs b/bin/node/cli/tests/build_spec_works.rs index 6d863ea7f949..aecabed60c84 100644 --- a/bin/node/cli/tests/build_spec_works.rs +++ b/bin/node/cli/tests/build_spec_works.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/cli/tests/check_block_works.rs b/bin/node/cli/tests/check_block_works.rs index 216bcc6d9fc1..ac853b201b8c 100644 --- a/bin/node/cli/tests/check_block_works.rs +++ b/bin/node/cli/tests/check_block_works.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/cli/tests/common.rs b/bin/node/cli/tests/common.rs index 85effc858e15..c17cabfa1d38 100644 --- a/bin/node/cli/tests/common.rs +++ b/bin/node/cli/tests/common.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -26,7 +26,6 @@ use nix::{ use node_primitives::Block; use remote_externalities::rpc_api; use std::{ - convert::TryInto, ops::{Deref, DerefMut}, path::Path, process::{Child, Command, ExitStatus}, diff --git a/bin/node/cli/tests/database_role_subdir_migration.rs b/bin/node/cli/tests/database_role_subdir_migration.rs deleted file mode 100644 index 9338d8a8e4f4..000000000000 --- a/bin/node/cli/tests/database_role_subdir_migration.rs +++ /dev/null @@ -1,116 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use sc_client_db::{ - light::LightStorage, DatabaseSettings, DatabaseSource, KeepBlocks, PruningMode, - TransactionStorageMode, -}; -use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper}; -use tempfile::tempdir; - -pub mod common; - -#[tokio::test] -#[cfg(unix)] -async fn database_role_subdir_migration() { - type Block = RawBlock>; - - let base_path = tempdir().expect("could not create a temp dir"); - let path = base_path.path().join("chains/dev/db"); - // create a dummy database dir - { - let _old_db = LightStorage::::new(DatabaseSettings { - state_cache_size: 0, - state_cache_child_ratio: None, - state_pruning: PruningMode::ArchiveAll, - source: DatabaseSource::RocksDb { path: path.to_path_buf(), cache_size: 128 }, - keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorageMode::BlockBody, - }) - .unwrap(); - } - - assert!(path.join("db_version").exists()); - assert!(!path.join("light").exists()); - - // start a light client - common::run_node_for_a_while( - base_path.path(), - &[ - "--dev", - "--light", - "--port", - "30335", - "--rpc-port", - "44444", - "--ws-port", - "44445", - "--no-prometheus", - ], - ) - .await; - - // check if the database dir had been migrated - assert!(!path.join("db_version").exists()); - assert!(path.join("light/db_version").exists()); -} - -#[test] -#[cfg(unix)] -fn database_role_subdir_migration_fail_on_different_role() { - type Block = RawBlock>; - - let base_path = tempdir().expect("could not create a temp dir"); - let path = base_path.path().join("chains/dev/db"); - - // create a database with the old layout - { - let _old_db = LightStorage::::new(DatabaseSettings { - state_cache_size: 0, - state_cache_child_ratio: None, - state_pruning: PruningMode::ArchiveAll, - source: DatabaseSource::RocksDb { path: path.to_path_buf(), cache_size: 128 }, - keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorageMode::BlockBody, - }) - .unwrap(); - } - - assert!(path.join("db_version").exists()); - assert!(!path.join("light/db_version").exists()); - - // start a client with a different role (full), it should fail and not change any files on disk - common::run_node_assert_fail( - &base_path.path(), - &[ - "--dev", - "--port", - "30334", - "--rpc-port", - "44446", - "--ws-port", - "44447", - "--no-prometheus", - ], - ); - - // check if the files are unchanged - assert!(path.join("db_version").exists()); - assert!(!path.join("light/db_version").exists()); - assert!(!path.join("full/db_version").exists()); -} diff --git a/bin/node/cli/tests/export_import_flow.rs b/bin/node/cli/tests/export_import_flow.rs index 937f03b8e5da..2a2133bbfe4f 100644 --- a/bin/node/cli/tests/export_import_flow.rs +++ b/bin/node/cli/tests/export_import_flow.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/cli/tests/inspect_works.rs b/bin/node/cli/tests/inspect_works.rs index 6f980d2acbfc..28ad88dd501d 100644 --- a/bin/node/cli/tests/inspect_works.rs +++ b/bin/node/cli/tests/inspect_works.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/cli/tests/purge_chain_works.rs b/bin/node/cli/tests/purge_chain_works.rs index 8a8601c863d9..1a62aec28744 100644 --- a/bin/node/cli/tests/purge_chain_works.rs +++ b/bin/node/cli/tests/purge_chain_works.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/cli/tests/running_the_node_and_interrupt.rs b/bin/node/cli/tests/running_the_node_and_interrupt.rs index fc5094c2d722..703123faf0e6 100644 --- a/bin/node/cli/tests/running_the_node_and_interrupt.rs +++ b/bin/node/cli/tests/running_the_node_and_interrupt.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -25,10 +25,7 @@ use nix::{ }, unistd::Pid, }; -use std::{ - convert::TryInto, - process::{Child, Command}, -}; +use std::process::{Child, Command}; use tempfile::tempdir; pub mod common; diff --git a/bin/node/cli/tests/telemetry.rs b/bin/node/cli/tests/telemetry.rs index 78a306284c4a..64da4bd4b68f 100644 --- a/bin/node/cli/tests/telemetry.rs +++ b/bin/node/cli/tests/telemetry.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,7 +21,7 @@ use nix::{ sys::signal::{kill, Signal::SIGINT}, unistd::Pid, }; -use std::{convert::TryInto, process}; +use std::process; pub mod common; pub mod websocket_server; diff --git a/bin/node/cli/tests/temp_base_path_works.rs b/bin/node/cli/tests/temp_base_path_works.rs index 5d8e6c9ec453..306c490c2f76 100644 --- a/bin/node/cli/tests/temp_base_path_works.rs +++ b/bin/node/cli/tests/temp_base_path_works.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -25,7 +25,6 @@ use nix::{ }; use regex::Regex; use std::{ - convert::TryInto, io::Read, path::PathBuf, process::{Command, Stdio}, diff --git a/bin/node/cli/tests/version.rs b/bin/node/cli/tests/version.rs index 5ed3a9a8800c..133eb65f4ace 100644 --- a/bin/node/cli/tests/version.rs +++ b/bin/node/cli/tests/version.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/cli/tests/websocket_server.rs b/bin/node/cli/tests/websocket_server.rs index 658b8de46345..6eecfaf6de53 100644 --- a/bin/node/cli/tests/websocket_server.rs +++ b/bin/node/cli/tests/websocket_server.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml index f283a913915f..3425a0df65d9 100644 --- a/bin/node/executor/Cargo.toml +++ b/bin/node/executor/Cargo.toml @@ -3,24 +3,25 @@ name = "node-executor" version = "3.0.0-dev" authors = ["Parity Technologies "] description = "Substrate node implementation in Rust." -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0" } -scale-info = { version = "1.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0" } +scale-info = { version = "2.0.1", features = ["derive"] } node-primitives = { version = "2.0.0", path = "../primitives" } node-runtime = { version = "3.0.0-dev", path = "../runtime" } sc-executor = { version = "0.10.0-dev", path = "../../../client/executor" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-keystore = { version = "0.10.0-dev", path = "../../../primitives/keystore" } -sp-state-machine = { version = "0.10.0-dev", path = "../../../primitives/state-machine" } -sp-trie = { version = "4.0.0-dev", path = "../../../primitives/trie" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" } +sp-state-machine = { version = "0.12.0", path = "../../../primitives/state-machine" } +sp-tracing = { version = "5.0.0", path = "../../../primitives/tracing" } +sp-trie = { version = "6.0.0", path = "../../../primitives/trie" } frame-benchmarking = { version = "4.0.0-dev", path = "../../../frame/benchmarking" } [dev-dependencies] @@ -33,12 +34,13 @@ pallet-contracts = { version = "4.0.0-dev", path = "../../../frame/contracts" } pallet-im-online = { version = "4.0.0-dev", path = "../../../frame/im-online" } pallet-timestamp = { version = "4.0.0-dev", path = "../../../frame/timestamp" } pallet-treasury = { version = "4.0.0-dev", path = "../../../frame/treasury" } -sp-application-crypto = { version = "4.0.0-dev", path = "../../../primitives/application-crypto" } +sp-application-crypto = { version = "6.0.0", path = "../../../primitives/application-crypto" } sp-consensus-babe = { version = "0.10.0-dev", path = "../../../primitives/consensus/babe" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -sp-externalities = { version = "0.10.0-dev", path = "../../../primitives/externalities" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } +sp-externalities = { version = "0.12.0", path = "../../../primitives/externalities" } +sp-keyring = { version = "6.0.0", path = "../../../primitives/keyring" } wat = "1.0" -futures = "0.3.9" +futures = "0.3.21" [features] wasmtime = ["sc-executor/wasmtime"] diff --git a/bin/node/executor/benches/bench.rs b/bin/node/executor/benches/bench.rs index 1a39c9decb32..3d7c264a89d1 100644 --- a/bin/node/executor/benches/bench.rs +++ b/bin/node/executor/benches/bench.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -53,7 +53,7 @@ const SPEC_VERSION: u32 = node_runtime::VERSION.spec_version; const HEAP_PAGES: u64 = 20; -type TestExternalities = CoreTestExternalities; +type TestExternalities = CoreTestExternalities; #[derive(Debug)] enum ExecutionMethod { @@ -83,14 +83,14 @@ fn construct_block( parent_hash: Hash, extrinsics: Vec, ) -> (Vec, Hash) { - use sp_trie::{trie_types::Layout, TrieConfiguration}; + use sp_trie::{LayoutV0, TrieConfiguration}; // sign extrinsics. let extrinsics = extrinsics.into_iter().map(sign).collect::>(); // calculate the header fields that we can. let extrinsics_root = - Layout::::ordered_trie_root(extrinsics.iter().map(Encode::encode)) + LayoutV0::::ordered_trie_root(extrinsics.iter().map(Encode::encode)) .to_fixed_bytes() .into(); @@ -188,13 +188,13 @@ fn bench_execute_block(c: &mut Criterion) { for strategy in execution_methods { group.bench_function(format!("{:?}", strategy), |b| { - let genesis_config = node_testing::genesis::config(false, Some(compact_code_unwrap())); + let genesis_config = node_testing::genesis::config(Some(compact_code_unwrap())); let (use_native, wasm_method) = match strategy { ExecutionMethod::Native => (true, WasmExecutionMethod::Interpreted), ExecutionMethod::Wasm(wasm_method) => (false, wasm_method), }; - let executor = NativeElseWasmExecutor::new(wasm_method, None, 8); + let executor = NativeElseWasmExecutor::new(wasm_method, None, 8, 2); let runtime_code = RuntimeCode { code_fetcher: &sp_core::traits::WrappedRuntimeCode(compact_code_unwrap().into()), hash: vec![1, 2, 3], diff --git a/bin/node/executor/src/lib.rs b/bin/node/executor/src/lib.rs index 9a7a0c4d3c11..9f87c7d12623 100644 --- a/bin/node/executor/src/lib.rs +++ b/bin/node/executor/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/node/executor/tests/basic.rs b/bin/node/executor/tests/basic.rs index c1ab5e5a0fe1..da0f4e6afb31 100644 --- a/bin/node/executor/tests/basic.rs +++ b/bin/node/executor/tests/basic.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,7 +29,7 @@ use sp_runtime::{ use node_primitives::{Balance, Hash}; use node_runtime::{ constants::{currency::*, time::SLOT_DURATION}, - Balances, Block, Call, CheckedExtrinsic, Event, Header, Runtime, System, TransactionPayment, + Balances, Call, CheckedExtrinsic, Event, Header, Runtime, System, TransactionPayment, UncheckedExtrinsic, }; use node_testing::keyring::*; @@ -78,7 +78,7 @@ fn set_heap_pages(ext: &mut E, heap_pages: u64) { fn changes_trie_block() -> (Vec, Hash) { let time = 42 * 1000; construct_block( - &mut new_test_ext(compact_code_unwrap(), true), + &mut new_test_ext(compact_code_unwrap()), 1, GENESIS_HASH.into(), vec![ @@ -102,7 +102,7 @@ fn changes_trie_block() -> (Vec, Hash) { /// are not guaranteed to be deterministic) and to ensure that the correct state is propagated /// from block1's execution to block2 to derive the correct storage_root. fn blocks() -> ((Vec, Hash), (Vec, Hash)) { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); let time1 = 42 * 1000; let block1 = construct_block( &mut t, @@ -160,7 +160,7 @@ fn blocks() -> ((Vec, Hash), (Vec, Hash)) { fn block_with_size(time: u64, nonce: u32, size: usize) -> (Vec, Hash) { construct_block( - &mut new_test_ext(compact_code_unwrap(), false), + &mut new_test_ext(compact_code_unwrap()), 1, GENESIS_HASH.into(), vec![ @@ -179,7 +179,7 @@ fn block_with_size(time: u64, nonce: u32, size: usize) -> (Vec, Hash) { #[test] fn panic_execution_with_foreign_code_gives_error() { - let mut t = new_test_ext(bloaty_code_unwrap(), false); + let mut t = new_test_ext(bloaty_code_unwrap()); t.insert( >::hashed_key_for(alice()), (69u128, 0u32, 0u128, 0u128, 0u128).encode(), @@ -211,7 +211,7 @@ fn panic_execution_with_foreign_code_gives_error() { #[test] fn bad_extrinsic_with_native_equivalent_code_gives_error() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); t.insert( >::hashed_key_for(alice()), (0u32, 0u32, 0u32, 69u128, 0u128, 0u128, 0u128).encode(), @@ -243,7 +243,7 @@ fn bad_extrinsic_with_native_equivalent_code_gives_error() { #[test] fn successful_execution_with_native_equivalent_code_gives_ok() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); t.insert( >::hashed_key_for(alice()), AccountInfo::<::Index, _> { @@ -296,7 +296,7 @@ fn successful_execution_with_native_equivalent_code_gives_ok() { #[test] fn successful_execution_with_foreign_code_gives_ok() { - let mut t = new_test_ext(bloaty_code_unwrap(), false); + let mut t = new_test_ext(bloaty_code_unwrap()); t.insert( >::hashed_key_for(alice()), AccountInfo::<::Index, _> { @@ -349,7 +349,7 @@ fn successful_execution_with_foreign_code_gives_ok() { #[test] fn full_native_block_import_works() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); let (block1, block2) = blocks(); @@ -378,33 +378,50 @@ fn full_native_block_import_works() { let events = vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: Event::System(frame_system::Event::ExtrinsicSuccess(DispatchInfo { - weight: timestamp_weight, - class: DispatchClass::Mandatory, - ..Default::default() - })), + event: Event::System(frame_system::Event::ExtrinsicSuccess { + dispatch_info: DispatchInfo { + weight: timestamp_weight, + class: DispatchClass::Mandatory, + ..Default::default() + }, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(1), + event: Event::Balances(pallet_balances::Event::Withdraw { + who: alice().into(), + amount: fees, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(1), + event: Event::Balances(pallet_balances::Event::Transfer { + from: alice().into(), + to: bob().into(), + amount: 69 * DOLLARS, + }), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::Balances(pallet_balances::Event::Transfer( - alice().into(), - bob().into(), - 69 * DOLLARS, - )), + event: Event::Balances(pallet_balances::Event::Deposit { + who: pallet_treasury::Pallet::::account_id(), + amount: fees * 8 / 10, + }), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::Treasury(pallet_treasury::Event::Deposit(fees * 8 / 10)), + event: Event::Treasury(pallet_treasury::Event::Deposit { value: fees * 8 / 10 }), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::System(frame_system::Event::ExtrinsicSuccess(DispatchInfo { - weight: transfer_weight, - ..Default::default() - })), + event: Event::System(frame_system::Event::ExtrinsicSuccess { + dispatch_info: DispatchInfo { weight: transfer_weight, ..Default::default() }, + }), topics: vec![], }, ]; @@ -432,55 +449,87 @@ fn full_native_block_import_works() { let events = vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: Event::System(frame_system::Event::ExtrinsicSuccess(DispatchInfo { - weight: timestamp_weight, - class: DispatchClass::Mandatory, - ..Default::default() - })), + event: Event::System(frame_system::Event::ExtrinsicSuccess { + dispatch_info: DispatchInfo { + weight: timestamp_weight, + class: DispatchClass::Mandatory, + ..Default::default() + }, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(1), + event: Event::Balances(pallet_balances::Event::Withdraw { + who: bob().into(), + amount: fees, + }), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::Balances(pallet_balances::Event::Transfer( - bob().into(), - alice().into(), - 5 * DOLLARS, - )), + event: Event::Balances(pallet_balances::Event::Transfer { + from: bob().into(), + to: alice().into(), + amount: 5 * DOLLARS, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(1), + event: Event::Balances(pallet_balances::Event::Deposit { + who: pallet_treasury::Pallet::::account_id(), + amount: fees * 8 / 10, + }), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::Treasury(pallet_treasury::Event::Deposit(fees * 8 / 10)), + event: Event::Treasury(pallet_treasury::Event::Deposit { value: fees * 8 / 10 }), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: Event::System(frame_system::Event::ExtrinsicSuccess(DispatchInfo { - weight: transfer_weight, - ..Default::default() - })), + event: Event::System(frame_system::Event::ExtrinsicSuccess { + dispatch_info: DispatchInfo { weight: transfer_weight, ..Default::default() }, + }), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(2), - event: Event::Balances(pallet_balances::Event::Transfer( - alice().into(), - bob().into(), - 15 * DOLLARS, - )), + event: Event::Balances(pallet_balances::Event::Withdraw { + who: alice().into(), + amount: fees, + }), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(2), - event: Event::Treasury(pallet_treasury::Event::Deposit(fees * 8 / 10)), + event: Event::Balances(pallet_balances::Event::Transfer { + from: alice().into(), + to: bob().into(), + amount: 15 * DOLLARS, + }), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(2), - event: Event::System(frame_system::Event::ExtrinsicSuccess(DispatchInfo { - weight: transfer_weight, - ..Default::default() - })), + event: Event::Balances(pallet_balances::Event::Deposit { + who: pallet_treasury::Pallet::::account_id(), + amount: fees * 8 / 10, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(2), + event: Event::Treasury(pallet_treasury::Event::Deposit { value: fees * 8 / 10 }), + topics: vec![], + }, + EventRecord { + phase: Phase::ApplyExtrinsic(2), + event: Event::System(frame_system::Event::ExtrinsicSuccess { + dispatch_info: DispatchInfo { weight: transfer_weight, ..Default::default() }, + }), topics: vec![], }, ]; @@ -490,7 +539,7 @@ fn full_native_block_import_works() { #[test] fn full_wasm_block_import_works() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); let (block1, block2) = blocks(); @@ -636,11 +685,9 @@ fn deploying_wasm_contract_should_work() { let addr = pallet_contracts::Pallet::::contract_address(&charlie(), &transfer_ch, &[]); - let subsistence = pallet_contracts::Pallet::::subsistence_threshold(); - let time = 42 * 1000; let b = construct_block( - &mut new_test_ext(compact_code_unwrap(), false), + &mut new_test_ext(compact_code_unwrap()), 1, GENESIS_HASH.into(), vec![ @@ -652,8 +699,9 @@ fn deploying_wasm_contract_should_work() { signed: Some((charlie(), signed_extra(0, 0))), function: Call::Contracts( pallet_contracts::Call::instantiate_with_code:: { - endowment: 1000 * DOLLARS + subsistence, + value: 0, gas_limit: 500_000_000, + storage_deposit_limit: None, code: transfer_code, data: Vec::new(), salt: Vec::new(), @@ -666,6 +714,7 @@ fn deploying_wasm_contract_should_work() { dest: sp_runtime::MultiAddress::Id(addr.clone()), value: 10, gas_limit: 500_000_000, + storage_deposit_limit: None, data: vec![0x00, 0x01, 0x02, 0x03], }), }, @@ -673,7 +722,7 @@ fn deploying_wasm_contract_should_work() { (time / SLOT_DURATION).into(), ); - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); executor_call:: _>(&mut t, "Core_execute_block", &b.0, false, None) .0 @@ -688,7 +737,7 @@ fn deploying_wasm_contract_should_work() { #[test] fn wasm_big_block_import_fails() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); set_heap_pages(&mut t.ext(), 4); @@ -705,7 +754,7 @@ fn wasm_big_block_import_fails() { #[test] fn native_big_block_import_succeeds() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); executor_call:: _>( &mut t, @@ -720,7 +769,7 @@ fn native_big_block_import_succeeds() { #[test] fn native_big_block_import_fails_on_fallback() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); // We set the heap pages to 8 because we know that should give an OOM in WASM with the given // block. @@ -739,7 +788,7 @@ fn native_big_block_import_fails_on_fallback() { #[test] fn panic_execution_gives_error() { - let mut t = new_test_ext(bloaty_code_unwrap(), false); + let mut t = new_test_ext(bloaty_code_unwrap()); t.insert( >::hashed_key_for(alice()), AccountInfo::<::Index, _> { @@ -776,7 +825,7 @@ fn panic_execution_gives_error() { #[test] fn successful_execution_gives_ok() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); t.insert( >::hashed_key_for(alice()), AccountInfo::<::Index, _> { @@ -835,44 +884,6 @@ fn successful_execution_gives_ok() { }); } -#[test] -fn full_native_block_import_works_with_changes_trie() { - let block1 = changes_trie_block(); - let block_data = block1.0; - let block = Block::decode(&mut &block_data[..]).unwrap(); - - let mut t = new_test_ext(compact_code_unwrap(), true); - executor_call:: _>( - &mut t, - "Core_execute_block", - &block.encode(), - true, - None, - ) - .0 - .unwrap(); - - assert!(t.ext().storage_changes_root(&GENESIS_HASH).unwrap().is_some()); -} - -#[test] -fn full_wasm_block_import_works_with_changes_trie() { - let block1 = changes_trie_block(); - - let mut t = new_test_ext(compact_code_unwrap(), true); - executor_call:: _>( - &mut t, - "Core_execute_block", - &block1.0, - false, - None, - ) - .0 - .unwrap(); - - assert!(t.ext().storage_changes_root(&GENESIS_HASH).unwrap().is_some()); -} - #[test] fn should_import_block_with_test_client() { use node_testing::client::{ diff --git a/bin/node/executor/tests/common.rs b/bin/node/executor/tests/common.rs index d1c24c83c836..a2bb91056f47 100644 --- a/bin/node/executor/tests/common.rs +++ b/bin/node/executor/tests/common.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -81,7 +81,7 @@ pub const SPEC_VERSION: u32 = node_runtime::VERSION.spec_version; pub const TRANSACTION_VERSION: u32 = node_runtime::VERSION.transaction_version; -pub type TestExternalities = CoreTestExternalities; +pub type TestExternalities = CoreTestExternalities; pub fn sign(xt: CheckedExtrinsic) -> UncheckedExtrinsic { node_testing::keyring::sign(xt, SPEC_VERSION, TRANSACTION_VERSION, GENESIS_HASH) @@ -96,7 +96,7 @@ pub fn from_block_number(n: u32) -> Header { } pub fn executor() -> NativeElseWasmExecutor { - NativeElseWasmExecutor::new(WasmExecutionMethod::Interpreted, None, 8) + NativeElseWasmExecutor::new(WasmExecutionMethod::Interpreted, None, 8, 2) } pub fn executor_call< @@ -119,18 +119,15 @@ pub fn executor_call< hash: sp_core::blake2_256(&code).to_vec(), heap_pages: heap_pages.and_then(|hp| Decode::decode(&mut &hp[..]).ok()), }; - + sp_tracing::try_init_simple(); executor().call::(&mut t, &runtime_code, method, data, use_native, native_call) } -pub fn new_test_ext(code: &[u8], support_changes_trie: bool) -> TestExternalities { - let mut ext = TestExternalities::new_with_code( +pub fn new_test_ext(code: &[u8]) -> TestExternalities { + let ext = TestExternalities::new_with_code( code, - node_testing::genesis::config(support_changes_trie, Some(code)) - .build_storage() - .unwrap(), + node_testing::genesis::config(Some(code)).build_storage().unwrap(), ); - ext.changes_trie_storage().insert(0, GENESIS_HASH.into(), Default::default()); ext } @@ -145,7 +142,7 @@ pub fn construct_block( extrinsics: Vec, babe_slot: Slot, ) -> (Vec, Hash) { - use sp_trie::{trie_types::Layout, TrieConfiguration}; + use sp_trie::{LayoutV1 as Layout, TrieConfiguration}; // sign extrinsics. let extrinsics = extrinsics.into_iter().map(sign).collect::>(); diff --git a/bin/node/executor/tests/fees.rs b/bin/node/executor/tests/fees.rs index 379cdda5b76a..a84ce1470d87 100644 --- a/bin/node/executor/tests/fees.rs +++ b/bin/node/executor/tests/fees.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,7 +36,7 @@ use self::common::{sign, *}; #[test] fn fee_multiplier_increases_and_decreases_on_big_weight() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); // initial fee multiplier must be one. let mut prev_multiplier = Multiplier::one(); @@ -45,7 +45,7 @@ fn fee_multiplier_increases_and_decreases_on_big_weight() { assert_eq!(TransactionPayment::next_fee_multiplier(), prev_multiplier); }); - let mut tt = new_test_ext(compact_code_unwrap(), false); + let mut tt = new_test_ext(compact_code_unwrap()); let time1 = 42 * 1000; // big one in terms of weight. @@ -151,7 +151,7 @@ fn transaction_fee_is_correct() { // - 1 MILLICENTS in substrate node. // - 1 milli-dot based on current polkadot runtime. // (this baed on assigning 0.1 CENT to the cheapest tx with `weight = 100`) - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); t.insert(>::hashed_key_for(alice()), new_account_info(100)); t.insert(>::hashed_key_for(bob()), new_account_info(10)); t.insert( @@ -226,9 +226,9 @@ fn block_weight_capacity_report() { use node_primitives::Index; // execution ext. - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); // setup ext. - let mut tt = new_test_ext(compact_code_unwrap(), false); + let mut tt = new_test_ext(compact_code_unwrap()); let factor = 50; let mut time = 10; @@ -241,7 +241,10 @@ fn block_weight_capacity_report() { let mut xts = (0..num_transfers) .map(|i| CheckedExtrinsic { signed: Some((charlie(), signed_extra(nonce + i as Index, 0))), - function: Call::Balances(pallet_balances::Call::transfer(bob().into(), 0)), + function: Call::Balances(pallet_balances::Call::transfer { + dest: bob().into(), + value: 0, + }), }) .collect::>(); @@ -249,7 +252,7 @@ fn block_weight_capacity_report() { 0, CheckedExtrinsic { signed: None, - function: Call::Timestamp(pallet_timestamp::Call::set(time * 1000)), + function: Call::Timestamp(pallet_timestamp::Call::set { now: time * 1000 }), }, ); @@ -300,9 +303,9 @@ fn block_length_capacity_report() { use node_primitives::Index; // execution ext. - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); // setup ext. - let mut tt = new_test_ext(compact_code_unwrap(), false); + let mut tt = new_test_ext(compact_code_unwrap()); let factor = 256 * 1024; let mut time = 10; @@ -319,7 +322,7 @@ fn block_length_capacity_report() { vec![ CheckedExtrinsic { signed: None, - function: Call::Timestamp(pallet_timestamp::Call::set(time * 1000)), + function: Call::Timestamp(pallet_timestamp::Call::set { now: time * 1000 }), }, CheckedExtrinsic { signed: Some((charlie(), signed_extra(nonce, 0))), diff --git a/bin/node/executor/tests/submit_transaction.rs b/bin/node/executor/tests/submit_transaction.rs index 19ca8e5677c4..7df13a577006 100644 --- a/bin/node/executor/tests/submit_transaction.rs +++ b/bin/node/executor/tests/submit_transaction.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,6 +20,7 @@ use frame_system::offchain::{SendSignedTransaction, Signer, SubmitTransaction}; use node_runtime::{Executive, Indices, Runtime, UncheckedExtrinsic}; use sp_application_crypto::AppKey; use sp_core::offchain::{testing::TestTransactionPoolExt, TransactionPoolExt}; +use sp_keyring::sr25519::Keyring::Alice; use sp_keystore::{testing::KeyStore, KeystoreExt, SyncCryptoStore}; use std::sync::Arc; @@ -28,12 +29,13 @@ use self::common::*; #[test] fn should_submit_unsigned_transaction() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); let (pool, state) = TestTransactionPoolExt::new(); t.register_extension(TransactionPoolExt::new(pool)); t.execute_with(|| { - let signature = Default::default(); + let signature = + pallet_im_online::sr25519::AuthoritySignature::try_from(vec![0; 64]).unwrap(); let heartbeat_data = pallet_im_online::Heartbeat { block_number: 1, network_state: Default::default(), @@ -56,7 +58,7 @@ const PHRASE: &str = "news slush supreme milk chapter athlete soap sausage put c #[test] fn should_submit_signed_transaction() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); let (pool, state) = TestTransactionPoolExt::new(); t.register_extension(TransactionPoolExt::new(pool)); @@ -85,7 +87,7 @@ fn should_submit_signed_transaction() { let results = Signer::::all_accounts().send_signed_transaction(|_| { pallet_balances::Call::transfer { - dest: Default::default(), + dest: Alice.to_account_id().into(), value: Default::default(), } }); @@ -99,7 +101,7 @@ fn should_submit_signed_transaction() { #[test] fn should_submit_signed_twice_from_the_same_account() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); let (pool, state) = TestTransactionPoolExt::new(); t.register_extension(TransactionPoolExt::new(pool)); @@ -122,7 +124,7 @@ fn should_submit_signed_twice_from_the_same_account() { let result = Signer::::any_account().send_signed_transaction(|_| { pallet_balances::Call::transfer { - dest: Default::default(), + dest: Alice.to_account_id().into(), value: Default::default(), } }); @@ -134,7 +136,7 @@ fn should_submit_signed_twice_from_the_same_account() { let result = Signer::::any_account().send_signed_transaction(|_| { pallet_balances::Call::transfer { - dest: Default::default(), + dest: Alice.to_account_id().into(), value: Default::default(), } }); @@ -146,7 +148,7 @@ fn should_submit_signed_twice_from_the_same_account() { let s = state.read(); fn nonce(tx: UncheckedExtrinsic) -> frame_system::CheckNonce { let extra = tx.signature.unwrap().2; - extra.4 + extra.5 } let nonce1 = nonce(UncheckedExtrinsic::decode(&mut &*s.transactions[0]).unwrap()); let nonce2 = nonce(UncheckedExtrinsic::decode(&mut &*s.transactions[1]).unwrap()); @@ -156,7 +158,7 @@ fn should_submit_signed_twice_from_the_same_account() { #[test] fn should_submit_signed_twice_from_all_accounts() { - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); let (pool, state) = TestTransactionPoolExt::new(); t.register_extension(TransactionPoolExt::new(pool)); @@ -172,7 +174,7 @@ fn should_submit_signed_twice_from_all_accounts() { t.execute_with(|| { let results = Signer::::all_accounts() .send_signed_transaction(|_| { - pallet_balances::Call::transfer { dest: Default::default(), value: Default::default() } + pallet_balances::Call::transfer { dest: Alice.to_account_id().into(), value: Default::default() } }); let len = results.len(); @@ -183,7 +185,7 @@ fn should_submit_signed_twice_from_all_accounts() { // submit another one from the same account. The nonce should be incremented. let results = Signer::::all_accounts() .send_signed_transaction(|_| { - pallet_balances::Call::transfer { dest: Default::default(), value: Default::default() } + pallet_balances::Call::transfer { dest: Alice.to_account_id().into(), value: Default::default() } }); let len = results.len(); @@ -195,7 +197,7 @@ fn should_submit_signed_twice_from_all_accounts() { let s = state.read(); fn nonce(tx: UncheckedExtrinsic) -> frame_system::CheckNonce { let extra = tx.signature.unwrap().2; - extra.4 + extra.5 } let nonce1 = nonce(UncheckedExtrinsic::decode(&mut &*s.transactions[0]).unwrap()); let nonce2 = nonce(UncheckedExtrinsic::decode(&mut &*s.transactions[1]).unwrap()); @@ -220,7 +222,7 @@ fn submitted_transaction_should_be_valid() { transaction_validity::{TransactionSource, TransactionTag}, }; - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); let (pool, state) = TestTransactionPoolExt::new(); t.register_extension(TransactionPoolExt::new(pool)); @@ -237,7 +239,7 @@ fn submitted_transaction_should_be_valid() { let results = Signer::::all_accounts().send_signed_transaction(|_| { pallet_balances::Call::transfer { - dest: Default::default(), + dest: Alice.to_account_id().into(), value: Default::default(), } }); @@ -249,7 +251,7 @@ fn submitted_transaction_should_be_valid() { // check that transaction is valid, but reset environment storage, // since CreateTransaction increments the nonce let tx0 = state.read().transactions[0].clone(); - let mut t = new_test_ext(compact_code_unwrap(), false); + let mut t = new_test_ext(compact_code_unwrap()); t.execute_with(|| { let source = TransactionSource::External; let extrinsic = UncheckedExtrinsic::decode(&mut &*tx0).unwrap(); diff --git a/bin/node/inspect/Cargo.toml b/bin/node/inspect/Cargo.toml index 1570e5dbf8e4..c41681d11be1 100644 --- a/bin/node/inspect/Cargo.toml +++ b/bin/node/inspect/Cargo.toml @@ -2,22 +2,22 @@ name = "node-inspect" version = "0.9.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0" } -derive_more = "0.99" +clap = { version = "3.1.6", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0" } +thiserror = "1.0" sc-cli = { version = "0.10.0-dev", path = "../../../client/cli" } sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" } sc-executor = { version = "0.10.0-dev", path = "../../../client/executor" } sc-service = { version = "0.10.0-dev", default-features = false, path = "../../../client/service" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -structopt = "0.3.8" +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } diff --git a/bin/node/inspect/src/cli.rs b/bin/node/inspect/src/cli.rs index c054fedaf57c..cc1f232e1fe0 100644 --- a/bin/node/inspect/src/cli.rs +++ b/bin/node/inspect/src/cli.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,27 +19,25 @@ //! Structs to easily compose inspect sub-command for CLI. use sc_cli::{ImportParams, SharedParams}; -use std::fmt::Debug; -use structopt::StructOpt; /// The `inspect` command used to print decoded chain data. -#[derive(Debug, StructOpt)] +#[derive(Debug, clap::Parser)] pub struct InspectCmd { #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(subcommand)] pub command: InspectSubCmd, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub shared_params: SharedParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub import_params: ImportParams, } /// A possible inspect sub-commands. -#[derive(Debug, StructOpt)] +#[derive(Debug, clap::Subcommand)] pub enum InspectSubCmd { /// Decode block with native version of runtime and print out the details. Block { @@ -48,7 +46,7 @@ pub enum InspectSubCmd { /// Can be either a block hash (no 0x prefix) or a number to retrieve existing block, /// or a 0x-prefixed bytes hex string, representing SCALE encoding of /// a block. - #[structopt(value_name = "HASH or NUMBER or BYTES")] + #[clap(value_name = "HASH or NUMBER or BYTES")] input: String, }, /// Decode extrinsic with native version of runtime and print out the details. @@ -58,7 +56,7 @@ pub enum InspectSubCmd { /// Can be either a block hash (no 0x prefix) or number and the index, in the form /// of `{block}:{index}` or a 0x-prefixed bytes hex string, /// representing SCALE encoding of an extrinsic. - #[structopt(value_name = "BLOCK:INDEX or BYTES")] + #[clap(value_name = "BLOCK:INDEX or BYTES")] input: String, }, } diff --git a/bin/node/inspect/src/command.rs b/bin/node/inspect/src/command.rs index 9bf69511689c..ce164e0768fb 100644 --- a/bin/node/inspect/src/command.rs +++ b/bin/node/inspect/src/command.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -41,6 +41,7 @@ impl InspectCmd { config.wasm_method, config.default_heap_pages, config.max_runtime_instances, + config.runtime_cache_size, ); let client = new_full_client::(&config, None, executor)?; diff --git a/bin/node/inspect/src/lib.rs b/bin/node/inspect/src/lib.rs index 30e7250ea2c6..b37c5aa7ca2e 100644 --- a/bin/node/inspect/src/lib.rs +++ b/bin/node/inspect/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. // -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // // This program is free software: you can redistribute it and/or modify @@ -78,26 +78,19 @@ impl PrettyPrinter for DebugPrinter { } /// Aggregated error for `Inspector` operations. -#[derive(Debug, derive_more::From, derive_more::Display)] +#[derive(Debug, thiserror::Error)] pub enum Error { /// Could not decode Block or Extrinsic. - Codec(codec::Error), + #[error(transparent)] + Codec(#[from] codec::Error), /// Error accessing blockchain DB. - Blockchain(sp_blockchain::Error), + #[error(transparent)] + Blockchain(#[from] sp_blockchain::Error), /// Given block has not been found. + #[error("{0}")] NotFound(String), } -impl std::error::Error for Error { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match *self { - Self::Codec(ref e) => Some(e), - Self::Blockchain(ref e) => Some(e), - Self::NotFound(_) => None, - } - } -} - /// A helper trait to access block headers and bodies. pub trait ChainAccess: HeaderBackend + BlockBackend {} @@ -261,7 +254,7 @@ impl FromStr for ExtrinsicAddres let index = it .next() - .ok_or_else(|| format!("Extrinsic index missing: example \"5:0\""))? + .ok_or("Extrinsic index missing: example \"5:0\"")? .parse() .map_err(|e| format!("Invalid index format: {}", e))?; diff --git a/bin/node/primitives/Cargo.toml b/bin/node/primitives/Cargo.toml index 12ec57e4d55b..711b30d33933 100644 --- a/bin/node/primitives/Cargo.toml +++ b/bin/node/primitives/Cargo.toml @@ -2,23 +2,23 @@ name = "node-primitives" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system" } -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/application-crypto" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../../../primitives/application-crypto" } +sp-core = { version = "6.0.0", default-features = false, path = "../../../primitives/core" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } [features] default = ["std"] diff --git a/bin/node/primitives/src/lib.rs b/bin/node/primitives/src/lib.rs index dade598c704d..feb9ee60d311 100644 --- a/bin/node/primitives/src/lib.rs +++ b/bin/node/primitives/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -57,41 +57,10 @@ pub type Hash = sp_core::H256; pub type Timestamp = u64; /// Digest item type. -pub type DigestItem = generic::DigestItem; +pub type DigestItem = generic::DigestItem; /// Header type. pub type Header = generic::Header; /// Block type. pub type Block = generic::Block; /// Block ID. pub type BlockId = generic::BlockId; - -/// App-specific crypto used for reporting equivocation/misbehavior in BABE and -/// GRANDPA. Any rewards for misbehavior reporting will be paid out to this -/// account. -pub mod report { - use super::{Signature, Verify}; - use frame_system::offchain::AppCrypto; - use sp_core::crypto::{key_types, KeyTypeId}; - - /// Key type for the reporting module. Used for reporting BABE and GRANDPA - /// equivocations. - pub const KEY_TYPE: KeyTypeId = key_types::REPORTING; - - mod app { - use sp_application_crypto::{app_crypto, sr25519}; - app_crypto!(sr25519, super::KEY_TYPE); - } - - /// Identity of the equivocation/misbehavior reporter. - pub type ReporterId = app::Public; - - /// An `AppCrypto` type to allow submitting signed transactions using the reporting - /// application key as signer. - pub struct ReporterAppCrypto; - - impl AppCrypto<::Signer, Signature> for ReporterAppCrypto { - type RuntimeAppPublic = ReporterId; - type GenericSignature = sp_core::sr25519::Signature; - type GenericPublic = sp_core::sr25519::Public; - } -} diff --git a/bin/node/rpc-client/Cargo.toml b/bin/node/rpc-client/Cargo.toml deleted file mode 100644 index a5255769158a..000000000000 --- a/bin/node/rpc-client/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "node-rpc-client" -version = "2.0.0" -authors = ["Parity Technologies "] -edition = "2018" -license = "Apache-2.0" -homepage = "https://substrate.dev" -repository = "https://github.com/paritytech/substrate/" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[dependencies] -futures = "0.3.16" -jsonrpc-core-client = { version = "18.0.0", default-features = false, features = [ - "http", -] } -node-primitives = { version = "2.0.0", path = "../primitives" } -sp-tracing = { version = "4.0.0-dev", path = "../../../primitives/tracing" } -sc-rpc = { version = "4.0.0-dev", path = "../../../client/rpc" } diff --git a/bin/node/rpc-client/src/main.rs b/bin/node/rpc-client/src/main.rs deleted file mode 100644 index 6d0b88799f54..000000000000 --- a/bin/node/rpc-client/src/main.rs +++ /dev/null @@ -1,63 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#![warn(missing_docs)] - -//! Example substrate RPC client code. -//! -//! This module shows how you can write a Rust RPC client that connects to a running -//! substrate node and use statically typed RPC wrappers. - -use futures::{Future, TryFutureExt}; -use jsonrpc_core_client::{transports::http, RpcError}; -use node_primitives::Hash; -use sc_rpc::author::{hash::ExtrinsicOrHash, AuthorClient}; - -fn main() -> Result<(), RpcError> { - sp_tracing::try_init_simple(); - - futures::executor::block_on(async { - let uri = "http://localhost:9933"; - - http::connect(uri) - .and_then(|client: AuthorClient| remove_all_extrinsics(client)) - .await - }) -} - -/// Remove all pending extrinsics from the node. -/// -/// The example code takes `AuthorClient` and first: -/// 1. Calls the `pending_extrinsics` method to get all extrinsics in the pool. -/// 2. Then calls `remove_extrinsic` passing the obtained raw extrinsics. -/// -/// As the result of running the code the entire content of the transaction pool is going -/// to be removed and the extrinsics are going to be temporarily banned. -fn remove_all_extrinsics( - client: AuthorClient, -) -> impl Future> { - client - .pending_extrinsics() - .and_then(move |pending| { - client.remove_extrinsic( - pending.into_iter().map(|tx| ExtrinsicOrHash::Extrinsic(tx.into())).collect(), - ) - }) - .map_ok(|removed| { - println!("Removed extrinsics: {:?}", removed); - }) -} diff --git a/bin/node/rpc/Cargo.toml b/bin/node/rpc/Cargo.toml index 06c0cee87886..6836f3c36cf0 100644 --- a/bin/node/rpc/Cargo.toml +++ b/bin/node/rpc/Cargo.toml @@ -2,9 +2,9 @@ name = "node-rpc" version = "3.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] @@ -30,9 +30,10 @@ sc-sync-state-rpc = { version = "0.10.0-dev", path = "../../../client/sync-state sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-block-builder = { version = "4.0.0-dev", path = "../../../primitives/block-builder" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sp-keystore = { version = "0.10.0-dev", path = "../../../primitives/keystore" } +sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" } sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } sp-consensus-babe = { version = "0.10.0-dev", path = "../../../primitives/consensus/babe" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../../client/transaction-pool/api" } substrate-frame-rpc-system = { version = "4.0.0-dev", path = "../../../utils/frame/rpc/system" } +substrate-state-trie-migration-rpc = { version = "4.0.0-dev", path = "../../../utils/frame/rpc/state-trie-migration-rpc/" } diff --git a/bin/node/rpc/src/lib.rs b/bin/node/rpc/src/lib.rs index 2f7862d3d264..b8349e26cd1d 100644 --- a/bin/node/rpc/src/lib.rs +++ b/bin/node/rpc/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -51,18 +51,6 @@ use sp_consensus::SelectChain; use sp_consensus_babe::BabeApi; use sp_keystore::SyncCryptoStorePtr; -/// Light client extra dependencies. -pub struct LightDeps { - /// The client instance to use. - pub client: Arc, - /// Transaction pool instance. - pub pool: Arc

, - /// Remote access to the blockchain (async). - pub remote_blockchain: Arc>, - /// Fetcher instance. - pub fetcher: Arc, -} - /// Extra dependencies for BABE. pub struct BabeDeps { /// BABE protocol config. @@ -111,9 +99,11 @@ pub type IoHandler = jsonrpc_core::IoHandler; /// Instantiate all Full RPC extensions. pub fn create_full( deps: FullDeps, + backend: Arc, ) -> Result, Box> where C: ProvideRuntimeApi + + sc_client_api::BlockBackend + HeaderBackend + AuxStore + HeaderMetadata @@ -134,6 +124,7 @@ where use pallet_contracts_rpc::{Contracts, ContractsApi}; use pallet_mmr_rpc::{Mmr, MmrApi}; use pallet_transaction_payment_rpc::{TransactionPayment, TransactionPaymentApi}; + use sc_rpc::dev::{Dev, DevApi}; use substrate_frame_rpc_system::{FullSystem, SystemApi}; let mut io = jsonrpc_core::IoHandler::default(); @@ -170,39 +161,18 @@ where subscription_executor, finality_provider, ))); - + io.extend_with(substrate_state_trie_migration_rpc::StateMigrationApi::to_delegate( + substrate_state_trie_migration_rpc::MigrationRpc::new(client.clone(), backend, deny_unsafe), + )); io.extend_with(sc_sync_state_rpc::SyncStateRpcApi::to_delegate( sc_sync_state_rpc::SyncStateRpcHandler::new( chain_spec, - client, + client.clone(), shared_authority_set, shared_epoch_changes, - deny_unsafe, )?, )); + io.extend_with(DevApi::to_delegate(Dev::new(client, deny_unsafe))); Ok(io) } - -/// Instantiate all Light RPC extensions. -pub fn create_light(deps: LightDeps) -> jsonrpc_core::IoHandler -where - C: sp_blockchain::HeaderBackend, - C: Send + Sync + 'static, - F: sc_client_api::light::Fetcher + 'static, - P: TransactionPool + 'static, - M: jsonrpc_core::Metadata + Default, -{ - use substrate_frame_rpc_system::{LightSystem, SystemApi}; - - let LightDeps { client, pool, remote_blockchain, fetcher } = deps; - let mut io = jsonrpc_core::IoHandler::default(); - io.extend_with(SystemApi::::to_delegate(LightSystem::new( - client, - remote_blockchain, - fetcher, - pool, - ))); - - io -} diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index d434be8f3c60..686508b47dba 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -2,10 +2,10 @@ name = "node-runtime" version = "3.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" build = "build.rs" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] @@ -14,13 +14,13 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] # third-party dependencies -codec = { package = "parity-scale-codec", version = "2.2.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", "max-encoded-len", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } static_assertions = "1.1.0" -hex-literal = { version = "0.3.1", optional = true } +hex-literal = { version = "0.3.4", optional = true } log = { version = "0.4.14", default-features = false } # primitives @@ -30,17 +30,16 @@ sp-block-builder = { path = "../../../primitives/block-builder", default-feature sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/inherents" } node-primitives = { version = "2.0.0", default-features = false, path = "../primitives" } sp-offchain = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/offchain" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/core" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/std" } +sp-core = { version = "6.0.0", default-features = false, path = "../../../primitives/core" } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/api" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/staking" } -sp-keyring = { version = "4.0.0-dev", optional = true, path = "../../../primitives/keyring" } sp-session = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/session" } sp-transaction-pool = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/transaction-pool" } -sp-version = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/version" } -sp-npos-elections = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/npos-elections" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/io" } +sp-version = { version = "5.0.0", default-features = false, path = "../../../primitives/version" } +sp-io = { version = "6.0.0", default-features = false, path = "../../../primitives/io" } +sp-sandbox = { version = "0.10.0-dev", default-features = false, path = "../../../primitives/sandbox" } # frame dependencies frame-executive = { version = "4.0.0-dev", default-features = false, path = "../../../frame/executive" } @@ -58,10 +57,12 @@ pallet-babe = { version = "4.0.0-dev", default-features = false, path = "../../. pallet-bags-list = { version = "4.0.0-dev", default-features = false, path = "../../../frame/bags-list" } pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../../frame/balances" } pallet-bounties = { version = "4.0.0-dev", default-features = false, path = "../../../frame/bounties" } +pallet-child-bounties = { version = "4.0.0-dev", default-features = false, path = "../../../frame/child-bounties" } pallet-collective = { version = "4.0.0-dev", default-features = false, path = "../../../frame/collective" } pallet-contracts = { version = "4.0.0-dev", default-features = false, path = "../../../frame/contracts" } -pallet-contracts-primitives = { version = "4.0.0-dev", default-features = false, path = "../../../frame/contracts/common/" } +pallet-contracts-primitives = { version = "6.0.0", default-features = false, path = "../../../frame/contracts/common/" } pallet-contracts-rpc-runtime-api = { version = "4.0.0-dev", default-features = false, path = "../../../frame/contracts/rpc/runtime-api/" } +pallet-conviction-voting = { version = "4.0.0-dev", default-features = false, path = "../../../frame/conviction-voting" } pallet-democracy = { version = "4.0.0-dev", default-features = false, path = "../../../frame/democracy" } pallet-election-provider-multi-phase = { version = "4.0.0-dev", default-features = false, path = "../../../frame/election-provider-multi-phase" } pallet-elections-phragmen = { version = "5.0.0-dev", default-features = false, path = "../../../frame/elections-phragmen" } @@ -76,15 +77,16 @@ pallet-mmr = { version = "4.0.0-dev", default-features = false, path = "../../.. pallet-multisig = { version = "4.0.0-dev", default-features = false, path = "../../../frame/multisig" } pallet-offences = { version = "4.0.0-dev", default-features = false, path = "../../../frame/offences" } pallet-offences-benchmarking = { version = "4.0.0-dev", path = "../../../frame/offences/benchmarking", default-features = false, optional = true } +pallet-preimage = { version = "4.0.0-dev", default-features = false, path = "../../../frame/preimage" } pallet-proxy = { version = "4.0.0-dev", default-features = false, path = "../../../frame/proxy" } pallet-randomness-collective-flip = { version = "4.0.0-dev", default-features = false, path = "../../../frame/randomness-collective-flip" } pallet-recovery = { version = "4.0.0-dev", default-features = false, path = "../../../frame/recovery" } -pallet-session = { version = "4.0.0-dev", features = [ - "historical", -], path = "../../../frame/session", default-features = false } +pallet-referenda = { version = "4.0.0-dev", default-features = false, path = "../../../frame/referenda" } +pallet-session = { version = "4.0.0-dev", features = [ "historical" ], path = "../../../frame/session", default-features = false } pallet-session-benchmarking = { version = "4.0.0-dev", path = "../../../frame/session/benchmarking", default-features = false, optional = true } pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../../../frame/staking" } pallet-staking-reward-curve = { version = "4.0.0-dev", default-features = false, path = "../../../frame/staking/reward-curve" } +pallet-state-trie-migration = { version = "4.0.0-dev", default-features = false, path = "../../../frame/state-trie-migration" } pallet-scheduler = { version = "4.0.0-dev", default-features = false, path = "../../../frame/scheduler" } pallet-society = { version = "4.0.0-dev", default-features = false, path = "../../../frame/society" } pallet-sudo = { version = "4.0.0-dev", default-features = false, path = "../../../frame/sudo" } @@ -94,9 +96,11 @@ pallet-treasury = { version = "4.0.0-dev", default-features = false, path = "../ pallet-utility = { version = "4.0.0-dev", default-features = false, path = "../../../frame/utility" } pallet-transaction-payment = { version = "4.0.0-dev", default-features = false, path = "../../../frame/transaction-payment" } pallet-transaction-payment-rpc-runtime-api = { version = "4.0.0-dev", default-features = false, path = "../../../frame/transaction-payment/rpc/runtime-api/" } +pallet-asset-tx-payment = { version = "4.0.0-dev", default-features = false, path = "../../../frame/transaction-payment/asset-tx-payment/" } pallet-transaction-storage = { version = "4.0.0-dev", default-features = false, path = "../../../frame/transaction-storage" } pallet-uniques = { version = "4.0.0-dev", default-features = false, path = "../../../frame/uniques" } pallet-vesting = { version = "4.0.0-dev", default-features = false, path = "../../../frame/vesting" } +pallet-whitelist = { version = "4.0.0-dev", default-features = false, path = "../../../frame/whitelist" } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } @@ -121,6 +125,7 @@ std = [ "pallet-contracts/std", "pallet-contracts-primitives/std", "pallet-contracts-rpc-runtime-api/std", + "pallet-conviction-voting/std", "pallet-democracy/std", "pallet-elections-phragmen/std", "frame-executive/std", @@ -138,6 +143,7 @@ std = [ "node-primitives/std", "sp-offchain/std", "pallet-offences/std", + "pallet-preimage/std", "pallet-proxy/std", "sp-core/std", "pallet-randomness-collective-flip/std", @@ -147,7 +153,7 @@ std = [ "sp-runtime/std", "sp-staking/std", "pallet-staking/std", - "sp-keyring", + "pallet-state-trie-migration/std", "sp-session/std", "pallet-sudo/std", "frame-support/std", @@ -165,28 +171,31 @@ std = [ "pallet-utility/std", "sp-version/std", "pallet-society/std", + "pallet-referenda/std", "pallet-recovery/std", "pallet-uniques/std", "pallet-vesting/std", "log/std", "frame-try-runtime/std", - "sp-npos-elections/std", - "sp-io/std" + "sp-io/std", + "pallet-child-bounties/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", - "pallet-election-provider-multi-phase/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "pallet-assets/runtime-benchmarks", "pallet-babe/runtime-benchmarks", "pallet-bags-list/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-bounties/runtime-benchmarks", + "pallet-child-bounties/runtime-benchmarks", "pallet-collective/runtime-benchmarks", "pallet-contracts/runtime-benchmarks", + "pallet-conviction-voting/runtime-benchmarks", "pallet-democracy/runtime-benchmarks", + "pallet-election-provider-multi-phase/runtime-benchmarks", "pallet-elections-phragmen/runtime-benchmarks", "pallet-gilt/runtime-benchmarks", "pallet-grandpa/runtime-benchmarks", @@ -197,10 +206,15 @@ runtime-benchmarks = [ "pallet-membership/runtime-benchmarks", "pallet-mmr/runtime-benchmarks", "pallet-multisig/runtime-benchmarks", + "pallet-offences-benchmarking", + "pallet-preimage/runtime-benchmarks", "pallet-proxy/runtime-benchmarks", "pallet-scheduler/runtime-benchmarks", + "pallet-referenda/runtime-benchmarks", + "pallet-session-benchmarking", "pallet-society/runtime-benchmarks", "pallet-staking/runtime-benchmarks", + "pallet-state-trie-migration/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", "pallet-tips/runtime-benchmarks", "pallet-transaction-storage/runtime-benchmarks", @@ -208,8 +222,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-uniques/runtime-benchmarks", "pallet-vesting/runtime-benchmarks", - "pallet-offences-benchmarking", - "pallet-session-benchmarking", + "pallet-whitelist/runtime-benchmarks", "frame-system-benchmarking", "hex-literal", ] @@ -223,37 +236,48 @@ try-runtime = [ "pallet-babe/try-runtime", "pallet-balances/try-runtime", "pallet-bounties/try-runtime", + "pallet-child-bounties/try-runtime", "pallet-collective/try-runtime", "pallet-contracts/try-runtime", + "pallet-conviction-voting/try-runtime", "pallet-democracy/try-runtime", + "pallet-election-provider-multi-phase/try-runtime", "pallet-elections-phragmen/try-runtime", + "pallet-gilt/try-runtime", "pallet-grandpa/try-runtime", + "pallet-identity/try-runtime", "pallet-im-online/try-runtime", "pallet-indices/try-runtime", "pallet-lottery/try-runtime", "pallet-membership/try-runtime", "pallet-mmr/try-runtime", "pallet-multisig/try-runtime", - "pallet-identity/try-runtime", - "pallet-scheduler/try-runtime", "pallet-offences/try-runtime", + "pallet-preimage/try-runtime", "pallet-proxy/try-runtime", "pallet-randomness-collective-flip/try-runtime", + "pallet-recovery/try-runtime", + "pallet-referenda/try-runtime", + "pallet-scheduler/try-runtime", "pallet-session/try-runtime", + "pallet-society/try-runtime", "pallet-staking/try-runtime", + "pallet-state-trie-migration/try-runtime", "pallet-sudo/try-runtime", - "pallet-election-provider-multi-phase/try-runtime", "pallet-timestamp/try-runtime", "pallet-tips/try-runtime", "pallet-transaction-payment/try-runtime", "pallet-treasury/try-runtime", - "pallet-utility/try-runtime", - "pallet-society/try-runtime", - "pallet-recovery/try-runtime", "pallet-uniques/try-runtime", + "pallet-utility/try-runtime", "pallet-vesting/try-runtime", - "pallet-gilt/try-runtime", + "pallet-whitelist/try-runtime", ] # Make contract callable functions marked as __unstable__ available. Do not enable # on live chains as those are subject to change. contracts-unstable-interface = ["pallet-contracts/unstable-interface"] +# Force `sp-sandbox` to call into the host resident executor. One still need to make sure +# that `sc-executor` gets the `wasmer-sandbox` feature which happens automatically when +# specified on the command line. +# Don't use that on a production chain. +wasmer-sandbox = ["sp-sandbox/wasmer-sandbox"] diff --git a/bin/node/runtime/build.rs b/bin/node/runtime/build.rs index a1c4b2d892cf..b773ed9cf6fb 100644 --- a/bin/node/runtime/build.rs +++ b/bin/node/runtime/build.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/node/runtime/src/constants.rs b/bin/node/runtime/src/constants.rs index 7533025a70b0..23fb13cfb049 100644 --- a/bin/node/runtime/src/constants.rs +++ b/bin/node/runtime/src/constants.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/node/runtime/src/impls.rs b/bin/node/runtime/src/impls.rs index e315a45e698c..f73443920c21 100644 --- a/bin/node/runtime/src/impls.rs +++ b/bin/node/runtime/src/impls.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,13 +17,31 @@ //! Some configurable implementations as associated type for the substrate runtime. -use crate::{Authorship, Balances, NegativeImbalance}; -use frame_support::traits::{Currency, OnUnbalanced}; +use crate::{AccountId, Assets, Authorship, Balances, NegativeImbalance, Runtime}; +use frame_support::traits::{ + fungibles::{Balanced, CreditOf}, + Currency, OnUnbalanced, +}; +use pallet_asset_tx_payment::HandleCredit; pub struct Author; impl OnUnbalanced for Author { fn on_nonzero_unbalanced(amount: NegativeImbalance) { - Balances::resolve_creating(&Authorship::author(), amount); + if let Some(author) = Authorship::author() { + Balances::resolve_creating(&author, amount); + } + } +} + +/// A `HandleCredit` implementation that naively transfers the fees to the block author. +/// Will drop and burn the assets in case the transfer fails. +pub struct CreditToBlockAuthor; +impl HandleCredit for CreditToBlockAuthor { + fn handle_credit(credit: CreditOf) { + if let Some(author) = pallet_authorship::Pallet::::author() { + // Drop the result which will trigger the `OnDrop` of the imbalance in case of error. + let _ = Assets::resolve(&author, credit); + } } } diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index ba82c3ab1c3b..0186794c540c 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,39 +23,40 @@ #![recursion_limit = "256"] use codec::{Decode, Encode, MaxEncodedLen}; +use frame_election_provider_support::{onchain, ExtendedBalance, SequentialPhragmen, VoteWeight}; use frame_support::{ - construct_runtime, parameter_types, + construct_runtime, + pallet_prelude::Get, + parameter_types, traits::{ - Currency, Everything, Imbalance, InstanceFilter, KeyOwnerProofSystem, LockIdentifier, - Nothing, OnUnbalanced, U128CurrencyToVote, + AsEnsureOriginWithArg, ConstU128, ConstU16, ConstU32, Currency, EnsureOneOf, + EqualPrivilegeOnly, Everything, Imbalance, InstanceFilter, KeyOwnerProofSystem, + LockIdentifier, Nothing, OnUnbalanced, U128CurrencyToVote, }, weights::{ constants::{BlockExecutionWeight, ExtrinsicBaseWeight, RocksDbWeight, WEIGHT_PER_SECOND}, - DispatchClass, IdentityFee, Weight, + ConstantMultiplier, DispatchClass, IdentityFee, Weight, }, PalletId, RuntimeDebug, }; use frame_system::{ limits::{BlockLength, BlockWeights}, - EnsureOneOf, EnsureRoot, + EnsureRoot, EnsureSigned, }; pub use node_primitives::{AccountId, Signature}; use node_primitives::{AccountIndex, Balance, BlockNumber, Hash, Index, Moment}; use pallet_contracts::weights::WeightInfo; +use pallet_election_provider_multi_phase::SolutionAccuracyOf; use pallet_grandpa::{ fg_primitives, AuthorityId as GrandpaId, AuthorityList as GrandpaAuthorityList, }; use pallet_im_online::sr25519::AuthorityId as ImOnlineId; -use pallet_session::historical as pallet_session_historical; +use pallet_session::historical::{self as pallet_session_historical}; pub use pallet_transaction_payment::{CurrencyAdapter, Multiplier, TargetedFeeAdjustment}; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use sp_api::impl_runtime_apis; use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; -use sp_core::{ - crypto::KeyTypeId, - u32_trait::{_1, _2, _3, _4, _5}, - OpaqueMetadata, -}; +use sp_core::{crypto::KeyTypeId, OpaqueMetadata}; use sp_inherents::{CheckInherentsResult, InherentData}; use sp_runtime::{ create_runtime_str, @@ -81,11 +82,13 @@ pub use pallet_balances::Call as BalancesCall; #[cfg(any(feature = "std", test))] pub use pallet_staking::StakerStatus; #[cfg(any(feature = "std", test))] +pub use pallet_sudo::Call as SudoCall; +#[cfg(any(feature = "std", test))] pub use sp_runtime::BuildStorage; /// Implementations of some helper traits passed into runtime modules as associated types. pub mod impls; -use impls::Author; +use impls::{Author, CreditToBlockAuthor}; /// Constant values used within the runtime. pub mod constants; @@ -119,10 +122,11 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to 0. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 267, - impl_version: 1, + spec_version: 268, + impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 2, + state_version: 1, }; /// The BABE epoch configuration at genesis. @@ -188,7 +192,6 @@ parameter_types! { }) .avg_block_initialization(AVERAGE_ON_INITIALIZE_RATIO) .build_or_panic(); - pub const SS58Prefix: u16 = 42; } const_assert!(NORMAL_DISPATCH_RATIO.deconstruct() >= AVERAGE_ON_INITIALIZE_RATIO.deconstruct()); @@ -215,8 +218,9 @@ impl frame_system::Config for Runtime { type OnNewAccount = (); type OnKilledAccount = (); type SystemWeightInfo = frame_system::weights::SubstrateWeight; - type SS58Prefix = SS58Prefix; + type SS58Prefix = ConstU16<42>; type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl pallet_randomness_collective_flip::Config for Runtime {} @@ -224,6 +228,7 @@ impl pallet_randomness_collective_flip::Config for Runtime {} impl pallet_utility::Config for Runtime { type Event = Event; type Call = Call; + type PalletsOrigin = OriginCaller; type WeightInfo = pallet_utility::weights::SubstrateWeight; } @@ -232,7 +237,6 @@ parameter_types! { pub const DepositBase: Balance = deposit(1, 88); // Additional storage item size of 32 bytes. pub const DepositFactor: Balance = deposit(0, 32); - pub const MaxSignatories: u16 = 100; } impl pallet_multisig::Config for Runtime { @@ -241,7 +245,7 @@ impl pallet_multisig::Config for Runtime { type Currency = Balances; type DepositBase = DepositBase; type DepositFactor = DepositFactor; - type MaxSignatories = MaxSignatories; + type MaxSignatories = ConstU16<100>; type WeightInfo = pallet_multisig::weights::SubstrateWeight; } @@ -250,10 +254,8 @@ parameter_types! { pub const ProxyDepositBase: Balance = deposit(1, 8); // Additional storage item size of 33 bytes. pub const ProxyDepositFactor: Balance = deposit(0, 33); - pub const MaxProxies: u16 = 32; pub const AnnouncementDepositBase: Balance = deposit(1, 8); pub const AnnouncementDepositFactor: Balance = deposit(0, 66); - pub const MaxPending: u16 = 32; } /// The type used to represent the kinds of proxying allowed. @@ -320,9 +322,9 @@ impl pallet_proxy::Config for Runtime { type ProxyType = ProxyType; type ProxyDepositBase = ProxyDepositBase; type ProxyDepositFactor = ProxyDepositFactor; - type MaxProxies = MaxProxies; + type MaxProxies = ConstU32<32>; type WeightInfo = pallet_proxy::weights::SubstrateWeight; - type MaxPending = MaxPending; + type MaxPending = ConstU32<32>; type CallHasher = BlakeTwo256; type AnnouncementDepositBase = AnnouncementDepositBase; type AnnouncementDepositFactor = AnnouncementDepositFactor; @@ -331,7 +333,8 @@ impl pallet_proxy::Config for Runtime { parameter_types! { pub MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * RuntimeBlockWeights::get().max_block; - pub const MaxScheduledPerBlock: u32 = 50; + // Retry a scheduled item every 10 blocks (1 minute) until the preimage exists. + pub const NoPreimagePostponement: Option = Some(10); } impl pallet_scheduler::Config for Runtime { @@ -341,8 +344,28 @@ impl pallet_scheduler::Config for Runtime { type Call = Call; type MaximumWeight = MaximumSchedulerWeight; type ScheduleOrigin = EnsureRoot; - type MaxScheduledPerBlock = MaxScheduledPerBlock; + type MaxScheduledPerBlock = ConstU32<50>; type WeightInfo = pallet_scheduler::weights::SubstrateWeight; + type OriginPrivilegeCmp = EqualPrivilegeOnly; + type PreimageProvider = Preimage; + type NoPreimagePostponement = NoPreimagePostponement; +} + +parameter_types! { + pub const PreimageMaxSize: u32 = 4096 * 1024; + pub const PreimageBaseDeposit: Balance = 1 * DOLLARS; + // One cent: $10,000 / MB + pub const PreimageByteDeposit: Balance = 1 * CENTS; +} + +impl pallet_preimage::Config for Runtime { + type WeightInfo = pallet_preimage::weights::SubstrateWeight; + type Event = Event; + type Currency = Balances; + type ManagerOrigin = EnsureRoot; + type MaxSize = PreimageMaxSize; + type BaseDeposit = PreimageBaseDeposit; + type ByteDeposit = PreimageByteDeposit; } parameter_types! { @@ -413,6 +436,7 @@ impl pallet_balances::Config for Runtime { parameter_types! { pub const TransactionByteFee: Balance = 10 * MILLICENTS; + pub const OperationalFeeMultiplier: u8 = 5; pub const TargetBlockFullness: Perquintill = Perquintill::from_percent(25); pub AdjustmentVariable: Multiplier = Multiplier::saturating_from_rational(1, 100_000); pub MinimumMultiplier: Multiplier = Multiplier::saturating_from_rational(1, 1_000_000_000u128); @@ -420,12 +444,21 @@ parameter_types! { impl pallet_transaction_payment::Config for Runtime { type OnChargeTransaction = CurrencyAdapter; - type TransactionByteFee = TransactionByteFee; + type OperationalFeeMultiplier = OperationalFeeMultiplier; type WeightToFee = IdentityFee; + type LengthToFee = ConstantMultiplier; type FeeMultiplierUpdate = TargetedFeeAdjustment; } +impl pallet_asset_tx_payment::Config for Runtime { + type Fungibles = Assets; + type OnChargeAssetTransaction = pallet_asset_tx_payment::FungiblesAdapter< + pallet_assets::BalanceToAssetBalance, + CreditToBlockAuthor, + >; +} + parameter_types! { pub const MinimumPeriod: Moment = SLOT_DURATION / 2; } @@ -457,10 +490,6 @@ impl_opaque_keys! { } } -parameter_types! { - pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(17); -} - impl pallet_session::Config for Runtime { type Event = Event; type ValidatorId = ::AccountId; @@ -470,7 +499,6 @@ impl pallet_session::Config for Runtime { type SessionManager = pallet_session::historical::NoteHistoricalRoot; type SessionHandler = ::KeyTypeIdProviders; type Keys = SessionKeys; - type DisabledValidatorsThreshold = DisabledValidatorsThreshold; type WeightInfo = pallet_session::weights::SubstrateWeight; } @@ -492,21 +520,22 @@ pallet_staking_reward_curve::build! { parameter_types! { pub const SessionsPerEra: sp_staking::SessionIndex = 6; - pub const BondingDuration: pallet_staking::EraIndex = 24 * 28; - pub const SlashDeferDuration: pallet_staking::EraIndex = 24 * 7; // 1/4 the bonding duration. + pub const BondingDuration: sp_staking::EraIndex = 24 * 28; + pub const SlashDeferDuration: sp_staking::EraIndex = 24 * 7; // 1/4 the bonding duration. pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; pub const MaxNominatorRewardedPerValidator: u32 = 256; + pub const OffendingValidatorsThreshold: Perbill = Perbill::from_percent(17); pub OffchainRepeat: BlockNumber = 5; } -use frame_election_provider_support::onchain; -impl onchain::Config for Runtime { - type Accuracy = Perbill; - type DataProvider = Staking; +pub struct StakingBenchmarkingConfig; +impl pallet_staking::BenchmarkingConfig for StakingBenchmarkingConfig { + type MaxNominators = ConstU32<1000>; + type MaxValidators = ConstU32<1000>; } impl pallet_staking::Config for Runtime { - const MAX_NOMINATIONS: u32 = MAX_NOMINATIONS; + type MaxNominations = MaxNominations; type Currency = Balances; type UnixTime = Timestamp; type CurrencyToVote = U128CurrencyToVote; @@ -519,20 +548,20 @@ impl pallet_staking::Config for Runtime { type SlashDeferDuration = SlashDeferDuration; /// A super-majority of the council can cancel the slash. type SlashCancelOrigin = EnsureOneOf< - AccountId, EnsureRoot, - pallet_collective::EnsureProportionAtLeast<_3, _4, AccountId, CouncilCollective>, + pallet_collective::EnsureProportionAtLeast, >; type SessionInterface = Self; type EraPayout = pallet_staking::ConvertCurve; type NextNewSession = Session; type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type OffendingValidatorsThreshold = OffendingValidatorsThreshold; type ElectionProvider = ElectionProviderMultiPhase; - type GenesisElectionProvider = onchain::OnChainSequentialPhragmen; - // Alternatively, use pallet_staking::UseNominatorsMap to just use the nominators map. - // Note that the aforementioned does not scale to a very large number of nominators. - type SortedListProvider = BagsList; + type GenesisElectionProvider = onchain::UnboundedExecution; + type VoterList = BagsList; + type MaxUnlockingChunks = ConstU32<32>; type WeightInfo = pallet_staking::weights::SubstrateWeight; + type BenchmarkingConfig = StakingBenchmarkingConfig; } parameter_types! { @@ -541,7 +570,6 @@ parameter_types! { pub const UnsignedPhase: u32 = EPOCH_DURATION_IN_BLOCKS / 4; // signed config - pub const SignedMaxSubmissions: u32 = 10; pub const SignedRewardBase: Balance = 1 * DOLLARS; pub const SignedDepositBase: Balance = 1 * DOLLARS; pub const SignedDepositByte: Balance = 1 * CENTS; @@ -559,29 +587,28 @@ parameter_types! { *RuntimeBlockLength::get() .max .get(DispatchClass::Normal); - - // BagsList allows a practically unbounded count of nominators to participate in NPoS elections. - // To ensure we respect memory limits when using the BagsList this must be set to a number of - // voters we know can fit into a single vec allocation. - pub const VoterSnapshotPerBlock: u32 = 10_000; } -sp_npos_elections::generate_solution_type!( +frame_election_provider_support::generate_solution_type!( #[compact] pub struct NposSolution16::< VoterIndex = u32, TargetIndex = u16, Accuracy = sp_runtime::PerU16, + MaxVoters = MaxElectingVoters, >(16) ); -pub const MAX_NOMINATIONS: u32 = ::LIMIT as u32; +parameter_types! { + pub MaxNominations: u32 = ::LIMIT as u32; + pub MaxElectingVoters: u32 = 10_000; +} /// The numbers configured here could always be more than the the maximum limits of staking pallet /// to ensure election snapshot will not run out of memory. For now, we set them to smaller values /// since the staking is bounded and the weight pipeline takes hours for this single pallet. -pub struct BenchmarkConfig; -impl pallet_election_provider_multi_phase::BenchmarkingConfig for BenchmarkConfig { +pub struct ElectionProviderBenchmarkConfig; +impl pallet_election_provider_multi_phase::BenchmarkingConfig for ElectionProviderBenchmarkConfig { const VOTERS: [u32; 2] = [1000, 2000]; const TARGETS: [u32; 2] = [500, 1000]; const ACTIVE_VOTERS: [u32; 2] = [500, 800]; @@ -597,10 +624,8 @@ pub const MINER_MAX_ITERATIONS: u32 = 10; /// A source of random balance for NposSolver, which is meant to be run by the OCW election miner. pub struct OffchainRandomBalancing; -impl frame_support::pallet_prelude::Get> - for OffchainRandomBalancing -{ - fn get() -> Option<(usize, sp_npos_elections::ExtendedBalance)> { +impl Get> for OffchainRandomBalancing { + fn get() -> Option<(usize, ExtendedBalance)> { use sp_runtime::traits::TrailingZeroInput; let iters = match MINER_MAX_ITERATIONS { 0 => 0, @@ -617,6 +642,21 @@ impl frame_support::pallet_prelude::Get, + >; + type DataProvider = ::DataProvider; +} + +impl onchain::BoundedExecutionConfig for OnChainSeqPhragmen { + type VotersBound = ConstU32<20_000>; + type TargetsBound = ConstU32<2_000>; +} + impl pallet_election_provider_multi_phase::Config for Runtime { type Event = Event; type Currency = Balances; @@ -628,7 +668,7 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type MinerMaxWeight = MinerMaxWeight; type MinerMaxLength = MinerMaxLength; type MinerTxPriority = MultiPhaseUnsignedPriority; - type SignedMaxSubmissions = SignedMaxSubmissions; + type SignedMaxSubmissions = ConstU32<10>; type SignedRewardBase = SignedRewardBase; type SignedDepositBase = SignedDepositBase; type SignedDepositByte = SignedDepositByte; @@ -638,16 +678,14 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type RewardHandler = (); // nothing to do upon rewards type DataProvider = Staking; type Solution = NposSolution16; - type Fallback = pallet_election_provider_multi_phase::NoFallback; - type Solver = frame_election_provider_support::SequentialPhragmen< - AccountId, - pallet_election_provider_multi_phase::SolutionAccuracyOf, - OffchainRandomBalancing, - >; - type WeightInfo = pallet_election_provider_multi_phase::weights::SubstrateWeight; + type Fallback = onchain::BoundedExecution; + type GovernanceFallback = onchain::BoundedExecution; + type Solver = SequentialPhragmen, OffchainRandomBalancing>; type ForceOrigin = EnsureRootOrHalfCouncil; - type BenchmarkingConfig = BenchmarkConfig; - type VoterSnapshotPerBlock = VoterSnapshotPerBlock; + type MaxElectableTargets = ConstU16<{ u16::MAX }>; + type MaxElectingVoters = MaxElectingVoters; + type BenchmarkingConfig = ElectionProviderBenchmarkConfig; + type WeightInfo = pallet_election_provider_multi_phase::weights::SubstrateWeight; } parameter_types! { @@ -656,22 +694,96 @@ parameter_types! { impl pallet_bags_list::Config for Runtime { type Event = Event; - type VoteWeightProvider = Staking; + type ScoreProvider = Staking; type WeightInfo = pallet_bags_list::weights::SubstrateWeight; type BagThresholds = BagThresholds; + type Score = VoteWeight; +} + +parameter_types! { + pub const VoteLockingPeriod: BlockNumber = 30 * DAYS; +} + +impl pallet_conviction_voting::Config for Runtime { + type WeightInfo = pallet_conviction_voting::weights::SubstrateWeight; + type Event = Event; + type Currency = Balances; + type VoteLockingPeriod = VoteLockingPeriod; + type MaxVotes = ConstU32<512>; + type MaxTurnout = frame_support::traits::TotalIssuanceOf; + type Polls = Referenda; +} + +parameter_types! { + pub const AlarmInterval: BlockNumber = 1; + pub const SubmissionDeposit: Balance = 100 * DOLLARS; + pub const UndecidingTimeout: BlockNumber = 28 * DAYS; +} + +pub struct TracksInfo; +impl pallet_referenda::TracksInfo for TracksInfo { + type Id = u8; + type Origin = ::PalletsOrigin; + fn tracks() -> &'static [(Self::Id, pallet_referenda::TrackInfo)] { + static DATA: [(u8, pallet_referenda::TrackInfo); 1] = [( + 0u8, + pallet_referenda::TrackInfo { + name: "root", + max_deciding: 1, + decision_deposit: 10, + prepare_period: 4, + decision_period: 4, + confirm_period: 2, + min_enactment_period: 4, + min_approval: pallet_referenda::Curve::LinearDecreasing { + begin: Perbill::from_percent(100), + delta: Perbill::from_percent(50), + }, + min_turnout: pallet_referenda::Curve::LinearDecreasing { + begin: Perbill::from_percent(100), + delta: Perbill::from_percent(100), + }, + }, + )]; + &DATA[..] + } + fn track_for(id: &Self::Origin) -> Result { + if let Ok(system_origin) = frame_system::RawOrigin::try_from(id.clone()) { + match system_origin { + frame_system::RawOrigin::Root => Ok(0), + _ => Err(()), + } + } else { + Err(()) + } + } +} + +impl pallet_referenda::Config for Runtime { + type WeightInfo = pallet_referenda::weights::SubstrateWeight; + type Call = Call; + type Event = Event; + type Scheduler = Scheduler; + type Currency = pallet_balances::Pallet; + type CancelOrigin = EnsureRoot; + type KillOrigin = EnsureRoot; + type Slash = (); + type Votes = pallet_conviction_voting::VotesOf; + type Tally = pallet_conviction_voting::TallyOf; + type SubmissionDeposit = SubmissionDeposit; + type MaxQueued = ConstU32<100>; + type UndecidingTimeout = UndecidingTimeout; + type AlarmInterval = AlarmInterval; + type Tracks = TracksInfo; } parameter_types! { pub const LaunchPeriod: BlockNumber = 28 * 24 * 60 * MINUTES; pub const VotingPeriod: BlockNumber = 28 * 24 * 60 * MINUTES; pub const FastTrackVotingPeriod: BlockNumber = 3 * 24 * 60 * MINUTES; - pub const InstantAllowed: bool = true; pub const MinimumDeposit: Balance = 100 * DOLLARS; pub const EnactmentPeriod: BlockNumber = 30 * 24 * 60 * MINUTES; pub const CooloffPeriod: BlockNumber = 28 * 24 * 60 * MINUTES; - // One cent: $10,000 / MB - pub const PreimageByteDeposit: Balance = 1 * CENTS; - pub const MaxVotes: u32 = 100; pub const MaxProposals: u32 = 100; } @@ -686,31 +798,30 @@ impl pallet_democracy::Config for Runtime { type MinimumDeposit = MinimumDeposit; /// A straight majority of the council can decide what their next motion is. type ExternalOrigin = - pallet_collective::EnsureProportionAtLeast<_1, _2, AccountId, CouncilCollective>; + pallet_collective::EnsureProportionAtLeast; /// A super-majority can have the next scheduled referendum be a straight majority-carries vote. type ExternalMajorityOrigin = - pallet_collective::EnsureProportionAtLeast<_3, _4, AccountId, CouncilCollective>; + pallet_collective::EnsureProportionAtLeast; /// A unanimous council can have the next scheduled referendum be a straight default-carries /// (NTB) vote. type ExternalDefaultOrigin = - pallet_collective::EnsureProportionAtLeast<_1, _1, AccountId, CouncilCollective>; + pallet_collective::EnsureProportionAtLeast; /// Two thirds of the technical committee can have an ExternalMajority/ExternalDefault vote /// be tabled immediately and with a shorter voting/enactment period. type FastTrackOrigin = - pallet_collective::EnsureProportionAtLeast<_2, _3, AccountId, TechnicalCollective>; + pallet_collective::EnsureProportionAtLeast; type InstantOrigin = - pallet_collective::EnsureProportionAtLeast<_1, _1, AccountId, TechnicalCollective>; - type InstantAllowed = InstantAllowed; + pallet_collective::EnsureProportionAtLeast; + type InstantAllowed = frame_support::traits::ConstBool; type FastTrackVotingPeriod = FastTrackVotingPeriod; // To cancel a proposal which has been passed, 2/3 of the council must agree to it. type CancellationOrigin = - pallet_collective::EnsureProportionAtLeast<_2, _3, AccountId, CouncilCollective>; + pallet_collective::EnsureProportionAtLeast; // To cancel a proposal before it has been passed, the technical committee must be unanimous or // Root must agree. type CancelProposalOrigin = EnsureOneOf< - AccountId, EnsureRoot, - pallet_collective::EnsureProportionAtLeast<_1, _1, AccountId, TechnicalCollective>, + pallet_collective::EnsureProportionAtLeast, >; type BlacklistOrigin = EnsureRoot; // Any single technical committee member may veto a coming council proposal, however they can @@ -722,7 +833,7 @@ impl pallet_democracy::Config for Runtime { type Slash = Treasury; type Scheduler = Scheduler; type PalletsOrigin = OriginCaller; - type MaxVotes = MaxVotes; + type MaxVotes = ConstU32<100>; type WeightInfo = pallet_democracy::weights::SubstrateWeight; type MaxProposals = MaxProposals; } @@ -799,9 +910,8 @@ impl pallet_collective::Config for Runtime { } type EnsureRootOrHalfCouncil = EnsureOneOf< - AccountId, EnsureRoot, - pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>, + pallet_collective::EnsureProportionMoreThan, >; impl pallet_membership::Config for Runtime { type Event = Event; @@ -825,13 +935,8 @@ parameter_types! { pub const TipFindersFee: Percent = Percent::from_percent(20); pub const TipReportDepositBase: Balance = 1 * DOLLARS; pub const DataDepositPerByte: Balance = 1 * CENTS; - pub const BountyDepositBase: Balance = 1 * DOLLARS; - pub const BountyDepositPayoutDelay: BlockNumber = 1 * DAYS; pub const TreasuryPalletId: PalletId = PalletId(*b"py/trsry"); - pub const BountyUpdatePeriod: BlockNumber = 14 * DAYS; - pub const MaximumReasonLength: u32 = 16384; - pub const BountyCuratorDeposit: Permill = Permill::from_percent(50); - pub const BountyValueMinimum: Balance = 5 * DOLLARS; + pub const MaximumReasonLength: u32 = 300; pub const MaxApprovals: u32 = 100; } @@ -839,19 +944,18 @@ impl pallet_treasury::Config for Runtime { type PalletId = TreasuryPalletId; type Currency = Balances; type ApproveOrigin = EnsureOneOf< - AccountId, EnsureRoot, - pallet_collective::EnsureProportionAtLeast<_3, _5, AccountId, CouncilCollective>, + pallet_collective::EnsureProportionAtLeast, >; type RejectOrigin = EnsureOneOf< - AccountId, EnsureRoot, - pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>, + pallet_collective::EnsureProportionMoreThan, >; type Event = Event; type OnSlash = (); type ProposalBond = ProposalBond; type ProposalBondMinimum = ProposalBondMinimum; + type ProposalBondMaximum = (); type SpendPeriod = SpendPeriod; type Burn = Burn; type BurnDestination = (); @@ -860,16 +964,41 @@ impl pallet_treasury::Config for Runtime { type MaxApprovals = MaxApprovals; } +parameter_types! { + pub const BountyCuratorDeposit: Permill = Permill::from_percent(50); + pub const BountyValueMinimum: Balance = 5 * DOLLARS; + pub const BountyDepositBase: Balance = 1 * DOLLARS; + pub const CuratorDepositMultiplier: Permill = Permill::from_percent(50); + pub const CuratorDepositMin: Balance = 1 * DOLLARS; + pub const CuratorDepositMax: Balance = 100 * DOLLARS; + pub const BountyDepositPayoutDelay: BlockNumber = 1 * DAYS; + pub const BountyUpdatePeriod: BlockNumber = 14 * DAYS; +} + impl pallet_bounties::Config for Runtime { type Event = Event; type BountyDepositBase = BountyDepositBase; type BountyDepositPayoutDelay = BountyDepositPayoutDelay; type BountyUpdatePeriod = BountyUpdatePeriod; - type BountyCuratorDeposit = BountyCuratorDeposit; + type CuratorDepositMultiplier = CuratorDepositMultiplier; + type CuratorDepositMin = CuratorDepositMin; + type CuratorDepositMax = CuratorDepositMax; type BountyValueMinimum = BountyValueMinimum; type DataDepositPerByte = DataDepositPerByte; type MaximumReasonLength = MaximumReasonLength; type WeightInfo = pallet_bounties::weights::SubstrateWeight; + type ChildBountyManager = ChildBounties; +} + +parameter_types! { + pub const ChildBountyValueMinimum: Balance = 1 * DOLLARS; +} + +impl pallet_child_bounties::Config for Runtime { + type Event = Event; + type MaxActiveChildBountyCount = ConstU32<5>; + type ChildBountyValueMinimum = ChildBountyValueMinimum; + type WeightInfo = pallet_child_bounties::weights::SubstrateWeight; } impl pallet_tips::Config for Runtime { @@ -884,10 +1013,8 @@ impl pallet_tips::Config for Runtime { } parameter_types! { - pub ContractDeposit: Balance = deposit( - 1, - >::contract_info_size(), - ); + pub const DepositPerItem: Balance = deposit(1, 0); + pub const DepositPerByte: Balance = deposit(0, 1); pub const MaxValueSize: u32 = 16 * 1024; // The lazy deletion runs inside on_initialize. pub DeletionWeightLimit: Weight = AVERAGE_ON_INITIALIZE_RATIO * @@ -914,7 +1041,8 @@ impl pallet_contracts::Config for Runtime { /// change because that would break already deployed contracts. The `Call` structure itself /// is not allowed to change the indices of existing pallets, too. type CallFilter = Nothing; - type ContractDeposit = ContractDeposit; + type DepositPerItem = DepositPerItem; + type DepositPerByte = DepositPerByte; type CallStack = [pallet_contracts::Frame; 31]; type WeightPrice = pallet_transaction_payment::Pallet; type WeightInfo = pallet_contracts::weights::SubstrateWeight; @@ -922,6 +1050,7 @@ impl pallet_contracts::Config for Runtime { type DeletionQueueDepth = DeletionQueueDepth; type DeletionWeightLimit = DeletionWeightLimit; type Schedule = Schedule; + type AddressGenerator = pallet_contracts::DefaultAddressGenerator; } impl pallet_sudo::Config for Runtime { @@ -960,13 +1089,14 @@ where .saturating_sub(1); let era = Era::mortal(period, current_block); let extra = ( + frame_system::CheckNonZeroSender::::new(), frame_system::CheckSpecVersion::::new(), frame_system::CheckTxVersion::::new(), frame_system::CheckGenesis::::new(), frame_system::CheckEra::::from(era), frame_system::CheckNonce::::from(nonce), frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(tip), + pallet_asset_tx_payment::ChargeAssetTxPayment::::from(tip, None), ); let raw_payload = SignedPayload::new(call, extra) .map_err(|e| { @@ -1106,7 +1236,7 @@ impl pallet_society::Config for Runtime { type RotationPeriod = RotationPeriod; type MaxLockDuration = MaxLockDuration; type FounderSetOrigin = - pallet_collective::EnsureProportionMoreThan<_1, _2, AccountId, CouncilCollective>; + pallet_collective::EnsureProportionMoreThan; type SuspensionJudgementOrigin = pallet_society::EnsureFounder; type MaxCandidateIntake = MaxCandidateIntake; type ChallengePeriod = ChallengePeriod; @@ -1165,11 +1295,12 @@ parameter_types! { impl pallet_assets::Config for Runtime { type Event = Event; - type Balance = u64; + type Balance = u128; type AssetId = u32; type Currency = Balances; type ForceOrigin = EnsureRoot; type AssetDeposit = AssetDeposit; + type AssetAccountDeposit = ConstU128; type MetadataDepositBase = MetadataDepositBase; type MetadataDepositPerByte = MetadataDepositPerByte; type ApprovalDeposit = ApprovalDeposit; @@ -1230,6 +1361,9 @@ impl pallet_uniques::Config for Runtime { type KeyLimit = KeyLimit; type ValueLimit = ValueLimit; type WeightInfo = pallet_uniques::weights::SubstrateWeight; + #[cfg(feature = "runtime-benchmarks")] + type Helper = (); + type CreateOrigin = AsEnsureOriginWithArg>; } impl pallet_transaction_storage::Config for Runtime { @@ -1240,53 +1374,90 @@ impl pallet_transaction_storage::Config for Runtime { type WeightInfo = pallet_transaction_storage::weights::SubstrateWeight; } +impl pallet_whitelist::Config for Runtime { + type Event = Event; + type Call = Call; + type WhitelistOrigin = EnsureRoot; + type DispatchWhitelistedOrigin = EnsureRoot; + type PreimageProvider = Preimage; + type WeightInfo = pallet_whitelist::weights::SubstrateWeight; +} + +parameter_types! { + pub const MigrationSignedDepositPerItem: Balance = 1 * CENTS; + pub const MigrationSignedDepositBase: Balance = 20 * DOLLARS; +} + +impl pallet_state_trie_migration::Config for Runtime { + type Event = Event; + type ControlOrigin = EnsureRoot; + type Currency = Balances; + type SignedDepositPerItem = MigrationSignedDepositPerItem; + type SignedDepositBase = MigrationSignedDepositBase; + // Warning: this is not advised, as it might allow the chain to be temporarily DOS-ed. + // Preferably, if the chain's governance/maintenance team is planning on using a specific + // account for the migration, put it here to make sure only that account can trigger the signed + // migrations. + type SignedFilter = EnsureSigned; + type WeightInfo = (); +} + construct_runtime!( pub enum Runtime where Block = Block, NodeBlock = node_primitives::Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: frame_system::{Pallet, Call, Config, Storage, Event}, - Utility: pallet_utility::{Pallet, Call, Event}, - Babe: pallet_babe::{Pallet, Call, Storage, Config, ValidateUnsigned}, - Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent}, - Authorship: pallet_authorship::{Pallet, Call, Storage, Inherent}, - Indices: pallet_indices::{Pallet, Call, Storage, Config, Event}, - Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, - TransactionPayment: pallet_transaction_payment::{Pallet, Storage}, - ElectionProviderMultiPhase: pallet_election_provider_multi_phase::{Pallet, Call, Storage, Event, ValidateUnsigned}, - Staking: pallet_staking::{Pallet, Call, Config, Storage, Event}, - Session: pallet_session::{Pallet, Call, Storage, Event, Config}, - Democracy: pallet_democracy::{Pallet, Call, Storage, Config, Event}, - Council: pallet_collective::::{Pallet, Call, Storage, Origin, Event, Config}, - TechnicalCommittee: pallet_collective::::{Pallet, Call, Storage, Origin, Event, Config}, - Elections: pallet_elections_phragmen::{Pallet, Call, Storage, Event, Config}, - TechnicalMembership: pallet_membership::::{Pallet, Call, Storage, Event, Config}, - Grandpa: pallet_grandpa::{Pallet, Call, Storage, Config, Event, ValidateUnsigned}, - Treasury: pallet_treasury::{Pallet, Call, Storage, Config, Event}, - Contracts: pallet_contracts::{Pallet, Call, Storage, Event}, - Sudo: pallet_sudo::{Pallet, Call, Config, Storage, Event}, - ImOnline: pallet_im_online::{Pallet, Call, Storage, Event, ValidateUnsigned, Config}, - AuthorityDiscovery: pallet_authority_discovery::{Pallet, Config}, - Offences: pallet_offences::{Pallet, Storage, Event}, + System: frame_system, + Utility: pallet_utility, + Babe: pallet_babe, + Timestamp: pallet_timestamp, + // Authorship must be before session in order to note author in the correct session and era + // for im-online and staking. + Authorship: pallet_authorship, + Indices: pallet_indices, + Balances: pallet_balances, + TransactionPayment: pallet_transaction_payment, + AssetTxPayment: pallet_asset_tx_payment, + ElectionProviderMultiPhase: pallet_election_provider_multi_phase, + Staking: pallet_staking, + Session: pallet_session, + Democracy: pallet_democracy, + Council: pallet_collective::, + TechnicalCommittee: pallet_collective::, + Elections: pallet_elections_phragmen, + TechnicalMembership: pallet_membership::, + Grandpa: pallet_grandpa, + Treasury: pallet_treasury, + Contracts: pallet_contracts, + Sudo: pallet_sudo, + ImOnline: pallet_im_online, + AuthorityDiscovery: pallet_authority_discovery, + Offences: pallet_offences, Historical: pallet_session_historical::{Pallet}, - RandomnessCollectiveFlip: pallet_randomness_collective_flip::{Pallet, Storage}, - Identity: pallet_identity::{Pallet, Call, Storage, Event}, - Society: pallet_society::{Pallet, Call, Storage, Event, Config}, - Recovery: pallet_recovery::{Pallet, Call, Storage, Event}, - Vesting: pallet_vesting::{Pallet, Call, Storage, Event, Config}, - Scheduler: pallet_scheduler::{Pallet, Call, Storage, Event}, - Proxy: pallet_proxy::{Pallet, Call, Storage, Event}, - Multisig: pallet_multisig::{Pallet, Call, Storage, Event}, - Bounties: pallet_bounties::{Pallet, Call, Storage, Event}, - Tips: pallet_tips::{Pallet, Call, Storage, Event}, - Assets: pallet_assets::{Pallet, Call, Storage, Event}, - Mmr: pallet_mmr::{Pallet, Storage}, - Lottery: pallet_lottery::{Pallet, Call, Storage, Event}, - Gilt: pallet_gilt::{Pallet, Call, Storage, Event, Config}, - Uniques: pallet_uniques::{Pallet, Call, Storage, Event}, - TransactionStorage: pallet_transaction_storage::{Pallet, Call, Storage, Inherent, Config, Event}, - BagsList: pallet_bags_list::{Pallet, Call, Storage, Event}, + RandomnessCollectiveFlip: pallet_randomness_collective_flip, + Identity: pallet_identity, + Society: pallet_society, + Recovery: pallet_recovery, + Vesting: pallet_vesting, + Scheduler: pallet_scheduler, + Preimage: pallet_preimage, + Proxy: pallet_proxy, + Multisig: pallet_multisig, + Bounties: pallet_bounties, + Tips: pallet_tips, + Assets: pallet_assets, + Mmr: pallet_mmr, + Lottery: pallet_lottery, + Gilt: pallet_gilt, + Uniques: pallet_uniques, + TransactionStorage: pallet_transaction_storage, + BagsList: pallet_bags_list, + StateTrieMigration: pallet_state_trie_migration, + ChildBounties: pallet_child_bounties, + Referenda: pallet_referenda, + ConvictionVoting: pallet_conviction_voting, + Whitelist: pallet_whitelist, } ); @@ -1306,13 +1477,14 @@ pub type BlockId = generic::BlockId; /// /// [`sign`]: <../../testing/src/keyring.rs.html> pub type SignedExtra = ( + frame_system::CheckNonZeroSender, frame_system::CheckSpecVersion, frame_system::CheckTxVersion, frame_system::CheckGenesis, frame_system::CheckEra, frame_system::CheckNonce, frame_system::CheckWeight, - pallet_transaction_payment::ChargeTransactionPayment, + pallet_asset_tx_payment::ChargeAssetTxPayment, ); /// Unchecked extrinsic type as expected by this runtime. pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; @@ -1326,8 +1498,8 @@ pub type Executive = frame_executive::Executive< Block, frame_system::ChainContext, Runtime, - AllPallets, - (), + AllPalletsWithSystem, + pallet_bags_list::migrations::CheckCounterPrefix, >; /// MMR helper types. @@ -1340,6 +1512,55 @@ mod mmr { pub type Hashing = ::Hashing; } +#[cfg(feature = "runtime-benchmarks")] +#[macro_use] +extern crate frame_benchmarking; + +#[cfg(feature = "runtime-benchmarks")] +mod benches { + define_benchmarks!( + [frame_benchmarking, BaselineBench::] + [pallet_assets, Assets] + [pallet_babe, Babe] + [pallet_bags_list, BagsList] + [pallet_balances, Balances] + [pallet_bounties, Bounties] + [pallet_child_bounties, ChildBounties] + [pallet_collective, Council] + [pallet_conviction_voting, ConvictionVoting] + [pallet_contracts, Contracts] + [pallet_democracy, Democracy] + [pallet_election_provider_multi_phase, ElectionProviderMultiPhase] + [pallet_elections_phragmen, Elections] + [pallet_gilt, Gilt] + [pallet_grandpa, Grandpa] + [pallet_identity, Identity] + [pallet_im_online, ImOnline] + [pallet_indices, Indices] + [pallet_lottery, Lottery] + [pallet_membership, TechnicalMembership] + [pallet_mmr, Mmr] + [pallet_multisig, Multisig] + [pallet_offences, OffencesBench::] + [pallet_preimage, Preimage] + [pallet_proxy, Proxy] + [pallet_referenda, Referenda] + [pallet_scheduler, Scheduler] + [pallet_session, SessionBench::] + [pallet_staking, Staking] + [pallet_state_trie_migration, StateTrieMigration] + [frame_system, SystemBench::] + [pallet_timestamp, Timestamp] + [pallet_tips, Tips] + [pallet_transaction_storage, TransactionStorage] + [pallet_treasury, Treasury] + [pallet_uniques, Uniques] + [pallet_utility, Utility] + [pallet_vesting, Vesting] + [pallet_whitelist, Whitelist] + ); +} + impl_runtime_apis! { impl sp_api::Core for Runtime { fn version() -> RuntimeVersion { @@ -1506,21 +1727,32 @@ impl_runtime_apis! { dest: AccountId, value: Balance, gas_limit: u64, + storage_deposit_limit: Option, input_data: Vec, - ) -> pallet_contracts_primitives::ContractExecResult { - Contracts::bare_call(origin, dest, value, gas_limit, input_data, true) + ) -> pallet_contracts_primitives::ContractExecResult { + Contracts::bare_call(origin, dest, value, gas_limit, storage_deposit_limit, input_data, true) } fn instantiate( origin: AccountId, - endowment: Balance, + value: Balance, gas_limit: u64, + storage_deposit_limit: Option, code: pallet_contracts_primitives::Code, data: Vec, salt: Vec, - ) -> pallet_contracts_primitives::ContractInstantiateResult + ) -> pallet_contracts_primitives::ContractInstantiateResult + { + Contracts::bare_instantiate(origin, value, gas_limit, storage_deposit_limit, code, data, salt, true) + } + + fn upload_code( + origin: AccountId, + code: Vec, + storage_deposit_limit: Option, + ) -> pallet_contracts_primitives::CodeUploadResult { - Contracts::bare_instantiate(origin, endowment, gas_limit, code, data, salt, true) + Contracts::bare_upload_code(origin, code, storage_deposit_limit) } fn get_storage( @@ -1547,7 +1779,7 @@ impl_runtime_apis! { Block, mmr::Hash, > for Runtime { - fn generate_proof(leaf_index: u64) + fn generate_proof(leaf_index: pallet_mmr::primitives::LeafIndex) -> Result<(mmr::EncodableOpaqueLeaf, mmr::Proof), mmr::Error> { Mmr::generate_proof(leaf_index) @@ -1616,7 +1848,7 @@ impl_runtime_apis! { Vec, Vec, ) { - use frame_benchmarking::{list_benchmark, Benchmarking, BenchmarkList}; + use frame_benchmarking::{baseline, Benchmarking, BenchmarkList}; use frame_support::traits::StorageInfoTrait; // Trying to add benchmarks directly to the Session Pallet caused cyclic dependency @@ -1625,41 +1857,10 @@ impl_runtime_apis! { use pallet_session_benchmarking::Pallet as SessionBench; use pallet_offences_benchmarking::Pallet as OffencesBench; use frame_system_benchmarking::Pallet as SystemBench; + use baseline::Pallet as BaselineBench; let mut list = Vec::::new(); - - list_benchmark!(list, extra, pallet_assets, Assets); - list_benchmark!(list, extra, pallet_babe, Babe); - list_benchmark!(list, extra, pallet_bags_list, BagsList); - list_benchmark!(list, extra, pallet_balances, Balances); - list_benchmark!(list, extra, pallet_bounties, Bounties); - list_benchmark!(list, extra, pallet_collective, Council); - list_benchmark!(list, extra, pallet_contracts, Contracts); - list_benchmark!(list, extra, pallet_democracy, Democracy); - list_benchmark!(list, extra, pallet_election_provider_multi_phase, ElectionProviderMultiPhase); - list_benchmark!(list, extra, pallet_elections_phragmen, Elections); - list_benchmark!(list, extra, pallet_gilt, Gilt); - list_benchmark!(list, extra, pallet_grandpa, Grandpa); - list_benchmark!(list, extra, pallet_identity, Identity); - list_benchmark!(list, extra, pallet_im_online, ImOnline); - list_benchmark!(list, extra, pallet_indices, Indices); - list_benchmark!(list, extra, pallet_lottery, Lottery); - list_benchmark!(list, extra, pallet_membership, TechnicalMembership); - list_benchmark!(list, extra, pallet_mmr, Mmr); - list_benchmark!(list, extra, pallet_multisig, Multisig); - list_benchmark!(list, extra, pallet_offences, OffencesBench::); - list_benchmark!(list, extra, pallet_proxy, Proxy); - list_benchmark!(list, extra, pallet_scheduler, Scheduler); - list_benchmark!(list, extra, pallet_session, SessionBench::); - list_benchmark!(list, extra, pallet_staking, Staking); - list_benchmark!(list, extra, frame_system, SystemBench::); - list_benchmark!(list, extra, pallet_timestamp, Timestamp); - list_benchmark!(list, extra, pallet_tips, Tips); - list_benchmark!(list, extra, pallet_transaction_storage, TransactionStorage); - list_benchmark!(list, extra, pallet_treasury, Treasury); - list_benchmark!(list, extra, pallet_uniques, Uniques); - list_benchmark!(list, extra, pallet_utility, Utility); - list_benchmark!(list, extra, pallet_vesting, Vesting); + list_benchmarks!(list, extra); let storage_info = AllPalletsWithSystem::storage_info(); @@ -1669,7 +1870,7 @@ impl_runtime_apis! { fn dispatch_benchmark( config: frame_benchmarking::BenchmarkConfig ) -> Result, sp_runtime::RuntimeString> { - use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark, TrackedStorageKey}; + use frame_benchmarking::{baseline, Benchmarking, BenchmarkBatch, TrackedStorageKey}; // Trying to add benchmarks directly to the Session Pallet caused cyclic dependency // issues. To get around that, we separated the Session benchmarks into its own crate, @@ -1677,10 +1878,12 @@ impl_runtime_apis! { use pallet_session_benchmarking::Pallet as SessionBench; use pallet_offences_benchmarking::Pallet as OffencesBench; use frame_system_benchmarking::Pallet as SystemBench; + use baseline::Pallet as BaselineBench; impl pallet_session_benchmarking::Config for Runtime {} impl pallet_offences_benchmarking::Config for Runtime {} impl frame_system_benchmarking::Config for Runtime {} + impl baseline::Config for Runtime {} let whitelist: Vec = vec![ // Block Number @@ -1701,41 +1904,8 @@ impl_runtime_apis! { let mut batches = Vec::::new(); let params = (&config, &whitelist); + add_benchmarks!(params, batches); - add_benchmark!(params, batches, pallet_assets, Assets); - add_benchmark!(params, batches, pallet_babe, Babe); - add_benchmark!(params, batches, pallet_balances, Balances); - add_benchmark!(params, batches, pallet_bags_list, BagsList); - add_benchmark!(params, batches, pallet_bounties, Bounties); - add_benchmark!(params, batches, pallet_collective, Council); - add_benchmark!(params, batches, pallet_contracts, Contracts); - add_benchmark!(params, batches, pallet_democracy, Democracy); - add_benchmark!(params, batches, pallet_election_provider_multi_phase, ElectionProviderMultiPhase); - add_benchmark!(params, batches, pallet_elections_phragmen, Elections); - add_benchmark!(params, batches, pallet_gilt, Gilt); - add_benchmark!(params, batches, pallet_grandpa, Grandpa); - add_benchmark!(params, batches, pallet_identity, Identity); - add_benchmark!(params, batches, pallet_im_online, ImOnline); - add_benchmark!(params, batches, pallet_indices, Indices); - add_benchmark!(params, batches, pallet_lottery, Lottery); - add_benchmark!(params, batches, pallet_membership, TechnicalMembership); - add_benchmark!(params, batches, pallet_mmr, Mmr); - add_benchmark!(params, batches, pallet_multisig, Multisig); - add_benchmark!(params, batches, pallet_offences, OffencesBench::); - add_benchmark!(params, batches, pallet_proxy, Proxy); - add_benchmark!(params, batches, pallet_scheduler, Scheduler); - add_benchmark!(params, batches, pallet_session, SessionBench::); - add_benchmark!(params, batches, pallet_staking, Staking); - add_benchmark!(params, batches, frame_system, SystemBench::); - add_benchmark!(params, batches, pallet_timestamp, Timestamp); - add_benchmark!(params, batches, pallet_tips, Tips); - add_benchmark!(params, batches, pallet_transaction_storage, TransactionStorage); - add_benchmark!(params, batches, pallet_treasury, Treasury); - add_benchmark!(params, batches, pallet_uniques, Uniques); - add_benchmark!(params, batches, pallet_utility, Utility); - add_benchmark!(params, batches, pallet_vesting, Vesting); - - if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) } Ok(batches) } } @@ -1744,6 +1914,7 @@ impl_runtime_apis! { #[cfg(test)] mod tests { use super::*; + use frame_election_provider_support::NposSolution; use frame_system::offchain::CreateSignedTransaction; use sp_runtime::UpperOf; @@ -1760,8 +1931,9 @@ mod tests { #[test] fn perbill_as_onchain_accuracy() { - type OnChainAccuracy = ::Accuracy; - let maximum_chain_accuracy: Vec> = (0..MAX_NOMINATIONS) + type OnChainAccuracy = + <::Solution as NposSolution>::Accuracy; + let maximum_chain_accuracy: Vec> = (0..MaxNominations::get()) .map(|_| >::from(OnChainAccuracy::one().deconstruct())) .collect(); let _: UpperOf = @@ -1770,11 +1942,13 @@ mod tests { #[test] fn call_size() { + let size = core::mem::size_of::(); assert!( - core::mem::size_of::() <= 200, - "size of Call is more than 200 bytes: some calls have too big arguments, use Box to reduce the + size <= 208, + "size of Call {} is more than 208 bytes: some calls have too big arguments, use Box to reduce the size of Call. If the limit is too strong, maybe consider increase the limit to 300.", + size, ); } } diff --git a/bin/node/runtime/src/voter_bags.rs b/bin/node/runtime/src/voter_bags.rs index c4c731a58bad..93790f028f45 100644 --- a/bin/node/runtime/src/voter_bags.rs +++ b/bin/node/runtime/src/voter_bags.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/bin/node/test-runner-example/Cargo.toml b/bin/node/test-runner-example/Cargo.toml deleted file mode 100644 index 96c4c2047ac4..000000000000 --- a/bin/node/test-runner-example/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -name = "test-runner-example" -version = "0.1.0" -authors = ["Parity Technologies "] -edition = "2018" -publish = false - -[dependencies] -test-runner = { path = "../../../test-utils/test-runner" } - -frame-system = { path = "../../../frame/system" } -frame-benchmarking = { path = "../../../frame/benchmarking" } -pallet-transaction-payment = { path = "../../../frame/transaction-payment" } - -node-runtime = { path = "../runtime" } -node-primitives = { path = "../primitives" } -node-cli = { path = "../cli" } - -grandpa = { package = "sc-finality-grandpa", path = "../../../client/finality-grandpa" } -sp-consensus-babe = { path = "../../../primitives/consensus/babe" } -sc-consensus-babe = { path = "../../../client/consensus/babe" } -sc-consensus-manual-seal = { path = "../../../client/consensus/manual-seal" } -sc-service = { default-features = false, path = "../../../client/service" } -sc-executor = { path = "../../../client/executor" } -sc-consensus = { path = "../../../client/consensus/common" } - -sp-runtime = { path = "../../../primitives/runtime" } -sp-keyring = { path = "../../../primitives/keyring" } diff --git a/bin/node/test-runner-example/src/lib.rs b/bin/node/test-runner-example/src/lib.rs deleted file mode 100644 index 0de7f5a4e2b7..000000000000 --- a/bin/node/test-runner-example/src/lib.rs +++ /dev/null @@ -1,131 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -#![deny(unused_extern_crates, missing_docs)] - -//! Basic example of end to end runtime tests. - -use grandpa::GrandpaBlockImport; -use sc_consensus_babe::BabeBlockImport; -use sc_consensus_manual_seal::consensus::babe::SlotTimestampProvider; -use sc_executor::NativeElseWasmExecutor; -use sc_service::{TFullBackend, TFullClient}; -use sp_runtime::generic::Era; -use test_runner::{ChainInfo, SignatureVerificationOverride}; - -type BlockImport = BabeBlockImport>; - -/// A unit struct which implements `NativeExecutionDispatch` feeding in the -/// hard-coded runtime. -pub struct ExecutorDispatch; - -impl sc_executor::NativeExecutionDispatch for ExecutorDispatch { - type ExtendHostFunctions = - (frame_benchmarking::benchmarking::HostFunctions, SignatureVerificationOverride); - - fn dispatch(method: &str, data: &[u8]) -> Option> { - node_runtime::api::dispatch(method, data) - } - - fn native_version() -> sc_executor::NativeVersion { - node_runtime::native_version() - } -} - -/// ChainInfo implementation. -struct NodeTemplateChainInfo; - -impl ChainInfo for NodeTemplateChainInfo { - type Block = node_primitives::Block; - type ExecutorDispatch = ExecutorDispatch; - type Runtime = node_runtime::Runtime; - type RuntimeApi = node_runtime::RuntimeApi; - type SelectChain = sc_consensus::LongestChain, Self::Block>; - type BlockImport = BlockImport< - Self::Block, - TFullBackend, - TFullClient>, - Self::SelectChain, - >; - type SignedExtras = node_runtime::SignedExtra; - type InherentDataProviders = - (SlotTimestampProvider, sp_consensus_babe::inherents::InherentDataProvider); - - fn signed_extras( - from: ::AccountId, - ) -> Self::SignedExtras { - ( - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(Era::Immortal), - frame_system::CheckNonce::::from( - frame_system::Pallet::::account_nonce(from), - ), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(0), - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use node_cli::chain_spec::development_config; - use sp_keyring::sr25519::Keyring::Alice; - use sp_runtime::{traits::IdentifyAccount, MultiSigner}; - use test_runner::{build_runtime, client_parts, ConfigOrChainSpec, Node}; - - #[test] - fn test_runner() { - let tokio_runtime = build_runtime().unwrap(); - let (rpc, task_manager, client, pool, command_sink, backend) = - client_parts::(ConfigOrChainSpec::ChainSpec( - Box::new(development_config()), - tokio_runtime.handle().clone(), - )) - .unwrap(); - let node = Node::::new( - rpc, - task_manager, - client, - pool, - command_sink, - backend, - ); - - tokio_runtime.block_on(async { - // seals blocks - node.seal_blocks(1).await; - // submit extrinsics - let alice = MultiSigner::from(Alice.public()).into_account(); - let _hash = node - .submit_extrinsic( - frame_system::Call::remark { remark: (b"hello world").to_vec() }, - Some(alice), - ) - .await - .unwrap(); - - // look ma, I can read state. - let _events = - node.with_state(|| frame_system::Pallet::::events()); - // get access to the underlying client. - let _client = node.client(); - }) - } -} diff --git a/bin/node/testing/Cargo.toml b/bin/node/testing/Cargo.toml index d05d815121f8..02d13f0c6471 100644 --- a/bin/node/testing/Cargo.toml +++ b/bin/node/testing/Cargo.toml @@ -3,9 +3,9 @@ name = "node-testing" version = "3.0.0-dev" authors = ["Parity Technologies "] description = "Test utilities for Substrate node." -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" publish = true @@ -23,14 +23,14 @@ sc-client-db = { version = "0.10.0-dev", path = "../../../client/db/", features ] } sc-client-api = { version = "4.0.0-dev", path = "../../../client/api/" } sc-consensus = { version = "0.10.0-dev", path = "../../../client/consensus/common" } -codec = { package = "parity-scale-codec", version = "2.0.0" } -sp-keyring = { version = "4.0.0-dev", path = "../../../primitives/keyring" } +codec = { package = "parity-scale-codec", version = "3.0.0" } +sp-keyring = { version = "6.0.0", path = "../../../primitives/keyring" } node-executor = { version = "3.0.0-dev", path = "../executor" } node-primitives = { version = "2.0.0", path = "../primitives" } node-runtime = { version = "3.0.0-dev", path = "../runtime" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../../primitives/io" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } sc-executor = { version = "0.10.0-dev", path = "../../../client/executor", features = [ "wasmtime", ] } @@ -38,6 +38,7 @@ sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/c frame-system = { version = "4.0.0-dev", path = "../../../frame/system" } substrate-test-client = { version = "2.0.0", path = "../../../test-utils/client" } pallet-transaction-payment = { version = "4.0.0-dev", path = "../../../frame/transaction-payment" } +pallet-asset-tx-payment = { version = "4.0.0-dev", path = "../../../frame/transaction-payment/asset-tx-payment/" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-timestamp = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/timestamp" } sp-block-builder = { version = "4.0.0-dev", path = "../../../primitives/block-builder" } @@ -47,4 +48,4 @@ sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" log = "0.4.8" tempfile = "3.1.0" fs_extra = "1" -futures = "0.3.1" +futures = "0.3.21" diff --git a/bin/node/testing/src/bench.rs b/bin/node/testing/src/bench.rs index cf0a463cc3e9..8227582d88a4 100644 --- a/bin/node/testing/src/bench.rs +++ b/bin/node/testing/src/bench.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -243,11 +243,21 @@ impl TaskExecutor { } impl SpawnNamed for TaskExecutor { - fn spawn(&self, _: &'static str, future: futures::future::BoxFuture<'static, ()>) { + fn spawn( + &self, + _: &'static str, + _: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ) { self.pool.spawn_ok(future); } - fn spawn_blocking(&self, _: &'static str, future: futures::future::BoxFuture<'static, ()>) { + fn spawn_blocking( + &self, + _: &'static str, + _: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ) { self.pool.spawn_ok(future); } } @@ -383,14 +393,13 @@ impl BenchDb { state_pruning: PruningMode::ArchiveAll, source: database_type.into_settings(dir.into()), keep_blocks: sc_client_db::KeepBlocks::All, - transaction_storage: sc_client_db::TransactionStorageMode::BlockBody, }; let task_executor = TaskExecutor::new(); let backend = sc_service::new_db_backend(db_config).expect("Should not fail"); let client = sc_service::new_client( backend.clone(), - NativeElseWasmExecutor::new(WasmExecutionMethod::Compiled, None, 8), + NativeElseWasmExecutor::new(WasmExecutionMethod::Compiled, None, 8, 2), &keyring.generate_genesis(), None, None, @@ -581,7 +590,6 @@ impl BenchKeyring { /// Generate genesis with accounts from this keyring endowed with some balance. pub fn generate_genesis(&self) -> node_runtime::GenesisConfig { crate::genesis::config_endowed( - false, Some(node_runtime::wasm_binary_unwrap()), self.collect_account_ids(), ) diff --git a/bin/node/testing/src/client.rs b/bin/node/testing/src/client.rs index 8bd75834c549..8cb98511098f 100644 --- a/bin/node/testing/src/client.rs +++ b/bin/node/testing/src/client.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -42,13 +42,11 @@ pub type Transaction = sc_client_api::backend::TransactionFor Storage { - crate::genesis::config(self.support_changes_trie, None).build_storage().unwrap() + crate::genesis::config(None).build_storage().unwrap() } } diff --git a/bin/node/testing/src/genesis.rs b/bin/node/testing/src/genesis.rs index 50c1e6f9d20b..8d2b53b0b721 100644 --- a/bin/node/testing/src/genesis.rs +++ b/bin/node/testing/src/genesis.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -24,22 +24,17 @@ use node_runtime::{ GenesisConfig, GrandpaConfig, IndicesConfig, SessionConfig, SocietyConfig, StakerStatus, StakingConfig, SystemConfig, BABE_GENESIS_EPOCH_CONFIG, }; -use sp_core::ChangesTrieConfiguration; use sp_keyring::{Ed25519Keyring, Sr25519Keyring}; use sp_runtime::Perbill; /// Create genesis runtime configuration for tests. -pub fn config(support_changes_trie: bool, code: Option<&[u8]>) -> GenesisConfig { - config_endowed(support_changes_trie, code, Default::default()) +pub fn config(code: Option<&[u8]>) -> GenesisConfig { + config_endowed(code, Default::default()) } /// Create genesis runtime configuration for tests with some extra /// endowed accounts. -pub fn config_endowed( - support_changes_trie: bool, - code: Option<&[u8]>, - extra_endowed: Vec, -) -> GenesisConfig { +pub fn config_endowed(code: Option<&[u8]>, extra_endowed: Vec) -> GenesisConfig { let mut endowed = vec![ (alice(), 111 * DOLLARS), (bob(), 100 * DOLLARS), @@ -53,22 +48,17 @@ pub fn config_endowed( GenesisConfig { system: SystemConfig { - changes_trie_config: if support_changes_trie { - Some(ChangesTrieConfiguration { digest_interval: 2, digest_levels: 2 }) - } else { - None - }, code: code.map(|x| x.to_vec()).unwrap_or_else(|| wasm_binary_unwrap().to_vec()), }, indices: IndicesConfig { indices: vec![] }, balances: BalancesConfig { balances: endowed }, session: SessionConfig { keys: vec![ - (dave(), alice(), to_session_keys(&Ed25519Keyring::Alice, &Sr25519Keyring::Alice)), - (eve(), bob(), to_session_keys(&Ed25519Keyring::Bob, &Sr25519Keyring::Bob)), + (alice(), dave(), to_session_keys(&Ed25519Keyring::Alice, &Sr25519Keyring::Alice)), + (bob(), eve(), to_session_keys(&Ed25519Keyring::Bob, &Sr25519Keyring::Bob)), ( - ferdie(), charlie(), + ferdie(), to_session_keys(&Ed25519Keyring::Charlie, &Sr25519Keyring::Charlie), ), ], @@ -98,7 +88,9 @@ pub fn config_endowed( treasury: Default::default(), society: SocietyConfig { members: vec![alice(), bob()], pot: 0, max_members: 999 }, vesting: Default::default(), + assets: Default::default(), gilt: Default::default(), transaction_storage: Default::default(), + transaction_payment: Default::default(), } } diff --git a/bin/node/testing/src/keyring.rs b/bin/node/testing/src/keyring.rs index 4e2d88b4bba3..41dd28bb8998 100644 --- a/bin/node/testing/src/keyring.rs +++ b/bin/node/testing/src/keyring.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -70,13 +70,14 @@ pub fn to_session_keys( /// Returns transaction extra. pub fn signed_extra(nonce: Index, extra_fee: Balance) -> SignedExtra { ( + frame_system::CheckNonZeroSender::new(), frame_system::CheckSpecVersion::new(), frame_system::CheckTxVersion::new(), frame_system::CheckGenesis::new(), frame_system::CheckEra::from(Era::mortal(256, 0)), frame_system::CheckNonce::from(nonce), frame_system::CheckWeight::new(), - pallet_transaction_payment::ChargeTransactionPayment::from(extra_fee), + pallet_asset_tx_payment::ChargeAssetTxPayment::from(extra_fee, None), ) } diff --git a/bin/node/testing/src/lib.rs b/bin/node/testing/src/lib.rs index a3392bcb29d5..ce3471e5b1f8 100644 --- a/bin/node/testing/src/lib.rs +++ b/bin/node/testing/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/utils/chain-spec-builder/Cargo.toml b/bin/utils/chain-spec-builder/Cargo.toml index 5bdf01badc3f..1ea1c402151d 100644 --- a/bin/utils/chain-spec-builder/Cargo.toml +++ b/bin/utils/chain-spec-builder/Cargo.toml @@ -2,10 +2,10 @@ name = "chain-spec-builder" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" build = "build.rs" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" publish = false @@ -15,10 +15,11 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] ansi_term = "0.12.1" +clap = { version = "3.1.6", features = ["derive"] } +rand = "0.8" + sc-keystore = { version = "4.0.0-dev", path = "../../../client/keystore" } sc-chain-spec = { version = "4.0.0-dev", path = "../../../client/chain-spec" } node-cli = { version = "3.0.0-dev", path = "../../node/cli" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-keystore = { version = "0.10.0-dev", path = "../../../primitives/keystore" } -rand = "0.7.2" -structopt = "0.3.8" +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" } diff --git a/bin/utils/chain-spec-builder/build.rs b/bin/utils/chain-spec-builder/build.rs index 57424f016f3e..b700b28e322c 100644 --- a/bin/utils/chain-spec-builder/build.rs +++ b/bin/utils/chain-spec-builder/build.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/bin/utils/chain-spec-builder/src/main.rs b/bin/utils/chain-spec-builder/src/main.rs index bf5f1a149578..3e8b1f4ea752 100644 --- a/bin/utils/chain-spec-builder/src/main.rs +++ b/bin/utils/chain-spec-builder/src/main.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,65 +23,65 @@ use std::{ }; use ansi_term::Style; +use clap::Parser; use rand::{distributions::Alphanumeric, rngs::OsRng, Rng}; -use structopt::StructOpt; use node_cli::chain_spec::{self, AccountId}; use sc_keystore::LocalKeystore; use sp_core::{ - crypto::{Public, Ss58Codec}, + crypto::{ByteArray, Ss58Codec}, sr25519, }; use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; /// A utility to easily create a testnet chain spec definition with a given set /// of authorities and endowed accounts and/or generate random accounts. -#[derive(StructOpt)] -#[structopt(rename_all = "kebab-case")] +#[derive(Parser)] +#[clap(rename_all = "kebab-case")] enum ChainSpecBuilder { /// Create a new chain spec with the given authorities, endowed and sudo /// accounts. New { /// Authority key seed. - #[structopt(long, short, required = true)] + #[clap(long, short, required = true)] authority_seeds: Vec, /// Active nominators (SS58 format), each backing a random subset of the aforementioned /// authorities. - #[structopt(long, short, default_value = "0")] + #[clap(long, short, default_value = "0")] nominator_accounts: Vec, /// Endowed account address (SS58 format). - #[structopt(long, short)] + #[clap(long, short)] endowed_accounts: Vec, /// Sudo account address (SS58 format). - #[structopt(long, short)] + #[clap(long, short)] sudo_account: String, /// The path where the chain spec should be saved. - #[structopt(long, short, default_value = "./chain_spec.json")] + #[clap(long, short, default_value = "./chain_spec.json")] chain_spec_path: PathBuf, }, /// Create a new chain spec with the given number of authorities and endowed /// accounts. Random keys will be generated as required. Generate { /// The number of authorities. - #[structopt(long, short)] + #[clap(long, short)] authorities: usize, /// The number of nominators backing the aforementioned authorities. /// /// Will nominate a random subset of `authorities`. - #[structopt(long, short, default_value = "0")] + #[clap(long, short, default_value = "0")] nominators: usize, /// The number of endowed accounts. - #[structopt(long, short, default_value = "0")] + #[clap(long, short, default_value = "0")] endowed: usize, /// The path where the chain spec should be saved. - #[structopt(long, short, default_value = "./chain_spec.json")] + #[clap(long, short, default_value = "./chain_spec.json")] chain_spec_path: PathBuf, /// Path to use when saving generated keystores for each authority. /// /// At this path, a new folder will be created for each authority's /// keystore named `auth-$i` where `i` is the authority index, i.e. /// `auth-0`, `auth-1`, etc. - #[structopt(long, short)] + #[clap(long, short)] keystore_path: Option, }, } @@ -155,6 +155,7 @@ fn generate_chain_spec( None, None, None, + None, Default::default(), ); @@ -235,13 +236,15 @@ fn main() -> Result<(), String> { the chain spec builder binary in `--release` mode.\n", ); - let builder = ChainSpecBuilder::from_args(); + let builder = ChainSpecBuilder::parse(); let chain_spec_path = builder.chain_spec_path().to_path_buf(); let (authority_seeds, nominator_accounts, endowed_accounts, sudo_account) = match builder { ChainSpecBuilder::Generate { authorities, nominators, endowed, keystore_path, .. } => { let authorities = authorities.max(1); - let rand_str = || -> String { OsRng.sample_iter(&Alphanumeric).take(32).collect() }; + let rand_str = || -> String { + OsRng.sample_iter(&Alphanumeric).take(32).map(char::from).collect() + }; let authority_seeds = (0..authorities).map(|_| rand_str()).collect::>(); let nominator_seeds = (0..nominators).map(|_| rand_str()).collect::>(); diff --git a/bin/utils/subkey/Cargo.toml b/bin/utils/subkey/Cargo.toml index 9bd38a21a664..c8132fe7b5b4 100644 --- a/bin/utils/subkey/Cargo.toml +++ b/bin/utils/subkey/Cargo.toml @@ -2,9 +2,9 @@ name = "subkey" version = "2.0.1" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" publish = false @@ -17,5 +17,5 @@ path = "src/main.rs" name = "subkey" [dependencies] +clap = { version = "3.1.6", features = ["derive"] } sc-cli = { version = "0.10.0-dev", path = "../../../client/cli" } -structopt = "0.3.14" diff --git a/bin/utils/subkey/README.md b/bin/utils/subkey/README.md index fbb486247a77..e762a42c3e79 100644 --- a/bin/utils/subkey/README.md +++ b/bin/utils/subkey/README.md @@ -18,15 +18,37 @@ If you save any output of `subkey` into a file, make sure to apply proper permis The following guide explains *some* of the `subkey` commands. For the full list and the most up to date documentation, make sure to check the integrated help with `subkey --help`. +### Install with Cargo + +You will need to have the Substrate build dependencies to install Subkey. Use the following two commands to install the dependencies and Subkey, respectively: + +Command: + +```bash +# Use the `--fast` flag to get the dependencies without needing to install the Substrate and Subkey binary +curl https://getsubstrate.io -sSf | bash -s -- --fast +# Install only `subkey`, at a specific version of the subkey crate +cargo install --force subkey --git https://github.com/paritytech/substrate --version --locked +``` + +### Run in a container + +```bash +# Use `--pull=always` with the `latest` tag, or specify a version in a tag +docker run -it --pull=always docker.io/parity/subkey:latest +``` + ### Generate a random account Generating a new key is as simple as running: - subkey generate +```bash +subkey generate +``` The output looks similar to: -``` +```text Secret phrase `hotel forest jar hover kite book view eight stuff angle legend defense` is account: Secret seed: 0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d Public key (hex): 0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515 @@ -47,9 +69,10 @@ The output above also show the **public key** and the **Account ID**. Those are The **SS58 address** (or **Public Address**) of a new account is a reprensentation of the public keys of an account for a given network (for instance Kusama or Polkadot). -You can read more about the SS58 format in the [substrate wiki](https://github.com/paritytech/substrate/wiki/External-Address-Format-(SS58)) and see the list of reserved prefixes in the [Polkadot wiki](https://wiki.polkadot.network/docs/build-ss58-registry). +You can read more about the SS58 format in the [Substrate Docs](https://docs.substrate.io/v3/advanced/ss58/) and see the list of reserved prefixes in the [SS58 Registry](https://github.com/paritytech/ss58-registry). For instance, considering the previous seed `0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d` the SS58 addresses are: + - Polkadot: `16m4J167Mptt8UXL8aGSAi7U2FnPpPxZHPrCgMG9KJzVoFqM` - Kusama: `JLNozAv8QeLSbLFwe2UvWeKKE4yvmDbfGxTuiYkF2BUMx4M` @@ -58,12 +81,14 @@ For instance, considering the previous seed `0xa05c75731970cc7868a2fb7cb577353cd `subkey` can calso generate the output as *json*. This is useful for automation. command: -``` + +```bash subkey generate --output-type json ``` output: -``` + +```json { "accountId": "0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515", "publicKey": "0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515", @@ -76,12 +101,14 @@ output: So if you only want to get the `secretSeed` for instance, you can use: command: -``` + +```bash subkey generate --output-type json | jq -r .secretSeed ``` output: -``` + +```text 0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d ``` @@ -89,10 +116,13 @@ output: `subkey` supports an additional user-defined secret that will be appended to the seed. Let's see the following example: - subkey generate --password extra_secret +```bash +subkey generate --password extra_secret +``` output: -``` + +```text Secret phrase `soup lyrics media market way crouch elevator put moon useful question wide` is account: Secret seed: 0xe7cfd179d6537a676cb94bac3b5c5c9cb1550e846ac4541040d077dfbac2e7fd Public key (hex): 0xf6a233c3e1de1a2ae0486100b460b3ce3d7231ddfe9dadabbd35ab968c70905d @@ -102,11 +132,15 @@ Secret phrase `soup lyrics media market way crouch elevator put moon useful ques Using the `inspect` command (see more details below), we see that knowning only the **secret seed** is no longer sufficient to recover the account: - subkey inspect "soup lyrics media market way crouch elevator put moon useful question wide" +```bash +subkey inspect "soup lyrics media market way crouch elevator put moon useful question wide" +``` which recovers the account `5Fe4sqj2K4fRuzEGvToi4KATqZfiDU7TqynjXG6PZE2dxwyh` and not `5He5pZpc7AJ8evPuab37vJF6KkFDqq9uDq2WXh877Qw6iaVC` as we expected. The additional user-defined **password** (`extra_secret` in our example) is now required to fully recover the account. Let's inspect the the previous mnemonic, this time passing also the required `password` as shown below: - subkey inspect --password extra_secret "soup lyrics media market way crouch elevator put moon useful question wide" +```bash +subkey inspect --password extra_secret "soup lyrics media market way crouch elevator put moon useful question wide" +``` This time, we properly recovered `5He5pZpc7AJ8evPuab37vJF6KkFDqq9uDq2WXh877Qw6iaVC`. @@ -116,23 +150,29 @@ If you have *some data* about a key, `subkey inpsect` will help you discover mor If you have **secrets** that you would like to verify for instance, you can use: - subkey inspect < mnemonic | seed > +```bash +subkey inspect < mnemonic | seed > +``` If you have only **public data**, you can see a subset of the information: - subkey inspect --public < pubkey | address > +```bash +subkey inspect --public < pubkey | address > +``` **NOTE**: While you will be able to recover the secret seed from the mnemonic, the opposite is not possible. **NOTE**: For obvious reasons, the **secrets** cannot be recovered from passing **public data** such as `pubkey` or `address` as input. command: -``` + +```bash subkey inspect 0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d ``` output: -``` + +```text Secret Key URI `0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d` is account: Secret seed: 0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d Public key (hex): 0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515 @@ -144,17 +184,23 @@ Secret Key URI `0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c9 `subkey` allows using a **secret key** to sign a random message. The signature can then be verified by anyone using your **public key**: - echo -n | subkey sign --suri +```bash +echo -n | subkey sign --suri +``` example: - MESSAGE=hello - SURI=0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d - echo -n $MESSAGE | subkey sign --suri $SURI +```text +MESSAGE=hello +SURI=0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d +echo -n $MESSAGE | subkey sign --suri $SURI +``` output: - 9201af3788ad4f986b800853c79da47155f2e08fde2070d866be4c27ab060466fea0623dc2b51f4392f4c61f25381a62848dd66c5d8217fae3858e469ebd668c +```text +9201af3788ad4f986b800853c79da47155f2e08fde2070d866be4c27ab060466fea0623dc2b51f4392f4c61f25381a62848dd66c5d8217fae3858e469ebd668c +``` **NOTE**: Each run of the `sign` command will yield a different output. While each signature is different, they are all valid. @@ -162,34 +208,44 @@ output: Given a message, a signature and an address, `subkey` can verify whether the **message** has been digitally signed by the holder (or one of the holders) of the **private key** for the given **address**: - echo -n | subkey verify

+```bash +echo -n | subkey verify
+``` example: - MESSAGE=hello - URI=0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515 - SIGNATURE=9201af3788ad4f986b800853c79da47155f2e08fde2070d866be4c27ab060466fea0623dc2b51f4392f4c61f25381a62848dd66c5d8217fae3858e469ebd668c - echo -n $MESSAGE | subkey verify $SIGNATURE $URI +```bash +MESSAGE=hello +URI=0xfec70cfbf1977c6965b5af10a4534a6a35d548eb14580594d0bc543286892515 +SIGNATURE=9201af3788ad4f986b800853c79da47155f2e08fde2070d866be4c27ab060466fea0623dc2b51f4392f4c61f25381a62848dd66c5d8217fae3858e469ebd668c +echo -n $MESSAGE | subkey verify $SIGNATURE $URI +``` output: - Signature verifies correctly. +```text +Signature verifies correctly. +``` A failure looks like: - Error: SignatureInvalid +```text +Error: SignatureInvalid +``` ### Using the vanity generator You can use the included vanity generator to find a seed that provides an address which includes the desired pattern. Be warned, depending on your hardware this may take a while. command: -``` + +```bash subkey vanity --network polkadot --pattern bob ``` output: -``` + +```text Generating key containing pattern 'bob' best: 190 == top: 189 Secret Key URI `0x8c9a73097f235b84021a446bc2826a00c690ea0be3e0d81a84931cb4146d6691` is account: diff --git a/bin/utils/subkey/src/lib.rs b/bin/utils/subkey/src/lib.rs index 5052d1b104c2..3731d9f3ec75 100644 --- a/bin/utils/subkey/src/lib.rs +++ b/bin/utils/subkey/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,17 +16,18 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +use clap::Parser; use sc_cli::{ Error, GenerateCmd, GenerateNodeKeyCmd, InspectKeyCmd, InspectNodeKeyCmd, SignCmd, VanityCmd, VerifyCmd, }; -use structopt::StructOpt; -#[derive(Debug, StructOpt)] -#[structopt( +#[derive(Debug, Parser)] +#[clap( name = "subkey", author = "Parity Team ", - about = "Utility for generating and restoring with Substrate keys" + about = "Utility for generating and restoring with Substrate keys", + version )] pub enum Subkey { /// Generate a random node libp2p key, save it to file or print it to stdout @@ -54,7 +55,7 @@ pub enum Subkey { /// Run the subkey command, given the appropriate runtime. pub fn run() -> Result<(), Error> { - match Subkey::from_args() { + match Subkey::parse() { Subkey::GenerateNodeKey(cmd) => cmd.run(), Subkey::Generate(cmd) => cmd.run(), Subkey::Inspect(cmd) => cmd.run(), diff --git a/bin/utils/subkey/src/main.rs b/bin/utils/subkey/src/main.rs index 2a0f0850713f..d85c6dbe9d04 100644 --- a/bin/utils/subkey/src/main.rs +++ b/bin/utils/subkey/src/main.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/allocator/Cargo.toml b/client/allocator/Cargo.toml index 5ebab6cf9d61..8375c39ba4a1 100644 --- a/client/allocator/Cargo.toml +++ b/client/allocator/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sc-allocator" -version = "4.0.0-dev" +version = "4.1.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Collection of allocator implementations." documentation = "https://docs.rs/sc-allocator" @@ -14,7 +14,7 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-wasm-interface = { version = "4.0.0-dev", path = "../../primitives/wasm-interface" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-wasm-interface = { version = "6.0.0", path = "../../primitives/wasm-interface" } log = "0.4.11" -thiserror = "1.0.21" +thiserror = "1.0.30" diff --git a/client/allocator/src/error.rs b/client/allocator/src/error.rs index 9b9a55325f75..d9fc483224ad 100644 --- a/client/allocator/src/error.rs +++ b/client/allocator/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/client/allocator/src/freeing_bump.rs b/client/allocator/src/freeing_bump.rs index 741f4012cdcb..7eeda45370b8 100644 --- a/client/allocator/src/freeing_bump.rs +++ b/client/allocator/src/freeing_bump.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -71,7 +71,6 @@ use crate::Error; pub use sp_core::MAX_POSSIBLE_ALLOCATION; use sp_wasm_interface::{Pointer, WordSize}; use std::{ - convert::{TryFrom, TryInto}, mem, ops::{Index, IndexMut, Range}, }; diff --git a/client/allocator/src/lib.rs b/client/allocator/src/lib.rs index 4493db3c7d14..2c3e3b2ae841 100644 --- a/client/allocator/src/lib.rs +++ b/client/allocator/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/client/api/Cargo.toml b/client/api/Cargo.toml index 772f22e822eb..d209a311f45b 100644 --- a/client/api/Cargo.toml +++ b/client/api/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-client-api" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate client interfaces." documentation = "https://docs.rs/sc-client-api" @@ -14,31 +14,31 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } sc-executor = { version = "0.10.0-dev", path = "../executor" } -sp-externalities = { version = "0.10.0-dev", path = "../../primitives/externalities" } +sp-externalities = { version = "0.12.0", path = "../../primitives/externalities" } fnv = "1.0.6" -futures = "0.3.1" +futures = "0.3.21" hash-db = { version = "0.15.2", default-features = false } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } log = "0.4.8" -parking_lot = "0.11.1" +parking_lot = "0.12.0" sp-database = { version = "4.0.0-dev", path = "../../primitives/database" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-keystore = { version = "0.10.0-dev", default-features = false, path = "../../primitives/keystore" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-keystore = { version = "0.12.0", default-features = false, path = "../../primitives/keystore" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } sc-utils = { version = "4.0.0-dev", path = "../utils" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-state-machine = { version = "0.10.0-dev", path = "../../primitives/state-machine" } -sp-trie = { version = "4.0.0-dev", path = "../../primitives/trie" } -sp-storage = { version = "4.0.0-dev", path = "../../primitives/storage" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-state-machine = { version = "0.12.0", path = "../../primitives/state-machine" } +sp-trie = { version = "6.0.0", path = "../../primitives/trie" } +sp-storage = { version = "6.0.0", path = "../../primitives/storage" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../transaction-pool/api" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.9.0", path = "../../utils/prometheus" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.10.0-dev", path = "../../utils/prometheus" } [dev-dependencies] sp-test-primitives = { version = "2.0.0", path = "../../primitives/test-primitives" } substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" } -thiserror = "1.0.21" +thiserror = "1.0.30" diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index 8b5bd50ffa61..e96616d5416e 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,29 +20,24 @@ use crate::{ blockchain::{well_known_cache_keys, Backend as BlockchainBackend}, - light::RemoteBlockchain, UsageInfo, }; use parking_lot::RwLock; use sp_blockchain; use sp_consensus::BlockOrigin; -use sp_core::{offchain::OffchainStorage, ChangesTrieConfigurationRange}; +use sp_core::offchain::OffchainStorage; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, HashFor, NumberFor}, - Justification, Justifications, Storage, + Justification, Justifications, StateVersion, Storage, }; use sp_state_machine::{ - ChangesTrieState, ChangesTrieStorage as StateChangesTrieStorage, ChangesTrieTransaction, ChildStorageCollection, IndexOperation, OffchainChangesCollection, StorageCollection, }; -use sp_storage::{ChildInfo, PrefixedStorageKey, StorageData, StorageKey}; -use std::{ - collections::{HashMap, HashSet}, - sync::Arc, -}; +use sp_storage::{ChildInfo, StorageData, StorageKey}; +use std::collections::{HashMap, HashSet}; -pub use sp_state_machine::Backend as StateBackend; +pub use sp_state_machine::{Backend as StateBackend, KeyValueStates}; use std::marker::PhantomData; /// Extracts the state backend type for the given backend. @@ -75,14 +70,28 @@ pub struct ImportSummary { pub tree_route: Option>, } -/// Import operation wrapper +/// Finalization operation summary. +/// +/// Contains information about the block that just got finalized, +/// including tree heads that became stale at the moment of finalization. +pub struct FinalizeSummary { + /// Last finalized block header. + pub header: Block::Header, + /// Blocks that were finalized. + /// The last entry is the one that has been explicitly finalized. + pub finalized: Vec, + /// Heads that became stale during this finalization operation. + pub stale_heads: Vec, +} + +/// Import operation wrapper. pub struct ClientImportOperation> { /// DB Operation. pub op: B::BlockImportOperation, /// Summary of imported block. pub notify_imported: Option>, - /// A list of hashes of blocks that got finalized. - pub notify_finalized: Vec, + /// Summary of finalized block. + pub notify_finalized: Option>, } /// Helper function to apply auxiliary data insertion into an operation. @@ -171,10 +180,15 @@ pub trait BlockImportOperation { &mut self, storage: Storage, commit: bool, + state_version: StateVersion, ) -> sp_blockchain::Result; /// Inject storage data into the database replacing any existing data. - fn reset_storage(&mut self, storage: Storage) -> sp_blockchain::Result; + fn reset_storage( + &mut self, + storage: Storage, + state_version: StateVersion, + ) -> sp_blockchain::Result; /// Set storage changes. fn update_storage( @@ -191,12 +205,6 @@ pub trait BlockImportOperation { Ok(()) } - /// Inject changes trie data into the database. - fn update_changes_trie( - &mut self, - update: ChangesTrieTransaction, NumberFor>, - ) -> sp_blockchain::Result<()>; - /// Insert auxiliary keys. /// /// Values are `None` if should be deleted. @@ -270,6 +278,10 @@ pub trait Finalizer> { } /// Provides access to an auxiliary database. +/// +/// This is a simple global database not aware of forks. Can be used for storing auxiliary +/// information like total block weight/difficulty for fork resolution purposes as a common use +/// case. pub trait AuxStore { /// Insert auxiliary data into key-value store. /// @@ -418,28 +430,6 @@ pub trait StorageProvider> { child_info: &ChildInfo, key: &StorageKey, ) -> sp_blockchain::Result>; - - /// Get longest range within [first; last] that is possible to use in `key_changes` - /// and `key_changes_proof` calls. - /// Range could be shortened from the beginning if some changes tries have been pruned. - /// Returns Ok(None) if changes tries are not supported. - fn max_key_changes_range( - &self, - first: NumberFor, - last: BlockId, - ) -> sp_blockchain::Result, BlockId)>>; - - /// Get pairs of (block, extrinsic) where key has been changed at given blocks range. - /// Works only for runtimes that are supporting changes tries. - /// - /// Changes are returned in descending order (i.e. last block comes first). - fn key_changes( - &self, - first: NumberFor, - last: BlockId, - storage_key: Option<&PrefixedStorageKey>, - key: &StorageKey, - ) -> sp_blockchain::Result, u32)>>; } /// Client backend. @@ -504,9 +494,6 @@ pub trait Backend: AuxStore + Send + Sync { /// Returns current usage statistics. fn usage_info(&self) -> Option; - /// Returns reference to changes trie storage. - fn changes_trie_storage(&self) -> Option<&dyn PrunableStateChangesTrieStorage>; - /// Returns a handle to offchain storage. fn offchain_storage(&self) -> Option; @@ -561,72 +548,5 @@ pub trait Backend: AuxStore + Send + Sync { fn get_import_lock(&self) -> &RwLock<()>; } -/// Changes trie storage that supports pruning. -pub trait PrunableStateChangesTrieStorage: - StateChangesTrieStorage, NumberFor> -{ - /// Get reference to StateChangesTrieStorage. - fn storage(&self) -> &dyn StateChangesTrieStorage, NumberFor>; - /// Get configuration at given block. - fn configuration_at( - &self, - at: &BlockId, - ) -> sp_blockchain::Result, Block::Hash>>; - /// Get end block (inclusive) of oldest pruned max-level (or skewed) digest trie blocks range. - /// It is guaranteed that we have no any changes tries before (and including) this block. - /// It is guaranteed that all existing changes tries after this block are not yet pruned (if - /// created). - fn oldest_pruned_digest_range_end(&self) -> NumberFor; -} - /// Mark for all Backend implementations, that are making use of state data, stored locally. pub trait LocalBackend: Backend {} - -/// Mark for all Backend implementations, that are fetching required state data from remote nodes. -pub trait RemoteBackend: Backend { - /// Returns true if the state for given block is available locally. - fn is_local_state_available(&self, block: &BlockId) -> bool; - - /// Returns reference to blockchain backend. - /// - /// Returned backend either resolves blockchain data - /// locally, or prepares request to fetch that data from remote node. - fn remote_blockchain(&self) -> Arc>; -} - -/// Return changes tries state at given block. -pub fn changes_tries_state_at_block<'a, Block: BlockT>( - block: &BlockId, - maybe_storage: Option<&'a dyn PrunableStateChangesTrieStorage>, -) -> sp_blockchain::Result, NumberFor>>> { - let storage = match maybe_storage { - Some(storage) => storage, - None => return Ok(None), - }; - - let config_range = storage.configuration_at(block)?; - match config_range.config { - Some(config) => - Ok(Some(ChangesTrieState::new(config, config_range.zero.0, storage.storage()))), - None => Ok(None), - } -} - -/// Provide CHT roots. These are stored on a light client and generated dynamically on a full -/// client. -pub trait ProvideChtRoots { - /// Get headers CHT root for given block. Returns None if the block is not a part of any CHT. - fn header_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result>; - - /// Get changes trie CHT root for given block. Returns None if the block is not a part of any - /// CHT. - fn changes_trie_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result>; -} diff --git a/client/api/src/call_executor.rs b/client/api/src/call_executor.rs index 22af495c0654..738f932a47bf 100644 --- a/client/api/src/call_executor.rs +++ b/client/api/src/call_executor.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,7 +19,7 @@ //! A method call executor interface. use codec::{Decode, Encode}; -use sc_executor::RuntimeVersion; +use sc_executor::{RuntimeVersion, RuntimeVersionOf}; use sp_core::NativeOrEncoded; use sp_externalities::Extensions; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; @@ -42,7 +42,7 @@ pub trait ExecutorProvider { } /// Method call executor. -pub trait CallExecutor { +pub trait CallExecutor: RuntimeVersionOf { /// Externalities error type. type Error: sp_state_machine::Error; diff --git a/client/api/src/cht.rs b/client/api/src/cht.rs deleted file mode 100644 index ee7854b5d829..000000000000 --- a/client/api/src/cht.rs +++ /dev/null @@ -1,474 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Canonical hash trie definitions and helper functions. -//! -//! Each CHT is a trie mapping block numbers to canonical hash. -//! One is generated for every `SIZE` blocks, allowing us to discard those blocks in -//! favor of the trie root. When the "ancient" blocks need to be accessed, we simply -//! request an inclusion proof of a specific block number against the trie with the -//! root hash. A correct proof implies that the claimed block is identical to the one -//! we discarded. - -use codec::Encode; -use hash_db; -use sp_trie; - -use sp_core::{convert_hash, H256}; -use sp_runtime::traits::{AtLeast32Bit, Header as HeaderT, One, Zero}; -use sp_state_machine::{ - prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend, - Backend as StateBackend, InMemoryBackend, MemoryDB, StorageProof, TrieBackend, -}; - -use sp_blockchain::{Error as ClientError, Result as ClientResult}; - -/// The size of each CHT. This value is passed to every CHT-related function from -/// production code. Other values are passed from tests. -const SIZE: u32 = 2048; - -/// Gets default CHT size. -pub fn size>() -> N { - SIZE.into() -} - -/// Returns Some(cht_number) if CHT is need to be built when the block with given number is -/// canonized. -pub fn is_build_required(cht_size: N, block_num: N) -> Option -where - N: Clone + AtLeast32Bit, -{ - let block_cht_num = block_to_cht_number(cht_size.clone(), block_num.clone())?; - let two = N::one() + N::one(); - if block_cht_num < two { - return None - } - let cht_start = start_number(cht_size, block_cht_num.clone()); - if cht_start != block_num { - return None - } - - Some(block_cht_num - two) -} - -/// Returns Some(max_cht_number) if CHT has ever been built given maximal canonical block number. -pub fn max_cht_number(cht_size: N, max_canonical_block: N) -> Option -where - N: Clone + AtLeast32Bit, -{ - let max_cht_number = block_to_cht_number(cht_size, max_canonical_block)?; - let two = N::one() + N::one(); - if max_cht_number < two { - return None - } - Some(max_cht_number - two) -} - -/// Compute a CHT root from an iterator of block hashes. Fails if shorter than -/// SIZE items. The items are assumed to proceed sequentially from `start_number(cht_num)`. -/// Discards the trie's nodes. -pub fn compute_root( - cht_size: Header::Number, - cht_num: Header::Number, - hashes: I, -) -> ClientResult -where - Header: HeaderT, - Hasher: hash_db::Hasher, - Hasher::Out: Ord, - I: IntoIterator>>, -{ - use sp_trie::TrieConfiguration; - Ok(sp_trie::trie_types::Layout::::trie_root(build_pairs::( - cht_size, cht_num, hashes, - )?)) -} - -/// Build CHT-based header proof. -pub fn build_proof( - cht_size: Header::Number, - cht_num: Header::Number, - blocks: BlocksI, - hashes: HashesI, -) -> ClientResult -where - Header: HeaderT, - Hasher: hash_db::Hasher, - Hasher::Out: Ord + codec::Codec, - BlocksI: IntoIterator, - HashesI: IntoIterator>>, -{ - let transaction = build_pairs::(cht_size, cht_num, hashes)? - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect::>(); - let storage = InMemoryBackend::::default().update(vec![(None, transaction)]); - let trie_storage = storage - .as_trie_backend() - .expect("InMemoryState::as_trie_backend always returns Some; qed"); - prove_read_on_trie_backend( - trie_storage, - blocks.into_iter().map(|number| encode_cht_key(number)), - ) - .map_err(ClientError::from_state) -} - -/// Check CHT-based header proof. -pub fn check_proof( - local_root: Header::Hash, - local_number: Header::Number, - remote_hash: Header::Hash, - remote_proof: StorageProof, -) -> ClientResult<()> -where - Header: HeaderT, - Hasher: hash_db::Hasher, - Hasher::Out: Ord + codec::Codec, -{ - do_check_proof::( - local_root, - local_number, - remote_hash, - move |local_root, local_cht_key| { - read_proof_check::( - local_root, - remote_proof, - ::std::iter::once(local_cht_key), - ) - .map(|mut map| map.remove(local_cht_key).expect("checked proof of local_cht_key; qed")) - .map_err(ClientError::from_state) - }, - ) -} - -/// Check CHT-based header proof on pre-created proving backend. -pub fn check_proof_on_proving_backend( - local_root: Header::Hash, - local_number: Header::Number, - remote_hash: Header::Hash, - proving_backend: &TrieBackend, Hasher>, -) -> ClientResult<()> -where - Header: HeaderT, - Hasher: hash_db::Hasher, - Hasher::Out: Ord + codec::Codec, -{ - do_check_proof::( - local_root, - local_number, - remote_hash, - |_, local_cht_key| { - read_proof_check_on_proving_backend::(proving_backend, local_cht_key) - .map_err(ClientError::from_state) - }, - ) -} - -/// Check CHT-based header proof using passed checker function. -fn do_check_proof( - local_root: Header::Hash, - local_number: Header::Number, - remote_hash: Header::Hash, - checker: F, -) -> ClientResult<()> -where - Header: HeaderT, - Hasher: hash_db::Hasher, - Hasher::Out: Ord, - F: FnOnce(Hasher::Out, &[u8]) -> ClientResult>>, -{ - let root: Hasher::Out = convert_hash(&local_root); - let local_cht_key = encode_cht_key(local_number); - let local_cht_value = checker(root, &local_cht_key)?; - let local_cht_value = local_cht_value.ok_or_else(|| ClientError::InvalidCHTProof)?; - let local_hash = - decode_cht_value(&local_cht_value).ok_or_else(|| ClientError::InvalidCHTProof)?; - match &local_hash[..] == remote_hash.as_ref() { - true => Ok(()), - false => Err(ClientError::InvalidCHTProof.into()), - } -} - -/// Group ordered blocks by CHT number and call functor with blocks of each group. -pub fn for_each_cht_group( - cht_size: Header::Number, - blocks: I, - mut functor: F, - mut functor_param: P, -) -> ClientResult<()> -where - Header: HeaderT, - I: IntoIterator, - F: FnMut(P, Header::Number, Vec) -> ClientResult

, -{ - let mut current_cht_num = None; - let mut current_cht_blocks = Vec::new(); - for block in blocks { - let new_cht_num = block_to_cht_number(cht_size, block).ok_or_else(|| { - ClientError::Backend(format!("Cannot compute CHT root for the block #{}", block)) - })?; - - let advance_to_next_cht = current_cht_num.is_some() && current_cht_num != Some(new_cht_num); - if advance_to_next_cht { - let current_cht_num = current_cht_num.expect( - "advance_to_next_cht is true; - it is true only when current_cht_num is Some; qed", - ); - assert!( - new_cht_num > current_cht_num, - "for_each_cht_group only supports ordered iterators" - ); - - functor_param = - functor(functor_param, current_cht_num, std::mem::take(&mut current_cht_blocks))?; - } - - current_cht_blocks.push(block); - current_cht_num = Some(new_cht_num); - } - - if let Some(current_cht_num) = current_cht_num { - functor(functor_param, current_cht_num, std::mem::take(&mut current_cht_blocks))?; - } - - Ok(()) -} - -/// Build pairs for computing CHT. -fn build_pairs( - cht_size: Header::Number, - cht_num: Header::Number, - hashes: I, -) -> ClientResult, Vec)>> -where - Header: HeaderT, - I: IntoIterator>>, -{ - let start_num = start_number(cht_size, cht_num); - let mut pairs = Vec::new(); - let mut hash_index = Header::Number::zero(); - for hash in hashes.into_iter() { - let hash = - hash?.ok_or_else(|| ClientError::from(ClientError::MissingHashRequiredForCHT))?; - pairs.push((encode_cht_key(start_num + hash_index).to_vec(), encode_cht_value(hash))); - hash_index += Header::Number::one(); - if hash_index == cht_size { - break - } - } - - if hash_index == cht_size { - Ok(pairs) - } else { - Err(ClientError::MissingHashRequiredForCHT) - } -} - -/// Get the starting block of a given CHT. -/// CHT 0 includes block 1...SIZE, -/// CHT 1 includes block SIZE + 1 ... 2*SIZE -/// More generally: CHT N includes block (1 + N*SIZE)...((N+1)*SIZE). -/// This is because the genesis hash is assumed to be known -/// and including it would be redundant. -pub fn start_number(cht_size: N, cht_num: N) -> N { - (cht_num * cht_size) + N::one() -} - -/// Get the ending block of a given CHT. -pub fn end_number(cht_size: N, cht_num: N) -> N { - (cht_num + N::one()) * cht_size -} - -/// Convert a block number to a CHT number. -/// Returns `None` for `block_num` == 0, `Some` otherwise. -pub fn block_to_cht_number(cht_size: N, block_num: N) -> Option { - if block_num == N::zero() { - None - } else { - Some((block_num - N::one()) / cht_size) - } -} - -/// Convert header number into CHT key. -pub fn encode_cht_key(number: N) -> Vec { - number.encode() -} - -/// Convert header hash into CHT value. -fn encode_cht_value>(hash: Hash) -> Vec { - hash.as_ref().to_vec() -} - -/// Convert CHT value into block header hash. -pub fn decode_cht_value(value: &[u8]) -> Option { - match value.len() { - 32 => Some(H256::from_slice(&value[0..32])), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use sp_runtime::{generic, traits::BlakeTwo256}; - - type Header = generic::Header; - - #[test] - fn is_build_required_works() { - assert_eq!(is_build_required(SIZE, 0u32.into()), None); - assert_eq!(is_build_required(SIZE, 1u32.into()), None); - assert_eq!(is_build_required(SIZE, SIZE), None); - assert_eq!(is_build_required(SIZE, SIZE + 1), None); - assert_eq!(is_build_required(SIZE, 2 * SIZE), None); - assert_eq!(is_build_required(SIZE, 2 * SIZE + 1), Some(0)); - assert_eq!(is_build_required(SIZE, 2 * SIZE + 2), None); - assert_eq!(is_build_required(SIZE, 3 * SIZE), None); - assert_eq!(is_build_required(SIZE, 3 * SIZE + 1), Some(1)); - assert_eq!(is_build_required(SIZE, 3 * SIZE + 2), None); - } - - #[test] - fn max_cht_number_works() { - assert_eq!(max_cht_number(SIZE, 0u32.into()), None); - assert_eq!(max_cht_number(SIZE, 1u32.into()), None); - assert_eq!(max_cht_number(SIZE, SIZE), None); - assert_eq!(max_cht_number(SIZE, SIZE + 1), None); - assert_eq!(max_cht_number(SIZE, 2 * SIZE), None); - assert_eq!(max_cht_number(SIZE, 2 * SIZE + 1), Some(0)); - assert_eq!(max_cht_number(SIZE, 2 * SIZE + 2), Some(0)); - assert_eq!(max_cht_number(SIZE, 3 * SIZE), Some(0)); - assert_eq!(max_cht_number(SIZE, 3 * SIZE + 1), Some(1)); - assert_eq!(max_cht_number(SIZE, 3 * SIZE + 2), Some(1)); - } - - #[test] - fn start_number_works() { - assert_eq!(start_number(SIZE, 0u32), 1u32); - assert_eq!(start_number(SIZE, 1u32), SIZE + 1); - assert_eq!(start_number(SIZE, 2u32), SIZE + SIZE + 1); - } - - #[test] - fn end_number_works() { - assert_eq!(end_number(SIZE, 0u32), SIZE); - assert_eq!(end_number(SIZE, 1u32), SIZE + SIZE); - assert_eq!(end_number(SIZE, 2u32), SIZE + SIZE + SIZE); - } - - #[test] - fn build_pairs_fails_when_no_enough_blocks() { - assert!(build_pairs::( - SIZE as _, - 0, - ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize / 2) - ) - .is_err()); - } - - #[test] - fn build_pairs_fails_when_missing_block() { - assert!(build_pairs::( - SIZE as _, - 0, - ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))) - .take(SIZE as usize / 2) - .chain(::std::iter::once(Ok(None))) - .chain( - ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(2)))) - .take(SIZE as usize / 2 - 1) - ) - ) - .is_err()); - } - - #[test] - fn compute_root_works() { - assert!(compute_root::( - SIZE as _, - 42, - ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize) - ) - .is_ok()); - } - - #[test] - #[should_panic] - fn build_proof_panics_when_querying_wrong_block() { - assert!(build_proof::( - SIZE as _, - 0, - vec![(SIZE * 1000) as u64], - ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize) - ) - .is_err()); - } - - #[test] - fn build_proof_works() { - assert!(build_proof::( - SIZE as _, - 0, - vec![(SIZE / 2) as u64], - ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize) - ) - .is_ok()); - } - - #[test] - #[should_panic] - fn for_each_cht_group_panics() { - let cht_size = SIZE as u64; - let _ = for_each_cht_group::( - cht_size, - vec![cht_size * 5, cht_size * 2], - |_, _, _| Ok(()), - (), - ); - } - - #[test] - fn for_each_cht_group_works() { - let cht_size = SIZE as u64; - let _ = for_each_cht_group::( - cht_size, - vec![ - cht_size * 2 + 1, - cht_size * 2 + 2, - cht_size * 2 + 5, - cht_size * 4 + 1, - cht_size * 4 + 7, - cht_size * 6 + 1, - ], - |_, cht_num, blocks| { - match cht_num { - 2 => assert_eq!( - blocks, - vec![cht_size * 2 + 1, cht_size * 2 + 2, cht_size * 2 + 5] - ), - 4 => assert_eq!(blocks, vec![cht_size * 4 + 1, cht_size * 4 + 7]), - 6 => assert_eq!(blocks, vec![cht_size * 6 + 1]), - _ => unreachable!(), - } - - Ok(()) - }, - (), - ); - } -} diff --git a/client/api/src/client.rs b/client/api/src/client.rs index 21f8aecad053..9c55be323813 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -25,9 +25,10 @@ use sp_runtime::{ traits::{Block as BlockT, NumberFor}, Justifications, }; -use std::{collections::HashSet, convert::TryFrom, fmt, sync::Arc}; +use std::{collections::HashSet, fmt, sync::Arc}; + +use crate::{blockchain::Info, notifications::StorageEventStream, FinalizeSummary, ImportSummary}; -use crate::{blockchain::Info, notifications::StorageEventStream}; use sc_transaction_pool_api::ChainEvent; use sc_utils::mpsc::TracingUnboundedReceiver; use sp_blockchain; @@ -76,6 +77,34 @@ pub trait BlockchainEvents { ) -> sp_blockchain::Result>; } +/// List of operations to be performed on storage aux data. +/// First tuple element is the encoded data key. +/// Second tuple element is the encoded optional data to write. +/// If `None`, the key and the associated data are deleted from storage. +pub type AuxDataOperations = Vec<(Vec, Option>)>; + +/// Callback invoked before committing the operations created during block import. +/// This gives the opportunity to perform auxiliary pre-commit actions and optionally +/// enqueue further storage write operations to be atomically performed on commit. +pub type OnImportAction = + Box) -> AuxDataOperations) + Send>; + +/// Callback invoked before committing the operations created during block finalization. +/// This gives the opportunity to perform auxiliary pre-commit actions and optionally +/// enqueue further storage write operations to be atomically performed on commit. +pub type OnFinalityAction = + Box) -> AuxDataOperations) + Send>; + +/// Interface to perform auxiliary actions before committing a block import or +/// finality operation. +pub trait PreCommitActions { + /// Actions to be performed on block import. + fn register_import_action(&self, op: OnImportAction); + + /// Actions to be performed on block finalization. + fn register_finality_action(&self, op: OnFinalityAction); +} + /// Interface for fetching block data. pub trait BlockBackend { /// Get block body by ID. Returns `None` if the body is not stored. @@ -273,10 +302,14 @@ pub struct BlockImportNotification { /// Summary of a finalized block. #[derive(Clone, Debug)] pub struct FinalityNotification { - /// Imported block header hash. + /// Finalized block header hash. pub hash: Block::Hash, - /// Imported block header. + /// Finalized block header. pub header: Block::Header, + /// Path from the old finalized to new finalized parent (implicitly finalized blocks). + pub tree_route: Arc<[Block::Hash]>, + /// Stale branches heads. + pub stale_heads: Arc<[Block::Hash]>, } impl TryFrom> for ChainEvent { @@ -293,6 +326,30 @@ impl TryFrom> for ChainEvent { impl From> for ChainEvent { fn from(n: FinalityNotification) -> Self { - Self::Finalized { hash: n.hash } + Self::Finalized { hash: n.hash, tree_route: n.tree_route } + } +} + +impl From> for FinalityNotification { + fn from(mut summary: FinalizeSummary) -> Self { + let hash = summary.finalized.pop().unwrap_or_default(); + FinalityNotification { + hash, + header: summary.header, + tree_route: Arc::from(summary.finalized), + stale_heads: Arc::from(summary.stale_heads), + } + } +} + +impl From> for BlockImportNotification { + fn from(summary: ImportSummary) -> Self { + BlockImportNotification { + hash: summary.hash, + origin: summary.origin, + header: summary.header, + is_new_best: summary.is_new_best, + tree_route: summary.tree_route.map(Arc::new), + } } } diff --git a/client/api/src/execution_extensions.rs b/client/api/src/execution_extensions.rs index ec44294b8a96..92efafe91a17 100644 --- a/client/api/src/execution_extensions.rs +++ b/client/api/src/execution_extensions.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -161,13 +161,13 @@ impl ExecutionExtensions { let mut extensions = self.extensions_factory.read().extensions_for(capabilities); - if capabilities.has(offchain::Capability::Keystore) { + if capabilities.contains(offchain::Capabilities::KEYSTORE) { if let Some(ref keystore) = self.keystore { extensions.register(KeystoreExt(keystore.clone())); } } - if capabilities.has(offchain::Capability::TransactionPool) { + if capabilities.contains(offchain::Capabilities::TRANSACTION_POOL) { if let Some(pool) = self.transaction_pool.read().as_ref().and_then(|x| x.upgrade()) { extensions .register(TransactionPoolExt( @@ -176,8 +176,8 @@ impl ExecutionExtensions { } } - if capabilities.has(offchain::Capability::OffchainDbRead) || - capabilities.has(offchain::Capability::OffchainDbWrite) + if capabilities.contains(offchain::Capabilities::OFFCHAIN_DB_READ) || + capabilities.contains(offchain::Capabilities::OFFCHAIN_DB_WRITE) { if let Some(offchain_db) = self.offchain_db.as_ref() { extensions.register(OffchainDbExt::new(offchain::LimitedExternalities::new( @@ -210,7 +210,7 @@ impl ExecutionExtensions { ExecutionContext::BlockConstruction => self.strategies.block_construction.get_manager(), ExecutionContext::Syncing => self.strategies.syncing.get_manager(), ExecutionContext::Importing => self.strategies.importing.get_manager(), - ExecutionContext::OffchainCall(Some((_, capabilities))) if capabilities.has_all() => + ExecutionContext::OffchainCall(Some((_, capabilities))) if capabilities.is_all() => self.strategies.offchain_worker.get_manager(), ExecutionContext::OffchainCall(_) => self.strategies.other.get_manager(), }; diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index e8fce19f8124..d989004ee217 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -26,11 +26,11 @@ use sp_core::{ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor, Zero}, - Justification, Justifications, Storage, + Justification, Justifications, StateVersion, Storage, }; use sp_state_machine::{ - Backend as StateBackend, ChangesTrieTransaction, ChildStorageCollection, InMemoryBackend, - IndexOperation, StorageCollection, + Backend as StateBackend, ChildStorageCollection, InMemoryBackend, IndexOperation, + StorageCollection, }; use std::{ collections::{HashMap, HashSet}, @@ -39,10 +39,10 @@ use std::{ }; use crate::{ - backend::{self, NewBlockState, ProvideChtRoots}, + backend::{self, NewBlockState}, blockchain::{self, well_known_cache_keys::Id as CacheKeyId, BlockStatus, HeaderBackend}, leaves::LeafSet, - light, UsageInfo, + UsageInfo, }; struct PendingBlock { @@ -109,7 +109,6 @@ struct BlockchainStorage { finalized_number: NumberFor, genesis_hash: Block::Hash, header_cht_roots: HashMap, Block::Hash>, - changes_trie_cht_roots: HashMap, Block::Hash>, leaves: LeafSet>, aux: HashMap, Vec>, } @@ -152,7 +151,6 @@ impl Blockchain { finalized_number: Zero::zero(), genesis_hash: Default::default(), header_cht_roots: HashMap::new(), - changes_trie_cht_roots: HashMap::new(), leaves: LeafSet::new(), aux: HashMap::new(), })); @@ -369,6 +367,7 @@ impl HeaderBackend for Blockchain { None }, number_leaves: storage.leaves.count(), + block_gap: None, } } @@ -441,10 +440,6 @@ impl blockchain::Backend for Blockchain { Ok(self.storage.read().finalized_hash.clone()) } - fn cache(&self) -> Option>> { - None - } - fn leaves(&self) -> sp_blockchain::Result> { Ok(self.storage.read().leaves.hashes()) } @@ -465,12 +460,6 @@ impl blockchain::Backend for Blockchain { } } -impl blockchain::ProvideCache for Blockchain { - fn cache(&self) -> Option>> { - None - } -} - impl backend::AuxStore for Blockchain { fn insert_aux< 'a, @@ -498,82 +487,6 @@ impl backend::AuxStore for Blockchain { } } -impl light::Storage for Blockchain -where - Block::Hash: From<[u8; 32]>, -{ - fn import_header( - &self, - header: Block::Header, - _cache: HashMap>, - state: NewBlockState, - aux_ops: Vec<(Vec, Option>)>, - ) -> sp_blockchain::Result<()> { - let hash = header.hash(); - self.insert(hash, header, None, None, state)?; - - self.write_aux(aux_ops); - Ok(()) - } - - fn set_head(&self, id: BlockId) -> sp_blockchain::Result<()> { - Blockchain::set_head(self, id) - } - - fn last_finalized(&self) -> sp_blockchain::Result { - Ok(self.storage.read().finalized_hash.clone()) - } - - fn finalize_header(&self, id: BlockId) -> sp_blockchain::Result<()> { - Blockchain::finalize_header(self, id, None) - } - - fn cache(&self) -> Option>> { - None - } - - fn usage_info(&self) -> Option { - None - } -} - -impl ProvideChtRoots for Blockchain { - fn header_cht_root( - &self, - _cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result> { - self.storage - .read() - .header_cht_roots - .get(&block) - .cloned() - .ok_or_else(|| { - sp_blockchain::Error::Backend(format!("Header CHT for block {} not exists", block)) - }) - .map(Some) - } - - fn changes_trie_cht_root( - &self, - _cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result> { - self.storage - .read() - .changes_trie_cht_roots - .get(&block) - .cloned() - .ok_or_else(|| { - sp_blockchain::Error::Backend(format!( - "Changes trie CHT for block {} not exists", - block - )) - }) - .map(Some) - } -} - /// In-memory operation. pub struct BlockImportOperation { pending_block: Option>, @@ -593,6 +506,7 @@ where &mut self, storage: Storage, commit: bool, + state_version: StateVersion, ) -> sp_blockchain::Result { check_genesis_storage(&storage)?; @@ -606,6 +520,7 @@ where let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, + state_version, ); if commit { @@ -649,23 +564,21 @@ where Ok(()) } - fn update_changes_trie( - &mut self, - _update: ChangesTrieTransaction, NumberFor>, - ) -> sp_blockchain::Result<()> { - Ok(()) - } - fn set_genesis_state( &mut self, storage: Storage, commit: bool, + state_version: StateVersion, ) -> sp_blockchain::Result { - self.apply_storage(storage, commit) + self.apply_storage(storage, commit, state_version) } - fn reset_storage(&mut self, storage: Storage) -> sp_blockchain::Result { - self.apply_storage(storage, true) + fn reset_storage( + &mut self, + storage: Storage, + state_version: StateVersion, + ) -> sp_blockchain::Result { + self.apply_storage(storage, true, state_version) } fn insert_aux(&mut self, ops: I) -> sp_blockchain::Result<()> @@ -845,10 +758,6 @@ where None } - fn changes_trie_storage(&self) -> Option<&dyn backend::PrunableStateChangesTrieStorage> { - None - } - fn offchain_storage(&self) -> Option { None } @@ -884,22 +793,6 @@ where impl backend::LocalBackend for Backend where Block::Hash: Ord {} -impl backend::RemoteBackend for Backend -where - Block::Hash: Ord, -{ - fn is_local_state_available(&self, block: &BlockId) -> bool { - self.blockchain - .expect_block_number_from_id(block) - .map(|num| num.is_zero()) - .unwrap_or(false) - } - - fn remote_blockchain(&self) -> Arc> { - unimplemented!() - } -} - /// Check that genesis storage is valid. pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> { if storage.top.iter().any(|(k, _)| well_known_keys::is_child_storage_key(k)) { diff --git a/client/api/src/leaves.rs b/client/api/src/leaves.rs index 80216bc4664b..2a3b95188e68 100644 --- a/client/api/src/leaves.rs +++ b/client/api/src/leaves.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/api/src/lib.rs b/client/api/src/lib.rs index 16935b1e846c..aab2fabd5e25 100644 --- a/client/api/src/lib.rs +++ b/client/api/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,25 +21,22 @@ pub mod backend; pub mod call_executor; -pub mod cht; pub mod client; pub mod execution_extensions; pub mod in_mem; pub mod leaves; -pub mod light; pub mod notifications; pub mod proof_provider; pub use backend::*; pub use call_executor::*; pub use client::*; -pub use light::*; pub use notifications::*; pub use proof_provider::*; pub use sp_blockchain as blockchain; pub use sp_blockchain::HeaderBackend; -pub use sp_state_machine::{ExecutionStrategy, StorageProof}; +pub use sp_state_machine::{CompactProof, ExecutionStrategy, StorageProof}; pub use sp_storage::{ChildInfo, PrefixedStorageKey, StorageData, StorageKey}; /// Usage Information Provider interface diff --git a/client/api/src/light.rs b/client/api/src/light.rs deleted file mode 100644 index 8638ddf741f3..000000000000 --- a/client/api/src/light.rs +++ /dev/null @@ -1,372 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Substrate light client interfaces - -use std::{ - collections::{BTreeMap, HashMap}, - future::Future, - sync::Arc, -}; - -use crate::{ - backend::{AuxStore, NewBlockState}, - ProvideChtRoots, UsageInfo, -}; -use sp_blockchain::{ - well_known_cache_keys, Cache as BlockchainCache, Error as ClientError, HeaderBackend, - HeaderMetadata, Result as ClientResult, -}; -use sp_core::{storage::PrefixedStorageKey, ChangesTrieConfigurationRange}; -use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, Header as HeaderT, NumberFor}, -}; -use sp_state_machine::StorageProof; - -/// Remote call request. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct RemoteCallRequest { - /// Call at state of given block. - pub block: Header::Hash, - /// Header of block at which call is performed. - pub header: Header, - /// Method to call. - pub method: String, - /// Call data. - pub call_data: Vec, - /// Number of times to retry request. None means that default RETRY_COUNT is used. - pub retry_count: Option, -} - -/// Remote canonical header request. -#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] -pub struct RemoteHeaderRequest { - /// The root of CHT this block is included in. - pub cht_root: Header::Hash, - /// Number of the header to query. - pub block: Header::Number, - /// Number of times to retry request. None means that default RETRY_COUNT is used. - pub retry_count: Option, -} - -/// Remote storage read request. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct RemoteReadRequest { - /// Read at state of given block. - pub block: Header::Hash, - /// Header of block at which read is performed. - pub header: Header, - /// Storage key to read. - pub keys: Vec>, - /// Number of times to retry request. None means that default RETRY_COUNT is used. - pub retry_count: Option, -} - -/// Remote storage read child request. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct RemoteReadChildRequest { - /// Read at state of given block. - pub block: Header::Hash, - /// Header of block at which read is performed. - pub header: Header, - /// Storage key for child. - pub storage_key: PrefixedStorageKey, - /// Child storage key to read. - pub keys: Vec>, - /// Number of times to retry request. None means that default RETRY_COUNT is used. - pub retry_count: Option, -} - -/// Remote key changes read request. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RemoteChangesRequest { - /// All changes trie configurations that are valid within [first_block; last_block]. - pub changes_trie_configs: Vec>, - /// Query changes from range of blocks, starting (and including) with this hash... - pub first_block: (Header::Number, Header::Hash), - /// ...ending (and including) with this hash. Should come after first_block and - /// be the part of the same fork. - pub last_block: (Header::Number, Header::Hash), - /// Only use digests from blocks up to this hash. Should be last_block OR come - /// after this block and be the part of the same fork. - pub max_block: (Header::Number, Header::Hash), - /// Known changes trie roots for the range of blocks [tries_roots.0..max_block]. - /// Proofs for roots of ascendants of tries_roots.0 are provided by the remote node. - pub tries_roots: (Header::Number, Header::Hash, Vec), - /// Optional Child Storage key to read. - pub storage_key: Option, - /// Storage key to read. - pub key: Vec, - /// Number of times to retry request. None means that default RETRY_COUNT is used. - pub retry_count: Option, -} - -/// Key changes read proof. -#[derive(Debug, PartialEq, Eq)] -pub struct ChangesProof { - /// Max block that has been used in changes query. - pub max_block: Header::Number, - /// All touched nodes of all changes tries. - pub proof: Vec>, - /// All changes tries roots that have been touched AND are missing from - /// the requester' node. It is a map of block number => changes trie root. - pub roots: BTreeMap, - /// The proofs for all changes tries roots that have been touched AND are - /// missing from the requester' node. It is a map of CHT number => proof. - pub roots_proof: StorageProof, -} - -/// Remote block body request -#[derive(Clone, Default, Debug, PartialEq, Eq, Hash)] -pub struct RemoteBodyRequest { - /// Header of the requested block body - pub header: Header, - /// Number of times to retry request. None means that default RETRY_COUNT is used. - pub retry_count: Option, -} - -/// Light client data fetcher. Implementations of this trait must check if remote data -/// is correct (see FetchedDataChecker) and return already checked data. -pub trait Fetcher: Send + Sync { - /// Remote header future. - type RemoteHeaderResult: Future> - + Unpin - + Send - + 'static; - /// Remote storage read future. - type RemoteReadResult: Future, Option>>, ClientError>> - + Unpin - + Send - + 'static; - /// Remote call result future. - type RemoteCallResult: Future, ClientError>> + Unpin + Send + 'static; - /// Remote changes result future. - type RemoteChangesResult: Future, u32)>, ClientError>> - + Unpin - + Send - + 'static; - /// Remote block body result future. - type RemoteBodyResult: Future, ClientError>> - + Unpin - + Send - + 'static; - - /// Fetch remote header. - fn remote_header( - &self, - request: RemoteHeaderRequest, - ) -> Self::RemoteHeaderResult; - /// Fetch remote storage value. - fn remote_read(&self, request: RemoteReadRequest) -> Self::RemoteReadResult; - /// Fetch remote storage child value. - fn remote_read_child( - &self, - request: RemoteReadChildRequest, - ) -> Self::RemoteReadResult; - /// Fetch remote call result. - fn remote_call(&self, request: RemoteCallRequest) -> Self::RemoteCallResult; - /// Fetch remote changes ((block number, extrinsic index)) where given key has been changed - /// at a given blocks range. - fn remote_changes( - &self, - request: RemoteChangesRequest, - ) -> Self::RemoteChangesResult; - /// Fetch remote block body - fn remote_body(&self, request: RemoteBodyRequest) -> Self::RemoteBodyResult; -} - -/// Light client remote data checker. -/// -/// Implementations of this trait should not use any prunable blockchain data -/// except that is passed to its methods. -pub trait FetchChecker: Send + Sync { - /// Check remote header proof. - fn check_header_proof( - &self, - request: &RemoteHeaderRequest, - header: Option, - remote_proof: StorageProof, - ) -> ClientResult; - /// Check remote storage read proof. - fn check_read_proof( - &self, - request: &RemoteReadRequest, - remote_proof: StorageProof, - ) -> ClientResult, Option>>>; - /// Check remote storage read proof. - fn check_read_child_proof( - &self, - request: &RemoteReadChildRequest, - remote_proof: StorageProof, - ) -> ClientResult, Option>>>; - /// Check remote method execution proof. - fn check_execution_proof( - &self, - request: &RemoteCallRequest, - remote_proof: StorageProof, - ) -> ClientResult>; - /// Check remote changes query proof. - fn check_changes_proof( - &self, - request: &RemoteChangesRequest, - proof: ChangesProof, - ) -> ClientResult, u32)>>; - /// Check remote body proof. - fn check_body_proof( - &self, - request: &RemoteBodyRequest, - body: Vec, - ) -> ClientResult>; -} - -/// Light client blockchain storage. -pub trait Storage: - AuxStore - + HeaderBackend - + HeaderMetadata - + ProvideChtRoots -{ - /// Store new header. Should refuse to revert any finalized blocks. - /// - /// Takes new authorities, the leaf state of the new block, and - /// any auxiliary storage updates to place in the same operation. - fn import_header( - &self, - header: Block::Header, - cache: HashMap>, - state: NewBlockState, - aux_ops: Vec<(Vec, Option>)>, - ) -> ClientResult<()>; - - /// Set an existing block as new best block. - fn set_head(&self, block: BlockId) -> ClientResult<()>; - - /// Mark historic header as finalized. - fn finalize_header(&self, block: BlockId) -> ClientResult<()>; - - /// Get last finalized header. - fn last_finalized(&self) -> ClientResult; - - /// Get storage cache. - fn cache(&self) -> Option>>; - - /// Get storage usage statistics. - fn usage_info(&self) -> Option; -} - -/// Remote header. -#[derive(Debug)] -pub enum LocalOrRemote { - /// When data is available locally, it is returned. - Local(Data), - /// When data is unavailable locally, the request to fetch it from remote node is returned. - Remote(Request), - /// When data is unknown. - Unknown, -} - -/// Futures-based blockchain backend that either resolves blockchain data -/// locally, or fetches required data from remote node. -pub trait RemoteBlockchain: Send + Sync { - /// Get block header. - fn header( - &self, - id: BlockId, - ) -> ClientResult>>; -} - -/// Returns future that resolves header either locally, or remotely. -pub fn future_header>( - blockchain: &dyn RemoteBlockchain, - fetcher: &F, - id: BlockId, -) -> impl Future, ClientError>> { - use futures::future::{ready, Either, FutureExt}; - - match blockchain.header(id) { - Ok(LocalOrRemote::Remote(request)) => - Either::Left(fetcher.remote_header(request).then(|header| ready(header.map(Some)))), - Ok(LocalOrRemote::Unknown) => Either::Right(ready(Ok(None))), - Ok(LocalOrRemote::Local(local_header)) => Either::Right(ready(Ok(Some(local_header)))), - Err(err) => Either::Right(ready(Err(err))), - } -} - -#[cfg(test)] -pub mod tests { - use super::*; - use futures::future::Ready; - use parking_lot::Mutex; - use sp_blockchain::Error as ClientError; - use sp_test_primitives::{Block, Extrinsic, Header}; - - #[derive(Debug, thiserror::Error)] - #[error("Not implemented on test node")] - struct MockError; - - impl Into for MockError { - fn into(self) -> ClientError { - ClientError::Application(Box::new(self)) - } - } - - pub type OkCallFetcher = Mutex>; - - fn not_implemented_in_tests() -> Ready> { - futures::future::ready(Err(MockError.into())) - } - - impl Fetcher for OkCallFetcher { - type RemoteHeaderResult = Ready>; - type RemoteReadResult = Ready, Option>>, ClientError>>; - type RemoteCallResult = Ready, ClientError>>; - type RemoteChangesResult = Ready, u32)>, ClientError>>; - type RemoteBodyResult = Ready, ClientError>>; - - fn remote_header(&self, _request: RemoteHeaderRequest

) -> Self::RemoteHeaderResult { - not_implemented_in_tests() - } - - fn remote_read(&self, _request: RemoteReadRequest
) -> Self::RemoteReadResult { - not_implemented_in_tests() - } - - fn remote_read_child( - &self, - _request: RemoteReadChildRequest
, - ) -> Self::RemoteReadResult { - not_implemented_in_tests() - } - - fn remote_call(&self, _request: RemoteCallRequest
) -> Self::RemoteCallResult { - futures::future::ready(Ok((*self.lock()).clone())) - } - - fn remote_changes( - &self, - _request: RemoteChangesRequest
, - ) -> Self::RemoteChangesResult { - not_implemented_in_tests() - } - - fn remote_body(&self, _request: RemoteBodyRequest
) -> Self::RemoteBodyResult { - not_implemented_in_tests() - } - } -} diff --git a/client/api/src/notifications.rs b/client/api/src/notifications.rs index 1346afd5e54d..36798abc5bde 100644 --- a/client/api/src/notifications.rs +++ b/client/api/src/notifications.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,24 +20,55 @@ use std::{ collections::{HashMap, HashSet}, + pin::Pin, sync::Arc, + task::Poll, }; -use fnv::{FnvHashMap, FnvHashSet}; -use prometheus_endpoint::{register, CounterVec, Opts, Registry, U64}; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use futures::Stream; + +use prometheus_endpoint::Registry as PrometheusRegistry; + +use sc_utils::pubsub::{Hub, Receiver}; use sp_core::storage::{StorageData, StorageKey}; use sp_runtime::traits::Block as BlockT; +mod registry; + +use registry::Registry; + +#[cfg(test)] +mod tests; + +/// A type of a message delivered to the subscribers +#[derive(Debug)] +pub struct StorageNotification { + /// The hash of the block + pub block: Hash, + + /// The set of changes + pub changes: StorageChangeSet, +} + /// Storage change set #[derive(Debug)] pub struct StorageChangeSet { - changes: Arc)>>, - child_changes: Arc)>)>>, - filter: Option>, - child_filters: Option>>>, + changes: Arc<[(StorageKey, Option)]>, + child_changes: Arc<[(StorageKey, Vec<(StorageKey, Option)>)]>, + filter: Keys, + child_filters: ChildKeys, } +/// Manages storage listeners. +#[derive(Debug)] +pub struct StorageNotifications(Hub, Registry>); + +/// Type that implements `futures::Stream` of storage change events. +pub struct StorageEventStream(Receiver, Registry>); + +type Keys = Option>; +type ChildKeys = Option>>>; + impl StorageChangeSet { /// Convert the change set into iterator over storage items. pub fn iter<'a>( @@ -73,481 +104,49 @@ impl StorageChangeSet { } } -/// Type that implements `futures::Stream` of storage change events. -pub type StorageEventStream = TracingUnboundedReceiver<(H, StorageChangeSet)>; - -type SubscriberId = u64; - -type SubscribersGauge = CounterVec; - -/// Manages storage listeners. -#[derive(Debug)] -pub struct StorageNotifications { - metrics: Option, - next_id: SubscriberId, - wildcard_listeners: FnvHashSet, - listeners: HashMap>, - child_listeners: HashMap< - StorageKey, - (HashMap>, FnvHashSet), - >, - sinks: FnvHashMap< - SubscriberId, - ( - TracingUnboundedSender<(Block::Hash, StorageChangeSet)>, - Option>, - Option>>>, - ), - >, -} - -impl Default for StorageNotifications { - fn default() -> Self { - Self { - metrics: Default::default(), - next_id: Default::default(), - wildcard_listeners: Default::default(), - listeners: Default::default(), - child_listeners: Default::default(), - sinks: Default::default(), - } +impl Stream for StorageEventStream { + type Item = StorageNotification; + fn poll_next( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + Stream::poll_next(Pin::new(&mut self.get_mut().0), cx) } } impl StorageNotifications { /// Initialize a new StorageNotifications /// optionally pass a prometheus registry to send subscriber metrics to - pub fn new(prometheus_registry: Option) -> Self { - let metrics = prometheus_registry.and_then(|r| { - CounterVec::new( - Opts::new( - "storage_notification_subscribers", - "Number of subscribers in storage notification sytem", - ), - &["action"], // added | removed - ) - .and_then(|g| register(g, &r)) - .ok() - }); + pub fn new(prometheus_registry: Option) -> Self { + let registry = Registry::new(prometheus_registry); + let hub = Hub::new_with_registry("mpsc_storage_notification_items", registry); - StorageNotifications { - metrics, - next_id: Default::default(), - wildcard_listeners: Default::default(), - listeners: Default::default(), - child_listeners: Default::default(), - sinks: Default::default(), - } + StorageNotifications(hub) } + /// Trigger notification to all listeners. /// /// Note the changes are going to be filtered by listener's filter key. /// In fact no event might be sent if clients are not interested in the changes. pub fn trigger( - &mut self, + &self, hash: &Block::Hash, changeset: impl Iterator, Option>)>, child_changeset: impl Iterator< Item = (Vec, impl Iterator, Option>)>), >, ) { - let has_wildcard = !self.wildcard_listeners.is_empty(); - - // early exit if no listeners - if !has_wildcard && self.listeners.is_empty() && self.child_listeners.is_empty() { - return - } - - let mut subscribers = self.wildcard_listeners.clone(); - let mut changes = Vec::new(); - let mut child_changes = Vec::new(); - - // Collect subscribers and changes - for (k, v) in changeset { - let k = StorageKey(k); - let listeners = self.listeners.get(&k); - - if let Some(ref listeners) = listeners { - subscribers.extend(listeners.iter()); - } - - if has_wildcard || listeners.is_some() { - changes.push((k, v.map(StorageData))); - } - } - for (sk, changeset) in child_changeset { - let sk = StorageKey(sk); - if let Some((cl, cw)) = self.child_listeners.get(&sk) { - let mut changes = Vec::new(); - for (k, v) in changeset { - let k = StorageKey(k); - let listeners = cl.get(&k); - - if let Some(ref listeners) = listeners { - subscribers.extend(listeners.iter()); - } - - subscribers.extend(cw.iter()); - - if !cw.is_empty() || listeners.is_some() { - changes.push((k, v.map(StorageData))); - } - } - if !changes.is_empty() { - child_changes.push((sk, changes)); - } - } - } - - // Don't send empty notifications - if changes.is_empty() && child_changes.is_empty() { - return - } - - let changes = Arc::new(changes); - let child_changes = Arc::new(child_changes); - // Trigger the events - - let to_remove = self - .sinks - .iter() - .filter_map(|(subscriber, &(ref sink, ref filter, ref child_filters))| { - let should_remove = { - if subscribers.contains(subscriber) { - sink.unbounded_send(( - hash.clone(), - StorageChangeSet { - changes: changes.clone(), - child_changes: child_changes.clone(), - filter: filter.clone(), - child_filters: child_filters.clone(), - }, - )) - .is_err() - } else { - sink.is_closed() - } - }; - - if should_remove { - Some(subscriber.clone()) - } else { - None - } - }) - .collect::>(); - - for sub_id in to_remove { - self.remove_subscriber(sub_id); - } - } - - fn remove_subscriber_from( - subscriber: &SubscriberId, - filters: &Option>, - listeners: &mut HashMap>, - wildcards: &mut FnvHashSet, - ) { - match filters { - None => { - wildcards.remove(subscriber); - }, - Some(filters) => - for key in filters.iter() { - let remove_key = match listeners.get_mut(key) { - Some(ref mut set) => { - set.remove(subscriber); - set.is_empty() - }, - None => false, - }; - - if remove_key { - listeners.remove(key); - } - }, - } - } - - fn remove_subscriber(&mut self, subscriber: SubscriberId) { - if let Some((_, filters, child_filters)) = self.sinks.remove(&subscriber) { - Self::remove_subscriber_from( - &subscriber, - &filters, - &mut self.listeners, - &mut self.wildcard_listeners, - ); - if let Some(child_filters) = child_filters.as_ref() { - for (c_key, filters) in child_filters { - if let Some((listeners, wildcards)) = self.child_listeners.get_mut(&c_key) { - Self::remove_subscriber_from( - &subscriber, - &filters, - &mut *listeners, - &mut *wildcards, - ); - - if listeners.is_empty() && wildcards.is_empty() { - self.child_listeners.remove(&c_key); - } - } - } - } - if let Some(m) = self.metrics.as_ref() { - m.with_label_values(&[&"removed"]).inc(); - } - } - } - - fn listen_from( - current_id: SubscriberId, - filter_keys: &Option>, - listeners: &mut HashMap>, - wildcards: &mut FnvHashSet, - ) -> Option> { - match filter_keys { - None => { - wildcards.insert(current_id); - None - }, - Some(keys) => Some( - keys.as_ref() - .iter() - .map(|key| { - listeners - .entry(key.clone()) - .or_insert_with(Default::default) - .insert(current_id); - key.clone() - }) - .collect(), - ), - } + self.0.send((hash, changeset, child_changeset)) } /// Start listening for particular storage keys. pub fn listen( - &mut self, + &self, filter_keys: Option<&[StorageKey]>, filter_child_keys: Option<&[(StorageKey, Option>)]>, ) -> StorageEventStream { - self.next_id += 1; - let current_id = self.next_id; - - // add subscriber for every key - let keys = Self::listen_from( - current_id, - &filter_keys, - &mut self.listeners, - &mut self.wildcard_listeners, - ); - let child_keys = filter_child_keys.map(|filter_child_keys| { - filter_child_keys - .iter() - .map(|(c_key, o_keys)| { - let (c_listeners, c_wildcards) = - self.child_listeners.entry(c_key.clone()).or_insert_with(Default::default); - - ( - c_key.clone(), - Self::listen_from(current_id, o_keys, &mut *c_listeners, &mut *c_wildcards), - ) - }) - .collect() - }); - - // insert sink - let (tx, rx) = tracing_unbounded("mpsc_storage_notification_items"); - self.sinks.insert(current_id, (tx, keys, child_keys)); - - if let Some(m) = self.metrics.as_ref() { - m.with_label_values(&[&"added"]).inc(); - } - - rx - } -} - -#[cfg(test)] -mod tests { - use super::*; - use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper, H256 as Hash}; - use std::iter::{empty, Empty}; - - type TestChangeSet = ( - Vec<(StorageKey, Option)>, - Vec<(StorageKey, Vec<(StorageKey, Option)>)>, - ); - - #[cfg(test)] - impl From for StorageChangeSet { - fn from(changes: TestChangeSet) -> Self { - // warning hardcoded child trie wildcard to test upon - let child_filters = Some( - [(StorageKey(vec![4]), None), (StorageKey(vec![5]), None)] - .iter() - .cloned() - .collect(), - ); - StorageChangeSet { - changes: Arc::new(changes.0), - child_changes: Arc::new(changes.1), - filter: None, - child_filters, - } - } - } - - #[cfg(test)] - impl PartialEq for StorageChangeSet { - fn eq(&self, other: &Self) -> bool { - self.iter().eq(other.iter()) - } - } - - type Block = RawBlock>; - - #[test] - fn triggering_change_should_notify_wildcard_listeners() { - // given - let mut notifications = StorageNotifications::::default(); - let child_filter = [(StorageKey(vec![4]), None)]; - let mut recv = - futures::executor::block_on_stream(notifications.listen(None, Some(&child_filter[..]))); - - // when - let changeset = vec![(vec![2], Some(vec![3])), (vec![3], None)]; - let c_changeset_1 = vec![(vec![5], Some(vec![4])), (vec![6], None)]; - let c_changeset = vec![(vec![4], c_changeset_1)]; - notifications.trigger( - &Hash::from_low_u64_be(1), - changeset.into_iter(), - c_changeset.into_iter().map(|(a, b)| (a, b.into_iter())), - ); - - // then - assert_eq!( - recv.next().unwrap(), - ( - Hash::from_low_u64_be(1), - ( - vec![ - (StorageKey(vec![2]), Some(StorageData(vec![3]))), - (StorageKey(vec![3]), None), - ], - vec![( - StorageKey(vec![4]), - vec![ - (StorageKey(vec![5]), Some(StorageData(vec![4]))), - (StorageKey(vec![6]), None), - ] - )] - ) - .into() - ) - ); - } - - #[test] - fn should_only_notify_interested_listeners() { - // given - let mut notifications = StorageNotifications::::default(); - let child_filter = [(StorageKey(vec![4]), Some(vec![StorageKey(vec![5])]))]; - let mut recv1 = futures::executor::block_on_stream( - notifications.listen(Some(&[StorageKey(vec![1])]), None), - ); - let mut recv2 = futures::executor::block_on_stream( - notifications.listen(Some(&[StorageKey(vec![2])]), None), - ); - let mut recv3 = futures::executor::block_on_stream( - notifications.listen(Some(&[]), Some(&child_filter)), - ); - - // when - let changeset = vec![(vec![2], Some(vec![3])), (vec![1], None)]; - let c_changeset_1 = vec![(vec![5], Some(vec![4])), (vec![6], None)]; - - let c_changeset = vec![(vec![4], c_changeset_1)]; - notifications.trigger( - &Hash::from_low_u64_be(1), - changeset.into_iter(), - c_changeset.into_iter().map(|(a, b)| (a, b.into_iter())), - ); - - // then - assert_eq!( - recv1.next().unwrap(), - (Hash::from_low_u64_be(1), (vec![(StorageKey(vec![1]), None),], vec![]).into()) - ); - assert_eq!( - recv2.next().unwrap(), - ( - Hash::from_low_u64_be(1), - (vec![(StorageKey(vec![2]), Some(StorageData(vec![3]))),], vec![]).into() - ) - ); - assert_eq!( - recv3.next().unwrap(), - ( - Hash::from_low_u64_be(1), - ( - vec![], - vec![( - StorageKey(vec![4]), - vec![(StorageKey(vec![5]), Some(StorageData(vec![4])))] - ),] - ) - .into() - ) - ); - } - - #[test] - fn should_cleanup_subscribers_if_dropped() { - // given - let mut notifications = StorageNotifications::::default(); - { - let child_filter = [(StorageKey(vec![4]), Some(vec![StorageKey(vec![5])]))]; - let _recv1 = futures::executor::block_on_stream( - notifications.listen(Some(&[StorageKey(vec![1])]), None), - ); - let _recv2 = futures::executor::block_on_stream( - notifications.listen(Some(&[StorageKey(vec![2])]), None), - ); - let _recv3 = futures::executor::block_on_stream(notifications.listen(None, None)); - let _recv4 = - futures::executor::block_on_stream(notifications.listen(None, Some(&child_filter))); - assert_eq!(notifications.listeners.len(), 2); - assert_eq!(notifications.wildcard_listeners.len(), 2); - assert_eq!(notifications.child_listeners.len(), 1); - } - - // when - let changeset = vec![(vec![2], Some(vec![3])), (vec![1], None)]; - let c_changeset = empty::<(_, Empty<_>)>(); - notifications.trigger(&Hash::from_low_u64_be(1), changeset.into_iter(), c_changeset); - - // then - assert_eq!(notifications.listeners.len(), 0); - assert_eq!(notifications.wildcard_listeners.len(), 0); - assert_eq!(notifications.child_listeners.len(), 0); - } - - #[test] - fn should_not_send_empty_notifications() { - // given - let mut recv = { - let mut notifications = StorageNotifications::::default(); - let recv = futures::executor::block_on_stream(notifications.listen(None, None)); - - // when - let changeset = vec![]; - let c_changeset = empty::<(_, Empty<_>)>(); - notifications.trigger(&Hash::from_low_u64_be(1), changeset.into_iter(), c_changeset); - recv - }; + let receiver = self.0.subscribe(registry::SubscribeOp { filter_keys, filter_child_keys }); - // then - assert_eq!(recv.next(), None); + StorageEventStream(receiver) } } diff --git a/client/api/src/notifications/registry.rs b/client/api/src/notifications/registry.rs new file mode 100644 index 000000000000..b34d5a6b6711 --- /dev/null +++ b/client/api/src/notifications/registry.rs @@ -0,0 +1,365 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use super::*; + +use sp_core::hexdisplay::HexDisplay; + +use fnv::{FnvHashMap, FnvHashSet}; +use prometheus_endpoint::{register, CounterVec, Opts, U64}; + +use sc_utils::{ + id_sequence::SeqID as SubscriberId, + pubsub::{Dispatch, Subscribe, Unsubscribe}, +}; + +type SubscribersGauge = CounterVec; + +/// A command to subscribe with the specified filters. +/// +/// Used by the implementation of [`Subscribe`] trait for [`Registry]. +pub(super) struct SubscribeOp<'a> { + pub filter_keys: Option<&'a [StorageKey]>, + pub filter_child_keys: Option<&'a [(StorageKey, Option>)]>, +} + +#[derive(Debug, Default)] +pub(super) struct Registry { + pub(super) metrics: Option, + pub(super) wildcard_listeners: FnvHashSet, + pub(super) listeners: HashMap>, + pub(super) child_listeners: HashMap< + StorageKey, + (HashMap>, FnvHashSet), + >, + pub(super) sinks: FnvHashMap, +} + +#[derive(Debug)] +pub(super) struct SubscriberSink { + subs_id: SubscriberId, + keys: Keys, + child_keys: ChildKeys, + was_triggered: bool, +} + +impl Drop for SubscriberSink { + fn drop(&mut self) { + if !self.was_triggered { + log::trace!( + target: "storage_notifications", + "Listener was never triggered: id={}, keys={:?}, child_keys={:?}", + self.subs_id, + PrintKeys(&self.keys), + PrintChildKeys(&self.child_keys), + ); + } + } +} + +impl SubscriberSink { + fn new(subs_id: SubscriberId, keys: Keys, child_keys: ChildKeys) -> Self { + Self { subs_id, keys, child_keys, was_triggered: false } + } +} + +impl Registry { + pub(super) fn new(prometheus_registry: Option) -> Self { + let metrics = prometheus_registry.and_then(|r| { + CounterVec::new( + Opts::new( + "substrate_storage_notification_subscribers", + "Number of subscribers in storage notification sytem", + ), + &["action"], // added | removed + ) + .and_then(|g| register(g, &r)) + .ok() + }); + + Registry { metrics, ..Default::default() } + } +} + +impl Unsubscribe for Registry { + fn unsubscribe(&mut self, subs_id: SubscriberId) { + self.remove_subscriber(subs_id); + } +} + +impl<'a> Subscribe> for Registry { + fn subscribe(&mut self, subs_op: SubscribeOp<'a>, subs_id: SubscriberId) { + let SubscribeOp { filter_keys, filter_child_keys } = subs_op; + + let keys = Self::listen_from( + subs_id, + filter_keys.as_ref(), + &mut self.listeners, + &mut self.wildcard_listeners, + ); + + let child_keys = filter_child_keys.map(|filter_child_keys| { + filter_child_keys + .iter() + .map(|(c_key, o_keys)| { + let (c_listeners, c_wildcards) = + self.child_listeners.entry(c_key.clone()).or_default(); + + ( + c_key.clone(), + Self::listen_from( + subs_id, + o_keys.as_ref(), + &mut *c_listeners, + &mut *c_wildcards, + ), + ) + }) + .collect() + }); + + if let Some(m) = self.metrics.as_ref() { + m.with_label_values(&[&"added"]).inc(); + } + + if self + .sinks + .insert(subs_id, SubscriberSink::new(subs_id, keys, child_keys)) + .is_some() + { + log::warn!("The `subscribe`-method has been passed a non-unique subs_id (in `sc-client-api::notifications`)"); + } + } +} + +impl<'a, Hash, CS, CCS, CCSI> Dispatch<(&'a Hash, CS, CCS)> for Registry +where + Hash: Clone, + CS: Iterator, Option>)>, + CCS: Iterator, CCSI)>, + CCSI: Iterator, Option>)>, +{ + type Item = StorageNotification; + type Ret = (); + + fn dispatch(&mut self, message: (&'a Hash, CS, CCS), dispatch: F) -> Self::Ret + where + F: FnMut(&SubscriberId, Self::Item), + { + let (hash, changeset, child_changeset) = message; + self.trigger(hash, changeset, child_changeset, dispatch); + } +} + +impl Registry { + pub(super) fn trigger( + &mut self, + hash: &Hash, + changeset: impl Iterator, Option>)>, + child_changeset: impl Iterator< + Item = (Vec, impl Iterator, Option>)>), + >, + mut dispatch: F, + ) where + Hash: Clone, + F: FnMut(&SubscriberId, StorageNotification), + { + let has_wildcard = !self.wildcard_listeners.is_empty(); + + // early exit if no listeners + if !has_wildcard && self.listeners.is_empty() && self.child_listeners.is_empty() { + return + } + + let mut subscribers = self.wildcard_listeners.clone(); + let mut changes = Vec::new(); + let mut child_changes = Vec::new(); + + // Collect subscribers and changes + for (k, v) in changeset { + let k = StorageKey(k); + let listeners = self.listeners.get(&k); + + if let Some(ref listeners) = listeners { + subscribers.extend(listeners.iter()); + } + + if has_wildcard || listeners.is_some() { + changes.push((k, v.map(StorageData))); + } + } + for (sk, changeset) in child_changeset { + let sk = StorageKey(sk); + if let Some((cl, cw)) = self.child_listeners.get(&sk) { + let mut changes = Vec::new(); + for (k, v) in changeset { + let k = StorageKey(k); + let listeners = cl.get(&k); + + if let Some(ref listeners) = listeners { + subscribers.extend(listeners.iter()); + } + + subscribers.extend(cw.iter()); + + if !cw.is_empty() || listeners.is_some() { + changes.push((k, v.map(StorageData))); + } + } + if !changes.is_empty() { + child_changes.push((sk, changes)); + } + } + } + + // Don't send empty notifications + if changes.is_empty() && child_changes.is_empty() { + return + } + + let changes = Arc::<[_]>::from(changes); + let child_changes = Arc::<[_]>::from(child_changes); + + // Trigger the events + self.sinks.iter_mut().for_each(|(subs_id, sink)| { + if subscribers.contains(subs_id) { + sink.was_triggered = true; + + let storage_change_set = StorageChangeSet { + changes: changes.clone(), + child_changes: child_changes.clone(), + filter: sink.keys.clone(), + child_filters: sink.child_keys.clone(), + }; + + let notification = + StorageNotification { block: hash.clone(), changes: storage_change_set }; + + dispatch(subs_id, notification); + } + }); + } +} + +impl Registry { + fn remove_subscriber(&mut self, subscriber: SubscriberId) -> Option<(Keys, ChildKeys)> { + let sink = self.sinks.remove(&subscriber)?; + + Self::remove_subscriber_from( + subscriber, + &sink.keys, + &mut self.listeners, + &mut self.wildcard_listeners, + ); + if let Some(child_filters) = &sink.child_keys { + for (c_key, filters) in child_filters { + if let Some((listeners, wildcards)) = self.child_listeners.get_mut(&c_key) { + Self::remove_subscriber_from( + subscriber, + &filters, + &mut *listeners, + &mut *wildcards, + ); + + if listeners.is_empty() && wildcards.is_empty() { + self.child_listeners.remove(&c_key); + } + } + } + } + if let Some(m) = self.metrics.as_ref() { + m.with_label_values(&[&"removed"]).inc(); + } + + Some((sink.keys.clone(), sink.child_keys.clone())) + } + + fn remove_subscriber_from( + subscriber: SubscriberId, + filters: &Keys, + listeners: &mut HashMap>, + wildcards: &mut FnvHashSet, + ) { + match filters { + None => { + wildcards.remove(&subscriber); + }, + Some(filters) => + for key in filters.iter() { + let remove_key = match listeners.get_mut(key) { + Some(ref mut set) => { + set.remove(&subscriber); + set.is_empty() + }, + None => false, + }; + + if remove_key { + listeners.remove(key); + } + }, + } + } + + fn listen_from( + current_id: SubscriberId, + filter_keys: Option>, + listeners: &mut HashMap>, + wildcards: &mut FnvHashSet, + ) -> Keys { + match filter_keys { + None => { + wildcards.insert(current_id); + None + }, + Some(keys) => Some( + keys.as_ref() + .iter() + .map(|key| { + listeners.entry(key.clone()).or_default().insert(current_id); + key.clone() + }) + .collect(), + ), + } + } +} + +pub(super) struct PrintKeys<'a>(pub &'a Keys); +impl<'a> std::fmt::Debug for PrintKeys<'a> { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + if let Some(keys) = self.0 { + fmt.debug_list().entries(keys.iter().map(HexDisplay::from)).finish() + } else { + write!(fmt, "None") + } + } +} + +pub(super) struct PrintChildKeys<'a>(pub &'a ChildKeys); +impl<'a> std::fmt::Debug for PrintChildKeys<'a> { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + if let Some(map) = self.0 { + fmt.debug_map() + .entries(map.iter().map(|(key, values)| (HexDisplay::from(key), PrintKeys(values)))) + .finish() + } else { + write!(fmt, "None") + } + } +} diff --git a/client/api/src/notifications/tests.rs b/client/api/src/notifications/tests.rs new file mode 100644 index 000000000000..2c728de7428d --- /dev/null +++ b/client/api/src/notifications/tests.rs @@ -0,0 +1,221 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use super::*; + +use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper, H256 as Hash}; +use std::iter::{empty, Empty}; + +type TestChangeSet = ( + Vec<(StorageKey, Option)>, + Vec<(StorageKey, Vec<(StorageKey, Option)>)>, +); + +impl From for StorageChangeSet { + fn from(changes: TestChangeSet) -> Self { + // warning hardcoded child trie wildcard to test upon + let child_filters = Some( + [(StorageKey(vec![4]), None), (StorageKey(vec![5]), None)] + .iter() + .cloned() + .collect(), + ); + StorageChangeSet { + changes: From::from(changes.0), + child_changes: From::from(changes.1), + filter: None, + child_filters, + } + } +} + +impl PartialEq for StorageChangeSet { + fn eq(&self, other: &Self) -> bool { + self.iter().eq(other.iter()) + } +} + +type Block = RawBlock>; + +#[test] +fn triggering_change_should_notify_wildcard_listeners() { + // given + let notifications = StorageNotifications::::new(None); + let child_filter = [(StorageKey(vec![4]), None)]; + let mut recv = + futures::executor::block_on_stream(notifications.listen(None, Some(&child_filter[..]))); + + // when + let changeset = vec![(vec![2], Some(vec![3])), (vec![3], None)]; + let c_changeset_1 = vec![(vec![5], Some(vec![4])), (vec![6], None)]; + let c_changeset = vec![(vec![4], c_changeset_1)]; + notifications.trigger( + &Hash::from_low_u64_be(1), + changeset.into_iter(), + c_changeset.into_iter().map(|(a, b)| (a, b.into_iter())), + ); + + // then + assert_eq!( + recv.next().map(StorageNotification::into_fields).unwrap(), + ( + Hash::from_low_u64_be(1), + ( + vec![ + (StorageKey(vec![2]), Some(StorageData(vec![3]))), + (StorageKey(vec![3]), None), + ], + vec![( + StorageKey(vec![4]), + vec![ + (StorageKey(vec![5]), Some(StorageData(vec![4]))), + (StorageKey(vec![6]), None), + ] + )] + ) + .into() + ) + ); +} + +#[test] +fn should_only_notify_interested_listeners() { + // given + let notifications = StorageNotifications::::new(None); + let child_filter = [(StorageKey(vec![4]), Some(vec![StorageKey(vec![5])]))]; + let mut recv1 = futures::executor::block_on_stream( + notifications.listen(Some(&[StorageKey(vec![1])]), None), + ); + let mut recv2 = futures::executor::block_on_stream( + notifications.listen(Some(&[StorageKey(vec![2])]), None), + ); + let mut recv3 = + futures::executor::block_on_stream(notifications.listen(Some(&[]), Some(&child_filter))); + + // when + let changeset = vec![(vec![2], Some(vec![3])), (vec![1], None)]; + let c_changeset_1 = vec![(vec![5], Some(vec![4])), (vec![6], None)]; + + let c_changeset = vec![(vec![4], c_changeset_1)]; + notifications.trigger( + &Hash::from_low_u64_be(1), + changeset.into_iter(), + c_changeset.into_iter().map(|(a, b)| (a, b.into_iter())), + ); + + // then + assert_eq!( + recv1.next().map(StorageNotification::into_fields).unwrap(), + (Hash::from_low_u64_be(1), (vec![(StorageKey(vec![1]), None),], vec![]).into()) + ); + assert_eq!( + recv2.next().map(StorageNotification::into_fields).unwrap(), + ( + Hash::from_low_u64_be(1), + (vec![(StorageKey(vec![2]), Some(StorageData(vec![3]))),], vec![]).into() + ) + ); + assert_eq!( + recv3.next().map(StorageNotification::into_fields).unwrap(), + ( + Hash::from_low_u64_be(1), + ( + vec![], + vec![( + StorageKey(vec![4]), + vec![(StorageKey(vec![5]), Some(StorageData(vec![4])))] + ),] + ) + .into() + ) + ); +} + +#[test] +fn should_cleanup_subscribers_if_dropped() { + // given + let notifications = StorageNotifications::::new(None); + { + let child_filter = [(StorageKey(vec![4]), Some(vec![StorageKey(vec![5])]))]; + let _recv1 = futures::executor::block_on_stream( + notifications.listen(Some(&[StorageKey(vec![1])]), None), + ); + let _recv2 = futures::executor::block_on_stream( + notifications.listen(Some(&[StorageKey(vec![2])]), None), + ); + let _recv3 = futures::executor::block_on_stream(notifications.listen(None, None)); + let _recv4 = + futures::executor::block_on_stream(notifications.listen(None, Some(&child_filter))); + assert_eq!(notifications.map_registry(|r| r.listeners.len()), 2); + assert_eq!(notifications.map_registry(|r| r.wildcard_listeners.len()), 2); + assert_eq!(notifications.map_registry(|r| r.child_listeners.len()), 1); + } + + // when + let changeset = vec![(vec![2], Some(vec![3])), (vec![1], None)]; + let c_changeset = empty::<(_, Empty<_>)>(); + notifications.trigger(&Hash::from_low_u64_be(1), changeset.into_iter(), c_changeset); + + // then + assert_eq!(notifications.map_registry(|r| r.listeners.len()), 0); + assert_eq!(notifications.map_registry(|r| r.wildcard_listeners.len()), 0); + assert_eq!(notifications.map_registry(|r| r.child_listeners.len()), 0); +} + +#[test] +fn should_cleanup_subscriber_if_stream_is_dropped() { + let notifications = StorageNotifications::::new(None); + let stream = notifications.listen(None, None); + assert_eq!(notifications.map_registry(|r| r.sinks.len()), 1); + std::mem::drop(stream); + assert_eq!(notifications.map_registry(|r| r.sinks.len()), 0); +} + +#[test] +fn should_not_send_empty_notifications() { + // given + let mut recv = { + let notifications = StorageNotifications::::new(None); + let recv = futures::executor::block_on_stream(notifications.listen(None, None)); + + // when + let changeset = vec![]; + let c_changeset = empty::<(_, Empty<_>)>(); + notifications.trigger(&Hash::from_low_u64_be(1), changeset.into_iter(), c_changeset); + recv + }; + + // then + assert_eq!(recv.next().map(StorageNotification::into_fields), None); +} + +impl StorageNotifications { + fn map_registry(&self, map: MapF) -> Ret + where + MapF: FnOnce(&Registry) -> Ret, + { + self.0.map_registry_for_tests(map) + } +} + +impl StorageNotification { + fn into_fields(self) -> (H, StorageChangeSet) { + let Self { block, changes } = self; + (block, changes) + } +} diff --git a/client/api/src/proof_provider.rs b/client/api/src/proof_provider.rs index 79444f006923..3aad4af1befb 100644 --- a/client/api/src/proof_provider.rs +++ b/client/api/src/proof_provider.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -17,9 +17,10 @@ // along with this program. If not, see . //! Proof utilities -use crate::{ChangesProof, StorageProof}; +use crate::{CompactProof, StorageProof}; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; -use sp_storage::{ChildInfo, PrefixedStorageKey, StorageKey}; +use sp_state_machine::{KeyValueStates, KeyValueStorageLevel}; +use sp_storage::ChildInfo; /// Interface for providing block proving utilities. pub trait ProofProvider { @@ -49,53 +50,44 @@ pub trait ProofProvider { method: &str, call_data: &[u8], ) -> sp_blockchain::Result<(Vec, StorageProof)>; - /// Reads given header and generates CHT-based header proof. - fn header_proof( - &self, - id: &BlockId, - ) -> sp_blockchain::Result<(Block::Header, StorageProof)>; - - /// Get proof for computation of (block, extrinsic) pairs where key has been changed at given - /// blocks range. `min` is the hash of the first block, which changes trie root is known to the - /// requester - when we're using changes tries from ascendants of this block, we should provide - /// proofs for changes tries roots `max` is the hash of the last block known to the requester - - /// we can't use changes tries from descendants of this block. - /// Works only for runtimes that are supporting changes tries. - fn key_changes_proof( - &self, - first: Block::Hash, - last: Block::Hash, - min: Block::Hash, - max: Block::Hash, - storage_key: Option<&PrefixedStorageKey>, - key: &StorageKey, - ) -> sp_blockchain::Result>; - /// Given a `BlockId` iterate over all storage values starting at `start_key` exclusively, - /// building proofs until size limit is reached. Returns combined proof and the number of - /// collected keys. + /// Given a `BlockId` iterate over all storage values starting at `start_keys`. + /// Last `start_keys` element contains last accessed key value. + /// With multiple `start_keys`, first `start_keys` element is + /// the current storage key of of the last accessed child trie. + /// at last level the value to start at exclusively. + /// Proofs is build until size limit is reached and always include at + /// least one key following `start_keys`. + /// Returns combined proof and the numbers of collected keys. fn read_proof_collection( &self, id: &BlockId, - start_key: &[u8], + start_keys: &[Vec], size_limit: usize, - ) -> sp_blockchain::Result<(StorageProof, u32)>; + ) -> sp_blockchain::Result<(CompactProof, u32)>; /// Given a `BlockId` iterate over all storage values starting at `start_key`. /// Returns collected keys and values. + /// Returns the collected keys values content of the top trie followed by the + /// collected keys values of child tries. + /// Only child tries with their root part of the collected content or + /// related to `start_key` are attached. + /// For each collected state a boolean indicates if state reach + /// end. fn storage_collection( &self, id: &BlockId, - start_key: &[u8], + start_key: &[Vec], size_limit: usize, - ) -> sp_blockchain::Result, Vec)>>; + ) -> sp_blockchain::Result>; /// Verify read storage proof for a set of keys. - /// Returns collected key-value pairs and a flag indicating if iteration is complete. + /// Returns collected key-value pairs and a the nested state + /// depth of current iteration or 0 if completed. fn verify_range_proof( &self, root: Block::Hash, - proof: StorageProof, - start_key: &[u8], - ) -> sp_blockchain::Result<(Vec<(Vec, Vec)>, bool)>; + proof: CompactProof, + start_keys: &[Vec], + ) -> sp_blockchain::Result<(KeyValueStates, usize)>; } diff --git a/client/authority-discovery/Cargo.toml b/client/authority-discovery/Cargo.toml index 8d5ed20730f0..ba432d073698 100644 --- a/client/authority-discovery/Cargo.toml +++ b/client/authority-discovery/Cargo.toml @@ -2,10 +2,10 @@ name = "sc-authority-discovery" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" build = "build.rs" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate authority discovery." readme = "README.md" @@ -14,30 +14,30 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [build-dependencies] -prost-build = "0.8" +prost-build = "0.9" [dependencies] async-trait = "0.1" -codec = { package = "parity-scale-codec", default-features = false, version = "2.0.0" } -derive_more = "0.99.2" -futures = "0.3.9" +codec = { package = "parity-scale-codec", default-features = false, version = "3.0.0" } +thiserror = "1.0" +futures = "0.3.21" futures-timer = "3.0.1" -ip_network = "0.4.0" -libp2p = { version = "0.39.1", default-features = false, features = ["kad"] } +ip_network = "0.4.1" +libp2p = { version = "0.40.0", default-features = false, features = ["kad"] } log = "0.4.8" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.9.0" } -prost = "0.8" +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.10.0-dev" } +prost = "0.9" rand = "0.7.2" sc-client-api = { version = "4.0.0-dev", path = "../api" } sc-network = { version = "0.10.0-dev", path = "../network" } sp-authority-discovery = { version = "4.0.0-dev", path = "../../primitives/authority-discovery" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-keystore = { version = "0.12.0", path = "../../primitives/keystore" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } [dev-dependencies] quickcheck = "1.0.3" -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } diff --git a/client/authority-discovery/README.md b/client/authority-discovery/README.md index 54c51d5ba04f..042e8f5982cd 100644 --- a/client/authority-discovery/README.md +++ b/client/authority-discovery/README.md @@ -1,4 +1,4 @@ -Substrate authority discovery. +# Substrate authority discovery This crate enables Substrate authorities to discover and directly connect to other authorities. It is split into two components the [`Worker`] and the @@ -6,4 +6,4 @@ other authorities. It is split into two components the [`Worker`] and the See [`Worker`] and [`Service`] for more documentation. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/client/authority-discovery/build.rs b/client/authority-discovery/build.rs index c44fe8578ba2..00d45f07ae15 100644 --- a/client/authority-discovery/build.rs +++ b/client/authority-discovery/build.rs @@ -1,3 +1,7 @@ fn main() { - prost_build::compile_protos(&["src/worker/schema/dht.proto"], &["src/worker/schema"]).unwrap(); + prost_build::compile_protos( + &["src/worker/schema/dht-v1.proto", "src/worker/schema/dht-v2.proto"], + &["src/worker/schema"], + ) + .unwrap(); } diff --git a/client/authority-discovery/src/error.rs b/client/authority-discovery/src/error.rs index b271f7b9d62b..bce39069ef7c 100644 --- a/client/authority-discovery/src/error.rs +++ b/client/authority-discovery/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -24,32 +24,56 @@ use sp_core::crypto::CryptoTypePublicPair; pub type Result = std::result::Result; /// Error type for the authority discovery module. -#[derive(Debug, derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] pub enum Error { - /// Received dht value found event with records with different keys. + #[error("Received dht value found event with records with different keys.")] ReceivingDhtValueFoundEventWithDifferentKeys, - /// Received dht value found event with no records. + + #[error("Received dht value found event with no records.")] ReceivingDhtValueFoundEventWithNoRecords, - /// Failed to verify a dht payload with the given signature. + + #[error("Failed to verify a dht payload with the given signature.")] VerifyingDhtPayload, - /// Failed to hash the authority id to be used as a dht key. - HashingAuthorityId(libp2p::core::multiaddr::multihash::Error), - /// Failed calling into the Substrate runtime. - CallingRuntime(sp_blockchain::Error), - /// Received a dht record with a key that does not match any in-flight awaited keys. + + #[error("Failed to hash the authority id to be used as a dht key.")] + HashingAuthorityId(#[from] libp2p::core::multiaddr::multihash::Error), + + #[error("Failed calling into the Substrate runtime: {0}")] + CallingRuntime(#[from] sp_blockchain::Error), + + #[error("Received a dht record with a key that does not match any in-flight awaited keys.")] ReceivingUnexpectedRecord, - /// Failed to encode a protobuf payload. - EncodingProto(prost::EncodeError), - /// Failed to decode a protobuf payload. - DecodingProto(prost::DecodeError), - /// Failed to encode or decode scale payload. - EncodingDecodingScale(codec::Error), - /// Failed to parse a libp2p multi address. - ParsingMultiaddress(libp2p::core::multiaddr::Error), - /// Failed to sign using a specific public key. + + #[error("Failed to encode a protobuf payload.")] + EncodingProto(#[from] prost::EncodeError), + + #[error("Failed to decode a protobuf payload.")] + DecodingProto(#[from] prost::DecodeError), + + #[error("Failed to encode or decode scale payload.")] + EncodingDecodingScale(#[from] codec::Error), + + #[error("Failed to parse a libp2p multi address.")] + ParsingMultiaddress(#[from] libp2p::core::multiaddr::Error), + + #[error("Failed to parse a libp2p key.")] + ParsingLibp2pIdentity(#[from] sc_network::DecodingError), + + #[error("Failed to sign using a specific public key.")] MissingSignature(CryptoTypePublicPair), - /// Failed to sign using all public keys. + + #[error("Failed to sign using all public keys.")] Signing, - /// Failed to register Prometheus metric. - Prometheus(prometheus_endpoint::PrometheusError), + + #[error("Failed to register Prometheus metric.")] + Prometheus(#[from] prometheus_endpoint::PrometheusError), + + #[error("Received authority record that contains addresses with multiple peer ids")] + ReceivingDhtValueFoundEventWithDifferentPeerIds, + + #[error("Received authority record without any addresses having a peer id")] + ReceivingDhtValueFoundEventWithNoPeerIds, + + #[error("Received authority record without a valid signature for the remote peer id.")] + MissingPeerIdSignature, } diff --git a/client/authority-discovery/src/interval.rs b/client/authority-discovery/src/interval.rs index f4e7c43e60d2..5ddf81fbccc3 100644 --- a/client/authority-discovery/src/interval.rs +++ b/client/authority-discovery/src/interval.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/authority-discovery/src/lib.rs b/client/authority-discovery/src/lib.rs index 800f683aa0ae..8522da9984a6 100644 --- a/client/authority-discovery/src/lib.rs +++ b/client/authority-discovery/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,6 +18,7 @@ #![warn(missing_docs)] #![recursion_limit = "1024"] + //! Substrate authority discovery. //! //! This crate enables Substrate authorities to discover and directly connect to @@ -31,7 +32,7 @@ pub use crate::{ worker::{NetworkProvider, Role, Worker}, }; -use std::{sync::Arc, time::Duration}; +use std::{collections::HashSet, sync::Arc, time::Duration}; use futures::{ channel::{mpsc, oneshot}, @@ -58,11 +59,13 @@ pub struct WorkerConfig { /// /// By default this is set to 1 hour. pub max_publish_interval: Duration, + /// Interval at which the keystore is queried. If the keys have changed, unconditionally /// re-publish its addresses on the DHT. /// /// By default this is set to 1 minute. pub keystore_refresh_interval: Duration, + /// The maximum interval in which the node will query the DHT for new entries. /// /// By default this is set to 10 minutes. @@ -75,6 +78,11 @@ pub struct WorkerConfig { /// /// Defaults to `true` to avoid the surprise factor. pub publish_non_global_ips: bool, + + /// Reject authority discovery records that are not signed by their network identity (PeerId) + /// + /// Defaults to `false` to provide compatibility with old versions + pub strict_record_validation: bool, } impl Default for WorkerConfig { @@ -95,6 +103,7 @@ impl Default for WorkerConfig { // `authority_discovery_dht_event_received`. max_query_interval: Duration::from_secs(10 * 60), publish_non_global_ips: true, + strict_record_validation: false, } } } @@ -156,7 +165,7 @@ where /// Message send from the [`Service`] to the [`Worker`]. pub(crate) enum ServicetoWorkerMsg { /// See [`Service::get_addresses_by_authority_id`]. - GetAddressesByAuthorityId(AuthorityId, oneshot::Sender>>), - /// See [`Service::get_authority_id_by_peer_id`]. - GetAuthorityIdByPeerId(PeerId, oneshot::Sender>), + GetAddressesByAuthorityId(AuthorityId, oneshot::Sender>>), + /// See [`Service::get_authority_ids_by_peer_id`]. + GetAuthorityIdsByPeerId(PeerId, oneshot::Sender>>), } diff --git a/client/authority-discovery/src/service.rs b/client/authority-discovery/src/service.rs index 2e5ae66e4dd4..c240e5d0c228 100644 --- a/client/authority-discovery/src/service.rs +++ b/client/authority-discovery/src/service.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,7 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use std::fmt::Debug; +use std::{collections::HashSet, fmt::Debug}; use crate::ServicetoWorkerMsg; @@ -62,7 +62,7 @@ impl Service { pub async fn get_addresses_by_authority_id( &mut self, authority: AuthorityId, - ) -> Option> { + ) -> Option> { let (tx, rx) = oneshot::channel(); self.to_worker @@ -78,11 +78,14 @@ impl Service { /// /// Returns `None` if no entry was present or connection to the /// [`crate::Worker`] failed. - pub async fn get_authority_id_by_peer_id(&mut self, peer_id: PeerId) -> Option { + pub async fn get_authority_ids_by_peer_id( + &mut self, + peer_id: PeerId, + ) -> Option> { let (tx, rx) = oneshot::channel(); self.to_worker - .send(ServicetoWorkerMsg::GetAuthorityIdByPeerId(peer_id, tx)) + .send(ServicetoWorkerMsg::GetAuthorityIdsByPeerId(peer_id, tx)) .await .ok()?; diff --git a/client/authority-discovery/src/tests.rs b/client/authority-discovery/src/tests.rs index 3784b4c83426..e9f94b6a186d 100644 --- a/client/authority-discovery/src/tests.rs +++ b/client/authority-discovery/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -29,7 +29,7 @@ use libp2p::core::{ multiaddr::{Multiaddr, Protocol}, PeerId, }; -use std::sync::Arc; +use std::{collections::HashSet, sync::Arc}; use sp_authority_discovery::AuthorityId; use sp_core::crypto::key_types; @@ -73,12 +73,41 @@ fn get_addresses_and_authority_id() { pool.run_until(async { assert_eq!( - Some(vec![remote_addr]), + Some(HashSet::from([remote_addr])), service.get_addresses_by_authority_id(remote_authority_id.clone()).await, ); assert_eq!( - Some(remote_authority_id), - service.get_authority_id_by_peer_id(remote_peer_id).await, + Some(HashSet::from([remote_authority_id])), + service.get_authority_ids_by_peer_id(remote_peer_id).await, ); }); } + +#[test] +fn cryptos_are_compatible() { + use sp_core::crypto::Pair; + + let libp2p_secret = sc_network::Keypair::generate_ed25519(); + let libp2p_public = libp2p_secret.public(); + + let sp_core_secret = { + let libp2p_ed_secret = match libp2p_secret.clone() { + sc_network::Keypair::Ed25519(x) => x, + _ => panic!("generate_ed25519 should have generated an Ed25519 key ¯\\_(ツ)_/¯"), + }; + sp_core::ed25519::Pair::from_seed_slice(&libp2p_ed_secret.secret().as_ref()).unwrap() + }; + let sp_core_public = sp_core_secret.public(); + + let message = b"we are more powerful than not to be better"; + + let libp2p_signature = libp2p_secret.sign(message).unwrap(); + let sp_core_signature = sp_core_secret.sign(message); // no error expected... + + assert!(sp_core::ed25519::Pair::verify( + &sp_core::ed25519::Signature::from_slice(&libp2p_signature).unwrap(), + message, + &sp_core_public + )); + assert!(libp2p_public.verify(message, sp_core_signature.as_ref())); +} diff --git a/client/authority-discovery/src/worker.rs b/client/authority-discovery/src/worker.rs index a689d0bafd26..726c032281cd 100644 --- a/client/authority-discovery/src/worker.rs +++ b/client/authority-discovery/src/worker.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,12 +19,11 @@ use crate::{ error::{Error, Result}, interval::ExpIncInterval, - ServicetoWorkerMsg, + ServicetoWorkerMsg, WorkerConfig, }; use std::{ collections::{HashMap, HashSet}, - convert::TryInto, marker::PhantomData, sync::Arc, time::Duration, @@ -57,7 +56,10 @@ use sp_runtime::{generic::BlockId, traits::Block as BlockT}; mod addr_cache; /// Dht payload schemas generated from Protobuf definitions via Prost crate in build.rs. mod schema { - include!(concat!(env!("OUT_DIR"), "/authority_discovery.rs")); + #[cfg(test)] + mod tests; + + include!(concat!(env!("OUT_DIR"), "/authority_discovery_v2.rs")); } #[cfg(test)] pub mod tests; @@ -111,6 +113,7 @@ pub struct Worker { client: Arc, network: Arc, + /// Channel we receive Dht events on. dht_event_rx: DhtEventStream, @@ -124,6 +127,8 @@ pub struct Worker { latest_published_keys: HashSet, /// Same value as in the configuration. publish_non_global_ips: bool, + /// Same value as in the configuration. + strict_record_validation: bool, /// Interval at which to request addresses of authorities, refilling the pending lookups queue. query_interval: ExpIncInterval, @@ -131,7 +136,7 @@ pub struct Worker { /// Queue of throttled lookups pending to be passed to the network. pending_lookups: Vec, /// Set of in-flight lookups. - in_flight_lookups: HashMap, + in_flight_lookups: HashMap, addr_cache: addr_cache::AddrCache, @@ -158,7 +163,7 @@ where dht_event_rx: DhtEventStream, role: Role, prometheus_registry: Option, - config: crate::WorkerConfig, + config: WorkerConfig, ) -> Self { // When a node starts up publishing and querying might fail due to various reasons, for // example due to being not yet fully bootstrapped on the DHT. Thus one should retry rather @@ -181,7 +186,7 @@ where Some(registry) => match Metrics::register(®istry) { Ok(metrics) => Some(metrics), Err(e) => { - error!(target: LOG_TARGET, "Failed to register metrics: {:?}", e); + error!(target: LOG_TARGET, "Failed to register metrics: {}", e); None }, }, @@ -197,6 +202,7 @@ where publish_if_changed_interval, latest_published_keys: HashSet::new(), publish_non_global_ips: config.publish_non_global_ips, + strict_record_validation: config.strict_record_validation, query_interval, pending_lookups: Vec::new(), in_flight_lookups: HashMap::new(), @@ -235,7 +241,7 @@ where if let Err(e) = self.publish_ext_addresses(only_if_changed).await { error!( target: LOG_TARGET, - "Failed to publish external addresses: {:?}", e, + "Failed to publish external addresses: {}", e, ); } }, @@ -244,7 +250,7 @@ where if let Err(e) = self.refill_pending_lookups_queue().await { error!( target: LOG_TARGET, - "Failed to request addresses of authorities: {:?}", e, + "Failed to request addresses of authorities: {}", e, ); } }, @@ -259,9 +265,9 @@ where self.addr_cache.get_addresses_by_authority_id(&authority).map(Clone::clone), ); }, - ServicetoWorkerMsg::GetAuthorityIdByPeerId(peer_id, sender) => { + ServicetoWorkerMsg::GetAuthorityIdsByPeerId(peer_id, sender) => { let _ = sender - .send(self.addr_cache.get_authority_id_by_peer_id(&peer_id).map(Clone::clone)); + .send(self.addr_cache.get_authority_ids_by_peer_id(&peer_id).map(Clone::clone)); }, } } @@ -313,7 +319,7 @@ where return Ok(()) } - let addresses = self.addresses_to_publish().map(|a| a.to_vec()).collect::>(); + let addresses = serialize_addresses(self.addresses_to_publish()); if let Some(metrics) = &self.metrics { metrics.publish.inc(); @@ -322,32 +328,21 @@ where .set(addresses.len().try_into().unwrap_or(std::u64::MAX)); } - let mut serialized_addresses = vec![]; - schema::AuthorityAddresses { addresses } - .encode(&mut serialized_addresses) - .map_err(Error::EncodingProto)?; + let serialized_record = serialize_authority_record(addresses)?; + let peer_signature = sign_record_with_peer_id(&serialized_record, self.network.as_ref())?; let keys_vec = keys.iter().cloned().collect::>(); - let signatures = key_store - .sign_with_all( - key_types::AUTHORITY_DISCOVERY, - keys_vec.clone(), - serialized_addresses.as_slice(), - ) - .await - .map_err(|_| Error::Signing)?; - - for (sign_result, key) in signatures.into_iter().zip(keys_vec.iter()) { - let mut signed_addresses = vec![]; - // Verify that all signatures exist for all provided keys. - let signature = - sign_result.ok().flatten().ok_or_else(|| Error::MissingSignature(key.clone()))?; - schema::SignedAuthorityAddresses { addresses: serialized_addresses.clone(), signature } - .encode(&mut signed_addresses) - .map_err(Error::EncodingProto)?; + let kv_pairs = sign_record_with_authority_ids( + serialized_record, + Some(peer_signature), + key_store.as_ref(), + keys_vec, + ) + .await?; - self.network.put_value(hash_authority_id(key.1.as_ref()), signed_addresses); + for (key, value) in kv_pairs.into_iter() { + self.network.put_value(key, value); } self.latest_published_keys = keys; @@ -374,7 +369,7 @@ where .map_err(|e| Error::CallingRuntime(e.into()))? .into_iter() .filter(|id| !local_keys.contains(id.as_ref())) - .collect(); + .collect::>(); self.addr_cache.retain_ids(&authorities); @@ -430,7 +425,7 @@ where metrics.handle_value_found_event_failure.inc(); } - debug!(target: LOG_TARGET, "Failed to handle Dht value found event: {:?}", e); + debug!(target: LOG_TARGET, "Failed to handle Dht value found event: {}", e); } }, DhtEvent::ValueNotFound(hash) => { @@ -471,18 +466,11 @@ where fn handle_dht_value_found_event( &mut self, - values: Vec<(libp2p::kad::record::Key, Vec)>, + values: Vec<(sc_network::KademliaKey, Vec)>, ) -> Result<()> { // Ensure `values` is not empty and all its keys equal. - let remote_key = values - .iter() - .fold(Ok(None), |acc, (key, _)| match acc { - Ok(None) => Ok(Some(key.clone())), - Ok(Some(ref prev_key)) if prev_key != key => - Err(Error::ReceivingDhtValueFoundEventWithDifferentKeys), - x @ Ok(_) => x, - Err(e) => Err(e), - })? + let remote_key = single(values.iter().map(|(key, _)| key.clone())) + .map_err(|_| Error::ReceivingDhtValueFoundEventWithDifferentKeys)? .ok_or(Error::ReceivingDhtValueFoundEventWithNoRecords)?; let authority_id: AuthorityId = self @@ -495,18 +483,18 @@ where let remote_addresses: Vec = values .into_iter() .map(|(_k, v)| { - let schema::SignedAuthorityAddresses { signature, addresses } = - schema::SignedAuthorityAddresses::decode(v.as_slice()) + let schema::SignedAuthorityRecord { record, auth_signature, peer_signature } = + schema::SignedAuthorityRecord::decode(v.as_slice()) .map_err(Error::DecodingProto)?; - let signature = AuthoritySignature::decode(&mut &signature[..]) + let auth_signature = AuthoritySignature::decode(&mut &auth_signature[..]) .map_err(Error::EncodingDecodingScale)?; - if !AuthorityPair::verify(&signature, &addresses, &authority_id) { + if !AuthorityPair::verify(&auth_signature, &record, &authority_id) { return Err(Error::VerifyingDhtPayload) } - let addresses = schema::AuthorityAddresses::decode(addresses.as_slice()) + let addresses: Vec = schema::AuthorityRecord::decode(record.as_slice()) .map(|a| a.addresses) .map_err(Error::DecodingProto)? .into_iter() @@ -514,32 +502,49 @@ where .collect::>() .map_err(Error::ParsingMultiaddress)?; + let get_peer_id = |a: &Multiaddr| match a.iter().last() { + Some(multiaddr::Protocol::P2p(key)) => PeerId::from_multihash(key).ok(), + _ => None, + }; + + // Ignore [`Multiaddr`]s without [`PeerId`] or with own addresses. + let addresses: Vec = addresses + .into_iter() + .filter(|a| get_peer_id(&a).filter(|p| *p != local_peer_id).is_some()) + .collect(); + + let remote_peer_id = single(addresses.iter().map(get_peer_id)) + .map_err(|_| Error::ReceivingDhtValueFoundEventWithDifferentPeerIds)? // different peer_id in records + .flatten() + .ok_or(Error::ReceivingDhtValueFoundEventWithNoPeerIds)?; // no records with peer_id in them + + // At this point we know all the valid multiaddresses from the record, know that + // each of them belong to the same PeerId, we just need to check if the record is + // properly signed by the owner of the PeerId + + if let Some(peer_signature) = peer_signature { + let public_key = + sc_network::PublicKey::from_protobuf_encoding(&peer_signature.public_key) + .map_err(|e| Error::ParsingLibp2pIdentity(e))?; + let signature = + sc_network::Signature { public_key, bytes: peer_signature.signature }; + + if !signature.verify(record, &remote_peer_id) { + return Err(Error::VerifyingDhtPayload) + } + } else if self.strict_record_validation { + return Err(Error::MissingPeerIdSignature) + } else { + debug!( + target: LOG_TARGET, + "Received unsigned authority discovery record from {}", authority_id + ); + } Ok(addresses) }) .collect::>>>()? .into_iter() .flatten() - // Ignore [`Multiaddr`]s without [`PeerId`] and own addresses. - .filter(|addr| { - addr.iter().any(|protocol| { - // Parse to PeerId first as Multihashes of old and new PeerId - // representation don't equal. - // - // See https://github.com/libp2p/rust-libp2p/issues/555 for - // details. - if let multiaddr::Protocol::P2p(hash) = protocol { - let peer_id = match PeerId::from_multihash(hash) { - Ok(peer_id) => peer_id, - Err(_) => return false, // Discard address. - }; - - // Discard if equal to local peer id, keep if it differs. - return !(peer_id == local_peer_id) - } - - false // `protocol` is not a [`Protocol::P2p`], let's keep looking. - }) - }) .take(MAX_ADDRESSES_PER_AUTHORITY) .collect(); @@ -548,7 +553,7 @@ where if let Some(metrics) = &self.metrics { metrics .known_authorities_count - .set(self.addr_cache.num_ids().try_into().unwrap_or(std::u64::MAX)); + .set(self.addr_cache.num_authority_ids().try_into().unwrap_or(std::u64::MAX)); } } Ok(()) @@ -575,29 +580,47 @@ where .authorities(&id) .map_err(|e| Error::CallingRuntime(e.into()))? .into_iter() - .map(std::convert::Into::into) + .map(Into::into) .collect::>(); - let intersection = local_pub_keys - .intersection(&authorities) - .cloned() - .map(std::convert::Into::into) - .collect(); + let intersection = + local_pub_keys.intersection(&authorities).cloned().map(Into::into).collect(); Ok(intersection) } } +pub trait NetworkSigner { + /// Sign a message in the name of `self.local_peer_id()` + fn sign_with_local_identity( + &self, + msg: impl AsRef<[u8]>, + ) -> std::result::Result; +} + /// NetworkProvider provides [`Worker`] with all necessary hooks into the /// underlying Substrate networking. Using this trait abstraction instead of /// [`sc_network::NetworkService`] directly is necessary to unit test [`Worker`]. #[async_trait] -pub trait NetworkProvider: NetworkStateInfo { +pub trait NetworkProvider: NetworkStateInfo + NetworkSigner { /// Start putting a value in the Dht. - fn put_value(&self, key: libp2p::kad::record::Key, value: Vec); + fn put_value(&self, key: sc_network::KademliaKey, value: Vec); /// Start getting a value from the Dht. - fn get_value(&self, key: &libp2p::kad::record::Key); + fn get_value(&self, key: &sc_network::KademliaKey); +} + +impl NetworkSigner for sc_network::NetworkService +where + B: BlockT + 'static, + H: ExHashT, +{ + fn sign_with_local_identity( + &self, + msg: impl AsRef<[u8]>, + ) -> std::result::Result { + self.sign_with_local_identity(msg) + } } #[async_trait::async_trait] @@ -606,16 +629,87 @@ where B: BlockT + 'static, H: ExHashT, { - fn put_value(&self, key: libp2p::kad::record::Key, value: Vec) { + fn put_value(&self, key: sc_network::KademliaKey, value: Vec) { self.put_value(key, value) } - fn get_value(&self, key: &libp2p::kad::record::Key) { + fn get_value(&self, key: &sc_network::KademliaKey) { self.get_value(key) } } -fn hash_authority_id(id: &[u8]) -> libp2p::kad::record::Key { - libp2p::kad::record::Key::new(&libp2p::multihash::Sha2_256::digest(id)) +fn hash_authority_id(id: &[u8]) -> sc_network::KademliaKey { + sc_network::KademliaKey::new(&libp2p::multihash::Sha2_256::digest(id)) +} + +// Makes sure all values are the same and returns it +// +// Returns Err(_) if not all values are equal. Returns Ok(None) if there are +// no values. +fn single(values: impl IntoIterator) -> std::result::Result, ()> +where + T: PartialEq, +{ + values.into_iter().try_fold(None, |acc, item| match acc { + None => Ok(Some(item)), + Some(ref prev) if *prev != item => Err(()), + Some(x) => Ok(Some(x)), + }) +} + +fn serialize_addresses(addresses: impl Iterator) -> Vec> { + addresses.map(|a| a.to_vec()).collect() +} + +fn serialize_authority_record(addresses: Vec>) -> Result> { + let mut serialized_record = vec![]; + schema::AuthorityRecord { addresses } + .encode(&mut serialized_record) + .map_err(Error::EncodingProto)?; + Ok(serialized_record) +} + +fn sign_record_with_peer_id( + serialized_record: &[u8], + network: &impl NetworkSigner, +) -> Result { + let signature = network + .sign_with_local_identity(serialized_record) + .map_err(|_| Error::Signing)?; + let public_key = signature.public_key.to_protobuf_encoding(); + let signature = signature.bytes; + Ok(schema::PeerSignature { signature, public_key }) +} + +async fn sign_record_with_authority_ids( + serialized_record: Vec, + peer_signature: Option, + key_store: &dyn CryptoStore, + keys: Vec, +) -> Result)>> { + let signatures = key_store + .sign_with_all(key_types::AUTHORITY_DISCOVERY, keys.clone(), &serialized_record) + .await + .map_err(|_| Error::Signing)?; + + let mut result = vec![]; + for (sign_result, key) in signatures.into_iter().zip(keys.iter()) { + let mut signed_record = vec![]; + + // Verify that all signatures exist for all provided keys. + let auth_signature = + sign_result.ok().flatten().ok_or_else(|| Error::MissingSignature(key.clone()))?; + schema::SignedAuthorityRecord { + record: serialized_record.clone(), + auth_signature, + peer_signature: peer_signature.clone(), + } + .encode(&mut signed_record) + .map_err(Error::EncodingProto)?; + + result.push((hash_authority_id(key.1.as_ref()), signed_record)); + } + + Ok(result) } /// Prometheus metrics for a [`Worker`]. @@ -635,14 +729,14 @@ impl Metrics { Ok(Self { publish: register( Counter::new( - "authority_discovery_times_published_total", + "substrate_authority_discovery_times_published_total", "Number of times authority discovery has published external addresses.", )?, registry, )?, amount_addresses_last_published: register( Gauge::new( - "authority_discovery_amount_external_addresses_last_published", + "substrate_authority_discovery_amount_external_addresses_last_published", "Number of external addresses published when authority discovery last \ published addresses.", )?, @@ -650,7 +744,7 @@ impl Metrics { )?, requests: register( Counter::new( - "authority_discovery_authority_addresses_requested_total", + "substrate_authority_discovery_authority_addresses_requested_total", "Number of times authority discovery has requested external addresses of a \ single authority.", )?, @@ -658,7 +752,7 @@ impl Metrics { )?, requests_pending: register( Gauge::new( - "authority_discovery_authority_address_requests_pending", + "substrate_authority_discovery_authority_address_requests_pending", "Number of pending authority address requests.", )?, registry, @@ -666,7 +760,7 @@ impl Metrics { dht_event_received: register( CounterVec::new( Opts::new( - "authority_discovery_dht_event_received", + "substrate_authority_discovery_dht_event_received", "Number of dht events received by authority discovery.", ), &["name"], @@ -675,14 +769,14 @@ impl Metrics { )?, handle_value_found_event_failure: register( Counter::new( - "authority_discovery_handle_value_found_event_failure", + "substrate_authority_discovery_handle_value_found_event_failure", "Number of times handling a dht value found event failed.", )?, registry, )?, known_authorities_count: register( Gauge::new( - "authority_discovery_known_authorities_count", + "substrate_authority_discovery_known_authorities_count", "Number of authorities known by authority discovery.", )?, registry, diff --git a/client/authority-discovery/src/worker/addr_cache.rs b/client/authority-discovery/src/worker/addr_cache.rs index e770297f6f3b..3cac5a6bf034 100644 --- a/client/authority-discovery/src/worker/addr_cache.rs +++ b/client/authority-discovery/src/worker/addr_cache.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -17,79 +17,94 @@ // along with this program. If not, see . use libp2p::core::multiaddr::{Multiaddr, Protocol}; -use std::collections::HashMap; use sc_network::PeerId; use sp_authority_discovery::AuthorityId; +use std::collections::{hash_map::Entry, HashMap, HashSet}; -/// Cache for [`AuthorityId`] -> [`Vec`] and [`PeerId`] -> [`AuthorityId`] mappings. +/// Cache for [`AuthorityId`] -> [`HashSet`] and [`PeerId`] -> [`HashSet`] +/// mappings. pub(super) struct AddrCache { - // The addresses found in `authority_id_to_addresses` are guaranteed to always match - // the peerids found in `peer_id_to_authority_id`. In other words, these two hashmaps - // are similar to a bi-directional map. - authority_id_to_addresses: HashMap>, - peer_id_to_authority_id: HashMap, + /// The addresses found in `authority_id_to_addresses` are guaranteed to always match + /// the peerids found in `peer_id_to_authority_ids`. In other words, these two hashmaps + /// are similar to a bi-directional map. + /// + /// Since we may store the mapping across several sessions, a single + /// `PeerId` might correspond to multiple `AuthorityId`s. However, + /// it's not expected that a single `AuthorityId` can have multiple `PeerId`s. + authority_id_to_addresses: HashMap>, + peer_id_to_authority_ids: HashMap>, } impl AddrCache { pub fn new() -> Self { AddrCache { authority_id_to_addresses: HashMap::new(), - peer_id_to_authority_id: HashMap::new(), + peer_id_to_authority_ids: HashMap::new(), } } /// Inserts the given [`AuthorityId`] and [`Vec`] pair for future lookups by /// [`AuthorityId`] or [`PeerId`]. - pub fn insert(&mut self, authority_id: AuthorityId, mut addresses: Vec) { - addresses.sort_unstable_by(|a, b| a.as_ref().cmp(b.as_ref())); + pub fn insert(&mut self, authority_id: AuthorityId, addresses: Vec) { + let addresses = addresses.into_iter().collect::>(); + let peer_ids = addresses_to_peer_ids(&addresses); + + if peer_ids.is_empty() { + log::debug!( + target: super::LOG_TARGET, + "Authority({:?}) provides no addresses or addresses without peer ids. Adresses: {:?}", + authority_id, + addresses, + ); - // Insert into `self.peer_id_to_authority_id`. - let peer_ids = addresses - .iter() - .map(|a| peer_id_from_multiaddr(a)) - .filter_map(|peer_id| peer_id); - for peer_id in peer_ids.clone() { - let former_auth = - match self.peer_id_to_authority_id.insert(peer_id, authority_id.clone()) { - Some(a) if a != authority_id => a, - _ => continue, - }; - - // PeerId was associated to a different authority id before. - // Remove corresponding authority from `self.authority_id_to_addresses`. - let former_auth_addrs = match self.authority_id_to_addresses.get_mut(&former_auth) { - Some(a) => a, - None => { - debug_assert!(false); - continue - }, - }; - former_auth_addrs.retain(|a| peer_id_from_multiaddr(a).map_or(true, |p| p != peer_id)); + return + } else if peer_ids.len() > 1 { + log::warn!( + target: super::LOG_TARGET, + "Authority({:?}) can be reached through multiple peer ids: {:?}", + authority_id, + peer_ids + ); } - // Insert into `self.authority_id_to_addresses`. - for former_addr in self - .authority_id_to_addresses - .insert(authority_id.clone(), addresses.clone()) - .unwrap_or_default() - { - // Must remove from `self.peer_id_to_authority_id` any PeerId formerly associated - // to that authority but that can't be found in its new addresses. - - let peer_id = match peer_id_from_multiaddr(&former_addr) { - Some(p) => p, - None => continue, - }; + let old_addresses = self.authority_id_to_addresses.insert(authority_id.clone(), addresses); + let old_peer_ids = addresses_to_peer_ids(&old_addresses.unwrap_or_default()); - if !peer_ids.clone().any(|p| p == peer_id) { - self.peer_id_to_authority_id.remove(&peer_id); + // Add the new peer ids + peer_ids.difference(&old_peer_ids).for_each(|new_peer_id| { + self.peer_id_to_authority_ids + .entry(*new_peer_id) + .or_default() + .insert(authority_id.clone()); + }); + + // Remove the old peer ids + self.remove_authority_id_from_peer_ids(&authority_id, old_peer_ids.difference(&peer_ids)); + } + + /// Remove the given `authority_id` from the `peer_id` to `authority_ids` mapping. + /// + /// If a `peer_id` doesn't have any `authority_id` assigned anymore, it is removed. + fn remove_authority_id_from_peer_ids<'a>( + &mut self, + authority_id: &AuthorityId, + peer_ids: impl Iterator, + ) { + peer_ids.for_each(|peer_id| { + if let Entry::Occupied(mut e) = self.peer_id_to_authority_ids.entry(*peer_id) { + e.get_mut().remove(authority_id); + + // If there are no more entries, remove the peer id. + if e.get().is_empty() { + e.remove(); + } } - } + }) } /// Returns the number of authority IDs in the cache. - pub fn num_ids(&self) -> usize { + pub fn num_authority_ids(&self) -> usize { self.authority_id_to_addresses.len() } @@ -97,18 +112,21 @@ impl AddrCache { pub fn get_addresses_by_authority_id( &self, authority_id: &AuthorityId, - ) -> Option<&Vec> { - self.authority_id_to_addresses.get(&authority_id) + ) -> Option<&HashSet> { + self.authority_id_to_addresses.get(authority_id) } - /// Returns the [`AuthorityId`] for the given [`PeerId`]. - pub fn get_authority_id_by_peer_id(&self, peer_id: &PeerId) -> Option<&AuthorityId> { - self.peer_id_to_authority_id.get(peer_id) + /// Returns the [`AuthorityId`]s for the given [`PeerId`]. + /// + /// As the authority id can change between sessions, one [`PeerId`] can be mapped to + /// multiple authority ids. + pub fn get_authority_ids_by_peer_id(&self, peer_id: &PeerId) -> Option<&HashSet> { + self.peer_id_to_authority_ids.get(peer_id) } /// Removes all [`PeerId`]s and [`Multiaddr`]s from the cache that are not related to the given /// [`AuthorityId`]s. - pub fn retain_ids(&mut self, authority_ids: &Vec) { + pub fn retain_ids(&mut self, authority_ids: &[AuthorityId]) { // The below logic could be replaced by `BtreeMap::drain_filter` once it stabilized. let authority_ids_to_remove = self .authority_id_to_addresses @@ -120,19 +138,18 @@ impl AddrCache { for authority_id_to_remove in authority_ids_to_remove { // Remove other entries from `self.authority_id_to_addresses`. - let addresses = self.authority_id_to_addresses.remove(&authority_id_to_remove); - - // Remove other entries from `self.peer_id_to_authority_id`. - let peer_ids = addresses - .iter() - .flatten() - .map(|a| peer_id_from_multiaddr(a)) - .filter_map(|peer_id| peer_id); - for peer_id in peer_ids { - if let Some(id) = self.peer_id_to_authority_id.remove(&peer_id) { - debug_assert_eq!(authority_id_to_remove, id); - } - } + let addresses = if let Some(addresses) = + self.authority_id_to_addresses.remove(&authority_id_to_remove) + { + addresses + } else { + continue + }; + + self.remove_authority_id_from_peer_ids( + &authority_id_to_remove, + addresses_to_peer_ids(&addresses).iter(), + ); } } } @@ -147,6 +164,13 @@ fn peer_id_from_multiaddr(addr: &Multiaddr) -> Option { }) } +fn addresses_to_peer_ids(addresses: &HashSet) -> HashSet { + addresses + .iter() + .filter_map(|a| peer_id_from_multiaddr(a)) + .collect::>() +} + #[cfg(test)] mod tests { use super::*; @@ -226,27 +250,27 @@ mod tests { cache.insert(third.0.clone(), vec![third.1.clone()]); assert_eq!( - Some(&vec![third.1.clone()]), + Some(&HashSet::from([third.1.clone()])), cache.get_addresses_by_authority_id(&third.0), - "Expect `get_addresses_by_authority_id` to return addresses of third authority." + "Expect `get_addresses_by_authority_id` to return addresses of third authority.", ); assert_eq!( - Some(&third.0), - cache.get_authority_id_by_peer_id(&peer_id_from_multiaddr(&third.1).unwrap()), - "Expect `get_authority_id_by_peer_id` to return `AuthorityId` of third authority." + Some(&HashSet::from([third.0.clone()])), + cache.get_authority_ids_by_peer_id(&peer_id_from_multiaddr(&third.1).unwrap()), + "Expect `get_authority_id_by_peer_id` to return `AuthorityId` of third authority.", ); - cache.retain_ids(&vec![first.0, second.0]); + cache.retain_ids(&vec![first.0.clone(), second.0]); assert_eq!( None, cache.get_addresses_by_authority_id(&third.0), - "Expect `get_addresses_by_authority_id` to not return `None` for third authority." + "Expect `get_addresses_by_authority_id` to not return `None` for third authority.", ); assert_eq!( None, - cache.get_authority_id_by_peer_id(&peer_id_from_multiaddr(&third.1).unwrap()), - "Expect `get_authority_id_by_peer_id` to return `None` for third authority." + cache.get_authority_ids_by_peer_id(&peer_id_from_multiaddr(&third.1).unwrap()), + "Expect `get_authority_id_by_peer_id` to return `None` for third authority.", ); TestResult::passed() @@ -282,44 +306,47 @@ mod tests { assert_eq!( None, - cache.get_authority_id_by_peer_id(&peer_id_from_multiaddr(&multiaddr1).unwrap()) + cache.get_authority_ids_by_peer_id(&peer_id_from_multiaddr(&multiaddr1).unwrap()) ); assert_eq!( - Some(&authority1), - cache.get_authority_id_by_peer_id(&peer_id_from_multiaddr(&multiaddr2).unwrap()) + Some(&HashSet::from([authority1.clone()])), + cache.get_authority_ids_by_peer_id(&peer_id_from_multiaddr(&multiaddr2).unwrap()) ); assert_eq!( - Some(&authority1), - cache.get_authority_id_by_peer_id(&peer_id_from_multiaddr(&multiaddr3).unwrap()) + Some(&HashSet::from([authority1.clone()])), + cache.get_authority_ids_by_peer_id(&peer_id_from_multiaddr(&multiaddr3).unwrap()) ); assert_eq!( - Some(&authority1), - cache.get_authority_id_by_peer_id(&peer_id_from_multiaddr(&multiaddr4).unwrap()) + Some(&HashSet::from([authority1.clone()])), + cache.get_authority_ids_by_peer_id(&peer_id_from_multiaddr(&multiaddr4).unwrap()) ); cache.insert(authority2.clone(), vec![multiaddr2.clone()]); assert_eq!( - Some(&authority2), - cache.get_authority_id_by_peer_id(&peer_id_from_multiaddr(&multiaddr2).unwrap()) + Some(&HashSet::from([authority2.clone(), authority1.clone()])), + cache.get_authority_ids_by_peer_id(&peer_id_from_multiaddr(&multiaddr2).unwrap()) ); assert_eq!( - Some(&authority1), - cache.get_authority_id_by_peer_id(&peer_id_from_multiaddr(&multiaddr3).unwrap()) + Some(&HashSet::from([authority1.clone()])), + cache.get_authority_ids_by_peer_id(&peer_id_from_multiaddr(&multiaddr3).unwrap()) ); - assert_eq!(cache.get_addresses_by_authority_id(&authority1).unwrap().len(), 2); + assert_eq!(cache.get_addresses_by_authority_id(&authority1).unwrap().len(), 3); cache.insert(authority2.clone(), vec![multiaddr2.clone(), multiaddr3.clone()]); assert_eq!( - Some(&authority2), - cache.get_authority_id_by_peer_id(&peer_id_from_multiaddr(&multiaddr2).unwrap()) + Some(&HashSet::from([authority2.clone(), authority1.clone()])), + cache.get_authority_ids_by_peer_id(&peer_id_from_multiaddr(&multiaddr2).unwrap()) + ); + assert_eq!( + Some(&HashSet::from([authority2.clone(), authority1.clone()])), + cache.get_authority_ids_by_peer_id(&peer_id_from_multiaddr(&multiaddr3).unwrap()) ); assert_eq!( - Some(&authority2), - cache.get_authority_id_by_peer_id(&peer_id_from_multiaddr(&multiaddr3).unwrap()) + &HashSet::from([multiaddr2.clone(), multiaddr3.clone(), multiaddr4.clone()]), + cache.get_addresses_by_authority_id(&authority1).unwrap(), ); - assert!(cache.get_addresses_by_authority_id(&authority1).unwrap().is_empty()); TestResult::passed() } @@ -328,4 +355,31 @@ mod tests { .max_tests(10) .quickcheck(property as fn(_, _, _, _, _) -> TestResult) } + + /// As the runtime gives us the current + next authority ids, it can happen that some + /// authority changed its session keys. Changing the sessions keys leads to having two + /// authority ids that map to the same `PeerId` & addresses. + #[test] + fn adding_two_authority_ids_for_the_same_peer_id() { + let mut addr_cache = AddrCache::new(); + + let peer_id = PeerId::random(); + let addr = Multiaddr::empty().with(Protocol::P2p(peer_id.into())); + + let authority_id0 = AuthorityPair::generate().0.public(); + let authority_id1 = AuthorityPair::generate().0.public(); + + addr_cache.insert(authority_id0.clone(), vec![addr.clone()]); + addr_cache.insert(authority_id1.clone(), vec![addr.clone()]); + + assert_eq!(2, addr_cache.num_authority_ids()); + assert_eq!( + &HashSet::from([addr.clone()]), + addr_cache.get_addresses_by_authority_id(&authority_id0).unwrap() + ); + assert_eq!( + &HashSet::from([addr]), + addr_cache.get_addresses_by_authority_id(&authority_id1).unwrap() + ); + } } diff --git a/client/authority-discovery/src/worker/schema/dht.proto b/client/authority-discovery/src/worker/schema/dht-v1.proto similarity index 90% rename from client/authority-discovery/src/worker/schema/dht.proto rename to client/authority-discovery/src/worker/schema/dht-v1.proto index 9dbe9d559f4b..0ef628888c09 100644 --- a/client/authority-discovery/src/worker/schema/dht.proto +++ b/client/authority-discovery/src/worker/schema/dht-v1.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package authority_discovery; +package authority_discovery_v1; // First we need to serialize the addresses in order to be able to sign them. message AuthorityAddresses { @@ -11,4 +11,4 @@ message AuthorityAddresses { message SignedAuthorityAddresses { bytes addresses = 1; bytes signature = 2; -} +} \ No newline at end of file diff --git a/client/authority-discovery/src/worker/schema/dht-v2.proto b/client/authority-discovery/src/worker/schema/dht-v2.proto new file mode 100644 index 000000000000..c63f6c176735 --- /dev/null +++ b/client/authority-discovery/src/worker/schema/dht-v2.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package authority_discovery_v2; + +// First we need to serialize the addresses in order to be able to sign them. +message AuthorityRecord { + // Possibly multiple `MultiAddress`es through which the node can be + repeated bytes addresses = 1; +} + +message PeerSignature { + bytes signature = 1; + bytes public_key = 2; +} + +// Then we need to serialize the authority record and signature to send them over the wire. +message SignedAuthorityRecord { + bytes record = 1; + bytes auth_signature = 2; + // Even if there are multiple `record.addresses`, all of them have the same peer id. + // Old versions are missing this field, so `optional` will provide compatibility both ways. + optional PeerSignature peer_signature = 3; +} diff --git a/client/authority-discovery/src/worker/schema/tests.rs b/client/authority-discovery/src/worker/schema/tests.rs new file mode 100644 index 000000000000..b85a4ce37447 --- /dev/null +++ b/client/authority-discovery/src/worker/schema/tests.rs @@ -0,0 +1,90 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +mod schema_v1 { + include!(concat!(env!("OUT_DIR"), "/authority_discovery_v1.rs")); +} + +use super::*; +use libp2p::multiaddr::Multiaddr; +use prost::Message; +use sc_network::PeerId; + +#[test] +fn v2_decodes_v1() { + let peer_id = PeerId::random(); + let multiaddress: Multiaddr = + format!("/ip4/127.0.0.1/tcp/3003/p2p/{}", peer_id).parse().unwrap(); + let vec_addresses = vec![multiaddress.to_vec()]; + let vec_auth_signature = b"Totally valid signature, I promise!".to_vec(); + + let addresses_v1 = schema_v1::AuthorityAddresses { addresses: vec_addresses.clone() }; + let mut vec_addresses_v1 = vec![]; + addresses_v1.encode(&mut vec_addresses_v1).unwrap(); + let signed_addresses_v1 = schema_v1::SignedAuthorityAddresses { + addresses: vec_addresses_v1.clone(), + signature: vec_auth_signature.clone(), + }; + let mut vec_signed_addresses_v1 = vec![]; + signed_addresses_v1.encode(&mut vec_signed_addresses_v1).unwrap(); + + let signed_record_v2_decoded = + SignedAuthorityRecord::decode(vec_signed_addresses_v1.as_slice()).unwrap(); + + assert_eq!(&signed_record_v2_decoded.record, &vec_addresses_v1); + assert_eq!(&signed_record_v2_decoded.auth_signature, &vec_auth_signature); + assert_eq!(&signed_record_v2_decoded.peer_signature, &None); + + let record_v2_decoded = AuthorityRecord::decode(vec_addresses_v1.as_slice()).unwrap(); + assert_eq!(&record_v2_decoded.addresses, &vec_addresses); +} + +#[test] +fn v1_decodes_v2() { + let peer_secret = sc_network::Keypair::generate_ed25519(); + let peer_public = peer_secret.public(); + let peer_id = peer_public.to_peer_id(); + let multiaddress: Multiaddr = + format!("/ip4/127.0.0.1/tcp/3003/p2p/{}", peer_id).parse().unwrap(); + let vec_addresses = vec![multiaddress.to_vec()]; + let vec_auth_signature = b"Totally valid signature, I promise!".to_vec(); + let vec_peer_signature = b"Surprisingly hard to crack crypto".to_vec(); + + let record_v2 = AuthorityRecord { addresses: vec_addresses.clone() }; + let mut vec_record_v2 = vec![]; + record_v2.encode(&mut vec_record_v2).unwrap(); + let vec_peer_public = peer_public.to_protobuf_encoding(); + let peer_signature_v2 = + PeerSignature { public_key: vec_peer_public, signature: vec_peer_signature }; + let signed_record_v2 = SignedAuthorityRecord { + record: vec_record_v2.clone(), + auth_signature: vec_auth_signature.clone(), + peer_signature: Some(peer_signature_v2.clone()), + }; + let mut vec_signed_record_v2 = vec![]; + signed_record_v2.encode(&mut vec_signed_record_v2).unwrap(); + + let signed_addresses_v1_decoded = + schema_v1::SignedAuthorityAddresses::decode(vec_signed_record_v2.as_slice()).unwrap(); + + assert_eq!(&signed_addresses_v1_decoded.addresses, &vec_record_v2); + assert_eq!(&signed_addresses_v1_decoded.signature, &vec_auth_signature); + + let addresses_v2_decoded = AuthorityRecord::decode(vec_record_v2.as_slice()).unwrap(); + assert_eq!(&addresses_v2_decoded.addresses, &vec_addresses); +} diff --git a/client/authority-discovery/src/worker/tests.rs b/client/authority-discovery/src/worker/tests.rs index f10d2751ccd3..904c674d269b 100644 --- a/client/authority-discovery/src/worker/tests.rs +++ b/client/authority-discovery/src/worker/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,9 +16,8 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::worker::schema; - use std::{ + collections::HashSet, sync::{Arc, Mutex}, task::Poll, }; @@ -31,11 +30,10 @@ use futures::{ sink::SinkExt, task::LocalSpawn, }; -use libp2p::{core::multiaddr, kad, PeerId}; +use libp2p::{core::multiaddr, PeerId}; use prometheus_endpoint::prometheus::default_registry; use sp_api::{ApiRef, ProvideRuntimeApi}; -use sp_core::crypto::Public; use sp_keystore::{testing::KeyStore, CryptoStore}; use sp_runtime::traits::{Block as BlockT, NumberFor, Zero}; use substrate_test_runtime_client::runtime::Block; @@ -73,6 +71,7 @@ impl HeaderBackend for TestApi { genesis_hash: Default::default(), number_leaves: Default::default(), finalized_state: None, + block_gap: None, } } @@ -112,17 +111,18 @@ sp_api::mock_impl_runtime_apis! { #[derive(Debug)] pub enum TestNetworkEvent { - GetCalled(kad::record::Key), - PutCalled(kad::record::Key, Vec), + GetCalled(sc_network::KademliaKey), + PutCalled(sc_network::KademliaKey, Vec), } pub struct TestNetwork { peer_id: PeerId, + identity: sc_network::Keypair, external_addresses: Vec, // Whenever functions on `TestNetwork` are called, the function arguments are added to the // vectors below. - pub put_value_call: Arc)>>>, - pub get_value_call: Arc>>, + pub put_value_call: Arc)>>>, + pub get_value_call: Arc>>, event_sender: mpsc::UnboundedSender, event_receiver: Option>, } @@ -136,8 +136,10 @@ impl TestNetwork { impl Default for TestNetwork { fn default() -> Self { let (tx, rx) = mpsc::unbounded(); + let identity = sc_network::Keypair::generate_ed25519(); TestNetwork { - peer_id: PeerId::random(), + peer_id: identity.public().to_peer_id(), + identity, external_addresses: vec!["/ip6/2001:db8::/tcp/30333".parse().unwrap()], put_value_call: Default::default(), get_value_call: Default::default(), @@ -147,16 +149,25 @@ impl Default for TestNetwork { } } +impl NetworkSigner for TestNetwork { + fn sign_with_local_identity( + &self, + msg: impl AsRef<[u8]>, + ) -> std::result::Result { + sc_network::Signature::sign_message(msg, &self.identity) + } +} + #[async_trait] impl NetworkProvider for TestNetwork { - fn put_value(&self, key: kad::record::Key, value: Vec) { + fn put_value(&self, key: sc_network::KademliaKey, value: Vec) { self.put_value_call.lock().unwrap().push((key.clone(), value.clone())); self.event_sender .clone() .unbounded_send(TestNetworkEvent::PutCalled(key, value)) .unwrap(); } - fn get_value(&self, key: &kad::record::Key) { + fn get_value(&self, key: &sc_network::KademliaKey) { self.get_value_call.lock().unwrap().push(key.clone()); self.event_sender .clone() @@ -175,35 +186,35 @@ impl NetworkStateInfo for TestNetwork { } } -async fn build_dht_event( +impl NetworkSigner for sc_network::Keypair { + fn sign_with_local_identity( + &self, + msg: impl AsRef<[u8]>, + ) -> std::result::Result { + sc_network::Signature::sign_message(msg, self) + } +} + +async fn build_dht_event( addresses: Vec, public_key: AuthorityId, - key_store: &KeyStore, -) -> (libp2p::kad::record::Key, Vec) { - let mut serialized_addresses = vec![]; - schema::AuthorityAddresses { addresses: addresses.into_iter().map(|a| a.to_vec()).collect() } - .encode(&mut serialized_addresses) - .map_err(Error::EncodingProto) - .unwrap(); - - let signature = key_store - .sign_with( - key_types::AUTHORITY_DISCOVERY, - &public_key.clone().into(), - serialized_addresses.as_slice(), - ) - .await - .unwrap() - .unwrap(); - - let mut signed_addresses = vec![]; - schema::SignedAuthorityAddresses { addresses: serialized_addresses.clone(), signature } - .encode(&mut signed_addresses) - .unwrap(); - - let key = hash_authority_id(&public_key.to_raw_vec()); - let value = signed_addresses; - (key, value) + key_store: &dyn CryptoStore, + network: Option<&Signer>, +) -> Vec<(sc_network::KademliaKey, Vec)> { + let serialized_record = + serialize_authority_record(serialize_addresses(addresses.into_iter())).unwrap(); + + let peer_signature = network.map(|n| sign_record_with_peer_id(&serialized_record, n).unwrap()); + let kv_pairs = sign_record_with_authority_ids( + serialized_record, + peer_signature, + key_store, + vec![public_key.into()], + ) + .await + .unwrap(); + // There is always a single item in it, because we signed it with a single key + kv_pairs } #[test] @@ -451,13 +462,14 @@ fn dont_stop_polling_dht_event_stream_after_bogus_event() { // Make previously triggered lookup succeed. let dht_event = { - let (key, value) = build_dht_event( + let kv_pairs = build_dht_event::( vec![remote_multiaddr.clone()], remote_public_key.clone(), &remote_key_store, + None, ) .await; - sc_network::DhtEvent::ValueFound(vec![(key, value)]) + sc_network::DhtEvent::ValueFound(kv_pairs) }; dht_event_tx.send(dht_event).await.expect("Channel has capacity of 1."); @@ -468,101 +480,195 @@ fn dont_stop_polling_dht_event_stream_after_bogus_event() { .send(ServicetoWorkerMsg::GetAddressesByAuthorityId(remote_public_key, sender)) .await .expect("Channel has capacity of 1."); - assert_eq!(Some(vec![remote_multiaddr]), addresses.await.unwrap()); + assert_eq!(Some(HashSet::from([remote_multiaddr])), addresses.await.unwrap()); }); } -#[test] -fn limit_number_of_addresses_added_to_cache_per_authority() { - let remote_key_store = KeyStore::new(); - let remote_public = - block_on(remote_key_store.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None)) - .unwrap(); +struct DhtValueFoundTester { + pub remote_key_store: KeyStore, + pub remote_authority_public: sp_core::sr25519::Public, + pub remote_node_key: sc_network::Keypair, + pub local_worker: Option< + Worker< + TestApi, + TestNetwork, + sp_runtime::generic::Block< + sp_runtime::generic::Header, + substrate_test_runtime_client::runtime::Extrinsic, + >, + std::pin::Pin>>, + >, + >, +} - let addresses = (0..100) - .map(|_| { - let peer_id = PeerId::random(); - let address: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap(); - address.with(multiaddr::Protocol::P2p(peer_id.into())) - }) - .collect(); +impl DhtValueFoundTester { + fn new() -> Self { + let remote_key_store = KeyStore::new(); + let remote_authority_public = + block_on(remote_key_store.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None)) + .unwrap(); - let dht_event = block_on(build_dht_event(addresses, remote_public.into(), &remote_key_store)); + let remote_node_key = sc_network::Keypair::generate_ed25519(); + Self { remote_key_store, remote_authority_public, remote_node_key, local_worker: None } + } - let (_dht_event_tx, dht_event_rx) = channel(1); + fn multiaddr_with_peer_id(&self, idx: u16) -> Multiaddr { + let peer_id = self.remote_node_key.public().to_peer_id(); + let address: Multiaddr = + format!("/ip6/2001:db8:0:0:0:0:0:{:x}/tcp/30333", idx).parse().unwrap(); - let (_to_worker, from_service) = mpsc::channel(0); - let mut worker = Worker::new( - from_service, - Arc::new(TestApi { authorities: vec![remote_public.into()] }), - Arc::new(TestNetwork::default()), - Box::pin(dht_event_rx), - Role::Discover, + address.with(multiaddr::Protocol::P2p(peer_id.into())) + } + + fn process_value_found( + &mut self, + strict_record_validation: bool, + values: Vec<(sc_network::KademliaKey, Vec)>, + ) -> Option<&HashSet> { + let (_dht_event_tx, dht_event_rx) = channel(1); + let local_test_api = + Arc::new(TestApi { authorities: vec![self.remote_authority_public.clone().into()] }); + let local_network: Arc = Arc::new(Default::default()); + let local_key_store = KeyStore::new(); + + let (_to_worker, from_service) = mpsc::channel(0); + let mut local_worker = Worker::new( + from_service, + local_test_api, + local_network.clone(), + Box::pin(dht_event_rx), + Role::PublishAndDiscover(Arc::new(local_key_store)), + None, + WorkerConfig { strict_record_validation, ..Default::default() }, + ); + + block_on(local_worker.refill_pending_lookups_queue()).unwrap(); + local_worker.start_new_lookups(); + + drop(local_worker.handle_dht_value_found_event(values)); + + self.local_worker = Some(local_worker); + + self.local_worker + .as_ref() + .map(|w| { + w.addr_cache + .get_addresses_by_authority_id(&self.remote_authority_public.clone().into()) + }) + .unwrap() + } +} + +#[test] +fn limit_number_of_addresses_added_to_cache_per_authority() { + let mut tester = DhtValueFoundTester::new(); + assert!(MAX_ADDRESSES_PER_AUTHORITY < 100); + let addresses = (1..100).map(|i| tester.multiaddr_with_peer_id(i)).collect(); + let kv_pairs = block_on(build_dht_event::( + addresses, + tester.remote_authority_public.clone().into(), + &tester.remote_key_store, None, - Default::default(), - ); + )); + + let cached_remote_addresses = tester.process_value_found(false, kv_pairs); + assert_eq!(MAX_ADDRESSES_PER_AUTHORITY, cached_remote_addresses.unwrap().len()); +} + +#[test] +fn strict_accept_address_with_peer_signature() { + let mut tester = DhtValueFoundTester::new(); + let addr = tester.multiaddr_with_peer_id(1); + let kv_pairs = block_on(build_dht_event( + vec![addr.clone()], + tester.remote_authority_public.clone().into(), + &tester.remote_key_store, + Some(&tester.remote_node_key), + )); - block_on(worker.refill_pending_lookups_queue()).unwrap(); - worker.start_new_lookups(); + let cached_remote_addresses = tester.process_value_found(true, kv_pairs); - worker.handle_dht_value_found_event(vec![dht_event]).unwrap(); assert_eq!( - MAX_ADDRESSES_PER_AUTHORITY, - worker - .addr_cache - .get_addresses_by_authority_id(&remote_public.into()) - .unwrap() - .len(), + Some(&HashSet::from([addr])), + cached_remote_addresses, + "Expect worker to only cache `Multiaddr`s with `PeerId`s.", ); } #[test] -fn do_not_cache_addresses_without_peer_id() { - let remote_key_store = KeyStore::new(); - let remote_public = - block_on(remote_key_store.sr25519_generate_new(key_types::AUTHORITY_DISCOVERY, None)) - .unwrap(); - - let multiaddr_with_peer_id = { - let peer_id = PeerId::random(); - let address: Multiaddr = "/ip6/2001:db8:0:0:0:0:0:2/tcp/30333".parse().unwrap(); +fn reject_address_with_rogue_peer_signature() { + let mut tester = DhtValueFoundTester::new(); + let rogue_remote_node_key = sc_network::Keypair::generate_ed25519(); + let kv_pairs = block_on(build_dht_event( + vec![tester.multiaddr_with_peer_id(1)], + tester.remote_authority_public.clone().into(), + &tester.remote_key_store, + Some(&rogue_remote_node_key), + )); - address.with(multiaddr::Protocol::P2p(peer_id.into())) - }; + let cached_remote_addresses = tester.process_value_found(false, kv_pairs); - let multiaddr_without_peer_id: Multiaddr = - "/ip6/2001:db8:0:0:0:0:0:1/tcp/30333".parse().unwrap(); + assert!( + cached_remote_addresses.is_none(), + "Expected worker to ignore record signed by a different key.", + ); +} - let dht_event = block_on(build_dht_event( - vec![multiaddr_with_peer_id.clone(), multiaddr_without_peer_id], - remote_public.into(), - &remote_key_store, +#[test] +fn reject_address_with_invalid_peer_signature() { + let mut tester = DhtValueFoundTester::new(); + let mut kv_pairs = block_on(build_dht_event( + vec![tester.multiaddr_with_peer_id(1)], + tester.remote_authority_public.clone().into(), + &tester.remote_key_store, + Some(&tester.remote_node_key), )); + // tamper with the signature + let mut record = schema::SignedAuthorityRecord::decode(kv_pairs[0].1.as_slice()).unwrap(); + record.peer_signature.as_mut().map(|p| p.signature[1] += 1); + record.encode(&mut kv_pairs[0].1).unwrap(); - let (_dht_event_tx, dht_event_rx) = channel(1); - let local_test_api = Arc::new(TestApi { authorities: vec![remote_public.into()] }); - let local_network: Arc = Arc::new(Default::default()); - let local_key_store = KeyStore::new(); + let cached_remote_addresses = tester.process_value_found(false, kv_pairs); - let (_to_worker, from_service) = mpsc::channel(0); - let mut local_worker = Worker::new( - from_service, - local_test_api, - local_network.clone(), - Box::pin(dht_event_rx), - Role::PublishAndDiscover(Arc::new(local_key_store)), - None, - Default::default(), + assert!( + cached_remote_addresses.is_none(), + "Expected worker to ignore record with tampered signature.", ); +} + +#[test] +fn reject_address_without_peer_signature() { + let mut tester = DhtValueFoundTester::new(); + let kv_pairs = block_on(build_dht_event::( + vec![tester.multiaddr_with_peer_id(1)], + tester.remote_authority_public.clone().into(), + &tester.remote_key_store, + None, + )); - block_on(local_worker.refill_pending_lookups_queue()).unwrap(); - local_worker.start_new_lookups(); + let cached_remote_addresses = tester.process_value_found(true, kv_pairs); - local_worker.handle_dht_value_found_event(vec![dht_event]).unwrap(); + assert!(cached_remote_addresses.is_none(), "Expected worker to ignore unsigned record.",); +} + +#[test] +fn do_not_cache_addresses_without_peer_id() { + let mut tester = DhtValueFoundTester::new(); + let multiaddr_with_peer_id = tester.multiaddr_with_peer_id(1); + let multiaddr_without_peer_id: Multiaddr = + "/ip6/2001:db8:0:0:0:0:0:2/tcp/30333".parse().unwrap(); + let kv_pairs = block_on(build_dht_event::( + vec![multiaddr_with_peer_id.clone(), multiaddr_without_peer_id], + tester.remote_authority_public.clone().into(), + &tester.remote_key_store, + None, + )); + + let cached_remote_addresses = tester.process_value_found(false, kv_pairs); assert_eq!( - Some(&vec![multiaddr_with_peer_id]), - local_worker.addr_cache.get_addresses_by_authority_id(&remote_public.into()), + Some(&HashSet::from([multiaddr_with_peer_id])), + cached_remote_addresses, "Expect worker to only cache `Multiaddr`s with `PeerId`s.", ); } @@ -695,10 +801,14 @@ fn lookup_throttling() { let remote_hash = network.get_value_call.lock().unwrap().pop().unwrap(); let remote_key: AuthorityId = remote_hash_to_key.get(&remote_hash).unwrap().clone(); let dht_event = { - let (key, value) = - build_dht_event(vec![remote_multiaddr.clone()], remote_key, &remote_key_store) - .await; - sc_network::DhtEvent::ValueFound(vec![(key, value)]) + let kv_pairs = build_dht_event::( + vec![remote_multiaddr.clone()], + remote_key, + &remote_key_store, + None, + ) + .await; + sc_network::DhtEvent::ValueFound(kv_pairs) }; dht_event_tx.send(dht_event).await.expect("Channel has capacity of 1."); diff --git a/client/basic-authorship/Cargo.toml b/client/basic-authorship/Cargo.toml index 469df55cf023..20560d4c834e 100644 --- a/client/basic-authorship/Cargo.toml +++ b/client/basic-authorship/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-basic-authorship" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Basic implementation of block-authoring logic." readme = "README.md" @@ -13,14 +13,14 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0" } -futures = "0.3.9" +codec = { package = "parity-scale-codec", version = "3.0.0" } +futures = "0.3.21" futures-timer = "3.0.1" log = "0.4.8" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.9.0"} +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.10.0-dev"} sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sc-client-api = { version = "4.0.0-dev", path = "../api" } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } @@ -28,9 +28,9 @@ sp-inherents = { version = "4.0.0-dev", path = "../../primitives/inherents" } sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../client/transaction-pool/api" } sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" } -sc-proposer-metrics = { version = "0.9.0", path = "../proposer-metrics" } +sc-proposer-metrics = { version = "0.10.0-dev", path = "../proposer-metrics" } [dev-dependencies] sc-transaction-pool = { version = "4.0.0-dev", path = "../transaction-pool" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } -parking_lot = "0.11.1" +parking_lot = "0.12.0" diff --git a/client/basic-authorship/src/basic_authorship.rs b/client/basic-authorship/src/basic_authorship.rs index 144a3ab6850f..23725e513869 100644 --- a/client/basic-authorship/src/basic_authorship.rs +++ b/client/basic-authorship/src/basic_authorship.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -41,12 +41,13 @@ use sp_core::traits::SpawnNamed; use sp_inherents::InherentData; use sp_runtime::{ generic::BlockId, - traits::{BlakeTwo256, Block as BlockT, DigestFor, Hash as HashT, Header as HeaderT}, + traits::{BlakeTwo256, Block as BlockT, Hash as HashT, Header as HeaderT}, + Digest, Percent, SaturatedConversion, }; use std::{marker::PhantomData, pin::Pin, sync::Arc, time}; use prometheus_endpoint::Registry as PrometheusRegistry; -use sc_proposer_metrics::MetricsLink as PrometheusMetrics; +use sc_proposer_metrics::{EndProposingReason, MetricsLink as PrometheusMetrics}; /// Default block size limit in bytes used by [`Proposer`]. /// @@ -57,6 +58,8 @@ use sc_proposer_metrics::MetricsLink as PrometheusMetrics; /// transferred to other nodes. pub const DEFAULT_BLOCK_SIZE_LIMIT: usize = 4 * 1024 * 1024 + 512; +const DEFAULT_SOFT_DEADLINE_PERCENT: Percent = Percent::from_percent(50); + /// [`Proposer`] factory. pub struct ProposerFactory { spawn_handle: Box, @@ -71,6 +74,14 @@ pub struct ProposerFactory { /// If no `block_size_limit` is passed to [`sp_consensus::Proposer::propose`], this block size /// limit will be used. default_block_size_limit: usize, + /// Soft deadline percentage of hard deadline. + /// + /// The value is used to compute soft deadline during block production. + /// The soft deadline indicates where we should stop attempting to add transactions + /// to the block, which exhaust resources. After soft deadline is reached, + /// we switch to a fixed-amount mode, in which after we see `MAX_SKIPPED_TRANSACTIONS` + /// transactions which exhaust resrouces, we will conclude that the block is full. + soft_deadline_percent: Percent, telemetry: Option, /// When estimating the block size, should the proof be included? include_proof_in_block_size_estimation: bool, @@ -95,6 +106,7 @@ impl ProposerFactory { transaction_pool, metrics: PrometheusMetrics::new(prometheus), default_block_size_limit: DEFAULT_BLOCK_SIZE_LIMIT, + soft_deadline_percent: DEFAULT_SOFT_DEADLINE_PERCENT, telemetry, client, include_proof_in_block_size_estimation: false, @@ -123,6 +135,7 @@ impl ProposerFactory { transaction_pool, metrics: PrometheusMetrics::new(prometheus), default_block_size_limit: DEFAULT_BLOCK_SIZE_LIMIT, + soft_deadline_percent: DEFAULT_SOFT_DEADLINE_PERCENT, telemetry, include_proof_in_block_size_estimation: true, _phantom: PhantomData, @@ -146,6 +159,22 @@ impl ProposerFactory { pub fn set_default_block_size_limit(&mut self, limit: usize) { self.default_block_size_limit = limit; } + + /// Set soft deadline percentage. + /// + /// The value is used to compute soft deadline during block production. + /// The soft deadline indicates where we should stop attempting to add transactions + /// to the block, which exhaust resources. After soft deadline is reached, + /// we switch to a fixed-amount mode, in which after we see `MAX_SKIPPED_TRANSACTIONS` + /// transactions which exhaust resrouces, we will conclude that the block is full. + /// + /// Setting the value too low will significantly limit the amount of transactions + /// we try in case they exhaust resources. Setting the value too high can + /// potentially open a DoS vector, where many "exhaust resources" transactions + /// are being tried with no success, hence block producer ends up creating an empty block. + pub fn set_soft_deadline(&mut self, percent: Percent) { + self.soft_deadline_percent = percent; + } } impl ProposerFactory @@ -183,6 +212,7 @@ where now, metrics: self.metrics.clone(), default_block_size_limit: self.default_block_size_limit, + soft_deadline_percent: self.soft_deadline_percent, telemetry: self.telemetry.clone(), _phantom: PhantomData, include_proof_in_block_size_estimation: self.include_proof_in_block_size_estimation, @@ -228,6 +258,7 @@ pub struct Proposer { metrics: PrometheusMetrics, default_block_size_limit: usize, include_proof_in_block_size_estimation: bool, + soft_deadline_percent: Percent, telemetry: Option, _phantom: PhantomData<(B, PR)>, } @@ -261,7 +292,7 @@ where fn propose( self, inherent_data: InherentData, - inherent_digests: DigestFor, + inherent_digests: Digest, max_duration: time::Duration, block_size_limit: Option, ) -> Self::Proposal { @@ -270,6 +301,7 @@ where spawn_handle.spawn_blocking( "basic-authorship-proposer", + None, Box::pin(async move { // leave some time for evaluation and block finalization (33%) let deadline = (self.now)() + max_duration - max_duration / 3; @@ -286,6 +318,11 @@ where } } +/// If the block is full we will attempt to push at most +/// this number of transactions before quitting for real. +/// It allows us to increase block utilization. +const MAX_SKIPPED_TRANSACTIONS: usize = 8; + impl Proposer where A: TransactionPool, @@ -304,20 +341,28 @@ where async fn propose_with( self, inherent_data: InherentData, - inherent_digests: DigestFor, + inherent_digests: Digest, deadline: time::Instant, block_size_limit: Option, ) -> Result, PR::Proof>, sp_blockchain::Error> { - /// If the block is full we will attempt to push at most - /// this number of transactions before quitting for real. - /// It allows us to increase block utilization. - const MAX_SKIPPED_TRANSACTIONS: usize = 8; - + let propose_with_start = time::Instant::now(); let mut block_builder = self.client.new_block_at(&self.parent_id, inherent_digests, PR::ENABLED)?; - for inherent in block_builder.create_inherents(inherent_data)? { + let create_inherents_start = time::Instant::now(); + let inherents = block_builder.create_inherents(inherent_data)?; + let create_inherents_end = time::Instant::now(); + + self.metrics.report(|metrics| { + metrics.create_inherents_time.observe( + create_inherents_end + .saturating_duration_since(create_inherents_start) + .as_secs_f64(), + ); + }); + + for inherent in inherents { match block_builder.push(inherent) { Err(ApplyExtrinsicFailed(Validity(e))) if e.exhausted_resources() => { warn!("⚠️ Dropping non-mandatory inherent from overweight block.") @@ -336,6 +381,12 @@ where } // proceed with transactions + // We calculate soft deadline used only in case we start skipping transactions. + let now = (self.now)(); + let left = deadline.saturating_duration_since(now); + let left_micros: u64 = left.as_micros().saturated_into(); + let soft_deadline = + now + time::Duration::from_micros(self.soft_deadline_percent.mul_floor(left_micros)); let block_timer = time::Instant::now(); let mut skipped = 0; let mut unqueue_invalid = Vec::new(); @@ -344,7 +395,7 @@ where let mut t2 = futures_timer::Delay::new(deadline.saturating_duration_since((self.now)()) / 8).fuse(); - let pending_iterator = select! { + let mut pending_iterator = select! { res = t1 => res, _ = t2 => { log::warn!( @@ -361,15 +412,21 @@ where debug!("Attempting to push transactions from the pool."); debug!("Pool status: {:?}", self.transaction_pool.status()); let mut transaction_pushed = false; - let mut hit_block_size_limit = false; - for pending_tx in pending_iterator { - if (self.now)() > deadline { + let end_reason = loop { + let pending_tx = if let Some(pending_tx) = pending_iterator.next() { + pending_tx + } else { + break EndProposingReason::NoMoreTransactions + }; + + let now = (self.now)(); + if now > deadline { debug!( "Consensus deadline reached when pushing block transactions, \ proceeding with proposing." ); - break + break EndProposingReason::HitDeadline } let pending_tx_data = pending_tx.data().clone(); @@ -378,6 +435,7 @@ where let block_size = block_builder.estimate_block_size(self.include_proof_in_block_size_estimation); if block_size + pending_tx_data.encoded_size() > block_size_limit { + pending_iterator.report_invalid(&pending_tx); if skipped < MAX_SKIPPED_TRANSACTIONS { skipped += 1; debug!( @@ -386,10 +444,16 @@ where MAX_SKIPPED_TRANSACTIONS - skipped, ); continue + } else if now < soft_deadline { + debug!( + "Transaction would overflow the block size limit, \ + but we still have time before the soft deadline, so \ + we will try a bit more." + ); + continue } else { debug!("Reached block size limit, proceeding with proposing."); - hit_block_size_limit = true; - break + break EndProposingReason::HitBlockSizeLimit } } @@ -400,18 +464,25 @@ where debug!("[{:?}] Pushed to the block.", pending_tx_hash); }, Err(ApplyExtrinsicFailed(Validity(e))) if e.exhausted_resources() => { + pending_iterator.report_invalid(&pending_tx); if skipped < MAX_SKIPPED_TRANSACTIONS { skipped += 1; debug!( "Block seems full, but will try {} more transactions before quitting.", MAX_SKIPPED_TRANSACTIONS - skipped, ); + } else if (self.now)() < soft_deadline { + debug!( + "Block seems full, but we still have time before the soft deadline, \ + so we will try a bit more before quitting." + ); } else { - debug!("Block is full, proceed with proposing."); - break + debug!("Reached block weight limit, proceeding with proposing."); + break EndProposingReason::HitBlockWeightLimit } }, Err(e) if skipped > 0 => { + pending_iterator.report_invalid(&pending_tx); trace!( "[{:?}] Ignoring invalid transaction when skipping: {}", pending_tx_hash, @@ -419,13 +490,14 @@ where ); }, Err(e) => { + pending_iterator.report_invalid(&pending_tx); debug!("[{:?}] Invalid transaction: {}", pending_tx_hash, e); unqueue_invalid.push(pending_tx_hash); }, } - } + }; - if hit_block_size_limit && !transaction_pushed { + if matches!(end_reason, EndProposingReason::HitBlockSizeLimit) && !transaction_pushed { warn!( "Hit block size limit of `{}` without including any transaction!", block_size_limit, @@ -439,17 +511,20 @@ where self.metrics.report(|metrics| { metrics.number_of_transactions.set(block.extrinsics().len() as u64); metrics.block_constructed.observe(block_timer.elapsed().as_secs_f64()); + + metrics.report_end_proposing_reason(end_reason); }); info!( - "🎁 Prepared block for proposing at {} [hash: {:?}; parent_hash: {}; extrinsics ({}): [{}]]", + "🎁 Prepared block for proposing at {} ({} ms) [hash: {:?}; parent_hash: {}; extrinsics ({}): [{}]]", block.header().number(), + block_timer.elapsed().as_millis(), ::Hash::from(block.header().hash()), block.header().parent_hash(), block.extrinsics().len(), block.extrinsics() .iter() - .map(|xt| format!("{}", BlakeTwo256::hash_of(xt))) + .map(|xt| BlakeTwo256::hash_of(xt).to_string()) .collect::>() .join(", ") ); @@ -473,6 +548,14 @@ where let proof = PR::into_proof(proof).map_err(|e| sp_blockchain::Error::Application(Box::new(e)))?; + + let propose_with_end = time::Instant::now(); + self.metrics.report(|metrics| { + metrics.create_block_proposal_time.observe( + propose_with_end.saturating_duration_since(propose_with_start).as_secs_f64(), + ); + }); + Ok(Proposal { block, proof, storage_changes }) } } @@ -489,6 +572,7 @@ mod tests { use sp_api::Core; use sp_blockchain::HeaderBackend; use sp_consensus::{BlockOrigin, Environment, Proposer}; + use sp_core::Pair; use sp_runtime::traits::NumberFor; use substrate_test_runtime_client::{ prelude::*, @@ -503,11 +587,24 @@ mod tests { amount: Default::default(), nonce, from: AccountKeyring::Alice.into(), - to: Default::default(), + to: AccountKeyring::Bob.into(), } .into_signed_tx() } + fn exhausts_resources_extrinsic_from(who: usize) -> Extrinsic { + let pair = AccountKeyring::numeric(who); + let transfer = Transfer { + // increase the amount to bump priority + amount: 1, + nonce: 0, + from: pair.public(), + to: AccountKeyring::Bob.into(), + }; + let signature = pair.sign(&transfer.encode()).into(); + Extrinsic::Transfer { transfer, signature, exhaust_resources_when_not_first: true } + } + fn chain_event(header: B::Header) -> ChainEvent where NumberFor: From, @@ -553,7 +650,7 @@ mod tests { return value.1 } let old = value.1; - let new = old + time::Duration::from_secs(2); + let new = old + time::Duration::from_secs(1); *value = (true, new); old }), @@ -654,13 +751,8 @@ mod tests { api.execute_block(&block_id, proposal.block).unwrap(); let state = backend.state_at(block_id).unwrap(); - let changes_trie_state = - backend::changes_tries_state_at_block(&block_id, backend.changes_trie_storage()) - .unwrap(); - let storage_changes = api - .into_storage_changes(&state, changes_trie_state.as_ref(), genesis_hash) - .unwrap(); + let storage_changes = api.into_storage_changes(&state, genesis_hash).unwrap(); assert_eq!( proposal.storage_changes.transaction_storage_root, @@ -691,14 +783,14 @@ mod tests { amount: Default::default(), nonce: 2, from: AccountKeyring::Alice.into(), - to: Default::default(), + to: AccountKeyring::Bob.into(), }.into_resources_exhausting_tx(), extrinsic(3), Transfer { amount: Default::default(), nonce: 4, from: AccountKeyring::Alice.into(), - to: Default::default(), + to: AccountKeyring::Bob.into(), }.into_resources_exhausting_tx(), extrinsic(5), extrinsic(6), @@ -718,7 +810,7 @@ mod tests { ); // when - let deadline = time::Duration::from_secs(9); + let deadline = time::Duration::from_secs(900); let block = block_on(proposer.propose(Default::default(), Default::default(), deadline, None)) .map(|r| r.block) @@ -726,8 +818,8 @@ mod tests { // then // block should have some extrinsics although we have some more in the pool. - assert_eq!(block.extrinsics().len(), expected_block_extrinsics); assert_eq!(txpool.ready().count(), expected_pool_transactions); + assert_eq!(block.extrinsics().len(), expected_block_extrinsics); block }; @@ -740,6 +832,7 @@ mod tests { .expect("there should be header"), )), ); + assert_eq!(txpool.ready().count(), 7); // let's create one block and import it let block = propose_block(&client, 0, 2, 7); @@ -753,6 +846,7 @@ mod tests { .expect("there should be header"), )), ); + assert_eq!(txpool.ready().count(), 5); // now let's make sure that we can still make some progress let block = propose_block(&client, 1, 2, 5); @@ -845,4 +939,142 @@ mod tests { // block size and thus, one less transaction should fit into the limit. assert_eq!(block.extrinsics().len(), extrinsics_num - 2); } + + #[test] + fn should_keep_adding_transactions_after_exhausts_resources_before_soft_deadline() { + // given + let client = Arc::new(substrate_test_runtime_client::new()); + let spawner = sp_core::testing::TaskExecutor::new(); + let txpool = BasicPool::new_full( + Default::default(), + true.into(), + None, + spawner.clone(), + client.clone(), + ); + + block_on( + txpool.submit_at( + &BlockId::number(0), + SOURCE, + // add 2 * MAX_SKIPPED_TRANSACTIONS that exhaust resources + (0..MAX_SKIPPED_TRANSACTIONS * 2) + .into_iter() + .map(|i| exhausts_resources_extrinsic_from(i)) + // and some transactions that are okay. + .chain((0..MAX_SKIPPED_TRANSACTIONS).into_iter().map(|i| extrinsic(i as _))) + .collect(), + ), + ) + .unwrap(); + + block_on( + txpool.maintain(chain_event( + client + .header(&BlockId::Number(0u64)) + .expect("header get error") + .expect("there should be header"), + )), + ); + assert_eq!(txpool.ready().count(), MAX_SKIPPED_TRANSACTIONS * 3); + + let mut proposer_factory = + ProposerFactory::new(spawner.clone(), client.clone(), txpool.clone(), None, None); + + let cell = Mutex::new(time::Instant::now()); + let proposer = proposer_factory.init_with_now( + &client.header(&BlockId::number(0)).unwrap().unwrap(), + Box::new(move || { + let mut value = cell.lock(); + let old = *value; + *value = old + time::Duration::from_secs(1); + old + }), + ); + + // when + // give it enough time so that deadline is never triggered. + let deadline = time::Duration::from_secs(900); + let block = + block_on(proposer.propose(Default::default(), Default::default(), deadline, None)) + .map(|r| r.block) + .unwrap(); + + // then block should have all non-exhaust resources extrinsics (+ the first one). + assert_eq!(block.extrinsics().len(), MAX_SKIPPED_TRANSACTIONS + 1); + } + + #[test] + fn should_only_skip_up_to_some_limit_after_soft_deadline() { + // given + let client = Arc::new(substrate_test_runtime_client::new()); + let spawner = sp_core::testing::TaskExecutor::new(); + let txpool = BasicPool::new_full( + Default::default(), + true.into(), + None, + spawner.clone(), + client.clone(), + ); + + block_on( + txpool.submit_at( + &BlockId::number(0), + SOURCE, + (0..MAX_SKIPPED_TRANSACTIONS + 2) + .into_iter() + .map(|i| exhausts_resources_extrinsic_from(i)) + // and some transactions that are okay. + .chain((0..MAX_SKIPPED_TRANSACTIONS).into_iter().map(|i| extrinsic(i as _))) + .collect(), + ), + ) + .unwrap(); + + block_on( + txpool.maintain(chain_event( + client + .header(&BlockId::Number(0u64)) + .expect("header get error") + .expect("there should be header"), + )), + ); + assert_eq!(txpool.ready().count(), MAX_SKIPPED_TRANSACTIONS * 2 + 2); + + let mut proposer_factory = + ProposerFactory::new(spawner.clone(), client.clone(), txpool.clone(), None, None); + + let deadline = time::Duration::from_secs(600); + let cell = Arc::new(Mutex::new((0, time::Instant::now()))); + let cell2 = cell.clone(); + let proposer = proposer_factory.init_with_now( + &client.header(&BlockId::number(0)).unwrap().unwrap(), + Box::new(move || { + let mut value = cell.lock(); + let (called, old) = *value; + // add time after deadline is calculated internally (hence 1) + let increase = if called == 1 { + // we start after the soft_deadline should have already been reached. + deadline / 2 + } else { + // but we make sure to never reach the actual deadline + time::Duration::from_millis(0) + }; + *value = (called + 1, old + increase); + old + }), + ); + + let block = + block_on(proposer.propose(Default::default(), Default::default(), deadline, None)) + .map(|r| r.block) + .unwrap(); + + // then the block should have no transactions despite some in the pool + assert_eq!(block.extrinsics().len(), 1); + assert!( + cell2.lock().0 > MAX_SKIPPED_TRANSACTIONS, + "Not enough calls to current time, which indicates the test might have ended because of deadline, not soft deadline" + ); + } } diff --git a/client/basic-authorship/src/lib.rs b/client/basic-authorship/src/lib.rs index 2b2fe554efdf..4a26ebd9df97 100644 --- a/client/basic-authorship/src/lib.rs +++ b/client/basic-authorship/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/beefy/Cargo.toml b/client/beefy/Cargo.toml index d4541288a628..02be645b3fc0 100644 --- a/client/beefy/Cargo.toml +++ b/client/beefy/Cargo.toml @@ -2,37 +2,53 @@ name = "beefy-gadget" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" +repository = "https://github.com/paritytech/substrate" +description = "BEEFY Client gadget for substrate" [dependencies] fnv = "1.0.6" futures = "0.3" +futures-timer = "3.0.1" +hex = "0.4.2" log = "0.4" -parking_lot = "0.11" +parking_lot = "0.12.0" thiserror = "1.0" wasm-timer = "0.2.5" -codec = { version = "2.2.0", package = "parity-scale-codec", features = ["derive"] } -prometheus = { version = "0.9.0", package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" } +codec = { version = "3.0.0", package = "parity-scale-codec", features = ["derive"] } +prometheus = { version = "0.10.0-dev", package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } -sp-application-crypto = { version = "4.0.0-dev", path = "../../primitives/application-crypto" } -sp-arithmetic = { version = "4.0.0-dev", path = "../../primitives/arithmetic" } +sp-application-crypto = { version = "6.0.0", path = "../../primitives/application-crypto" } +sp-arithmetic = { version = "5.0.0", path = "../../primitives/arithmetic" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } +sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-keystore = { version = "0.12.0", path = "../../primitives/keystore" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } -sc-utils = { version = "4.0.0-dev", path = "../utils" } +sc-chain-spec = { version = "4.0.0-dev", path = "../../client/chain-spec" } sc-client-api = { version = "4.0.0-dev", path = "../api" } +sc-finality-grandpa = { version = "0.10.0-dev", path = "../../client/finality-grandpa" } sc-keystore = { version = "4.0.0-dev", path = "../keystore" } sc-network = { version = "0.10.0-dev", path = "../network" } sc-network-gossip = { version = "0.10.0-dev", path = "../network-gossip" } +sc-utils = { version = "4.0.0-dev", path = "../utils" } beefy-primitives = { version = "4.0.0-dev", path = "../../primitives/beefy" } [dev-dependencies] +sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" } sc-network-test = { version = "0.8.0", path = "../network/test" } -strum = { version = "0.21", features = ["derive"] } +sp-finality-grandpa = { version = "4.0.0-dev", path = "../../primitives/finality-grandpa" } +sp-keyring = { version = "6.0.0", path = "../../primitives/keyring" } +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } +substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } + +serde = "1.0.136" +strum = { version = "0.23", features = ["derive"] } +tokio = "1.15" +tempfile = "3.1.0" diff --git a/client/beefy/rpc/Cargo.toml b/client/beefy/rpc/Cargo.toml index 8af2fa3eac86..47f2558c5ee7 100644 --- a/client/beefy/rpc/Cargo.toml +++ b/client/beefy/rpc/Cargo.toml @@ -2,25 +2,38 @@ name = "beefy-gadget-rpc" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" +repository = "https://github.com/paritytech/substrate" +description = "RPC for the BEEFY Client gadget for substrate" [dependencies] -futures = "0.3.16" +futures = "0.3.21" log = "0.4" -serde = { version = "1.0.130", features = ["derive"] } +parking_lot = "0.12.0" +thiserror = "1.0" +serde = { version = "1.0.136", features = ["derive"] } jsonrpc-core = "18.0.0" jsonrpc-core-client = "18.0.0" jsonrpc-derive = "18.0.0" jsonrpc-pubsub = "18.0.0" -codec = { version = "2.2.0", package = "parity-scale-codec", features = ["derive"] } +codec = { version = "3.0.0", package = "parity-scale-codec", features = ["derive"] } sc-rpc = { version = "4.0.0-dev", path = "../../rpc" } +sc-utils = { version = "4.0.0-dev", path = "../../utils" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } beefy-gadget = { version = "4.0.0-dev", path = "../." } beefy-primitives = { version = "4.0.0-dev", path = "../../../primitives/beefy" } + +[dev-dependencies] +serde_json = "1.0.79" + +sc-rpc = { version = "4.0.0-dev", path = "../../rpc", features = [ + "test-helpers", +] } +substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } diff --git a/client/beefy/rpc/src/lib.rs b/client/beefy/rpc/src/lib.rs index c9a09525569b..4c1bc03e222e 100644 --- a/client/beefy/rpc/src/lib.rs +++ b/client/beefy/rpc/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,19 +20,62 @@ #![warn(missing_docs)] +use parking_lot::RwLock; use std::sync::Arc; use sp_runtime::traits::Block as BlockT; -use futures::{FutureExt, SinkExt, StreamExt}; +use futures::{task::SpawnError, FutureExt, SinkExt, StreamExt, TryFutureExt}; use jsonrpc_derive::rpc; use jsonrpc_pubsub::{manager::SubscriptionManager, typed::Subscriber, SubscriptionId}; use log::warn; -use beefy_gadget::notification::BeefySignedCommitmentStream; +use beefy_gadget::notification::{BeefyBestBlockStream, BeefySignedCommitmentStream}; mod notification; +type FutureResult = jsonrpc_core::BoxFuture>; + +#[derive(Debug, thiserror::Error)] +/// Top-level error type for the RPC handler +pub enum Error { + /// The BEEFY RPC endpoint is not ready. + #[error("BEEFY RPC endpoint not ready")] + EndpointNotReady, + /// The BEEFY RPC background task failed to spawn. + #[error("BEEFY RPC background task failed to spawn")] + RpcTaskFailure(#[from] SpawnError), +} + +/// The error codes returned by jsonrpc. +pub enum ErrorCode { + /// Returned when BEEFY RPC endpoint is not ready. + NotReady = 1, + /// Returned on BEEFY RPC background task failure. + TaskFailure = 2, +} + +impl From for ErrorCode { + fn from(error: Error) -> Self { + match error { + Error::EndpointNotReady => ErrorCode::NotReady, + Error::RpcTaskFailure(_) => ErrorCode::TaskFailure, + } + } +} + +impl From for jsonrpc_core::Error { + fn from(error: Error) -> Self { + let message = format!("{}", error); + let code = ErrorCode::from(error); + jsonrpc_core::Error { + message, + code: jsonrpc_core::ErrorCode::ServerError(code as i64), + data: None, + } + } +} + /// Provides RPC methods for interacting with BEEFY. #[rpc] pub trait BeefyApi { @@ -62,26 +105,57 @@ pub trait BeefyApi { metadata: Option, id: SubscriptionId, ) -> jsonrpc_core::Result; + + /// Returns hash of the latest BEEFY finalized block as seen by this client. + /// + /// The latest BEEFY block might not be available if the BEEFY gadget is not running + /// in the network or if the client is still initializing or syncing with the network. + /// In such case an error would be returned. + #[rpc(name = "beefy_getFinalizedHead")] + fn latest_finalized(&self) -> FutureResult; } /// Implements the BeefyApi RPC trait for interacting with BEEFY. pub struct BeefyRpcHandler { signed_commitment_stream: BeefySignedCommitmentStream, + beefy_best_block: Arc>>, manager: SubscriptionManager, } impl BeefyRpcHandler { /// Creates a new BeefyRpcHandler instance. - pub fn new(signed_commitment_stream: BeefySignedCommitmentStream, executor: E) -> Self + pub fn new( + signed_commitment_stream: BeefySignedCommitmentStream, + best_block_stream: BeefyBestBlockStream, + executor: E, + ) -> Result where E: futures::task::Spawn + Send + Sync + 'static, { + let beefy_best_block = Arc::new(RwLock::new(None)); + + let stream = best_block_stream.subscribe(); + let closure_clone = beefy_best_block.clone(); + let future = stream.for_each(move |best_beefy| { + let async_clone = closure_clone.clone(); + async move { + *async_clone.write() = Some(best_beefy); + } + }); + + executor + .spawn_obj(futures::task::FutureObj::new(Box::pin(future))) + .map_err(|e| { + log::error!("Failed to spawn BEEFY RPC background task; err: {}", e); + e + })?; + let manager = SubscriptionManager::new(Arc::new(executor)); - Self { signed_commitment_stream, manager } + Ok(Self { signed_commitment_stream, beefy_best_block, manager }) } } -impl BeefyApi for BeefyRpcHandler +impl BeefyApi for BeefyRpcHandler where Block: BlockT, { @@ -90,12 +164,12 @@ where fn subscribe_justifications( &self, _metadata: Self::Metadata, - subscriber: Subscriber, + subscriber: Subscriber, ) { let stream = self .signed_commitment_stream .subscribe() - .map(|x| Ok::<_, ()>(Ok(notification::SignedCommitment::new::(x)))); + .map(|x| Ok::<_, ()>(Ok(notification::EncodedSignedCommitment::new::(x)))); self.manager.add(subscriber, |sink| { stream @@ -111,4 +185,212 @@ where ) -> jsonrpc_core::Result { Ok(self.manager.cancel(id)) } + + fn latest_finalized(&self) -> FutureResult { + let result: Result = self + .beefy_best_block + .read() + .as_ref() + .cloned() + .ok_or(Error::EndpointNotReady.into()); + let future = async move { result }.boxed(); + future.map_err(jsonrpc_core::Error::from).boxed() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use jsonrpc_core::{types::Params, Notification, Output}; + + use beefy_gadget::notification::{BeefySignedCommitment, BeefySignedCommitmentSender}; + use beefy_primitives::{known_payload_ids, Payload}; + use codec::{Decode, Encode}; + use sp_runtime::traits::{BlakeTwo256, Hash}; + use substrate_test_runtime_client::runtime::Block; + + fn setup_io_handler( + ) -> (jsonrpc_core::MetaIoHandler, BeefySignedCommitmentSender) { + let (_, stream) = BeefyBestBlockStream::::channel(); + setup_io_handler_with_best_block_stream(stream) + } + + fn setup_io_handler_with_best_block_stream( + best_block_stream: BeefyBestBlockStream, + ) -> (jsonrpc_core::MetaIoHandler, BeefySignedCommitmentSender) { + let (commitment_sender, commitment_stream) = + BeefySignedCommitmentStream::::channel(); + + let handler: BeefyRpcHandler = BeefyRpcHandler::new( + commitment_stream, + best_block_stream, + sc_rpc::testing::TaskExecutor, + ) + .unwrap(); + + let mut io = jsonrpc_core::MetaIoHandler::default(); + io.extend_with(BeefyApi::to_delegate(handler)); + + (io, commitment_sender) + } + + fn setup_session() -> (sc_rpc::Metadata, futures::channel::mpsc::UnboundedReceiver) { + let (tx, rx) = futures::channel::mpsc::unbounded(); + let meta = sc_rpc::Metadata::new(tx); + (meta, rx) + } + + #[test] + fn uninitialized_rpc_handler() { + let (io, _) = setup_io_handler(); + + let request = r#"{"jsonrpc":"2.0","method":"beefy_getFinalizedHead","params":[],"id":1}"#; + let response = r#"{"jsonrpc":"2.0","error":{"code":1,"message":"BEEFY RPC endpoint not ready"},"id":1}"#; + + let meta = sc_rpc::Metadata::default(); + assert_eq!(Some(response.into()), io.handle_request_sync(request, meta)); + } + + #[test] + fn latest_finalized_rpc() { + let (sender, stream) = BeefyBestBlockStream::::channel(); + let (io, _) = setup_io_handler_with_best_block_stream(stream); + + let hash = BlakeTwo256::hash(b"42"); + let r: Result<(), ()> = sender.notify(|| Ok(hash)); + r.unwrap(); + + // Verify RPC `beefy_getFinalizedHead` returns expected hash. + let request = r#"{"jsonrpc":"2.0","method":"beefy_getFinalizedHead","params":[],"id":1}"#; + let expected = "{\ + \"jsonrpc\":\"2.0\",\ + \"result\":\"0x2f0039e93a27221fcf657fb877a1d4f60307106113e885096cb44a461cd0afbf\",\ + \"id\":1\ + }"; + let not_ready = "{\ + \"jsonrpc\":\"2.0\",\ + \"error\":{\"code\":1,\"message\":\"BEEFY RPC endpoint not ready\"},\ + \"id\":1\ + }"; + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + while std::time::Instant::now() < deadline { + let meta = sc_rpc::Metadata::default(); + let response = io.handle_request_sync(request, meta); + // Retry "not ready" responses. + if response != Some(not_ready.into()) { + assert_eq!(response, Some(expected.into())); + // Success + return + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + panic!( + "Deadline reached while waiting for best BEEFY block to update. Perhaps the background task is broken?" + ); + } + + #[test] + fn subscribe_and_unsubscribe_to_justifications() { + let (io, _) = setup_io_handler(); + let (meta, _) = setup_session(); + + // Subscribe + let sub_request = + r#"{"jsonrpc":"2.0","method":"beefy_subscribeJustifications","params":[],"id":1}"#; + let resp = io.handle_request_sync(sub_request, meta.clone()); + let resp: Output = serde_json::from_str(&resp.unwrap()).unwrap(); + + let sub_id = match resp { + Output::Success(success) => success.result, + _ => panic!(), + }; + + // Unsubscribe + let unsub_req = format!( + r#"{{"jsonrpc":"2.0","method":"beefy_unsubscribeJustifications","params":[{}],"id":1}}"#, + sub_id + ); + assert_eq!( + io.handle_request_sync(&unsub_req, meta.clone()), + Some(r#"{"jsonrpc":"2.0","result":true,"id":1}"#.into()), + ); + + // Unsubscribe again and fail + assert_eq!( + io.handle_request_sync(&unsub_req, meta), + Some(r#"{"jsonrpc":"2.0","error":{"code":-32602,"message":"Invalid subscription id."},"id":1}"#.into()), + ); + } + + #[test] + fn subscribe_and_unsubscribe_with_wrong_id() { + let (io, _) = setup_io_handler(); + let (meta, _) = setup_session(); + + // Subscribe + let sub_request = + r#"{"jsonrpc":"2.0","method":"beefy_subscribeJustifications","params":[],"id":1}"#; + let resp = io.handle_request_sync(sub_request, meta.clone()); + let resp: Output = serde_json::from_str(&resp.unwrap()).unwrap(); + assert!(matches!(resp, Output::Success(_))); + + // Unsubscribe with wrong ID + assert_eq!( + io.handle_request_sync( + r#"{"jsonrpc":"2.0","method":"beefy_unsubscribeJustifications","params":["FOO"],"id":1}"#, + meta.clone() + ), + Some(r#"{"jsonrpc":"2.0","error":{"code":-32602,"message":"Invalid subscription id."},"id":1}"#.into()) + ); + } + + fn create_commitment() -> BeefySignedCommitment { + let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode()); + BeefySignedCommitment:: { + commitment: beefy_primitives::Commitment { + payload, + block_number: 5, + validator_set_id: 0, + }, + signatures: vec![], + } + } + + #[test] + fn subscribe_and_listen_to_one_justification() { + let (io, commitment_sender) = setup_io_handler(); + let (meta, receiver) = setup_session(); + + // Subscribe + let sub_request = + r#"{"jsonrpc":"2.0","method":"beefy_subscribeJustifications","params":[],"id":1}"#; + + let resp = io.handle_request_sync(sub_request, meta.clone()); + let mut resp: serde_json::Value = serde_json::from_str(&resp.unwrap()).unwrap(); + let sub_id: String = serde_json::from_value(resp["result"].take()).unwrap(); + + // Notify with commitment + let commitment = create_commitment(); + let r: Result<(), ()> = commitment_sender.notify(|| Ok(commitment.clone())); + r.unwrap(); + + // Inspect what we received + let recv = futures::executor::block_on(receiver.take(1).collect::>()); + let recv: Notification = serde_json::from_str(&recv[0]).unwrap(); + let mut json_map = match recv.params { + Params::Map(json_map) => json_map, + _ => panic!(), + }; + + let recv_sub_id: String = serde_json::from_value(json_map["subscription"].take()).unwrap(); + let recv_commitment: sp_core::Bytes = + serde_json::from_value(json_map["result"].take()).unwrap(); + let recv_commitment: BeefySignedCommitment = + Decode::decode(&mut &recv_commitment[..]).unwrap(); + + assert_eq!(recv.method, "beefy_justifications"); + assert_eq!(recv_sub_id, sub_id); + assert_eq!(recv_commitment, commitment); + } } diff --git a/client/beefy/rpc/src/notification.rs b/client/beefy/rpc/src/notification.rs index 4830d72905a9..2f58c7c6bb5d 100644 --- a/client/beefy/rpc/src/notification.rs +++ b/client/beefy/rpc/src/notification.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -25,15 +25,15 @@ use sp_runtime::traits::Block as BlockT; /// The given bytes should be the SCALE-encoded representation of a /// `beefy_primitives::SignedCommitment`. #[derive(Clone, Serialize, Deserialize)] -pub struct SignedCommitment(sp_core::Bytes); +pub struct EncodedSignedCommitment(sp_core::Bytes); -impl SignedCommitment { +impl EncodedSignedCommitment { pub fn new( - signed_commitment: beefy_gadget::notification::SignedCommitment, + signed_commitment: beefy_gadget::notification::BeefySignedCommitment, ) -> Self where Block: BlockT, { - SignedCommitment(signed_commitment.encode().into()) + EncodedSignedCommitment(signed_commitment.encode().into()) } } diff --git a/client/beefy/src/error.rs b/client/beefy/src/error.rs index db532d34c1e3..eacadeb7613a 100644 --- a/client/beefy/src/error.rs +++ b/client/beefy/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/beefy/src/gossip.rs b/client/beefy/src/gossip.rs index d0199964b6eb..54d283fede32 100644 --- a/client/beefy/src/gossip.rs +++ b/client/beefy/src/gossip.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -30,18 +30,11 @@ use wasm_timer::Instant; use beefy_primitives::{ crypto::{Public, Signature}, - MmrRootHash, VoteMessage, + VoteMessage, }; use crate::keystore::BeefyKeystore; -#[cfg(test)] -#[path = "gossip_tests.rs"] -mod tests; - -// Limit BEEFY gossip by keeping only a bound number of voting rounds alive. -const MAX_LIVE_GOSSIP_ROUNDS: usize = 3; - // Timeout for rebroadcasting messages. const REBROADCAST_AFTER: Duration = Duration::from_secs(60 * 5); @@ -56,13 +49,50 @@ where /// A type that represents hash of the message. pub type MessageHash = [u8; 8]; -type KnownVotes = BTreeMap, fnv::FnvHashSet>; +struct KnownVotes { + last_done: Option>, + live: BTreeMap, fnv::FnvHashSet>, +} + +impl KnownVotes { + pub fn new() -> Self { + Self { last_done: None, live: BTreeMap::new() } + } + + /// Create new round votes set if not already present. + fn insert(&mut self, round: NumberFor) { + self.live.entry(round).or_default(); + } + + /// Remove `round` and older from live set, update `last_done` accordingly. + fn conclude(&mut self, round: NumberFor) { + self.live.retain(|&number, _| number > round); + self.last_done = self.last_done.max(Some(round)); + } + + /// Return true if `round` is newer than previously concluded rounds. + /// + /// Latest concluded round is still considered alive to allow proper gossiping for it. + fn is_live(&self, round: &NumberFor) -> bool { + Some(*round) >= self.last_done + } + + /// Add new _known_ `hash` to the round's known votes. + fn add_known(&mut self, round: &NumberFor, hash: MessageHash) { + self.live.get_mut(round).map(|known| known.insert(hash)); + } + + /// Check if `hash` is already part of round's known votes. + fn is_known(&self, round: &NumberFor, hash: &MessageHash) -> bool { + self.live.get(round).map(|known| known.contains(hash)).unwrap_or(false) + } +} /// BEEFY gossip validator /// /// Validate BEEFY gossip messages and limit the number of live BEEFY voting rounds. /// -/// Allows messages from last [`MAX_LIVE_GOSSIP_ROUNDS`] to flow, everything else gets +/// Allows messages for 'rounds >= last concluded' to flow, everything else gets /// rejected/expired. /// ///All messaging is handled in a single BEEFY global topic. @@ -82,57 +112,25 @@ where pub fn new() -> GossipValidator { GossipValidator { topic: topic::(), - known_votes: RwLock::new(BTreeMap::new()), + known_votes: RwLock::new(KnownVotes::new()), next_rebroadcast: Mutex::new(Instant::now() + REBROADCAST_AFTER), } } /// Note a voting round. /// - /// Noting `round` will keep `round` live. - /// - /// We retain the [`MAX_LIVE_GOSSIP_ROUNDS`] most **recent** voting rounds as live. - /// As long as a voting round is live, it will be gossiped to peer nodes. + /// Noting round will start a live `round`. pub(crate) fn note_round(&self, round: NumberFor) { - debug!(target: "beefy", "🥩 About to note round #{}", round); - - let mut live = self.known_votes.write(); - - if !live.contains_key(&round) { - live.insert(round, Default::default()); - } - - if live.len() > MAX_LIVE_GOSSIP_ROUNDS { - let to_remove = live.iter().next().map(|x| x.0).copied(); - if let Some(first) = to_remove { - live.remove(&first); - } - } - } - - fn add_known(known_votes: &mut KnownVotes, round: &NumberFor, hash: MessageHash) { - known_votes.get_mut(round).map(|known| known.insert(hash)); - } - - // Note that we will always keep the most recent unseen round alive. - // - // This is a preliminary fix and the detailed description why we are - // doing this can be found as part of the issue below - // - // https://github.com/paritytech/grandpa-bridge-gadget/issues/237 - // - fn is_live(known_votes: &KnownVotes, round: &NumberFor) -> bool { - let unseen_round = if let Some(max_known_round) = known_votes.keys().last() { - round > max_known_round - } else { - known_votes.is_empty() - }; - - known_votes.contains_key(round) || unseen_round + debug!(target: "beefy", "🥩 About to note gossip round #{}", round); + self.known_votes.write().insert(round); } - fn is_known(known_votes: &KnownVotes, round: &NumberFor, hash: &MessageHash) -> bool { - known_votes.get(round).map(|known| known.contains(hash)).unwrap_or(false) + /// Conclude a voting round. + /// + /// This can be called once round is complete so we stop gossiping for it. + pub(crate) fn conclude_round(&self, round: NumberFor) { + debug!(target: "beefy", "🥩 About to drop gossip round #{}", round); + self.known_votes.write().conclude(round); } } @@ -146,9 +144,7 @@ where sender: &PeerId, mut data: &[u8], ) -> ValidationResult { - if let Ok(msg) = - VoteMessage::, Public, Signature>::decode(&mut data) - { + if let Ok(msg) = VoteMessage::, Public, Signature>::decode(&mut data) { let msg_hash = twox_64(data); let round = msg.commitment.block_number; @@ -158,17 +154,17 @@ where { let known_votes = self.known_votes.read(); - if !GossipValidator::::is_live(&known_votes, &round) { + if !known_votes.is_live(&round) { return ValidationResult::Discard } - if GossipValidator::::is_known(&known_votes, &round, &msg_hash) { + if known_votes.is_known(&round, &msg_hash) { return ValidationResult::ProcessAndKeep(self.topic) } } if BeefyKeystore::verify(&msg.id, &msg.signature, &msg.commitment.encode()) { - GossipValidator::::add_known(&mut *self.known_votes.write(), &round, msg_hash); + self.known_votes.write().add_known(&round, msg_hash); return ValidationResult::ProcessAndKeep(self.topic) } else { // TODO: report peer @@ -182,15 +178,13 @@ where fn message_expired<'a>(&'a self) -> Box bool + 'a> { let known_votes = self.known_votes.read(); Box::new(move |_topic, mut data| { - let msg = match VoteMessage::, Public, Signature>::decode( - &mut data, - ) { + let msg = match VoteMessage::, Public, Signature>::decode(&mut data) { Ok(vote) => vote, Err(_) => return true, }; let round = msg.commitment.block_number; - let expired = !GossipValidator::::is_live(&known_votes, &round); + let expired = !known_votes.is_live(&round); trace!(target: "beefy", "🥩 Message for round #{} expired: {}", round, expired); @@ -218,15 +212,13 @@ where return do_rebroadcast } - let msg = match VoteMessage::, Public, Signature>::decode( - &mut data, - ) { + let msg = match VoteMessage::, Public, Signature>::decode(&mut data) { Ok(vote) => vote, - Err(_) => return true, + Err(_) => return false, }; let round = msg.commitment.block_number; - let allowed = GossipValidator::::is_live(&known_votes, &round); + let allowed = known_votes.is_live(&round); debug!(target: "beefy", "🥩 Message for round #{} allowed: {}", round, allowed); @@ -234,3 +226,237 @@ where }) } } + +#[cfg(test)] +mod tests { + use sc_keystore::LocalKeystore; + use sc_network_test::Block; + use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; + + use crate::keystore::{tests::Keyring, BeefyKeystore}; + use beefy_primitives::{ + crypto::Signature, known_payload_ids, Commitment, MmrRootHash, Payload, VoteMessage, + KEY_TYPE, + }; + + use super::*; + + #[test] + fn known_votes_insert_remove() { + let mut kv = KnownVotes::::new(); + + kv.insert(1); + kv.insert(1); + kv.insert(2); + assert_eq!(kv.live.len(), 2); + + let mut kv = KnownVotes::::new(); + kv.insert(1); + kv.insert(2); + kv.insert(3); + + assert!(kv.last_done.is_none()); + kv.conclude(2); + assert_eq!(kv.live.len(), 1); + assert!(!kv.live.contains_key(&2)); + assert_eq!(kv.last_done, Some(2)); + + kv.conclude(1); + assert_eq!(kv.last_done, Some(2)); + + kv.conclude(3); + assert_eq!(kv.last_done, Some(3)); + assert!(kv.live.is_empty()); + } + + #[test] + fn note_and_drop_round_works() { + let gv = GossipValidator::::new(); + + gv.note_round(1u64); + + assert!(gv.known_votes.read().is_live(&1u64)); + + gv.note_round(3u64); + gv.note_round(7u64); + gv.note_round(10u64); + + assert_eq!(gv.known_votes.read().live.len(), 4); + + gv.conclude_round(7u64); + + let votes = gv.known_votes.read(); + + // rounds 1 and 3 are outdated, don't gossip anymore + assert!(!votes.is_live(&1u64)); + assert!(!votes.is_live(&3u64)); + // latest concluded round is still gossiped + assert!(votes.is_live(&7u64)); + // round 10 is alive and in-progress + assert!(votes.is_live(&10u64)); + } + + #[test] + fn note_same_round_twice() { + let gv = GossipValidator::::new(); + + gv.note_round(3u64); + gv.note_round(7u64); + gv.note_round(10u64); + + assert_eq!(gv.known_votes.read().live.len(), 3); + + // note round #7 again -> should not change anything + gv.note_round(7u64); + + let votes = gv.known_votes.read(); + + assert_eq!(votes.live.len(), 3); + + assert!(votes.is_live(&3u64)); + assert!(votes.is_live(&7u64)); + assert!(votes.is_live(&10u64)); + } + + struct TestContext; + impl ValidatorContext for TestContext { + fn broadcast_topic(&mut self, _topic: B::Hash, _force: bool) { + todo!() + } + + fn broadcast_message(&mut self, _topic: B::Hash, _message: Vec, _force: bool) { + todo!() + } + + fn send_message(&mut self, _who: &sc_network::PeerId, _message: Vec) { + todo!() + } + + fn send_topic(&mut self, _who: &sc_network::PeerId, _topic: B::Hash, _force: bool) { + todo!() + } + } + + fn sign_commitment(who: &Keyring, commitment: &Commitment) -> Signature { + let store: SyncCryptoStorePtr = std::sync::Arc::new(LocalKeystore::in_memory()); + SyncCryptoStore::ecdsa_generate_new(&*store, KEY_TYPE, Some(&who.to_seed())).unwrap(); + let beefy_keystore: BeefyKeystore = Some(store).into(); + + beefy_keystore.sign(&who.public(), &commitment.encode()).unwrap() + } + + fn dummy_vote(block_number: u64) -> VoteMessage { + let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, MmrRootHash::default().encode()); + let commitment = Commitment { payload, block_number, validator_set_id: 0 }; + let signature = sign_commitment(&Keyring::Alice, &commitment); + + VoteMessage { commitment, id: Keyring::Alice.public(), signature } + } + + #[test] + fn should_avoid_verifying_signatures_twice() { + let gv = GossipValidator::::new(); + let sender = sc_network::PeerId::random(); + let mut context = TestContext; + + let vote = dummy_vote(3); + + gv.note_round(3u64); + gv.note_round(7u64); + gv.note_round(10u64); + + // first time the cache should be populated + let res = gv.validate(&mut context, &sender, &vote.encode()); + + assert!(matches!(res, ValidationResult::ProcessAndKeep(_))); + assert_eq!( + gv.known_votes.read().live.get(&vote.commitment.block_number).map(|x| x.len()), + Some(1) + ); + + // second time we should hit the cache + let res = gv.validate(&mut context, &sender, &vote.encode()); + + assert!(matches!(res, ValidationResult::ProcessAndKeep(_))); + + // next we should quickly reject if the round is not live + gv.conclude_round(7_u64); + + assert!(!gv.known_votes.read().is_live(&vote.commitment.block_number)); + + let res = gv.validate(&mut context, &sender, &vote.encode()); + + assert!(matches!(res, ValidationResult::Discard)); + } + + #[test] + fn messages_allowed_and_expired() { + let gv = GossipValidator::::new(); + let sender = sc_network::PeerId::random(); + let topic = Default::default(); + let intent = MessageIntent::Broadcast; + + // note round 2 and 3, then conclude 2 + gv.note_round(2u64); + gv.note_round(3u64); + gv.conclude_round(2u64); + let mut allowed = gv.message_allowed(); + let mut expired = gv.message_expired(); + + // check bad vote format + assert!(!allowed(&sender, intent, &topic, &mut [0u8; 16])); + assert!(expired(topic, &mut [0u8; 16])); + + // inactive round 1 -> expired + let vote = dummy_vote(1); + let mut encoded_vote = vote.encode(); + assert!(!allowed(&sender, intent, &topic, &mut encoded_vote)); + assert!(expired(topic, &mut encoded_vote)); + + // active round 2 -> !expired - concluded but still gossiped + let vote = dummy_vote(2); + let mut encoded_vote = vote.encode(); + assert!(allowed(&sender, intent, &topic, &mut encoded_vote)); + assert!(!expired(topic, &mut encoded_vote)); + + // in progress round 3 -> !expired + let vote = dummy_vote(3); + let mut encoded_vote = vote.encode(); + assert!(allowed(&sender, intent, &topic, &mut encoded_vote)); + assert!(!expired(topic, &mut encoded_vote)); + + // unseen round 4 -> !expired + let vote = dummy_vote(3); + let mut encoded_vote = vote.encode(); + assert!(allowed(&sender, intent, &topic, &mut encoded_vote)); + assert!(!expired(topic, &mut encoded_vote)); + } + + #[test] + fn messages_rebroadcast() { + let gv = GossipValidator::::new(); + let sender = sc_network::PeerId::random(); + let topic = Default::default(); + + let vote = dummy_vote(1); + let mut encoded_vote = vote.encode(); + + // re-broadcasting only allowed at `REBROADCAST_AFTER` intervals + let intent = MessageIntent::PeriodicRebroadcast; + let mut allowed = gv.message_allowed(); + + // rebroadcast not allowed so soon after GossipValidator creation + assert!(!allowed(&sender, intent, &topic, &mut encoded_vote)); + + // hack the inner deadline to be `now` + *gv.next_rebroadcast.lock() = Instant::now(); + + // still not allowed on old `allowed` closure result + assert!(!allowed(&sender, intent, &topic, &mut encoded_vote)); + + // renew closure result + let mut allowed = gv.message_allowed(); + // rebroadcast should be allowed now + assert!(allowed(&sender, intent, &topic, &mut encoded_vote)); + } +} diff --git a/client/beefy/src/gossip_tests.rs b/client/beefy/src/gossip_tests.rs deleted file mode 100644 index 2d46b873cb7b..000000000000 --- a/client/beefy/src/gossip_tests.rs +++ /dev/null @@ -1,182 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use sc_keystore::LocalKeystore; -use sc_network_test::Block; -use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; - -use beefy_primitives::{crypto::Signature, Commitment, MmrRootHash, VoteMessage, KEY_TYPE}; - -use crate::keystore::{tests::Keyring, BeefyKeystore}; - -use super::*; - -#[test] -fn note_round_works() { - let gv = GossipValidator::::new(); - - gv.note_round(1u64); - - let live = gv.known_votes.read(); - assert!(GossipValidator::::is_live(&live, &1u64)); - - drop(live); - - gv.note_round(3u64); - gv.note_round(7u64); - gv.note_round(10u64); - - let live = gv.known_votes.read(); - - assert_eq!(live.len(), MAX_LIVE_GOSSIP_ROUNDS); - - assert!(!GossipValidator::::is_live(&live, &1u64)); - assert!(GossipValidator::::is_live(&live, &3u64)); - assert!(GossipValidator::::is_live(&live, &7u64)); - assert!(GossipValidator::::is_live(&live, &10u64)); -} - -#[test] -fn keeps_most_recent_max_rounds() { - let gv = GossipValidator::::new(); - - gv.note_round(3u64); - gv.note_round(7u64); - gv.note_round(10u64); - gv.note_round(1u64); - - let live = gv.known_votes.read(); - - assert_eq!(live.len(), MAX_LIVE_GOSSIP_ROUNDS); - - assert!(GossipValidator::::is_live(&live, &3u64)); - assert!(!GossipValidator::::is_live(&live, &1u64)); - - drop(live); - - gv.note_round(23u64); - gv.note_round(15u64); - gv.note_round(20u64); - gv.note_round(2u64); - - let live = gv.known_votes.read(); - - assert_eq!(live.len(), MAX_LIVE_GOSSIP_ROUNDS); - - assert!(GossipValidator::::is_live(&live, &15u64)); - assert!(GossipValidator::::is_live(&live, &20u64)); - assert!(GossipValidator::::is_live(&live, &23u64)); -} - -#[test] -fn note_same_round_twice() { - let gv = GossipValidator::::new(); - - gv.note_round(3u64); - gv.note_round(7u64); - gv.note_round(10u64); - - let live = gv.known_votes.read(); - - assert_eq!(live.len(), MAX_LIVE_GOSSIP_ROUNDS); - - drop(live); - - // note round #7 again -> should not change anything - gv.note_round(7u64); - - let live = gv.known_votes.read(); - - assert_eq!(live.len(), MAX_LIVE_GOSSIP_ROUNDS); - - assert!(GossipValidator::::is_live(&live, &3u64)); - assert!(GossipValidator::::is_live(&live, &7u64)); - assert!(GossipValidator::::is_live(&live, &10u64)); -} - -struct TestContext; -impl ValidatorContext for TestContext { - fn broadcast_topic(&mut self, _topic: B::Hash, _force: bool) { - todo!() - } - - fn broadcast_message(&mut self, _topic: B::Hash, _message: Vec, _force: bool) { - todo!() - } - - fn send_message(&mut self, _who: &sc_network::PeerId, _message: Vec) { - todo!() - } - - fn send_topic(&mut self, _who: &sc_network::PeerId, _topic: B::Hash, _force: bool) { - todo!() - } -} - -fn sign_commitment( - who: &Keyring, - commitment: &Commitment, -) -> Signature { - let store: SyncCryptoStorePtr = std::sync::Arc::new(LocalKeystore::in_memory()); - SyncCryptoStore::ecdsa_generate_new(&*store, KEY_TYPE, Some(&who.to_seed())).unwrap(); - let beefy_keystore: BeefyKeystore = Some(store).into(); - - beefy_keystore.sign(&who.public(), &commitment.encode()).unwrap() -} - -#[test] -fn should_avoid_verifying_signatures_twice() { - let gv = GossipValidator::::new(); - let sender = sc_network::PeerId::random(); - let mut context = TestContext; - - let commitment = - Commitment { payload: MmrRootHash::default(), block_number: 3_u64, validator_set_id: 0 }; - - let signature = sign_commitment(&Keyring::Alice, &commitment); - - let vote = VoteMessage { commitment, id: Keyring::Alice.public(), signature }; - - gv.note_round(3u64); - gv.note_round(7u64); - gv.note_round(10u64); - - // first time the cache should be populated. - let res = gv.validate(&mut context, &sender, &vote.encode()); - - assert!(matches!(res, ValidationResult::ProcessAndKeep(_))); - assert_eq!(gv.known_votes.read().get(&vote.commitment.block_number).map(|x| x.len()), Some(1)); - - // second time we should hit the cache - let res = gv.validate(&mut context, &sender, &vote.encode()); - - assert!(matches!(res, ValidationResult::ProcessAndKeep(_))); - - // next we should quickly reject if the round is not live. - gv.note_round(11_u64); - gv.note_round(12_u64); - - assert!(!GossipValidator::::is_live( - &*gv.known_votes.read(), - &vote.commitment.block_number - )); - - let res = gv.validate(&mut context, &sender, &vote.encode()); - - assert!(matches!(res, ValidationResult::Discard)); -} diff --git a/client/beefy/src/keystore.rs b/client/beefy/src/keystore.rs index 88618b8a5a14..b0259a42075e 100644 --- a/client/beefy/src/keystore.rs +++ b/client/beefy/src/keystore.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,8 +16,6 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use std::convert::{From, TryInto}; - use sp_application_crypto::RuntimeAppPublic; use sp_core::keccak_256; use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; @@ -31,10 +29,6 @@ use beefy_primitives::{ use crate::error; -#[cfg(test)] -#[path = "keystore_tests.rs"] -pub mod tests; - /// A BEEFY specific keystore implemented as a `Newtype`. This is basically a /// wrapper around [`sp_keystore::SyncCryptoStore`] and allows to customize /// common cryptographic functionality. @@ -93,8 +87,8 @@ impl BeefyKeystore { let store = self.0.clone().ok_or_else(|| error::Error::Keystore("no Keystore".into()))?; let pk: Vec = SyncCryptoStore::ecdsa_public_keys(&*store, KEY_TYPE) - .iter() - .map(|k| Public::from(k.clone())) + .drain(..) + .map(Public::from) .collect(); Ok(pk) @@ -117,3 +111,268 @@ impl From> for BeefyKeystore { BeefyKeystore(store) } } + +#[cfg(test)] +pub mod tests { + use std::sync::Arc; + + use sc_keystore::LocalKeystore; + use sp_core::{ecdsa, keccak_256, Pair}; + use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; + + use beefy_primitives::{crypto, KEY_TYPE}; + + use super::BeefyKeystore; + use crate::error::Error; + + /// Set of test accounts using [`beefy_primitives::crypto`] types. + #[allow(missing_docs)] + #[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display, strum::EnumIter)] + pub(crate) enum Keyring { + Alice, + Bob, + Charlie, + Dave, + Eve, + Ferdie, + One, + Two, + } + + impl Keyring { + /// Sign `msg`. + pub fn sign(self, msg: &[u8]) -> crypto::Signature { + let msg = keccak_256(msg); + ecdsa::Pair::from(self).sign_prehashed(&msg).into() + } + + /// Return key pair. + pub fn pair(self) -> crypto::Pair { + ecdsa::Pair::from_string(self.to_seed().as_str(), None).unwrap().into() + } + + /// Return public key. + pub fn public(self) -> crypto::Public { + self.pair().public() + } + + /// Return seed string. + pub fn to_seed(self) -> String { + format!("//{}", self) + } + } + + impl From for crypto::Pair { + fn from(k: Keyring) -> Self { + k.pair() + } + } + + impl From for ecdsa::Pair { + fn from(k: Keyring) -> Self { + k.pair().into() + } + } + + fn keystore() -> SyncCryptoStorePtr { + Arc::new(LocalKeystore::in_memory()) + } + + #[test] + fn verify_should_work() { + let msg = keccak_256(b"I am Alice!"); + let sig = Keyring::Alice.sign(b"I am Alice!"); + + assert!(ecdsa::Pair::verify_prehashed( + &sig.clone().into(), + &msg, + &Keyring::Alice.public().into(), + )); + + // different public key -> fail + assert!(!ecdsa::Pair::verify_prehashed( + &sig.clone().into(), + &msg, + &Keyring::Bob.public().into(), + )); + + let msg = keccak_256(b"I am not Alice!"); + + // different msg -> fail + assert!( + !ecdsa::Pair::verify_prehashed(&sig.into(), &msg, &Keyring::Alice.public().into(),) + ); + } + + #[test] + fn pair_works() { + let want = crypto::Pair::from_string("//Alice", None).expect("Pair failed").to_raw_vec(); + let got = Keyring::Alice.pair().to_raw_vec(); + assert_eq!(want, got); + + let want = crypto::Pair::from_string("//Bob", None).expect("Pair failed").to_raw_vec(); + let got = Keyring::Bob.pair().to_raw_vec(); + assert_eq!(want, got); + + let want = crypto::Pair::from_string("//Charlie", None).expect("Pair failed").to_raw_vec(); + let got = Keyring::Charlie.pair().to_raw_vec(); + assert_eq!(want, got); + + let want = crypto::Pair::from_string("//Dave", None).expect("Pair failed").to_raw_vec(); + let got = Keyring::Dave.pair().to_raw_vec(); + assert_eq!(want, got); + + let want = crypto::Pair::from_string("//Eve", None).expect("Pair failed").to_raw_vec(); + let got = Keyring::Eve.pair().to_raw_vec(); + assert_eq!(want, got); + + let want = crypto::Pair::from_string("//Ferdie", None).expect("Pair failed").to_raw_vec(); + let got = Keyring::Ferdie.pair().to_raw_vec(); + assert_eq!(want, got); + + let want = crypto::Pair::from_string("//One", None).expect("Pair failed").to_raw_vec(); + let got = Keyring::One.pair().to_raw_vec(); + assert_eq!(want, got); + + let want = crypto::Pair::from_string("//Two", None).expect("Pair failed").to_raw_vec(); + let got = Keyring::Two.pair().to_raw_vec(); + assert_eq!(want, got); + } + + #[test] + fn authority_id_works() { + let store = keystore(); + + let alice: crypto::Public = + SyncCryptoStore::ecdsa_generate_new(&*store, KEY_TYPE, Some(&Keyring::Alice.to_seed())) + .ok() + .unwrap() + .into(); + + let bob = Keyring::Bob.public(); + let charlie = Keyring::Charlie.public(); + + let store: BeefyKeystore = Some(store).into(); + + let mut keys = vec![bob, charlie]; + + let id = store.authority_id(keys.as_slice()); + assert!(id.is_none()); + + keys.push(alice.clone()); + + let id = store.authority_id(keys.as_slice()).unwrap(); + assert_eq!(id, alice); + } + + #[test] + fn sign_works() { + let store = keystore(); + + let alice: crypto::Public = + SyncCryptoStore::ecdsa_generate_new(&*store, KEY_TYPE, Some(&Keyring::Alice.to_seed())) + .ok() + .unwrap() + .into(); + + let store: BeefyKeystore = Some(store).into(); + + let msg = b"are you involved or commited?"; + + let sig1 = store.sign(&alice, msg).unwrap(); + let sig2 = Keyring::Alice.sign(msg); + + assert_eq!(sig1, sig2); + } + + #[test] + fn sign_error() { + let store = keystore(); + + let _ = + SyncCryptoStore::ecdsa_generate_new(&*store, KEY_TYPE, Some(&Keyring::Bob.to_seed())) + .ok() + .unwrap(); + + let store: BeefyKeystore = Some(store).into(); + + let alice = Keyring::Alice.public(); + + let msg = b"are you involved or commited?"; + let sig = store.sign(&alice, msg).err().unwrap(); + let err = Error::Signature("ecdsa_sign_prehashed() failed".to_string()); + + assert_eq!(sig, err); + } + + #[test] + fn sign_no_keystore() { + let store: BeefyKeystore = None.into(); + + let alice = Keyring::Alice.public(); + let msg = b"are you involved or commited"; + + let sig = store.sign(&alice, msg).err().unwrap(); + let err = Error::Keystore("no Keystore".to_string()); + assert_eq!(sig, err); + } + + #[test] + fn verify_works() { + let store = keystore(); + + let alice: crypto::Public = + SyncCryptoStore::ecdsa_generate_new(&*store, KEY_TYPE, Some(&Keyring::Alice.to_seed())) + .ok() + .unwrap() + .into(); + + let store: BeefyKeystore = Some(store).into(); + + // `msg` and `sig` match + let msg = b"are you involved or commited?"; + let sig = store.sign(&alice, msg).unwrap(); + assert!(BeefyKeystore::verify(&alice, &sig, msg)); + + // `msg and `sig` don't match + let msg = b"you are just involved"; + assert!(!BeefyKeystore::verify(&alice, &sig, msg)); + } + + // Note that we use keys with and without a seed for this test. + #[test] + fn public_keys_works() { + const TEST_TYPE: sp_application_crypto::KeyTypeId = + sp_application_crypto::KeyTypeId(*b"test"); + + let store = keystore(); + + let add_key = |key_type, seed: Option<&str>| { + SyncCryptoStore::ecdsa_generate_new(&*store, key_type, seed).unwrap() + }; + + // test keys + let _ = add_key(TEST_TYPE, Some(Keyring::Alice.to_seed().as_str())); + let _ = add_key(TEST_TYPE, Some(Keyring::Bob.to_seed().as_str())); + + let _ = add_key(TEST_TYPE, None); + let _ = add_key(TEST_TYPE, None); + + // BEEFY keys + let _ = add_key(KEY_TYPE, Some(Keyring::Dave.to_seed().as_str())); + let _ = add_key(KEY_TYPE, Some(Keyring::Eve.to_seed().as_str())); + + let key1: crypto::Public = add_key(KEY_TYPE, None).into(); + let key2: crypto::Public = add_key(KEY_TYPE, None).into(); + + let store: BeefyKeystore = Some(store).into(); + + let keys = store.public_keys().ok().unwrap(); + + assert!(keys.len() == 4); + assert!(keys.contains(&Keyring::Dave.public())); + assert!(keys.contains(&Keyring::Eve.public())); + assert!(keys.contains(&key1)); + assert!(keys.contains(&key2)); + } +} diff --git a/client/beefy/src/keystore_tests.rs b/client/beefy/src/keystore_tests.rs deleted file mode 100644 index 99e3e42228df..000000000000 --- a/client/beefy/src/keystore_tests.rs +++ /dev/null @@ -1,275 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use std::sync::Arc; - -use sc_keystore::LocalKeystore; -use sp_core::{ecdsa, keccak_256, Pair}; -use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; - -use beefy_primitives::{crypto, KEY_TYPE}; - -use super::BeefyKeystore; -use crate::error::Error; - -/// Set of test accounts using [`beefy_primitives::crypto`] types. -#[allow(missing_docs)] -#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::Display, strum::EnumIter)] -pub(crate) enum Keyring { - Alice, - Bob, - Charlie, - Dave, - Eve, - Ferdie, - One, - Two, -} - -impl Keyring { - /// Sign `msg`. - pub fn sign(self, msg: &[u8]) -> crypto::Signature { - let msg = keccak_256(msg); - ecdsa::Pair::from(self).sign_prehashed(&msg).into() - } - - /// Return key pair. - pub fn pair(self) -> crypto::Pair { - ecdsa::Pair::from_string(self.to_seed().as_str(), None).unwrap().into() - } - - /// Return public key. - pub fn public(self) -> crypto::Public { - self.pair().public() - } - - /// Return seed string. - pub fn to_seed(self) -> String { - format!("//{}", self) - } -} - -impl From for crypto::Pair { - fn from(k: Keyring) -> Self { - k.pair() - } -} - -impl From for ecdsa::Pair { - fn from(k: Keyring) -> Self { - k.pair().into() - } -} - -fn keystore() -> SyncCryptoStorePtr { - Arc::new(LocalKeystore::in_memory()) -} - -#[test] -fn verify_should_work() { - let msg = keccak_256(b"I am Alice!"); - let sig = Keyring::Alice.sign(b"I am Alice!"); - - assert!(ecdsa::Pair::verify_prehashed( - &sig.clone().into(), - &msg, - &Keyring::Alice.public().into(), - )); - - // different public key -> fail - assert!(!ecdsa::Pair::verify_prehashed( - &sig.clone().into(), - &msg, - &Keyring::Bob.public().into(), - )); - - let msg = keccak_256(b"I am not Alice!"); - - // different msg -> fail - assert!(!ecdsa::Pair::verify_prehashed(&sig.into(), &msg, &Keyring::Alice.public().into(),)); -} - -#[test] -fn pair_works() { - let want = crypto::Pair::from_string("//Alice", None).expect("Pair failed").to_raw_vec(); - let got = Keyring::Alice.pair().to_raw_vec(); - assert_eq!(want, got); - - let want = crypto::Pair::from_string("//Bob", None).expect("Pair failed").to_raw_vec(); - let got = Keyring::Bob.pair().to_raw_vec(); - assert_eq!(want, got); - - let want = crypto::Pair::from_string("//Charlie", None).expect("Pair failed").to_raw_vec(); - let got = Keyring::Charlie.pair().to_raw_vec(); - assert_eq!(want, got); - - let want = crypto::Pair::from_string("//Dave", None).expect("Pair failed").to_raw_vec(); - let got = Keyring::Dave.pair().to_raw_vec(); - assert_eq!(want, got); - - let want = crypto::Pair::from_string("//Eve", None).expect("Pair failed").to_raw_vec(); - let got = Keyring::Eve.pair().to_raw_vec(); - assert_eq!(want, got); - - let want = crypto::Pair::from_string("//Ferdie", None).expect("Pair failed").to_raw_vec(); - let got = Keyring::Ferdie.pair().to_raw_vec(); - assert_eq!(want, got); - - let want = crypto::Pair::from_string("//One", None).expect("Pair failed").to_raw_vec(); - let got = Keyring::One.pair().to_raw_vec(); - assert_eq!(want, got); - - let want = crypto::Pair::from_string("//Two", None).expect("Pair failed").to_raw_vec(); - let got = Keyring::Two.pair().to_raw_vec(); - assert_eq!(want, got); -} - -#[test] -fn authority_id_works() { - let store = keystore(); - - let alice: crypto::Public = - SyncCryptoStore::ecdsa_generate_new(&*store, KEY_TYPE, Some(&Keyring::Alice.to_seed())) - .ok() - .unwrap() - .into(); - - let bob = Keyring::Bob.public(); - let charlie = Keyring::Charlie.public(); - - let store: BeefyKeystore = Some(store).into(); - - let mut keys = vec![bob, charlie]; - - let id = store.authority_id(keys.as_slice()); - assert!(id.is_none()); - - keys.push(alice.clone()); - - let id = store.authority_id(keys.as_slice()).unwrap(); - assert_eq!(id, alice); -} - -#[test] -fn sign_works() { - let store = keystore(); - - let alice: crypto::Public = - SyncCryptoStore::ecdsa_generate_new(&*store, KEY_TYPE, Some(&Keyring::Alice.to_seed())) - .ok() - .unwrap() - .into(); - - let store: BeefyKeystore = Some(store).into(); - - let msg = b"are you involved or commited?"; - - let sig1 = store.sign(&alice, msg).unwrap(); - let sig2 = Keyring::Alice.sign(msg); - - assert_eq!(sig1, sig2); -} - -#[test] -fn sign_error() { - let store = keystore(); - - let _ = SyncCryptoStore::ecdsa_generate_new(&*store, KEY_TYPE, Some(&Keyring::Bob.to_seed())) - .ok() - .unwrap(); - - let store: BeefyKeystore = Some(store).into(); - - let alice = Keyring::Alice.public(); - - let msg = b"are you involved or commited?"; - let sig = store.sign(&alice, msg).err().unwrap(); - let err = Error::Signature("ecdsa_sign_prehashed() failed".to_string()); - - assert_eq!(sig, err); -} - -#[test] -fn sign_no_keystore() { - let store: BeefyKeystore = None.into(); - - let alice = Keyring::Alice.public(); - let msg = b"are you involved or commited"; - - let sig = store.sign(&alice, msg).err().unwrap(); - let err = Error::Keystore("no Keystore".to_string()); - assert_eq!(sig, err); -} - -#[test] -fn verify_works() { - let store = keystore(); - - let alice: crypto::Public = - SyncCryptoStore::ecdsa_generate_new(&*store, KEY_TYPE, Some(&Keyring::Alice.to_seed())) - .ok() - .unwrap() - .into(); - - let store: BeefyKeystore = Some(store).into(); - - // `msg` and `sig` match - let msg = b"are you involved or commited?"; - let sig = store.sign(&alice, msg).unwrap(); - assert!(BeefyKeystore::verify(&alice, &sig, msg)); - - // `msg and `sig` don't match - let msg = b"you are just involved"; - assert!(!BeefyKeystore::verify(&alice, &sig, msg)); -} - -// Note that we use keys with and without a seed for this test. -#[test] -fn public_keys_works() { - const TEST_TYPE: sp_application_crypto::KeyTypeId = sp_application_crypto::KeyTypeId(*b"test"); - - let store = keystore(); - - let add_key = |key_type, seed: Option<&str>| { - SyncCryptoStore::ecdsa_generate_new(&*store, key_type, seed).unwrap() - }; - - // test keys - let _ = add_key(TEST_TYPE, Some(Keyring::Alice.to_seed().as_str())); - let _ = add_key(TEST_TYPE, Some(Keyring::Bob.to_seed().as_str())); - - let _ = add_key(TEST_TYPE, None); - let _ = add_key(TEST_TYPE, None); - - // BEEFY keys - let _ = add_key(KEY_TYPE, Some(Keyring::Dave.to_seed().as_str())); - let _ = add_key(KEY_TYPE, Some(Keyring::Eve.to_seed().as_str())); - - let key1: crypto::Public = add_key(KEY_TYPE, None).into(); - let key2: crypto::Public = add_key(KEY_TYPE, None).into(); - - let store: BeefyKeystore = Some(store).into(); - - let keys = store.public_keys().ok().unwrap(); - - assert!(keys.len() == 4); - assert!(keys.contains(&Keyring::Dave.public())); - assert!(keys.contains(&Keyring::Eve.public())); - assert!(keys.contains(&key1)); - assert!(keys.contains(&key2)); -} diff --git a/client/beefy/src/lib.rs b/client/beefy/src/lib.rs index b2372b2a6c51..8a6e175f5832 100644 --- a/client/beefy/src/lib.rs +++ b/client/beefy/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,19 +18,21 @@ use std::sync::Arc; -use log::debug; use prometheus::Registry; use sc_client_api::{Backend, BlockchainEvents, Finalizer}; -use sc_network_gossip::{GossipEngine, Network as GossipNetwork}; +use sc_network_gossip::Network as GossipNetwork; use sp_api::ProvideRuntimeApi; use sp_blockchain::HeaderBackend; +use sp_consensus::SyncOracle; use sp_keystore::SyncCryptoStorePtr; use sp_runtime::traits::Block; use beefy_primitives::BeefyApi; +use crate::notification::{BeefyBestBlockSender, BeefySignedCommitmentSender}; + mod error; mod gossip; mod keystore; @@ -40,14 +42,43 @@ mod worker; pub mod notification; -pub const BEEFY_PROTOCOL_NAME: &str = "/paritytech/beefy/1"; +#[cfg(test)] +mod tests; + +pub use beefy_protocol_name::standard_name as protocol_standard_name; + +pub(crate) mod beefy_protocol_name { + use sc_chain_spec::ChainSpec; + + const NAME: &'static str = "/beefy/1"; + /// Old names for the notifications protocol, used for backward compatibility. + pub(crate) const LEGACY_NAMES: [&'static str; 1] = ["/paritytech/beefy/1"]; + + /// Name of the notifications protocol used by BEEFY. + /// + /// Must be registered towards the networking in order for BEEFY to properly function. + pub fn standard_name>( + genesis_hash: &Hash, + chain_spec: &Box, + ) -> std::borrow::Cow<'static, str> { + let chain_prefix = match chain_spec.fork_id() { + Some(fork_id) => format!("/{}/{}", hex::encode(genesis_hash), fork_id), + None => format!("/{}", hex::encode(genesis_hash)), + }; + format!("{}{}", chain_prefix, NAME).into() + } +} /// Returns the configuration value to put in /// [`sc_network::config::NetworkConfiguration::extra_sets`]. -pub fn beefy_peers_set_config() -> sc_network::config::NonDefaultSetConfig { - let mut cfg = - sc_network::config::NonDefaultSetConfig::new(BEEFY_PROTOCOL_NAME.into(), 1024 * 1024); +/// For standard protocol name see [`beefy_protocol_name::standard_name`]. +pub fn beefy_peers_set_config( + protocol_name: std::borrow::Cow<'static, str>, +) -> sc_network::config::NonDefaultSetConfig { + let mut cfg = sc_network::config::NonDefaultSetConfig::new(protocol_name, 1024 * 1024); + cfg.allow_non_reserved(25, 25); + cfg.add_fallback_names(beefy_protocol_name::LEGACY_NAMES.iter().map(|&n| n.into()).collect()); cfg } @@ -85,7 +116,7 @@ where BE: Backend, C: Client, C::Api: BeefyApi, - N: GossipNetwork + Clone + Send + 'static, + N: GossipNetwork + Clone + SyncOracle + Send + Sync + 'static, { /// BEEFY client pub client: Arc, @@ -96,13 +127,18 @@ where /// Gossip network pub network: N, /// BEEFY signed commitment sender - pub signed_commitment_sender: notification::BeefySignedCommitmentSender, + pub signed_commitment_sender: BeefySignedCommitmentSender, + /// BEEFY best block sender + pub beefy_best_block_sender: BeefyBestBlockSender, /// Minimal delta between blocks, BEEFY should vote for pub min_block_delta: u32, /// Prometheus metric registry pub prometheus_registry: Option, + /// Chain specific GRANDPA protocol name. See [`beefy_protocol_name::standard_name`]. + pub protocol_name: std::borrow::Cow<'static, str>, } +#[cfg(not(test))] /// Start the BEEFY gadget. /// /// This is a thin shim around running and awaiting a BEEFY worker. @@ -112,7 +148,7 @@ where BE: Backend, C: Client, C::Api: BeefyApi, - N: GossipNetwork + Clone + Send + 'static, + N: GossipNetwork + Clone + SyncOracle + Send + Sync + 'static, { let BeefyParams { client, @@ -120,23 +156,30 @@ where key_store, network, signed_commitment_sender, + beefy_best_block_sender, min_block_delta, prometheus_registry, + protocol_name, } = beefy_params; + let sync_oracle = network.clone(); let gossip_validator = Arc::new(gossip::GossipValidator::new()); - let gossip_engine = - GossipEngine::new(network, BEEFY_PROTOCOL_NAME, gossip_validator.clone(), None); + let gossip_engine = sc_network_gossip::GossipEngine::new( + network, + protocol_name, + gossip_validator.clone(), + None, + ); let metrics = prometheus_registry.as_ref().map(metrics::Metrics::register).and_then( |result| match result { Ok(metrics) => { - debug!(target: "beefy", "🥩 Registered metrics"); + log::debug!(target: "beefy", "🥩 Registered metrics"); Some(metrics) }, Err(err) => { - debug!(target: "beefy", "🥩 Failed to register metrics: {:?}", err); + log::debug!(target: "beefy", "🥩 Failed to register metrics: {:?}", err); None }, }, @@ -147,13 +190,15 @@ where backend, key_store: key_store.into(), signed_commitment_sender, + beefy_best_block_sender, gossip_engine, gossip_validator, min_block_delta, metrics, + sync_oracle, }; - let worker = worker::BeefyWorker::<_, _, _>::new(worker_params); + let worker = worker::BeefyWorker::<_, _, _, _>::new(worker_params); worker.run().await } diff --git a/client/beefy/src/metrics.rs b/client/beefy/src/metrics.rs index 0fdc29f97c37..20fa98e52fdd 100644 --- a/client/beefy/src/metrics.rs +++ b/client/beefy/src/metrics.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,7 +18,9 @@ //! BEEFY Prometheus metrics definition -use prometheus::{register, Counter, Gauge, PrometheusError, Registry, U64}; +#[cfg(not(test))] +use prometheus::{register, PrometheusError, Registry}; +use prometheus::{Counter, Gauge, U64}; /// BEEFY metrics exposed through Prometheus pub(crate) struct Metrics { @@ -37,31 +39,38 @@ pub(crate) struct Metrics { } impl Metrics { + #[cfg(not(test))] pub(crate) fn register(registry: &Registry) -> Result { Ok(Self { beefy_validator_set_id: register( - Gauge::new("beefy_validator_set_id", "Current BEEFY active validator set id.")?, + Gauge::new( + "substrate_beefy_validator_set_id", + "Current BEEFY active validator set id.", + )?, registry, )?, beefy_votes_sent: register( - Counter::new("beefy_votes_sent", "Number of votes sent by this node")?, + Counter::new("substrate_beefy_votes_sent", "Number of votes sent by this node")?, registry, )?, beefy_round_concluded: register( - Gauge::new("beefy_round_concluded", "Voting round, that has been concluded")?, + Gauge::new( + "substrate_beefy_round_concluded", + "Voting round, that has been concluded", + )?, registry, )?, beefy_best_block: register( - Gauge::new("beefy_best_block", "Best block finalized by BEEFY")?, + Gauge::new("substrate_beefy_best_block", "Best block finalized by BEEFY")?, registry, )?, beefy_should_vote_on: register( - Gauge::new("beefy_should_vote_on", "Next block, BEEFY should vote on")?, + Gauge::new("substrate_beefy_should_vote_on", "Next block, BEEFY should vote on")?, registry, )?, beefy_skipped_sessions: register( Counter::new( - "beefy_skipped_sessions", + "substrate_beefy_skipped_sessions", "Number of sessions without a signed commitment", )?, registry, @@ -91,3 +100,11 @@ macro_rules! metric_inc { } }}; } + +#[cfg(test)] +#[macro_export] +macro_rules! metric_get { + ($self:ident, $m:ident) => {{ + $self.metrics.as_ref().map(|metrics| metrics.$m.clone()) + }}; +} diff --git a/client/beefy/src/notification.rs b/client/beefy/src/notification.rs index 6099c9681447..7c18d809f6ef 100644 --- a/client/beefy/src/notification.rs +++ b/client/beefy/src/notification.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,98 +16,41 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use std::sync::Arc; +use sc_utils::notification::{NotificationSender, NotificationStream, TracingKeyStr}; +use sp_runtime::traits::{Block as BlockT, NumberFor}; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; -use sp_runtime::traits::{Block, NumberFor}; +/// A commitment with matching BEEFY authorities' signatures. +pub type BeefySignedCommitment = + beefy_primitives::SignedCommitment, beefy_primitives::crypto::Signature>; -use parking_lot::Mutex; +/// The sending half of the notifications channel(s) used to send +/// notifications about best BEEFY block from the gadget side. +pub type BeefyBestBlockSender = NotificationSender<::Hash>; -/// Stream of signed commitments returned when subscribing. -pub type SignedCommitment = - beefy_primitives::SignedCommitment, beefy_primitives::MmrRootHash>; +/// The receiving half of a notifications channel used to receive +/// notifications about best BEEFY blocks determined on the gadget side. +pub type BeefyBestBlockStream = + NotificationStream<::Hash, BeefyBestBlockTracingKey>; -/// Stream of signed commitments returned when subscribing. -type SignedCommitmentStream = TracingUnboundedReceiver>; +/// The sending half of the notifications channel(s) used to send notifications +/// about signed commitments generated at the end of a BEEFY round. +pub type BeefySignedCommitmentSender = NotificationSender>; -/// Sending endpoint for notifying about signed commitments. -type SignedCommitmentSender = TracingUnboundedSender>; +/// The receiving half of a notifications channel used to receive notifications +/// about signed commitments generated at the end of a BEEFY round. +pub type BeefySignedCommitmentStream = + NotificationStream, BeefySignedCommitmentTracingKey>; -/// Collection of channel sending endpoints shared with the receiver side so they can register -/// themselves. -type SharedSignedCommitmentSenders = Arc>>>; - -/// The sending half of the signed commitment channel(s). -/// -/// Used to send notifications about signed commitments generated at the end of a BEEFY round. +/// Provides tracing key for BEEFY best block stream. #[derive(Clone)] -pub struct BeefySignedCommitmentSender -where - B: Block, -{ - subscribers: SharedSignedCommitmentSenders, -} - -impl BeefySignedCommitmentSender -where - B: Block, -{ - /// The `subscribers` should be shared with a corresponding `SignedCommitmentSender`. - fn new(subscribers: SharedSignedCommitmentSenders) -> Self { - Self { subscribers } - } - - /// Send out a notification to all subscribers that a new signed commitment is available for a - /// block. - pub fn notify(&self, signed_commitment: SignedCommitment) { - let mut subscribers = self.subscribers.lock(); - - // do an initial prune on closed subscriptions - subscribers.retain(|n| !n.is_closed()); - - if !subscribers.is_empty() { - subscribers.retain(|n| n.unbounded_send(signed_commitment.clone()).is_ok()); - } - } +pub struct BeefyBestBlockTracingKey; +impl TracingKeyStr for BeefyBestBlockTracingKey { + const TRACING_KEY: &'static str = "mpsc_beefy_best_block_notification_stream"; } -/// The receiving half of the signed commitments channel. -/// -/// Used to receive notifications about signed commitments generated at the end of a BEEFY round. -/// The `BeefySignedCommitmentStream` entity stores the `SharedSignedCommitmentSenders` so it can be -/// used to add more subscriptions. +/// Provides tracing key for BEEFY signed commitments stream. #[derive(Clone)] -pub struct BeefySignedCommitmentStream -where - B: Block, -{ - subscribers: SharedSignedCommitmentSenders, -} - -impl BeefySignedCommitmentStream -where - B: Block, -{ - /// Creates a new pair of receiver and sender of signed commitment notifications. - pub fn channel() -> (BeefySignedCommitmentSender, Self) { - let subscribers = Arc::new(Mutex::new(vec![])); - let receiver = BeefySignedCommitmentStream::new(subscribers.clone()); - let sender = BeefySignedCommitmentSender::new(subscribers); - (sender, receiver) - } - - /// Create a new receiver of signed commitment notifications. - /// - /// The `subscribers` should be shared with a corresponding `BeefySignedCommitmentSender`. - fn new(subscribers: SharedSignedCommitmentSenders) -> Self { - Self { subscribers } - } - - /// Subscribe to a channel through which signed commitments are sent at the end of each BEEFY - /// voting round. - pub fn subscribe(&self) -> SignedCommitmentStream { - let (sender, receiver) = tracing_unbounded("mpsc_signed_commitments_notification_stream"); - self.subscribers.lock().push(sender); - receiver - } +pub struct BeefySignedCommitmentTracingKey; +impl TracingKeyStr for BeefySignedCommitmentTracingKey { + const TRACING_KEY: &'static str = "mpsc_beefy_signed_commitments_notification_stream"; } diff --git a/client/beefy/src/round.rs b/client/beefy/src/round.rs index 7d443603b364..eba769b2356f 100644 --- a/client/beefy/src/round.rs +++ b/client/beefy/src/round.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,7 +16,10 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use std::{collections::BTreeMap, hash::Hash}; +use std::{ + collections::{BTreeMap, HashMap}, + hash::Hash, +}; use log::{debug, trace}; @@ -24,25 +27,33 @@ use beefy_primitives::{ crypto::{Public, Signature}, ValidatorSet, ValidatorSetId, }; -use sp_arithmetic::traits::AtLeast32BitUnsigned; -use sp_runtime::traits::MaybeDisplay; +use sp_runtime::traits::{Block, NumberFor}; +/// Tracks for each round which validators have voted/signed and +/// whether the local `self` validator has voted/signed. +/// +/// Does not do any validation on votes or signatures, layers above need to handle that (gossip). #[derive(Default)] struct RoundTracker { - votes: Vec<(Public, Signature)>, + self_vote: bool, + votes: HashMap, } impl RoundTracker { - fn add_vote(&mut self, vote: (Public, Signature)) -> bool { - // this needs to handle equivocations in the future - if self.votes.contains(&vote) { + fn add_vote(&mut self, vote: (Public, Signature), self_vote: bool) -> bool { + if self.votes.contains_key(&vote.0) { return false } - self.votes.push(vote); + self.self_vote = self.self_vote || self_vote; + self.votes.insert(vote.0, vote.1); true } + fn has_self_vote(&self) -> bool { + self.self_vote + } + fn is_done(&self, threshold: usize) -> bool { self.votes.len() >= threshold } @@ -53,69 +64,359 @@ fn threshold(authorities: usize) -> usize { authorities - faulty } -pub(crate) struct Rounds { - rounds: BTreeMap<(Hash, Number), RoundTracker>, +/// Keeps track of all voting rounds (block numbers) within a session. +/// Only round numbers > `best_done` are of interest, all others are considered stale. +/// +/// Does not do any validation on votes or signatures, layers above need to handle that (gossip). +pub(crate) struct Rounds { + rounds: BTreeMap<(Payload, NumberFor), RoundTracker>, + best_done: Option>, + session_start: NumberFor, validator_set: ValidatorSet, + prev_validator_set: ValidatorSet, } -impl Rounds +impl Rounds where - H: Ord + Hash, - N: Ord + AtLeast32BitUnsigned + MaybeDisplay, + P: Ord + Hash + Clone, + B: Block, { - pub(crate) fn new(validator_set: ValidatorSet) -> Self { - Rounds { rounds: BTreeMap::new(), validator_set } + pub(crate) fn new( + session_start: NumberFor, + validator_set: ValidatorSet, + prev_validator_set: ValidatorSet, + ) -> Self { + Rounds { + rounds: BTreeMap::new(), + best_done: None, + session_start, + validator_set, + prev_validator_set, + } } } -impl Rounds +impl Rounds where - H: Ord + Hash, - N: Ord + AtLeast32BitUnsigned + MaybeDisplay, + P: Ord + Hash + Clone, + B: Block, { - pub(crate) fn validator_set_id(&self) -> ValidatorSetId { - self.validator_set.id + pub(crate) fn validator_set_id_for(&self, block_number: NumberFor) -> ValidatorSetId { + if block_number > self.session_start { + self.validator_set.id() + } else { + self.prev_validator_set.id() + } } - pub(crate) fn validators(&self) -> Vec { - self.validator_set.validators.clone() + pub(crate) fn validators_for(&self, block_number: NumberFor) -> &[Public] { + if block_number > self.session_start { + self.validator_set.validators() + } else { + self.prev_validator_set.validators() + } } - pub(crate) fn add_vote(&mut self, round: (H, N), vote: (Public, Signature)) -> bool { - self.rounds.entry(round).or_default().add_vote(vote) + pub(crate) fn validator_set(&self) -> &ValidatorSet { + &self.validator_set + } + + pub(crate) fn session_start(&self) -> &NumberFor { + &self.session_start + } + + pub(crate) fn should_self_vote(&self, round: &(P, NumberFor)) -> bool { + Some(round.1.clone()) > self.best_done && + self.rounds.get(round).map(|tracker| !tracker.has_self_vote()).unwrap_or(true) + } + + pub(crate) fn add_vote( + &mut self, + round: &(P, NumberFor), + vote: (Public, Signature), + self_vote: bool, + ) -> bool { + if Some(round.1.clone()) <= self.best_done { + debug!( + target: "beefy", + "🥩 received vote for old stale round {:?}, ignoring", + round.1 + ); + false + } else if !self.validator_set.validators().iter().any(|id| vote.0 == *id) { + debug!( + target: "beefy", + "🥩 received vote {:?} from validator that is not in the validator set, ignoring", + vote + ); + false + } else { + self.rounds.entry(round.clone()).or_default().add_vote(vote, self_vote) + } } - pub(crate) fn is_done(&self, round: &(H, N)) -> bool { + pub(crate) fn try_conclude( + &mut self, + round: &(P, NumberFor), + ) -> Option>> { let done = self .rounds .get(round) - .map(|tracker| tracker.is_done(threshold(self.validator_set.validators.len()))) + .map(|tracker| tracker.is_done(threshold(self.validator_set.len()))) .unwrap_or(false); + trace!(target: "beefy", "🥩 Round #{} done: {}", round.1, done); + + if done { + // remove this and older (now stale) rounds + let signatures = self.rounds.remove(round)?.votes; + self.rounds.retain(|&(_, number), _| number > round.1); + self.best_done = self.best_done.clone().max(Some(round.1.clone())); + trace!(target: "beefy", "🥩 Concluded round #{}", round.1); + + Some( + self.validator_set + .validators() + .iter() + .map(|authority_id| signatures.get(authority_id).cloned()) + .collect(), + ) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use sc_network_test::Block; + use sp_core::H256; + + use beefy_primitives::{crypto::Public, ValidatorSet}; + + use super::{threshold, RoundTracker, Rounds}; + use crate::keystore::tests::Keyring; + + #[test] + fn round_tracker() { + let mut rt = RoundTracker::default(); + let bob_vote = (Keyring::Bob.public(), Keyring::Bob.sign(b"I am committed")); + let threshold = 2; + + // self vote not added yet + assert!(!rt.has_self_vote()); + + // adding new vote allowed + assert!(rt.add_vote(bob_vote.clone(), false)); + // adding existing vote not allowed + assert!(!rt.add_vote(bob_vote, false)); + + // self vote still not added yet + assert!(!rt.has_self_vote()); + + // vote is not done + assert!(!rt.is_done(threshold)); - debug!(target: "beefy", "🥩 Round #{} done: {}", round.1, done); + let alice_vote = (Keyring::Alice.public(), Keyring::Alice.sign(b"I am committed")); + // adding new vote (self vote this time) allowed + assert!(rt.add_vote(alice_vote, true)); - done + // self vote registered + assert!(rt.has_self_vote()); + // vote is now done + assert!(rt.is_done(threshold)); } - pub(crate) fn drop(&mut self, round: &(H, N)) -> Option>> { - trace!(target: "beefy", "🥩 About to drop round #{}", round.1); + #[test] + fn vote_threshold() { + assert_eq!(threshold(1), 1); + assert_eq!(threshold(2), 2); + assert_eq!(threshold(3), 3); + assert_eq!(threshold(4), 3); + assert_eq!(threshold(100), 67); + assert_eq!(threshold(300), 201); + } + + #[test] + fn new_rounds() { + sp_tracing::try_init_simple(); + + let validators = ValidatorSet::::new( + vec![Keyring::Alice.public(), Keyring::Bob.public(), Keyring::Charlie.public()], + 42, + ) + .unwrap(); + + let session_start = 1u64.into(); + let rounds = Rounds::::new(session_start, validators.clone(), validators); + + assert_eq!(42, rounds.validator_set_id_for(session_start)); + assert_eq!(1, *rounds.session_start()); + assert_eq!( + &vec![Keyring::Alice.public(), Keyring::Bob.public(), Keyring::Charlie.public()], + rounds.validators_for(session_start) + ); + } + + #[test] + fn add_and_conclude_votes() { + sp_tracing::try_init_simple(); + + let validators = ValidatorSet::::new( + vec![ + Keyring::Alice.public(), + Keyring::Bob.public(), + Keyring::Charlie.public(), + Keyring::Eve.public(), + ], + Default::default(), + ) + .unwrap(); + let round = (H256::from_low_u64_le(1), 1); + + let session_start = 1u64.into(); + let mut rounds = Rounds::::new(session_start, validators.clone(), validators); + + // no self vote yet, should self vote + assert!(rounds.should_self_vote(&round)); + + // add 1st good vote + assert!(rounds.add_vote( + &round, + (Keyring::Alice.public(), Keyring::Alice.sign(b"I am committed")), + true + )); + // round not concluded + assert!(rounds.try_conclude(&round).is_none()); + // self vote already present, should not self vote + assert!(!rounds.should_self_vote(&round)); - let signatures = self.rounds.remove(round)?.votes; + // double voting not allowed + assert!(!rounds.add_vote( + &round, + (Keyring::Alice.public(), Keyring::Alice.sign(b"I am committed")), + true + )); - Some( - self.validator_set - .validators - .iter() - .map(|authority_id| { - signatures.iter().find_map(|(id, sig)| { - if id == authority_id { - Some(sig.clone()) - } else { - None - } - }) - }) - .collect(), + // invalid vote (Dave is not a validator) + assert!(!rounds.add_vote( + &round, + (Keyring::Dave.public(), Keyring::Dave.sign(b"I am committed")), + false + )); + assert!(rounds.try_conclude(&round).is_none()); + + // add 2nd good vote + assert!(rounds.add_vote( + &round, + (Keyring::Bob.public(), Keyring::Bob.sign(b"I am committed")), + false + )); + // round not concluded + assert!(rounds.try_conclude(&round).is_none()); + + // add 3rd good vote + assert!(rounds.add_vote( + &round, + (Keyring::Charlie.public(), Keyring::Charlie.sign(b"I am committed")), + false + )); + // round concluded + assert!(rounds.try_conclude(&round).is_some()); + + // Eve is a validator, but round was concluded, adding vote disallowed + assert!(!rounds.add_vote( + &round, + (Keyring::Eve.public(), Keyring::Eve.sign(b"I am committed")), + false + )); + } + + #[test] + fn multiple_rounds() { + sp_tracing::try_init_simple(); + + let validators = ValidatorSet::::new( + vec![ + Keyring::Alice.public(), + Keyring::Bob.public(), + Keyring::Charlie.public(), + Keyring::Dave.public(), + ], + Default::default(), ) + .unwrap(); + + let session_start = 1u64.into(); + let mut rounds = Rounds::::new(session_start, validators.clone(), validators); + + // round 1 + assert!(rounds.add_vote( + &(H256::from_low_u64_le(1), 1), + (Keyring::Alice.public(), Keyring::Alice.sign(b"I am committed")), + true, + )); + assert!(rounds.add_vote( + &(H256::from_low_u64_le(1), 1), + (Keyring::Bob.public(), Keyring::Bob.sign(b"I am committed")), + false, + )); + assert!(rounds.add_vote( + &(H256::from_low_u64_le(1), 1), + (Keyring::Charlie.public(), Keyring::Charlie.sign(b"I am committed")), + false, + )); + + // round 2 + assert!(rounds.add_vote( + &(H256::from_low_u64_le(2), 2), + (Keyring::Alice.public(), Keyring::Alice.sign(b"I am again committed")), + true, + )); + assert!(rounds.add_vote( + &(H256::from_low_u64_le(2), 2), + (Keyring::Bob.public(), Keyring::Bob.sign(b"I am again committed")), + false, + )); + assert!(rounds.add_vote( + &(H256::from_low_u64_le(2), 2), + (Keyring::Charlie.public(), Keyring::Charlie.sign(b"I am again committed")), + false, + )); + + // round 3 + assert!(rounds.add_vote( + &(H256::from_low_u64_le(3), 3), + (Keyring::Alice.public(), Keyring::Alice.sign(b"I am still committed")), + true, + )); + assert!(rounds.add_vote( + &(H256::from_low_u64_le(3), 3), + (Keyring::Bob.public(), Keyring::Bob.sign(b"I am still committed")), + false, + )); + assert!(rounds.add_vote( + &(H256::from_low_u64_le(3), 3), + (Keyring::Charlie.public(), Keyring::Charlie.sign(b"I am still committed")), + false, + )); + assert_eq!(3, rounds.rounds.len()); + + // conclude unknown round + assert!(rounds.try_conclude(&(H256::from_low_u64_le(5), 5)).is_none()); + assert_eq!(3, rounds.rounds.len()); + + // conclude round 2 + let signatures = rounds.try_conclude(&(H256::from_low_u64_le(2), 2)).unwrap(); + assert_eq!(1, rounds.rounds.len()); + + assert_eq!( + signatures, + vec![ + Some(Keyring::Alice.sign(b"I am again committed")), + Some(Keyring::Bob.sign(b"I am again committed")), + Some(Keyring::Charlie.sign(b"I am again committed")), + None + ] + ); } } diff --git a/client/beefy/src/tests.rs b/client/beefy/src/tests.rs new file mode 100644 index 000000000000..92b5ad91c11e --- /dev/null +++ b/client/beefy/src/tests.rs @@ -0,0 +1,590 @@ +// This file is part of Substrate. + +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Tests and test helpers for BEEFY. + +use futures::{future, stream::FuturesUnordered, Future, StreamExt}; +use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; +use std::{sync::Arc, task::Poll}; +use tokio::{runtime::Runtime, time::Duration}; + +use sc_chain_spec::{ChainSpec, GenericChainSpec}; +use sc_client_api::HeaderBackend; +use sc_consensus::BoxJustificationImport; +use sc_keystore::LocalKeystore; +use sc_network::{config::ProtocolConfig, NetworkService}; +use sc_network_gossip::GossipEngine; +use sc_network_test::{ + Block, BlockImportAdapter, FullPeerConfig, PassThroughVerifier, Peer, PeersClient, + PeersFullClient, TestNetFactory, +}; +use sc_utils::notification::NotificationReceiver; + +use beefy_primitives::{ + crypto::AuthorityId, ConsensusLog, MmrRootHash, ValidatorSet, BEEFY_ENGINE_ID, + KEY_TYPE as BeefyKeyType, +}; +use sp_consensus::BlockOrigin; +use sp_core::H256; +use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; +use sp_runtime::{ + codec::Encode, generic::BlockId, traits::Header as HeaderT, BuildStorage, DigestItem, Storage, +}; + +use substrate_test_runtime_client::{runtime::Header, Backend, ClientExt}; + +use crate::{ + beefy_protocol_name, + keystore::tests::Keyring as BeefyKeyring, + notification::*, + worker::{tests::TestModifiers, BeefyWorker}, +}; + +const BEEFY_PROTOCOL_NAME: &'static str = "/beefy/1"; + +type BeefyValidatorSet = ValidatorSet; +type BeefyPeer = Peer; + +#[derive(Debug, Serialize, Deserialize)] +struct Genesis(std::collections::BTreeMap); +impl BuildStorage for Genesis { + fn assimilate_storage(&self, storage: &mut Storage) -> Result<(), String> { + storage + .top + .extend(self.0.iter().map(|(a, b)| (a.clone().into_bytes(), b.clone().into_bytes()))); + Ok(()) + } +} + +#[test] +fn beefy_protocol_name() { + let chain_spec = GenericChainSpec::::from_json_file(std::path::PathBuf::from( + "../chain-spec/res/chain_spec.json", + )) + .unwrap() + .cloned_box(); + + // Create protocol name using random genesis hash. + let genesis_hash = H256::random(); + let expected = format!("/{}/beefy/1", hex::encode(genesis_hash)); + let proto_name = beefy_protocol_name::standard_name(&genesis_hash, &chain_spec); + assert_eq!(proto_name.to_string(), expected); + + // Create protocol name using hardcoded genesis hash. Verify exact representation. + let genesis_hash = [ + 50, 4, 60, 123, 58, 106, 216, 246, 194, 188, 139, 193, 33, 212, 202, 171, 9, 55, 123, 94, + 8, 43, 12, 251, 187, 57, 173, 19, 188, 74, 205, 147, + ]; + let expected = + "/32043c7b3a6ad8f6c2bc8bc121d4caab09377b5e082b0cfbbb39ad13bc4acd93/beefy/1".to_string(); + let proto_name = beefy_protocol_name::standard_name(&genesis_hash, &chain_spec); + assert_eq!(proto_name.to_string(), expected); +} + +// TODO: compiler warns us about unused `signed_commitment_stream`, will use in later tests +#[allow(dead_code)] +#[derive(Clone)] +pub(crate) struct BeefyLinkHalf { + signed_commitment_stream: BeefySignedCommitmentStream, + beefy_best_block_stream: BeefyBestBlockStream, +} + +#[derive(Default)] +pub(crate) struct PeerData { + pub(crate) beefy_link_half: Mutex>, + pub(crate) test_modifiers: Option, +} + +impl PeerData { + pub(crate) fn use_validator_set(&mut self, validator_set: &ValidatorSet) { + if let Some(tm) = self.test_modifiers.as_mut() { + tm.active_validators = validator_set.clone(); + } else { + self.test_modifiers = Some(TestModifiers { + active_validators: validator_set.clone(), + corrupt_mmr_roots: false, + }); + } + } +} + +pub(crate) struct BeefyTestNet { + peers: Vec, +} + +impl BeefyTestNet { + pub(crate) fn new(n_authority: usize, n_full: usize) -> Self { + let mut net = BeefyTestNet { peers: Vec::with_capacity(n_authority + n_full) }; + for _ in 0..n_authority { + net.add_authority_peer(); + } + for _ in 0..n_full { + net.add_full_peer(); + } + net + } + + pub(crate) fn add_authority_peer(&mut self) { + self.add_full_peer_with_config(FullPeerConfig { + notifications_protocols: vec![BEEFY_PROTOCOL_NAME.into()], + is_authority: true, + ..Default::default() + }) + } + + pub(crate) fn generate_blocks( + &mut self, + count: usize, + session_length: u64, + validator_set: &BeefyValidatorSet, + ) { + self.peer(0).generate_blocks(count, BlockOrigin::File, |builder| { + let mut block = builder.build().unwrap().block; + + let block_num = *block.header.number(); + let num_byte = block_num.to_le_bytes().into_iter().next().unwrap(); + let mmr_root = MmrRootHash::repeat_byte(num_byte); + + add_mmr_digest(&mut block.header, mmr_root); + + if block_num % session_length == 0 { + add_auth_change_digest(&mut block.header, validator_set.clone()); + } + + block + }); + } +} + +impl TestNetFactory for BeefyTestNet { + type Verifier = PassThroughVerifier; + type BlockImport = PeersClient; + type PeerData = PeerData; + + /// Create new test network with peers and given config. + fn from_config(_config: &ProtocolConfig) -> Self { + BeefyTestNet { peers: Vec::new() } + } + + fn make_verifier( + &self, + _client: PeersClient, + _cfg: &ProtocolConfig, + _: &PeerData, + ) -> Self::Verifier { + PassThroughVerifier::new(false) // use non-instant finality. + } + + fn make_block_import( + &self, + client: PeersClient, + ) -> ( + BlockImportAdapter, + Option>, + Self::PeerData, + ) { + (client.as_block_import(), None, PeerData::default()) + } + + fn peer(&mut self, i: usize) -> &mut BeefyPeer { + &mut self.peers[i] + } + + fn peers(&self) -> &Vec { + &self.peers + } + + fn mut_peers)>(&mut self, closure: F) { + closure(&mut self.peers); + } + + fn add_full_peer(&mut self) { + self.add_full_peer_with_config(FullPeerConfig { + notifications_protocols: vec![BEEFY_PROTOCOL_NAME.into()], + is_authority: false, + ..Default::default() + }) + } +} + +fn add_mmr_digest(header: &mut Header, mmr_hash: MmrRootHash) { + header.digest_mut().push(DigestItem::Consensus( + BEEFY_ENGINE_ID, + ConsensusLog::::MmrRoot(mmr_hash).encode(), + )); +} + +fn add_auth_change_digest(header: &mut Header, new_auth_set: BeefyValidatorSet) { + header.digest_mut().push(DigestItem::Consensus( + BEEFY_ENGINE_ID, + ConsensusLog::::AuthoritiesChange(new_auth_set).encode(), + )); +} + +pub(crate) fn make_beefy_ids(keys: &[BeefyKeyring]) -> Vec { + keys.iter().map(|key| key.clone().public().into()).collect() +} + +pub(crate) fn create_beefy_keystore(authority: BeefyKeyring) -> SyncCryptoStorePtr { + let keystore = Arc::new(LocalKeystore::in_memory()); + SyncCryptoStore::ecdsa_generate_new(&*keystore, BeefyKeyType, Some(&authority.to_seed())) + .expect("Creates authority key"); + keystore +} + +pub(crate) fn create_beefy_worker( + peer: &BeefyPeer, + key: &BeefyKeyring, + min_block_delta: u32, +) -> BeefyWorker>> { + let keystore = create_beefy_keystore(*key); + + let (signed_commitment_sender, signed_commitment_stream) = + BeefySignedCommitmentStream::::channel(); + let (beefy_best_block_sender, beefy_best_block_stream) = + BeefyBestBlockStream::::channel(); + + let beefy_link_half = BeefyLinkHalf { signed_commitment_stream, beefy_best_block_stream }; + *peer.data.beefy_link_half.lock() = Some(beefy_link_half); + let test_modifiers = peer.data.test_modifiers.clone().unwrap(); + + let network = peer.network_service().clone(); + let sync_oracle = network.clone(); + let gossip_validator = Arc::new(crate::gossip::GossipValidator::new()); + let gossip_engine = + GossipEngine::new(network, BEEFY_PROTOCOL_NAME, gossip_validator.clone(), None); + let worker_params = crate::worker::WorkerParams { + client: peer.client().as_client(), + backend: peer.client().as_backend(), + key_store: Some(keystore).into(), + signed_commitment_sender, + beefy_best_block_sender, + gossip_engine, + gossip_validator, + min_block_delta, + metrics: None, + sync_oracle, + }; + + BeefyWorker::<_, _, _, _>::new(worker_params, test_modifiers) +} + +// Spawns beefy voters. Returns a future to spawn on the runtime. +fn initialize_beefy( + net: &mut BeefyTestNet, + peers: &[BeefyKeyring], + min_block_delta: u32, +) -> impl Future { + let voters = FuturesUnordered::new(); + + for (peer_id, key) in peers.iter().enumerate() { + let worker = create_beefy_worker(&net.peers[peer_id], key, min_block_delta); + let gadget = worker.run(); + + fn assert_send(_: &T) {} + assert_send(&gadget); + voters.push(gadget); + } + + voters.for_each(|_| async move {}) +} + +fn block_until(future: impl Future + Unpin, net: &Arc>, runtime: &mut Runtime) { + let drive_to_completion = futures::future::poll_fn(|cx| { + net.lock().poll(cx); + Poll::<()>::Pending + }); + runtime.block_on(future::select(future, drive_to_completion)); +} + +fn run_for(duration: Duration, net: &Arc>, runtime: &mut Runtime) { + let sleep = runtime.spawn(async move { tokio::time::sleep(duration).await }); + block_until(sleep, net, runtime); +} + +pub(crate) fn get_beefy_streams( + net: &mut BeefyTestNet, + peers: &[BeefyKeyring], +) -> (Vec>, Vec>>) { + let mut best_block_streams = Vec::new(); + let mut signed_commitment_streams = Vec::new(); + for peer_id in 0..peers.len() { + let beefy_link_half = + net.peer(peer_id).data.beefy_link_half.lock().as_ref().unwrap().clone(); + let BeefyLinkHalf { signed_commitment_stream, beefy_best_block_stream } = beefy_link_half; + best_block_streams.push(beefy_best_block_stream.subscribe()); + signed_commitment_streams.push(signed_commitment_stream.subscribe()); + } + (best_block_streams, signed_commitment_streams) +} + +fn wait_for_best_beefy_blocks( + streams: Vec>, + net: &Arc>, + runtime: &mut Runtime, + expected_beefy_blocks: &[u64], +) { + let mut wait_for = Vec::new(); + let len = expected_beefy_blocks.len(); + streams.into_iter().enumerate().for_each(|(i, stream)| { + let mut expected = expected_beefy_blocks.iter(); + wait_for.push(Box::pin(stream.take(len).for_each(move |best_beefy_hash| { + let expected = expected.next(); + async move { + let block_id = BlockId::hash(best_beefy_hash); + let header = + net.lock().peer(i).client().as_client().expect_header(block_id).unwrap(); + let best_beefy = *header.number(); + + assert_eq!(expected, Some(best_beefy).as_ref()); + } + }))); + }); + let wait_for = futures::future::join_all(wait_for); + block_until(wait_for, net, runtime); +} + +fn wait_for_beefy_signed_commitments( + streams: Vec>>, + net: &Arc>, + runtime: &mut Runtime, + expected_commitment_block_nums: &[u64], +) { + let mut wait_for = Vec::new(); + let len = expected_commitment_block_nums.len(); + streams.into_iter().for_each(|stream| { + let mut expected = expected_commitment_block_nums.iter(); + wait_for.push(Box::pin(stream.take(len).for_each(move |signed_commitment| { + let expected = expected.next(); + async move { + let commitment_block_num = signed_commitment.commitment.block_number; + assert_eq!(expected, Some(commitment_block_num).as_ref()); + // TODO: also verify commitment payload, validator set id, and signatures. + } + }))); + }); + let wait_for = futures::future::join_all(wait_for); + block_until(wait_for, net, runtime); +} + +fn streams_empty_after_timeout( + streams: Vec>, + net: &Arc>, + runtime: &mut Runtime, + timeout: Option, +) where + T: std::fmt::Debug, + T: std::cmp::PartialEq, +{ + if let Some(timeout) = timeout { + run_for(timeout, net, runtime); + } + streams.into_iter().for_each(|mut stream| { + runtime.block_on(future::poll_fn(move |cx| { + assert_eq!(stream.poll_next_unpin(cx), Poll::Pending); + Poll::Ready(()) + })); + }); +} + +fn finalize_block_and_wait_for_beefy( + net: &Arc>, + peers: &[BeefyKeyring], + runtime: &mut Runtime, + finalize_targets: &[u64], + expected_beefy: &[u64], +) { + let (best_blocks, signed_commitments) = get_beefy_streams(&mut *net.lock(), peers); + + for block in finalize_targets { + let finalize = BlockId::number(*block); + for i in 0..peers.len() { + net.lock().peer(i).client().as_client().finalize_block(finalize, None).unwrap(); + } + } + + if expected_beefy.is_empty() { + // run for 1 second then verify no new best beefy block available + let timeout = Some(Duration::from_millis(500)); + streams_empty_after_timeout(best_blocks, &net, runtime, timeout); + streams_empty_after_timeout(signed_commitments, &net, runtime, None); + } else { + // run until expected beefy blocks are received + wait_for_best_beefy_blocks(best_blocks, &net, runtime, expected_beefy); + wait_for_beefy_signed_commitments(signed_commitments, &net, runtime, expected_beefy); + } +} + +#[test] +fn beefy_finalizing_blocks() { + sp_tracing::try_init_simple(); + + let mut runtime = Runtime::new().unwrap(); + let peers = &[BeefyKeyring::Alice, BeefyKeyring::Bob]; + let validator_set = ValidatorSet::new(make_beefy_ids(peers), 0).unwrap(); + let session_len = 10; + let min_block_delta = 4; + + let mut net = BeefyTestNet::new(2, 0); + + for i in 0..peers.len() { + net.peer(i).data.use_validator_set(&validator_set); + } + runtime.spawn(initialize_beefy(&mut net, peers, min_block_delta)); + + // push 42 blocks including `AuthorityChange` digests every 10 blocks. + net.generate_blocks(42, session_len, &validator_set); + net.block_until_sync(); + + let net = Arc::new(Mutex::new(net)); + + // Minimum BEEFY block delta is 4. + + // finalize block #5 -> BEEFY should finalize #1 (mandatory) and #5 from diff-power-of-two rule. + finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[5], &[1, 5]); + + // GRANDPA finalize #10 -> BEEFY finalize #10 (mandatory) + finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[10], &[10]); + + // GRANDPA finalize #18 -> BEEFY finalize #14, then #18 (diff-power-of-two rule) + finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[18], &[14, 18]); + + // GRANDPA finalize #20 -> BEEFY finalize #20 (mandatory) + finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[20], &[20]); + + // GRANDPA finalize #21 -> BEEFY finalize nothing (yet) because min delta is 4 + finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[21], &[]); +} + +#[test] +fn lagging_validators() { + sp_tracing::try_init_simple(); + + let mut runtime = Runtime::new().unwrap(); + let peers = &[BeefyKeyring::Charlie, BeefyKeyring::Dave]; + let validator_set = ValidatorSet::new(make_beefy_ids(peers), 0).unwrap(); + let session_len = 30; + let min_block_delta = 1; + + let mut net = BeefyTestNet::new(2, 0); + for i in 0..peers.len() { + net.peer(i).data.use_validator_set(&validator_set); + } + runtime.spawn(initialize_beefy(&mut net, peers, min_block_delta)); + + // push 42 blocks including `AuthorityChange` digests every 30 blocks. + net.generate_blocks(42, session_len, &validator_set); + net.block_until_sync(); + + let net = Arc::new(Mutex::new(net)); + + // finalize block #15 -> BEEFY should finalize #1 (mandatory) and #9, #13, #14, #15 from + // diff-power-of-two rule. + finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[15], &[1, 9, 13, 14, 15]); + + // Charlie finalizes #25, Dave lags behind + let finalize = BlockId::number(25); + let (best_blocks, signed_commitments) = get_beefy_streams(&mut *net.lock(), peers); + net.lock().peer(0).client().as_client().finalize_block(finalize, None).unwrap(); + // verify nothing gets finalized by BEEFY + let timeout = Some(Duration::from_millis(500)); + streams_empty_after_timeout(best_blocks, &net, &mut runtime, timeout); + streams_empty_after_timeout(signed_commitments, &net, &mut runtime, None); + + // Dave catches up and also finalizes #25 + let (best_blocks, signed_commitments) = get_beefy_streams(&mut *net.lock(), peers); + net.lock().peer(1).client().as_client().finalize_block(finalize, None).unwrap(); + // expected beefy finalizes block #17 from diff-power-of-two + wait_for_best_beefy_blocks(best_blocks, &net, &mut runtime, &[23, 24, 25]); + wait_for_beefy_signed_commitments(signed_commitments, &net, &mut runtime, &[23, 24, 25]); + + // Both finalize #30 (mandatory session) and #32 -> BEEFY finalize #30 (mandatory), #31, #32 + finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[30, 32], &[30, 31, 32]); +} + +#[test] +fn correct_beefy_payload() { + sp_tracing::try_init_simple(); + + let mut runtime = Runtime::new().unwrap(); + let peers = + &[BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie, BeefyKeyring::Dave]; + let validator_set = ValidatorSet::new(make_beefy_ids(peers), 0).unwrap(); + let session_len = 20; + let min_block_delta = 2; + + let mut net = BeefyTestNet::new(4, 0); + for i in 0..peers.len() { + net.peer(i).data.use_validator_set(&validator_set); + } + + // Dave will vote on bad mmr roots + net.peer(3).data.test_modifiers.as_mut().map(|tm| tm.corrupt_mmr_roots = true); + runtime.spawn(initialize_beefy(&mut net, peers, min_block_delta)); + + // push 10 blocks + net.generate_blocks(12, session_len, &validator_set); + net.block_until_sync(); + + let net = Arc::new(Mutex::new(net)); + // with 3 good voters and 1 bad one, consensus should happen and best blocks produced. + finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[10], &[1, 9]); + + let (best_blocks, signed_commitments) = + get_beefy_streams(&mut *net.lock(), &[BeefyKeyring::Alice]); + + // now 2 good validators and 1 bad one are voting + net.lock() + .peer(0) + .client() + .as_client() + .finalize_block(BlockId::number(11), None) + .unwrap(); + net.lock() + .peer(1) + .client() + .as_client() + .finalize_block(BlockId::number(11), None) + .unwrap(); + net.lock() + .peer(3) + .client() + .as_client() + .finalize_block(BlockId::number(11), None) + .unwrap(); + + // verify consensus is _not_ reached + let timeout = Some(Duration::from_millis(500)); + streams_empty_after_timeout(best_blocks, &net, &mut runtime, timeout); + streams_empty_after_timeout(signed_commitments, &net, &mut runtime, None); + + // 3rd good validator catches up and votes as well + let (best_blocks, signed_commitments) = + get_beefy_streams(&mut *net.lock(), &[BeefyKeyring::Alice]); + net.lock() + .peer(2) + .client() + .as_client() + .finalize_block(BlockId::number(11), None) + .unwrap(); + + // verify consensus is reached + wait_for_best_beefy_blocks(best_blocks, &net, &mut runtime, &[11]); + wait_for_beefy_signed_commitments(signed_commitments, &net, &mut runtime, &[11]); +} diff --git a/client/beefy/src/worker.rs b/client/beefy/src/worker.rs index 3f5268693033..85674c09a278 100644 --- a/client/beefy/src/worker.rs +++ b/client/beefy/src/worker.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,11 +16,11 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use std::{collections::BTreeSet, fmt::Debug, marker::PhantomData, sync::Arc}; +use std::{collections::BTreeSet, fmt::Debug, marker::PhantomData, sync::Arc, time::Duration}; use codec::{Codec, Decode, Encode}; use futures::{future, FutureExt, StreamExt}; -use log::{debug, error, info, trace, warn}; +use log::{debug, error, info, log_enabled, trace, warn}; use parking_lot::Mutex; use sc_client_api::{Backend, FinalityNotification, FinalityNotifications}; @@ -28,6 +28,7 @@ use sc_network_gossip::GossipEngine; use sp_api::BlockId; use sp_arithmetic::traits::AtLeast32Bit; +use sp_consensus::SyncOracle; use sp_runtime::{ generic::OpaqueDigestItemId, traits::{Block, Header, NumberFor}, @@ -35,9 +36,9 @@ use sp_runtime::{ }; use beefy_primitives::{ - crypto::{AuthorityId, Public, Signature}, - BeefyApi, Commitment, ConsensusLog, MmrRootHash, SignedCommitment, ValidatorSet, - VersionedCommitment, VoteMessage, BEEFY_ENGINE_ID, GENESIS_AUTHORITY_SET_ID, + crypto::{AuthorityId, Signature}, + known_payload_ids, BeefyApi, Commitment, ConsensusLog, MmrRootHash, Payload, SignedCommitment, + ValidatorSet, VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID, GENESIS_AUTHORITY_SET_ID, }; use crate::{ @@ -46,57 +47,70 @@ use crate::{ keystore::BeefyKeystore, metric_inc, metric_set, metrics::Metrics, - notification, round, Client, + notification::{BeefyBestBlockSender, BeefySignedCommitmentSender}, + round::Rounds, + Client, }; -pub(crate) struct WorkerParams +pub(crate) struct WorkerParams where B: Block, { pub client: Arc, pub backend: Arc, pub key_store: BeefyKeystore, - pub signed_commitment_sender: notification::BeefySignedCommitmentSender, + pub signed_commitment_sender: BeefySignedCommitmentSender, + pub beefy_best_block_sender: BeefyBestBlockSender, pub gossip_engine: GossipEngine, pub gossip_validator: Arc>, pub min_block_delta: u32, pub metrics: Option, + pub sync_oracle: SO, } /// A BEEFY worker plays the BEEFY protocol -pub(crate) struct BeefyWorker +pub(crate) struct BeefyWorker where B: Block, BE: Backend, C: Client, + SO: SyncOracle + Send + Sync + Clone + 'static, { client: Arc, backend: Arc, key_store: BeefyKeystore, - signed_commitment_sender: notification::BeefySignedCommitmentSender, + signed_commitment_sender: BeefySignedCommitmentSender, gossip_engine: Arc>>, gossip_validator: Arc>, /// Min delta in block numbers between two blocks, BEEFY should vote on min_block_delta: u32, metrics: Option, - rounds: round::Rounds>, + rounds: Option>, finality_notifications: FinalityNotifications, /// Best block we received a GRANDPA notification for - best_grandpa_block: NumberFor, + best_grandpa_block_header: ::Header, /// Best block a BEEFY voting round has been concluded for best_beefy_block: Option>, + /// Used to keep RPC worker up to date on latest/best beefy + beefy_best_block_sender: BeefyBestBlockSender, /// Validator set id for the last signed commitment last_signed_id: u64, + /// Handle to the sync oracle + sync_oracle: SO, // keep rustc happy _backend: PhantomData, + #[cfg(test)] + // behavior modifiers used in tests + test_res: tests::TestModifiers, } -impl BeefyWorker +impl BeefyWorker where B: Block + Codec, BE: Backend, C: Client, C::Api: BeefyApi, + SO: SyncOracle + Send + Sync + Clone + 'static, { /// Return a new BEEFY worker instance. /// @@ -104,18 +118,29 @@ where /// BEEFY pallet has been deployed on-chain. /// /// The BEEFY pallet is needed in order to keep track of the BEEFY authority set. - pub(crate) fn new(worker_params: WorkerParams) -> Self { + pub(crate) fn new( + worker_params: WorkerParams, + #[cfg(test)] + // behavior modifiers used in tests + test_res: tests::TestModifiers, + ) -> Self { let WorkerParams { client, backend, key_store, signed_commitment_sender, + beefy_best_block_sender, gossip_engine, gossip_validator, min_block_delta, metrics, + sync_oracle, } = worker_params; + let last_finalized_header = client + .expect_header(BlockId::number(client.info().finalized_number)) + .expect("latest block always has header available; qed."); + BeefyWorker { client: client.clone(), backend, @@ -123,236 +148,366 @@ where signed_commitment_sender, gossip_engine: Arc::new(Mutex::new(gossip_engine)), gossip_validator, - min_block_delta, + // always target at least one block better than current best beefy + min_block_delta: min_block_delta.max(1), metrics, - rounds: round::Rounds::new(ValidatorSet::empty()), + rounds: None, finality_notifications: client.finality_notification_stream(), - best_grandpa_block: client.info().finalized_number, + best_grandpa_block_header: last_finalized_header, best_beefy_block: None, last_signed_id: 0, + beefy_best_block_sender, + sync_oracle, _backend: PhantomData, + #[cfg(test)] + test_res, } } } -impl BeefyWorker +impl BeefyWorker where B: Block, BE: Backend, C: Client, C::Api: BeefyApi, + SO: SyncOracle + Send + Sync + Clone + 'static, { - /// Return `true`, if we should vote on block `number` - fn should_vote_on(&self, number: NumberFor) -> bool { - let best_beefy_block = if let Some(block) = self.best_beefy_block { - block + /// Return `Some(number)` if we should be voting on block `number` now, + /// return `None` if there is no block we should vote on now. + fn current_vote_target(&self) -> Option> { + let rounds = if let Some(r) = &self.rounds { + r } else { - debug!(target: "beefy", "🥩 Missing best BEEFY block - won't vote for: {:?}", number); - return false + debug!(target: "beefy", "🥩 No voting round started"); + return None }; - let target = vote_target(self.best_grandpa_block, best_beefy_block, self.min_block_delta); - - trace!(target: "beefy", "🥩 should_vote_on: #{:?}, next_block_to_vote_on: #{:?}", number, target); - - metric_set!(self, beefy_should_vote_on, target); - - number == target - } - - /// Return the current active validator set at header `header`. - /// - /// Note that the validator set could be `None`. This is the case if we don't find - /// a BEEFY authority set change and we can't fetch the authority set from the - /// BEEFY on-chain state. - /// - /// Such a failure is usually an indication that the BEEFY pallet has not been deployed (yet). - fn validator_set(&self, header: &B::Header) -> Option> { - let new = if let Some(new) = find_authorities_change::(header) { - Some(new) - } else { - let at = BlockId::hash(header.hash()); - self.client.runtime_api().validator_set(&at).ok() - }; - - trace!(target: "beefy", "🥩 active validator set: {:?}", new); - - new + let best_finalized = *self.best_grandpa_block_header.number(); + // `target` is guaranteed > `best_beefy` since `min_block_delta` is at least `1`. + let target = vote_target( + best_finalized, + self.best_beefy_block, + *rounds.session_start(), + self.min_block_delta, + ); + trace!( + target: "beefy", + "🥩 best beefy: #{:?}, best finalized: #{:?}, current_vote_target: {:?}", + self.best_beefy_block, + best_finalized, + target + ); + if let Some(target) = &target { + metric_set!(self, beefy_should_vote_on, target); + } + target } /// Verify `active` validator set for `block` against the key store /// - /// The critical case is, if we do have a public key in the key store which is not - /// part of the active validator set. + /// We want to make sure that we have _at least one_ key in our keystore that + /// is part of the validator set, that's because if there are no local keys + /// then we can't perform our job as a validator. /// /// Note that for a non-authority node there will be no keystore, and we will /// return an error and don't check. The error can usually be ignored. fn verify_validator_set( &self, block: &NumberFor, - mut active: ValidatorSet, + active: &ValidatorSet, ) -> Result<(), error::Error> { - let active: BTreeSet = active.validators.drain(..).collect(); + let active: BTreeSet<&AuthorityId> = active.validators().iter().collect(); - let store: BTreeSet = self.key_store.public_keys()?.drain(..).collect(); + let public_keys = self.key_store.public_keys()?; + let store: BTreeSet<&AuthorityId> = public_keys.iter().collect(); - let missing: Vec<_> = store.difference(&active).cloned().collect(); - - if !missing.is_empty() { - debug!(target: "beefy", "🥩 for block {:?} public key missing in validator set: {:?}", block, missing); + if store.intersection(&active).count() == 0 { + let msg = "no authority public key found in store".to_string(); + debug!(target: "beefy", "🥩 for block {:?} {}", block, msg); + Err(error::Error::Keystore(msg)) + } else { + Ok(()) } - - Ok(()) } - fn handle_finality_notification(&mut self, notification: FinalityNotification) { - trace!(target: "beefy", "🥩 Finality notification: {:?}", notification); - - // update best GRANDPA finalized block we have seen - self.best_grandpa_block = *notification.header.number(); - - if let Some(active) = self.validator_set(¬ification.header) { - // Authority set change or genesis set id triggers new voting rounds - // - // TODO: (adoerr) Enacting a new authority set will also implicitly 'conclude' - // the currently active BEEFY voting round by starting a new one. This is - // temporary and needs to be replaced by proper round life cycle handling. - if active.id != self.rounds.validator_set_id() || - (active.id == GENESIS_AUTHORITY_SET_ID && self.best_beefy_block.is_none()) - { - debug!(target: "beefy", "🥩 New active validator set id: {:?}", active); - metric_set!(self, beefy_validator_set_id, active.id); - - // BEEFY should produce a signed commitment for each session - if active.id != self.last_signed_id + 1 && active.id != GENESIS_AUTHORITY_SET_ID { - metric_inc!(self, beefy_skipped_sessions); - } - - // verify the new validator set - let _ = self.verify_validator_set(notification.header.number(), active.clone()); - - self.rounds = round::Rounds::new(active.clone()); - - debug!(target: "beefy", "🥩 New Rounds for id: {:?}", active.id); - - self.best_beefy_block = Some(*notification.header.number()); - - // this metric is kind of 'fake'. Best BEEFY block should only be updated once we - // have a signed commitment for the block. Remove once the above TODO is done. - metric_set!(self, beefy_best_block, *notification.header.number()); - } - } - - if self.should_vote_on(*notification.header.number()) { - let authority_id = if let Some(id) = - self.key_store.authority_id(self.rounds.validators().as_slice()) - { - debug!(target: "beefy", "🥩 Local authority id: {:?}", id); - id - } else { - debug!(target: "beefy", "🥩 Missing validator id - can't vote for: {:?}", notification.header.hash()); - return + /// Set best BEEFY block to `block_num`. + /// + /// Also sends/updates the best BEEFY block hash to the RPC worker. + fn set_best_beefy_block(&mut self, block_num: NumberFor) { + if Some(block_num) > self.best_beefy_block { + // Try to get block hash ourselves. + let block_hash = match self.client.hash(block_num) { + Ok(h) => h, + Err(e) => { + error!(target: "beefy", "🥩 Failed to get hash for block number {}: {}", + block_num, e); + None + }, }; + // Update RPC worker with new best BEEFY block hash. + block_hash.map(|hash| { + self.beefy_best_block_sender + .notify(|| Ok::<_, ()>(hash)) + .expect("forwards closure result; the closure always returns Ok; qed.") + }); + // Set new best BEEFY block number. + self.best_beefy_block = Some(block_num); + metric_set!(self, beefy_best_block, block_num); + } else { + debug!(target: "beefy", "🥩 Can't set best beefy to older: {}", block_num); + } + } - let mmr_root = - if let Some(hash) = find_mmr_root_digest::(¬ification.header) { - hash - } else { - warn!(target: "beefy", "🥩 No MMR root digest found for: {:?}", notification.header.hash()); - return - }; - - let commitment = Commitment { - payload: mmr_root, - block_number: notification.header.number(), - validator_set_id: self.rounds.validator_set_id(), - }; - let encoded_commitment = commitment.encode(); + /// Handle session changes by starting new voting round for mandatory blocks. + fn init_session_at(&mut self, active: ValidatorSet, session_start: NumberFor) { + debug!(target: "beefy", "🥩 New active validator set: {:?}", active); + metric_set!(self, beefy_validator_set_id, active.id()); + // BEEFY should produce a signed commitment for each session + if active.id() != self.last_signed_id + 1 && active.id() != GENESIS_AUTHORITY_SET_ID { + metric_inc!(self, beefy_skipped_sessions); + } - let signature = match self.key_store.sign(&authority_id, &*encoded_commitment) { - Ok(sig) => sig, - Err(err) => { - warn!(target: "beefy", "🥩 Error signing commitment: {:?}", err); - return - }, - }; + if log_enabled!(target: "beefy", log::Level::Debug) { + // verify the new validator set - only do it if we're also logging the warning + let _ = self.verify_validator_set(&session_start, &active); + } - trace!( - target: "beefy", - "🥩 Produced signature using {:?}, is_valid: {:?}", - authority_id, - BeefyKeystore::verify(&authority_id, &signature, &*encoded_commitment) - ); + let prev_validator_set = if let Some(r) = &self.rounds { + r.validator_set().clone() + } else { + // no previous rounds present use new validator set instead (genesis case) + active.clone() + }; + let id = active.id(); + self.rounds = Some(Rounds::new(session_start, active, prev_validator_set)); + info!(target: "beefy", "🥩 New Rounds for validator set id: {:?} with session_start {:?}", id, session_start); + } - let message = VoteMessage { commitment, id: authority_id, signature }; + fn handle_finality_notification(&mut self, notification: &FinalityNotification) { + trace!(target: "beefy", "🥩 Finality notification: {:?}", notification); + let number = *notification.header.number(); - let encoded_message = message.encode(); + // On start-up ignore old finality notifications that we're not interested in. + if number <= *self.best_grandpa_block_header.number() { + debug!(target: "beefy", "🥩 Got unexpected finality for old block #{:?}", number); + return + } - metric_inc!(self, beefy_votes_sent); + // update best GRANDPA finalized block we have seen + self.best_grandpa_block_header = notification.header.clone(); - debug!(target: "beefy", "🥩 Sent vote message: {:?}", message); + self.handle_finality(¬ification.header); + } - self.handle_vote( - (message.commitment.payload, *message.commitment.block_number), - (message.id, message.signature), - ); + fn handle_finality(&mut self, header: &B::Header) { + // Check for and handle potential new session. + if let Some(new_validator_set) = find_authorities_change::(header) { + self.init_session_at(new_validator_set, *header.number()); + } - self.gossip_engine.lock().gossip_message(topic::(), encoded_message, false); + // Vote if there's now a new vote target. + if let Some(target_number) = self.current_vote_target() { + self.do_vote(target_number); } } - fn handle_vote(&mut self, round: (MmrRootHash, NumberFor), vote: (Public, Signature)) { + fn handle_vote( + &mut self, + round: (Payload, NumberFor), + vote: (AuthorityId, Signature), + self_vote: bool, + ) { self.gossip_validator.note_round(round.1); - let vote_added = self.rounds.add_vote(round, vote); + let rounds = if let Some(rounds) = self.rounds.as_mut() { + rounds + } else { + debug!(target: "beefy", "🥩 Missing validator set - can't handle vote {:?}", vote); + return + }; + + if rounds.add_vote(&round, vote, self_vote) { + if let Some(signatures) = rounds.try_conclude(&round) { + self.gossip_validator.conclude_round(round.1); - if vote_added && self.rounds.is_done(&round) { - if let Some(signatures) = self.rounds.drop(&round) { // id is stored for skipped session metric calculation - self.last_signed_id = self.rounds.validator_set_id(); + self.last_signed_id = rounds.validator_set_id_for(round.1); + let block_num = round.1; let commitment = Commitment { payload: round.0, - block_number: round.1, + block_number: block_num, validator_set_id: self.last_signed_id, }; let signed_commitment = SignedCommitment { commitment, signatures }; - metric_set!(self, beefy_round_concluded, round.1); + metric_set!(self, beefy_round_concluded, block_num); info!(target: "beefy", "🥩 Round #{} concluded, committed: {:?}.", round.1, signed_commitment); - if self - .backend - .append_justification( - BlockId::Number(round.1), - ( - BEEFY_ENGINE_ID, - VersionedCommitment::V1(signed_commitment.clone()).encode(), - ), - ) - .is_err() - { - // just a trace, because until the round lifecycle is improved, we will - // conclude certain rounds multiple times. - trace!(target: "beefy", "🥩 Failed to append justification: {:?}", signed_commitment); + if let Err(e) = self.backend.append_justification( + BlockId::Number(block_num), + ( + BEEFY_ENGINE_ID, + VersionedFinalityProof::V1(signed_commitment.clone()).encode(), + ), + ) { + trace!(target: "beefy", "🥩 Error {:?} on appending justification: {:?}", e, signed_commitment); } + self.signed_commitment_sender + .notify(|| Ok::<_, ()>(signed_commitment)) + .expect("forwards closure result; the closure always returns Ok; qed."); - self.signed_commitment_sender.notify(signed_commitment); - self.best_beefy_block = Some(round.1); + self.set_best_beefy_block(block_num); - metric_set!(self, beefy_best_block, round.1); + // Vote if there's now a new vote target. + if let Some(target_number) = self.current_vote_target() { + self.do_vote(target_number); + } } } } + /// Create and gossip Signed Commitment for block number `target_number`. + /// + /// Also handle this self vote by calling `self.handle_vote()` for it. + fn do_vote(&mut self, target_number: NumberFor) { + trace!(target: "beefy", "🥩 Try voting on {}", target_number); + + // Most of the time we get here, `target` is actually `best_grandpa`, + // avoid asking `client` for header in that case. + let target_header = if target_number == *self.best_grandpa_block_header.number() { + self.best_grandpa_block_header.clone() + } else { + match self.client.expect_header(BlockId::Number(target_number)) { + Ok(h) => h, + Err(err) => { + debug!( + target: "beefy", + "🥩 Could not get header for block #{:?} (error: {:?}), skipping vote..", + target_number, + err + ); + return + }, + } + }; + let target_hash = target_header.hash(); + + let mmr_root = if let Some(hash) = self.extract_mmr_root_digest(&target_header) { + hash + } else { + warn!(target: "beefy", "🥩 No MMR root digest found for: {:?}", target_hash); + return + }; + let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, mmr_root.encode()); + + let (validators, validator_set_id) = if let Some(rounds) = &self.rounds { + if !rounds.should_self_vote(&(payload.clone(), target_number)) { + debug!(target: "beefy", "🥩 Don't double vote for block number: {:?}", target_number); + return + } + (rounds.validators_for(target_number), rounds.validator_set_id_for(target_number)) + } else { + debug!(target: "beefy", "🥩 Missing validator set - can't vote for: {:?}", target_hash); + return + }; + let authority_id = if let Some(id) = self.key_store.authority_id(validators) { + debug!(target: "beefy", "🥩 Local authority id: {:?}", id); + id + } else { + debug!(target: "beefy", "🥩 Missing validator id - can't vote for: {:?}", target_hash); + return + }; + + let commitment = Commitment { payload, block_number: target_number, validator_set_id }; + let encoded_commitment = commitment.encode(); + + let signature = match self.key_store.sign(&authority_id, &*encoded_commitment) { + Ok(sig) => sig, + Err(err) => { + warn!(target: "beefy", "🥩 Error signing commitment: {:?}", err); + return + }, + }; + + trace!( + target: "beefy", + "🥩 Produced signature using {:?}, is_valid: {:?}", + authority_id, + BeefyKeystore::verify(&authority_id, &signature, &*encoded_commitment) + ); + + let message = VoteMessage { commitment, id: authority_id, signature }; + + let encoded_message = message.encode(); + + metric_inc!(self, beefy_votes_sent); + + debug!(target: "beefy", "🥩 Sent vote message: {:?}", message); + + self.handle_vote( + (message.commitment.payload, message.commitment.block_number), + (message.id, message.signature), + true, + ); + + self.gossip_engine.lock().gossip_message(topic::(), encoded_message, false); + } + + /// Wait for BEEFY runtime pallet to be available. + #[cfg(not(test))] + async fn wait_for_runtime_pallet(&mut self) { + self.client + .finality_notification_stream() + .take_while(|notif| { + let at = BlockId::hash(notif.header.hash()); + if let Some(active) = self.client.runtime_api().validator_set(&at).ok().flatten() { + if active.id() == GENESIS_AUTHORITY_SET_ID { + // When starting from genesis, there is no session boundary digest. + // Just initialize `rounds` to Block #1 as BEEFY mandatory block. + self.init_session_at(active, 1u32.into()); + } + // In all other cases, we just go without `rounds` initialized, meaning the + // worker won't vote until it witnesses a session change. + // Once we'll implement 'initial sync' (catch-up), the worker will be able to + // start voting right away. + self.handle_finality_notification(notif); + future::ready(false) + } else { + trace!(target: "beefy", "🥩 Finality notification: {:?}", notif); + trace!(target: "beefy", "🥩 Waiting for BEEFY pallet to become available..."); + future::ready(true) + } + }) + .for_each(|_| future::ready(())) + .await; + // get a new stream that provides _new_ notifications (from here on out) + self.finality_notifications = self.client.finality_notification_stream(); + } + + /// For tests don't use runtime pallet. Start rounds from block #1. + #[cfg(test)] + async fn wait_for_runtime_pallet(&mut self) { + let active = self.test_res.active_validators.clone(); + self.init_session_at(active, 1u32.into()); + } + + /// Main loop for BEEFY worker. + /// + /// Wait for BEEFY runtime pallet to be available, then start the main async loop + /// which is driven by finality notifications and gossiped votes. pub(crate) async fn run(mut self) { + info!(target: "beefy", "🥩 run BEEFY worker, best grandpa: #{:?}.", self.best_grandpa_block_header.number()); + self.wait_for_runtime_pallet().await; + let mut votes = Box::pin(self.gossip_engine.lock().messages_for(topic::()).filter_map( |notification| async move { debug!(target: "beefy", "🥩 Got vote message: {:?}", notification); - VoteMessage::, Public, Signature>::decode( + VoteMessage::, AuthorityId, Signature>::decode( &mut ¬ification.message[..], ) .ok() @@ -360,13 +515,18 @@ where )); loop { + while self.sync_oracle.is_major_syncing() { + debug!(target: "beefy", "Waiting for major sync to complete..."); + futures_timer::Delay::new(Duration::from_secs(5)).await; + } + let engine = self.gossip_engine.clone(); let gossip_engine = future::poll_fn(|cx| engine.lock().poll_unpin(cx)); futures::select! { notification = self.finality_notifications.next().fuse() => { if let Some(notification) = notification { - self.handle_finality_notification(notification); + self.handle_finality_notification(¬ification); } else { return; } @@ -376,6 +536,7 @@ where self.handle_vote( (vote.commitment.payload, vote.commitment.block_number), (vote.id, vote.signature), + false ); } else { return; @@ -388,20 +549,36 @@ where } } } + + /// Simple wrapper over mmr root extraction. + #[cfg(not(test))] + fn extract_mmr_root_digest(&self, header: &B::Header) -> Option { + find_mmr_root_digest::(header) + } + + /// For tests, have the option to modify mmr root. + #[cfg(test)] + fn extract_mmr_root_digest(&self, header: &B::Header) -> Option { + let mut mmr_root = find_mmr_root_digest::(header); + if self.test_res.corrupt_mmr_roots { + mmr_root.as_mut().map(|hash| *hash ^= MmrRootHash::random()); + } + mmr_root + } } /// Extract the MMR root hash from a digest in the given header, if it exists. -fn find_mmr_root_digest(header: &B::Header) -> Option +fn find_mmr_root_digest(header: &B::Header) -> Option where B: Block, - Id: Codec, { - header.digest().logs().iter().find_map(|log| { - match log.try_to::>(OpaqueDigestItemId::Consensus(&BEEFY_ENGINE_ID)) { - Some(ConsensusLog::MmrRoot(root)) => Some(root), - _ => None, - } - }) + let id = OpaqueDigestItemId::Consensus(&BEEFY_ENGINE_ID); + + let filter = |log: ConsensusLog| match log { + ConsensusLog::MmrRoot(root) => Some(root), + _ => None, + }; + header.digest().convert_first(|l| l.try_to(id).and_then(filter)) } /// Scan the `header` digest log for a BEEFY validator set change. Return either the new @@ -416,119 +593,402 @@ where ConsensusLog::AuthoritiesChange(validator_set) => Some(validator_set), _ => None, }; - header.digest().convert_first(|l| l.try_to(id).and_then(filter)) } -/// Calculate next block number to vote on -fn vote_target(best_grandpa: N, best_beefy: N, min_delta: u32) -> N +/// Calculate next block number to vote on. +/// +/// Return `None` if there is no voteable target yet. +fn vote_target( + best_grandpa: N, + best_beefy: Option, + session_start: N, + min_delta: u32, +) -> Option where N: AtLeast32Bit + Copy + Debug, { - let diff = best_grandpa.saturating_sub(best_beefy); - let diff = diff.saturated_into::(); - let target = best_beefy + min_delta.max(diff.next_power_of_two()).into(); - - trace!( - target: "beefy", - "🥩 vote target - diff: {:?}, next_power_of_two: {:?}, target block: #{:?}", - diff, - diff.next_power_of_two(), - target, - ); - - target + // if the mandatory block (session_start) does not have a beefy justification yet, + // we vote on it + let target = match best_beefy { + None => { + trace!( + target: "beefy", + "🥩 vote target - mandatory block: #{:?}", + session_start, + ); + session_start + }, + Some(bbb) if bbb < session_start => { + trace!( + target: "beefy", + "🥩 vote target - mandatory block: #{:?}", + session_start, + ); + session_start + }, + Some(bbb) => { + let diff = best_grandpa.saturating_sub(bbb) + 1u32.into(); + let diff = diff.saturated_into::() / 2; + let target = bbb + min_delta.max(diff.next_power_of_two()).into(); + + trace!( + target: "beefy", + "🥩 vote target - diff: {:?}, next_power_of_two: {:?}, target block: #{:?}", + diff, + diff.next_power_of_two(), + target, + ); + + target + }, + }; + + // Don't vote for targets until they've been finalized + // (`target` can be > `best_grandpa` when `min_delta` is big enough). + if target > best_grandpa { + None + } else { + Some(target) + } } #[cfg(test)] -mod tests { - use super::vote_target; +pub(crate) mod tests { + use super::*; + use crate::{ + keystore::tests::Keyring, + tests::{create_beefy_worker, get_beefy_streams, make_beefy_ids, BeefyTestNet}, + }; + + use futures::{executor::block_on, future::poll_fn, task::Poll}; + + use sc_client_api::HeaderBackend; + use sc_network::NetworkService; + use sc_network_test::{PeersFullClient, TestNetFactory}; + use sp_api::HeaderT; + use substrate_test_runtime_client::{ + runtime::{Block, Digest, DigestItem, Header, H256}, + Backend, + }; + + #[derive(Clone)] + pub struct TestModifiers { + pub active_validators: ValidatorSet, + pub corrupt_mmr_roots: bool, + } #[test] fn vote_on_min_block_delta() { - let t = vote_target(1u32, 0, 4); - assert_eq!(4, t); - let t = vote_target(2u32, 0, 4); - assert_eq!(4, t); - let t = vote_target(3u32, 0, 4); - assert_eq!(4, t); - let t = vote_target(4u32, 0, 4); - assert_eq!(4, t); - - let t = vote_target(4u32, 4, 4); - assert_eq!(8, t); - - let t = vote_target(10u32, 10, 4); - assert_eq!(14, t); - let t = vote_target(11u32, 10, 4); - assert_eq!(14, t); - let t = vote_target(12u32, 10, 4); - assert_eq!(14, t); - let t = vote_target(13u32, 10, 4); - assert_eq!(14, t); - - let t = vote_target(10u32, 10, 8); - assert_eq!(18, t); - let t = vote_target(11u32, 10, 8); - assert_eq!(18, t); - let t = vote_target(12u32, 10, 8); - assert_eq!(18, t); - let t = vote_target(13u32, 10, 8); - assert_eq!(18, t); + let t = vote_target(1u32, Some(1), 1, 4); + assert_eq!(None, t); + let t = vote_target(2u32, Some(1), 1, 4); + assert_eq!(None, t); + let t = vote_target(4u32, Some(2), 1, 4); + assert_eq!(None, t); + let t = vote_target(6u32, Some(2), 1, 4); + assert_eq!(Some(6), t); + + let t = vote_target(9u32, Some(4), 1, 4); + assert_eq!(Some(8), t); + + let t = vote_target(10u32, Some(10), 1, 8); + assert_eq!(None, t); + let t = vote_target(12u32, Some(10), 1, 8); + assert_eq!(None, t); + let t = vote_target(18u32, Some(10), 1, 8); + assert_eq!(Some(18), t); } #[test] fn vote_on_power_of_two() { - let t = vote_target(1008u32, 1000, 4); - assert_eq!(1008, t); + let t = vote_target(1008u32, Some(1000), 1, 4); + assert_eq!(Some(1004), t); - let t = vote_target(1016u32, 1000, 4); - assert_eq!(1016, t); + let t = vote_target(1016u32, Some(1000), 1, 4); + assert_eq!(Some(1008), t); - let t = vote_target(1032u32, 1000, 4); - assert_eq!(1032, t); + let t = vote_target(1032u32, Some(1000), 1, 4); + assert_eq!(Some(1016), t); - let t = vote_target(1064u32, 1000, 4); - assert_eq!(1064, t); + let t = vote_target(1064u32, Some(1000), 1, 4); + assert_eq!(Some(1032), t); - let t = vote_target(1128u32, 1000, 4); - assert_eq!(1128, t); + let t = vote_target(1128u32, Some(1000), 1, 4); + assert_eq!(Some(1064), t); - let t = vote_target(1256u32, 1000, 4); - assert_eq!(1256, t); + let t = vote_target(1256u32, Some(1000), 1, 4); + assert_eq!(Some(1128), t); - let t = vote_target(1512u32, 1000, 4); - assert_eq!(1512, t); + let t = vote_target(1512u32, Some(1000), 1, 4); + assert_eq!(Some(1256), t); - let t = vote_target(1024u32, 0, 4); - assert_eq!(1024, t); + let t = vote_target(1024u32, Some(1), 1, 4); + assert_eq!(Some(513), t); } #[test] fn vote_on_target_block() { - let t = vote_target(1008u32, 1002, 4); - assert_eq!(1010, t); - let t = vote_target(1010u32, 1002, 4); - assert_eq!(1010, t); - - let t = vote_target(1016u32, 1006, 4); - assert_eq!(1022, t); - let t = vote_target(1022u32, 1006, 4); - assert_eq!(1022, t); - - let t = vote_target(1032u32, 1012, 4); - assert_eq!(1044, t); - let t = vote_target(1044u32, 1012, 4); - assert_eq!(1044, t); - - let t = vote_target(1064u32, 1014, 4); - assert_eq!(1078, t); - let t = vote_target(1078u32, 1014, 4); - assert_eq!(1078, t); - - let t = vote_target(1128u32, 1008, 4); - assert_eq!(1136, t); - let t = vote_target(1136u32, 1008, 4); - assert_eq!(1136, t); + let t = vote_target(1008u32, Some(1002), 1, 4); + assert_eq!(Some(1006), t); + let t = vote_target(1010u32, Some(1002), 1, 4); + assert_eq!(Some(1006), t); + + let t = vote_target(1016u32, Some(1006), 1, 4); + assert_eq!(Some(1014), t); + let t = vote_target(1022u32, Some(1006), 1, 4); + assert_eq!(Some(1014), t); + + let t = vote_target(1032u32, Some(1012), 1, 4); + assert_eq!(Some(1028), t); + let t = vote_target(1044u32, Some(1012), 1, 4); + assert_eq!(Some(1028), t); + + let t = vote_target(1064u32, Some(1014), 1, 4); + assert_eq!(Some(1046), t); + let t = vote_target(1078u32, Some(1014), 1, 4); + assert_eq!(Some(1046), t); + + let t = vote_target(1128u32, Some(1008), 1, 4); + assert_eq!(Some(1072), t); + let t = vote_target(1136u32, Some(1008), 1, 4); + assert_eq!(Some(1072), t); + } + + #[test] + fn vote_on_mandatory_block() { + let t = vote_target(1008u32, Some(1002), 1004, 4); + assert_eq!(Some(1004), t); + let t = vote_target(1016u32, Some(1006), 1007, 4); + assert_eq!(Some(1007), t); + let t = vote_target(1064u32, Some(1014), 1063, 4); + assert_eq!(Some(1063), t); + let t = vote_target(1320u32, Some(1012), 1234, 4); + assert_eq!(Some(1234), t); + + let t = vote_target(1128u32, Some(1008), 1008, 4); + assert_eq!(Some(1072), t); + } + + #[test] + fn extract_authorities_change_digest() { + let mut header = Header::new( + 1u32.into(), + Default::default(), + Default::default(), + Default::default(), + Digest::default(), + ); + + // verify empty digest shows nothing + assert!(find_authorities_change::(&header).is_none()); + + let peers = &[Keyring::One, Keyring::Two]; + let id = 42; + let validator_set = ValidatorSet::new(make_beefy_ids(peers), id).unwrap(); + header.digest_mut().push(DigestItem::Consensus( + BEEFY_ENGINE_ID, + ConsensusLog::::AuthoritiesChange(validator_set.clone()).encode(), + )); + + // verify validator set is correctly extracted from digest + let extracted = find_authorities_change::(&header); + assert_eq!(extracted, Some(validator_set)); + } + + #[test] + fn extract_mmr_root_digest() { + let mut header = Header::new( + 1u32.into(), + Default::default(), + Default::default(), + Default::default(), + Digest::default(), + ); + + // verify empty digest shows nothing + assert!(find_mmr_root_digest::(&header).is_none()); + + let mmr_root_hash = H256::random(); + header.digest_mut().push(DigestItem::Consensus( + BEEFY_ENGINE_ID, + ConsensusLog::::MmrRoot(mmr_root_hash.clone()).encode(), + )); + + // verify validator set is correctly extracted from digest + let extracted = find_mmr_root_digest::(&header); + assert_eq!(extracted, Some(mmr_root_hash)); + } + + #[test] + fn should_vote_target() { + let keys = &[Keyring::Alice]; + let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap(); + let mut net = BeefyTestNet::new(1, 0); + net.peer(0).data.use_validator_set(&validator_set); + let mut worker = create_beefy_worker(&net.peer(0), &keys[0], 1); + + // rounds not initialized -> should vote: `None` + assert_eq!(worker.current_vote_target(), None); + + let set_up = |worker: &mut BeefyWorker< + Block, + PeersFullClient, + Backend, + Arc>, + >, + best_grandpa: u64, + best_beefy: Option, + session_start: u64, + min_delta: u32| { + let grandpa_header = Header::new( + best_grandpa, + Default::default(), + Default::default(), + Default::default(), + Default::default(), + ); + worker.best_grandpa_block_header = grandpa_header; + worker.best_beefy_block = best_beefy; + worker.min_block_delta = min_delta; + worker.rounds = + Some(Rounds::new(session_start, validator_set.clone(), validator_set.clone())); + }; + + // under min delta + set_up(&mut worker, 1, Some(1), 1, 4); + assert_eq!(worker.current_vote_target(), None); + set_up(&mut worker, 5, Some(2), 1, 4); + assert_eq!(worker.current_vote_target(), None); + + // vote on min delta + set_up(&mut worker, 9, Some(4), 1, 4); + assert_eq!(worker.current_vote_target(), Some(8)); + set_up(&mut worker, 18, Some(10), 1, 8); + assert_eq!(worker.current_vote_target(), Some(18)); + + // vote on power of two + set_up(&mut worker, 1008, Some(1000), 1, 1); + assert_eq!(worker.current_vote_target(), Some(1004)); + set_up(&mut worker, 1016, Some(1000), 1, 2); + assert_eq!(worker.current_vote_target(), Some(1008)); + + // nothing new to vote on + set_up(&mut worker, 1000, Some(1000), 1, 1); + assert_eq!(worker.current_vote_target(), None); + + // vote on mandatory + set_up(&mut worker, 1008, None, 1000, 8); + assert_eq!(worker.current_vote_target(), Some(1000)); + set_up(&mut worker, 1008, Some(1000), 1001, 8); + assert_eq!(worker.current_vote_target(), Some(1001)); + } + + #[test] + fn keystore_vs_validator_set() { + let keys = &[Keyring::Alice]; + let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap(); + let mut net = BeefyTestNet::new(1, 0); + net.peer(0).data.use_validator_set(&validator_set); + let mut worker = create_beefy_worker(&net.peer(0), &keys[0], 1); + + // keystore doesn't contain other keys than validators' + assert_eq!(worker.verify_validator_set(&1, &validator_set), Ok(())); + + // unknown `Bob` key + let keys = &[Keyring::Bob]; + let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap(); + let err_msg = "no authority public key found in store".to_string(); + let expected = Err(error::Error::Keystore(err_msg)); + assert_eq!(worker.verify_validator_set(&1, &validator_set), expected); + + // worker has no keystore + worker.key_store = None.into(); + let expected_err = Err(error::Error::Keystore("no Keystore".into())); + assert_eq!(worker.verify_validator_set(&1, &validator_set), expected_err); + } + + #[test] + fn setting_best_beefy_block() { + let keys = &[Keyring::Alice]; + let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap(); + let mut net = BeefyTestNet::new(1, 0); + net.peer(0).data.use_validator_set(&validator_set); + let mut worker = create_beefy_worker(&net.peer(0), &keys[0], 1); + + let (mut best_block_streams, _) = get_beefy_streams(&mut net, keys); + let mut best_block_stream = best_block_streams.drain(..).next().unwrap(); + + // no 'best beefy block' + assert_eq!(worker.best_beefy_block, None); + block_on(poll_fn(move |cx| { + assert_eq!(best_block_stream.poll_next_unpin(cx), Poll::Pending); + Poll::Ready(()) + })); + + // unknown hash for block #1 + let (mut best_block_streams, _) = get_beefy_streams(&mut net, keys); + let mut best_block_stream = best_block_streams.drain(..).next().unwrap(); + worker.set_best_beefy_block(1); + assert_eq!(worker.best_beefy_block, Some(1)); + block_on(poll_fn(move |cx| { + assert_eq!(best_block_stream.poll_next_unpin(cx), Poll::Pending); + Poll::Ready(()) + })); + + // generate 2 blocks, try again expect success + let (mut best_block_streams, _) = get_beefy_streams(&mut net, keys); + let mut best_block_stream = best_block_streams.drain(..).next().unwrap(); + net.generate_blocks(2, 10, &validator_set); + + worker.set_best_beefy_block(2); + assert_eq!(worker.best_beefy_block, Some(2)); + block_on(poll_fn(move |cx| { + match best_block_stream.poll_next_unpin(cx) { + // expect Some(hash-of-block-2) + Poll::Ready(Some(hash)) => { + let block_num = net.peer(0).client().as_client().number(hash).unwrap(); + assert_eq!(block_num, Some(2)); + }, + v => panic!("unexpected value: {:?}", v), + } + Poll::Ready(()) + })); + } + + #[test] + fn setting_initial_session() { + let keys = &[Keyring::Alice]; + let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap(); + let mut net = BeefyTestNet::new(1, 0); + net.peer(0).data.use_validator_set(&validator_set); + let mut worker = create_beefy_worker(&net.peer(0), &keys[0], 1); + + assert!(worker.rounds.is_none()); + + // verify setting the correct validator sets and boundary for genesis session + worker.init_session_at(validator_set.clone(), 1); + + let worker_rounds = worker.rounds.as_ref().unwrap(); + assert_eq!(worker_rounds.validator_set(), &validator_set); + assert_eq!(worker_rounds.session_start(), &1); + // in genesis case both current and prev validator sets are the same + assert_eq!(worker_rounds.validator_set_id_for(1), validator_set.id()); + assert_eq!(worker_rounds.validator_set_id_for(2), validator_set.id()); + + // new validator set + let keys = &[Keyring::Bob]; + let new_validator_set = ValidatorSet::new(make_beefy_ids(keys), 1).unwrap(); + + // verify setting the correct validator sets and boundary for non-genesis session + worker.init_session_at(new_validator_set.clone(), 11); + + let worker_rounds = worker.rounds.as_ref().unwrap(); + assert_eq!(worker_rounds.validator_set(), &new_validator_set); + assert_eq!(worker_rounds.session_start(), &11); + // mandatory block gets prev set, further blocks get new set + assert_eq!(worker_rounds.validator_set_id_for(11), validator_set.id()); + assert_eq!(worker_rounds.validator_set_id_for(12), new_validator_set.id()); + assert_eq!(worker_rounds.validator_set_id_for(13), new_validator_set.id()); } } diff --git a/client/block-builder/Cargo.toml b/client/block-builder/Cargo.toml index 6fef8498134e..fff7740ff8d0 100644 --- a/client/block-builder/Cargo.toml +++ b/client/block-builder/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-block-builder" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate block builder" readme = "README.md" @@ -14,15 +14,15 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-state-machine = { version = "0.10.0-dev", path = "../../primitives/state-machine" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } +sp-state-machine = { version = "0.12.0", path = "../../primitives/state-machine" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } sp-block-builder = { version = "4.0.0-dev", path = "../../primitives/block-builder" } sp-inherents = { version = "4.0.0-dev", path = "../../primitives/inherents" } sc-client-api = { version = "4.0.0-dev", path = "../api" } -codec = { package = "parity-scale-codec", version = "2.0.0", features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", features = [ "derive", ] } diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index e89421edfb16..3f9fecbccbb9 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -35,7 +35,9 @@ use sp_blockchain::{ApplyExtrinsicFailed, Error}; use sp_core::ExecutionContext; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, DigestFor, Hash, HashFor, Header as HeaderT, NumberFor, One}, + legacy, + traits::{Block as BlockT, Hash, HashFor, Header as HeaderT, NumberFor, One}, + Digest, }; pub use sp_block_builder::BlockBuilder as BlockBuilderApi; @@ -119,14 +121,14 @@ where fn new_block_at>( &self, parent: &BlockId, - inherent_digests: DigestFor, + inherent_digests: Digest, record_proof: R, ) -> sp_blockchain::Result>; /// Create a new block, built on the head of the chain. fn new_block( &self, - inherent_digests: DigestFor, + inherent_digests: Digest, ) -> sp_blockchain::Result>; } @@ -134,6 +136,7 @@ where pub struct BlockBuilder<'a, Block: BlockT, A: ProvideRuntimeApi, B> { extrinsics: Vec, api: ApiRef<'a, A::Api>, + version: u32, block_id: BlockId, parent_hash: Block::Hash, backend: &'a B, @@ -159,7 +162,7 @@ where parent_hash: Block::Hash, parent_number: NumberFor, record_proof: RecordProof, - inherent_digests: DigestFor, + inherent_digests: Digest, backend: &'a B, ) -> Result { let header = <::Header as HeaderT>::new( @@ -182,10 +185,15 @@ where api.initialize_block_with_context(&block_id, ExecutionContext::BlockConstruction, &header)?; + let version = api + .api_version::>(&block_id)? + .ok_or_else(|| Error::VersionInvalid("BlockBuilderApi".to_string()))?; + Ok(Self { parent_hash, extrinsics: Vec::new(), api, + version, block_id, backend, estimated_header_size, @@ -198,13 +206,26 @@ where pub fn push(&mut self, xt: ::Extrinsic) -> Result<(), Error> { let block_id = &self.block_id; let extrinsics = &mut self.extrinsics; + let version = self.version; self.api.execute_in_transaction(|api| { - match api.apply_extrinsic_with_context( - block_id, - ExecutionContext::BlockConstruction, - xt.clone(), - ) { + let res = if version < 6 { + #[allow(deprecated)] + api.apply_extrinsic_before_version_6_with_context( + block_id, + ExecutionContext::BlockConstruction, + xt.clone(), + ) + .map(legacy::byte_sized_error::convert_to_latest) + } else { + api.apply_extrinsic_with_context( + block_id, + ExecutionContext::BlockConstruction, + xt.clone(), + ) + }; + + match res { Ok(Ok(_)) => { extrinsics.push(xt); TransactionOutcome::Commit(Ok(())) @@ -231,21 +252,18 @@ where header.extrinsics_root().clone(), HashFor::::ordered_trie_root( self.extrinsics.iter().map(Encode::encode).collect(), + sp_runtime::StateVersion::V0, ), ); let proof = self.api.extract_proof(); let state = self.backend.state_at(self.block_id)?; - let changes_trie_state = backend::changes_tries_state_at_block( - &self.block_id, - self.backend.changes_trie_storage(), - )?; let parent_hash = self.parent_hash; let storage_changes = self .api - .into_storage_changes(&state, changes_trie_state.as_ref(), parent_hash) + .into_storage_changes(&state, parent_hash) .map_err(|e| sp_blockchain::Error::StorageChanges(e))?; Ok(BuiltBlock { diff --git a/client/chain-spec/Cargo.toml b/client/chain-spec/Cargo.toml index 8af2996e968d..bd19fd59d699 100644 --- a/client/chain-spec/Cargo.toml +++ b/client/chain-spec/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-chain-spec" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate chain configurations." readme = "README.md" @@ -14,11 +14,12 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sc-chain-spec-derive = { version = "4.0.0-dev", path = "./derive" } -impl-trait-for-tuples = "0.2.1" +impl-trait-for-tuples = "0.2.2" sc-network = { version = "0.10.0-dev", path = "../network" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -serde = { version = "1.0.126", features = ["derive"] } -serde_json = "1.0.68" -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +serde = { version = "1.0.136", features = ["derive"] } +serde_json = "1.0.79" +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" } -codec = { package = "parity-scale-codec", version = "2.0.0" } +codec = { package = "parity-scale-codec", version = "3.0.0" } +memmap2 = "0.5.0" diff --git a/client/chain-spec/derive/Cargo.toml b/client/chain-spec/derive/Cargo.toml index b210fa1320e0..9aa1d6c09405 100644 --- a/client/chain-spec/derive/Cargo.toml +++ b/client/chain-spec/derive/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-chain-spec-derive" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Macros to derive chain spec extension traits implementation." @@ -15,9 +15,9 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -proc-macro-crate = "1.0.0" -proc-macro2 = "1.0.29" -quote = "1.0.3" -syn = "1.0.58" +proc-macro-crate = "1.1.3" +proc-macro2 = "1.0.36" +quote = "1.0.10" +syn = "1.0.82" [dev-dependencies] diff --git a/client/chain-spec/derive/src/impls.rs b/client/chain-spec/derive/src/impls.rs index 8c56430e81d0..7af403d46ad1 100644 --- a/client/chain-spec/derive/src/impls.rs +++ b/client/chain-spec/derive/src/impls.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -52,7 +52,7 @@ pub fn extension_derive(ast: &DeriveInput) -> proc_macro::TokenStream { use std::any::{Any, TypeId}; match TypeId::of::() { - #( x if x == TypeId::of::<#field_types>() => Any::downcast_ref(&self.#field_names) ),*, + #( x if x == TypeId::of::<#field_types>() => ::downcast_ref(&self.#field_names) ),*, _ => None, } } diff --git a/client/chain-spec/derive/src/lib.rs b/client/chain-spec/derive/src/lib.rs index 53f0c69491ec..75356a225046 100644 --- a/client/chain-spec/derive/src/lib.rs +++ b/client/chain-spec/derive/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/chain-spec/res/chain_spec.json b/client/chain-spec/res/chain_spec.json index 673f35d50791..c3365a9192f6 100644 --- a/client/chain-spec/res/chain_spec.json +++ b/client/chain-spec/res/chain_spec.json @@ -19,7 +19,6 @@ ["wss://telemetry.polkadot.io/submit/", 0] ], "protocolId": "fir", - "consensusEngine": null, "genesis": { "raw": [ { diff --git a/client/chain-spec/res/chain_spec2.json b/client/chain-spec/res/chain_spec2.json index 950a7fc82749..00b9d603ae29 100644 --- a/client/chain-spec/res/chain_spec2.json +++ b/client/chain-spec/res/chain_spec2.json @@ -19,7 +19,6 @@ ["wss://telemetry.polkadot.io/submit/", 0] ], "protocolId": "fir", - "consensusEngine": null, "myProperty": "Test Extension", "genesis": { "raw": [ diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index ff3a99760bd2..efb40d46f216 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -29,7 +29,7 @@ use sp_core::{ Bytes, }; use sp_runtime::BuildStorage; -use std::{borrow::Cow, collections::HashMap, fs::File, path::PathBuf, sync::Arc}; +use std::{borrow::Cow, collections::BTreeMap, fs::File, path::PathBuf, sync::Arc}; enum GenesisSource { File(PathBuf), @@ -58,8 +58,9 @@ impl GenesisSource { match self { Self::File(path) => { - let file = - File::open(path).map_err(|e| format!("Error opening spec file: {}", e))?; + let file = File::open(path).map_err(|e| { + format!("Error opening spec file at `{}`: {}", path.display(), e) + })?; let genesis: GenesisContainer = json::from_reader(file) .map_err(|e| format!("Error parsing spec file: {}", e))?; Ok(genesis.genesis) @@ -118,6 +119,10 @@ impl BuildStorage for ChainSpec { }) .collect(), }), + // The `StateRootHash` variant exists as a way to keep note that other clients support + // it, but Substrate itself isn't capable of loading chain specs with just a hash at the + // moment. + Genesis::StateRootHash(_) => Err("Genesis storage in hash format not supported".into()), } } @@ -126,7 +131,7 @@ impl BuildStorage for ChainSpec { } } -pub type GenesisStorage = HashMap; +pub type GenesisStorage = BTreeMap; /// Raw storage content for genesis block. #[derive(Serialize, Deserialize)] @@ -134,7 +139,7 @@ pub type GenesisStorage = HashMap; #[serde(deny_unknown_fields)] pub struct RawGenesis { pub top: GenesisStorage, - pub children_default: HashMap, + pub children_default: BTreeMap, } #[derive(Serialize, Deserialize)] @@ -143,6 +148,8 @@ pub struct RawGenesis { enum Genesis { Runtime(G), Raw(RawGenesis), + /// State root hash of the genesis storage. + StateRootHash(StorageData), } /// A configuration of a client. Does not include runtime storage initialization. @@ -157,20 +164,27 @@ struct ClientSpec { boot_nodes: Vec, telemetry_endpoints: Option, protocol_id: Option, + /// Arbitrary string. Nodes will only synchronize with other nodes that have the same value + /// in their `fork_id`. This can be used in order to segregate nodes in cases when multiple + /// chains have the same genesis hash. + #[serde(default = "Default::default", skip_serializing_if = "Option::is_none")] + fork_id: Option, properties: Option, #[serde(flatten)] extensions: E, // Never used, left only for backward compatibility. + #[serde(default, skip_serializing)] + #[allow(unused)] consensus_engine: (), #[serde(skip_serializing)] #[allow(unused)] genesis: serde::de::IgnoredAny, - /// Mapping from `block_hash` to `wasm_code`. + /// Mapping from `block_number` to `wasm_code`. /// - /// The given `wasm_code` will be used to substitute the on-chain wasm code from the given - /// block hash onwards. + /// The given `wasm_code` will be used to substitute the on-chain wasm code starting with the + /// given block number until the `spec_version` on chain changes. #[serde(default)] - code_substitutes: HashMap, + code_substitutes: BTreeMap, } /// A type denoting empty extensions. @@ -213,7 +227,12 @@ impl ChainSpec { /// Network protocol id. pub fn protocol_id(&self) -> Option<&str> { - self.client_spec.protocol_id.as_ref().map(String::as_str) + self.client_spec.protocol_id.as_deref() + } + + /// Optional network fork identifier. + pub fn fork_id(&self) -> Option<&str> { + self.client_spec.fork_id.as_deref() } /// Additional loosly-typed properties of the chain. @@ -247,6 +266,7 @@ impl ChainSpec { boot_nodes: Vec, telemetry_endpoints: Option, protocol_id: Option<&str>, + fork_id: Option<&str>, properties: Option, extensions: E, ) -> Self { @@ -257,11 +277,12 @@ impl ChainSpec { boot_nodes, telemetry_endpoints, protocol_id: protocol_id.map(str::to_owned), + fork_id: fork_id.map(str::to_owned), properties, extensions, consensus_engine: (), genesis: Default::default(), - code_substitutes: HashMap::new(), + code_substitutes: BTreeMap::new(), }; ChainSpec { client_spec, genesis: GenesisSource::Factory(Arc::new(constructor)) } @@ -284,9 +305,20 @@ impl ChainSpec { /// Parse json file into a `ChainSpec` pub fn from_json_file(path: PathBuf) -> Result { - let file = File::open(&path).map_err(|e| format!("Error opening spec file: {}", e))?; + // We mmap the file into memory first, as this is *a lot* faster than using + // `serde_json::from_reader`. See https://github.com/serde-rs/json/issues/160 + let file = File::open(&path) + .map_err(|e| format!("Error opening spec file `{}`: {}", path.display(), e))?; + + // SAFETY: `mmap` is fundamentally unsafe since technically the file can change + // underneath us while it is mapped; in practice it's unlikely to be a problem + let bytes = unsafe { + memmap2::Mmap::map(&file) + .map_err(|e| format!("Error mmaping spec file `{}`: {}", path.display(), e))? + }; + let client_spec = - json::from_reader(file).map_err(|e| format!("Error parsing spec file: {}", e))?; + json::from_slice(&bytes).map_err(|e| format!("Error parsing spec file: {}", e))?; Ok(ChainSpec { client_spec, genesis: GenesisSource::File(path) }) } } @@ -363,6 +395,10 @@ where ChainSpec::protocol_id(self) } + fn fork_id(&self) -> Option<&str> { + ChainSpec::fork_id(self) + } + fn properties(&self) -> Properties { ChainSpec::properties(self) } @@ -395,7 +431,7 @@ where self.genesis = GenesisSource::Storage(storage); } - fn code_substitutes(&self) -> std::collections::HashMap> { + fn code_substitutes(&self) -> std::collections::BTreeMap> { self.client_spec .code_substitutes .iter() @@ -409,7 +445,7 @@ mod tests { use super::*; #[derive(Debug, Serialize, Deserialize)] - struct Genesis(HashMap); + struct Genesis(BTreeMap); impl BuildStorage for Genesis { fn assimilate_storage(&self, storage: &mut Storage) -> Result<(), String> { @@ -434,12 +470,28 @@ mod tests { assert_eq!(spec2.chain_type(), ChainType::Live) } - #[derive(Debug, Serialize, Deserialize)] + #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] struct Extension1 { my_property: String, } + impl crate::Extension for Extension1 { + type Forks = Option<()>; + + fn get(&self) -> Option<&T> { + None + } + + fn get_any(&self, _: std::any::TypeId) -> &dyn std::any::Any { + self + } + + fn get_any_mut(&mut self, _: std::any::TypeId) -> &mut dyn std::any::Any { + self + } + } + type TestSpec2 = ChainSpec; #[test] @@ -451,4 +503,35 @@ mod tests { assert_eq!(spec.extensions().my_property, "Test Extension"); } + + #[test] + fn chain_spec_raw_output_should_be_deterministic() { + let mut spec = TestSpec2::from_json_bytes(Cow::Owned( + include_bytes!("../res/chain_spec2.json").to_vec(), + )) + .unwrap(); + + let mut storage = spec.build_storage().unwrap(); + + // Add some extra data, so that storage "sorting" is tested. + let extra_data = &[("random_key", "val"), ("r@nd0m_key", "val"), ("aaarandom_key", "val")]; + storage + .top + .extend(extra_data.iter().map(|(k, v)| (k.as_bytes().to_vec(), v.as_bytes().to_vec()))); + crate::ChainSpec::set_storage(&mut spec, storage); + + let json = spec.as_json(true).unwrap(); + + // Check multiple times that decoding and encoding the chain spec leads always to the same + // output. + for _ in 0..10 { + assert_eq!( + json, + TestSpec2::from_json_bytes(json.as_bytes().to_vec()) + .unwrap() + .as_json(true) + .unwrap() + ); + } + } } diff --git a/client/chain-spec/src/extension.rs b/client/chain-spec/src/extension.rs index 4b59232cf577..c0b3e15a2df4 100644 --- a/client/chain-spec/src/extension.rs +++ b/client/chain-spec/src/extension.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index 334d8f8b3d7a..eb72592b54a9 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -165,6 +165,8 @@ pub trait ChainSpec: BuildStorage + Send + Sync { fn telemetry_endpoints(&self) -> &Option; /// Network protocol id. fn protocol_id(&self) -> Option<&str>; + /// Optional network fork identifier. `None` by default. + fn fork_id(&self) -> Option<&str>; /// Additional loosly-typed properties of the chain. /// /// Returns an empty JSON object if 'properties' not defined in config @@ -186,7 +188,7 @@ pub trait ChainSpec: BuildStorage + Send + Sync { /// This will be used as storage at genesis. fn set_storage(&mut self, storage: Storage); /// Returns code substitutes that should be used for the on chain wasm. - fn code_substitutes(&self) -> std::collections::HashMap>; + fn code_substitutes(&self) -> std::collections::BTreeMap>; } impl std::fmt::Debug for dyn ChainSpec { diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index e7a0330e76e0..1fabbca46968 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -3,9 +3,9 @@ name = "sc-cli" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "Substrate CLI interface." -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,42 +13,41 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -log = "0.4.11" -regex = "1.4.2" -tokio = { version = "1.10", features = [ "signal", "rt-multi-thread" ] } -futures = "0.3.9" +chrono = "0.4.10" +clap = { version = "3.1.6", features = ["derive"] } fdlimit = "0.2.1" -libp2p = "0.39.1" -parity-scale-codec = "2.0.0" +futures = "0.3.21" hex = "0.4.2" +libp2p = "0.40.0" +log = "0.4.11" +names = { version = "0.13.0", default-features = false } rand = "0.7.3" -tiny-bip39 = "0.8.0" -serde_json = "1.0.68" -sc-keystore = { version = "4.0.0-dev", path = "../keystore" } -sp-panic-handler = { version = "3.0.0", path = "../../primitives/panic-handler" } +regex = "1.5.5" +rpassword = "5.0.0" +serde = "1.0.136" +serde_json = "1.0.79" +thiserror = "1.0.30" +tiny-bip39 = "0.8.2" +tokio = { version = "1.17.0", features = ["signal", "rt-multi-thread", "parking_lot"] } + +parity-scale-codec = "3.0.0" sc-client-api = { version = "4.0.0-dev", path = "../api" } -sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } +sc-keystore = { version = "4.0.0-dev", path = "../keystore" } sc-network = { version = "0.10.0-dev", path = "../network" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sc-utils = { version = "4.0.0-dev", path = "../utils" } -sp-version = { version = "4.0.0-dev", path = "../../primitives/version" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore" } sc-service = { version = "0.10.0-dev", default-features = false, path = "../service" } sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" } -sp-keyring = { version = "4.0.0-dev", path = "../../primitives/keyring" } -names = { version = "0.12.0", default-features = false } -structopt = "0.3.8" sc-tracing = { version = "4.0.0-dev", path = "../tracing" } -chrono = "0.4.10" -serde = "1.0.126" -thiserror = "1.0.21" -rpassword = "5.0.0" +sc-utils = { version = "4.0.0-dev", path = "../utils" } +sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-keyring = { version = "6.0.0", path = "../../primitives/keyring" } +sp-keystore = { version = "0.12.0", path = "../../primitives/keystore" } +sp-panic-handler = { version = "4.0.0", path = "../../primitives/panic-handler" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } +sp-version = { version = "5.0.0", path = "../../primitives/version" } [dev-dependencies] tempfile = "3.1.0" [features] -wasmtime = [ - "sc-service/wasmtime", -] +wasmtime = ["sc-service/wasmtime"] diff --git a/client/cli/src/arg_enums.rs b/client/cli/src/arg_enums.rs index 5221500f08b3..a09b7f824c7b 100644 --- a/client/cli/src/arg_enums.rs +++ b/client/cli/src/arg_enums.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -15,10 +15,10 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -// NOTE: we allow missing docs here because arg_enum! creates the function variants without doc -#![allow(missing_docs)] -use structopt::clap::arg_enum; +//! Definitions of [`ArgEnum`] types. + +use clap::ArgEnum; /// How to execute Wasm runtime code. #[derive(Debug, Clone, Copy)] @@ -51,7 +51,7 @@ impl std::str::FromStr for WasmExecutionMethod { } #[cfg(not(feature = "wasmtime"))] { - Err(format!("`Compiled` variant requires the `wasmtime` feature to be enabled")) + Err("`Compiled` variant requires the `wasmtime` feature to be enabled".into()) } } else { Err(format!("Unknown variant `{}`, known variants: {:?}", s, Self::variants())) @@ -86,12 +86,20 @@ impl Into for WasmExecutionMethod { } } -arg_enum! { - #[allow(missing_docs)] - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - pub enum TracingReceiver { - Log, - } +/// The default [`WasmExecutionMethod`]. +#[cfg(feature = "wasmtime")] +pub const DEFAULT_WASM_EXECUTION_METHOD: &str = "Compiled"; + +/// The default [`WasmExecutionMethod`]. +#[cfg(not(feature = "wasmtime"))] +pub const DEFAULT_WASM_EXECUTION_METHOD: &str = "interpreted-i-know-what-i-do"; + +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, ArgEnum)] +#[clap(rename_all = "PascalCase")] +pub enum TracingReceiver { + /// Output the tracing records using the log. + Log, } impl Into for TracingReceiver { @@ -102,44 +110,48 @@ impl Into for TracingReceiver { } } -arg_enum! { - #[allow(missing_docs)] - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - pub enum NodeKeyType { - Ed25519 - } +/// The type of the node key. +#[derive(Debug, Copy, Clone, PartialEq, Eq, ArgEnum)] +#[clap(rename_all = "PascalCase")] +pub enum NodeKeyType { + /// Use ed25519. + Ed25519, } -arg_enum! { - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - pub enum CryptoScheme { - Ed25519, - Sr25519, - Ecdsa, - } +/// The crypto scheme to use. +#[derive(Debug, Copy, Clone, PartialEq, Eq, ArgEnum)] +#[clap(rename_all = "PascalCase")] +pub enum CryptoScheme { + /// Use ed25519. + Ed25519, + /// Use sr25519. + Sr25519, + /// Use + Ecdsa, } -arg_enum! { - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - pub enum OutputType { - Json, - Text, - } +/// The type of the output format. +#[derive(Debug, Copy, Clone, PartialEq, Eq, ArgEnum)] +#[clap(rename_all = "PascalCase")] +pub enum OutputType { + /// Output as json. + Json, + /// Output as text. + Text, } -arg_enum! { - /// How to execute blocks - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub enum ExecutionStrategy { - // Execute with native build (if available, WebAssembly otherwise). - Native, - // Only execute with the WebAssembly build. - Wasm, - // Execute with both native (where available) and WebAssembly builds. - Both, - // Execute with the native build if possible; if it fails, then execute with WebAssembly. - NativeElseWasm, - } +/// How to execute blocks +#[derive(Debug, Copy, Clone, PartialEq, Eq, ArgEnum)] +#[clap(rename_all = "PascalCase")] +pub enum ExecutionStrategy { + /// Execute with native build (if available, WebAssembly otherwise). + Native, + /// Only execute with the WebAssembly build. + Wasm, + /// Execute with both native (where available) and WebAssembly builds. + Both, + /// Execute with the native build if possible; if it fails, then execute with WebAssembly. + NativeElseWasm, } impl Into for ExecutionStrategy { @@ -165,19 +177,18 @@ impl ExecutionStrategy { } } -arg_enum! { - /// Available RPC methods. - #[allow(missing_docs)] - #[derive(Debug, Copy, Clone, PartialEq)] - pub enum RpcMethods { - // Expose every RPC method only when RPC is listening on `localhost`, - // otherwise serve only safe RPC methods. - Auto, - // Allow only a safe subset of RPC methods. - Safe, - // Expose every RPC method (even potentially unsafe ones). - Unsafe, - } +/// Available RPC methods. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone, PartialEq, ArgEnum)] +#[clap(rename_all = "PascalCase")] +pub enum RpcMethods { + /// Expose every RPC method only when RPC is listening on `localhost`, + /// otherwise serve only safe RPC methods. + Auto, + /// Allow only a safe subset of RPC methods. + Safe, + /// Expose every RPC method (even potentially unsafe ones). + Unsafe, } impl Into for RpcMethods { @@ -191,15 +202,17 @@ impl Into for RpcMethods { } /// Database backend -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, PartialEq, Copy)] pub enum Database { /// Facebooks RocksDB RocksDb, /// ParityDb. ParityDb, /// Detect whether there is an existing database. Use it, if there is, if not, create new - /// instance of paritydb + /// instance of ParityDb Auto, + /// ParityDb. + ParityDbDeprecated, } impl std::str::FromStr for Database { @@ -209,6 +222,8 @@ impl std::str::FromStr for Database { if s.eq_ignore_ascii_case("rocksdb") { Ok(Self::RocksDb) } else if s.eq_ignore_ascii_case("paritydb-experimental") { + Ok(Self::ParityDbDeprecated) + } else if s.eq_ignore_ascii_case("paritydb") { Ok(Self::ParityDb) } else if s.eq_ignore_ascii_case("auto") { Ok(Self::Auto) @@ -221,35 +236,35 @@ impl std::str::FromStr for Database { impl Database { /// Returns all the variants of this enum to be shown in the cli. pub fn variants() -> &'static [&'static str] { - &["rocksdb", "paritydb-experimental", "auto"] + &["rocksdb", "paritydb", "paritydb-experimental", "auto"] } } -arg_enum! { - /// Whether off-chain workers are enabled. - #[allow(missing_docs)] - #[derive(Debug, Clone)] - pub enum OffchainWorkerEnabled { - Always, - Never, - WhenValidating, - } +/// Whether off-chain workers are enabled. +#[allow(missing_docs)] +#[derive(Debug, Clone, ArgEnum)] +#[clap(rename_all = "PascalCase")] +pub enum OffchainWorkerEnabled { + /// Always have offchain worker enabled. + Always, + /// Never enable the offchain worker. + Never, + /// Only enable the offchain worker when running as validator. + WhenValidating, } -arg_enum! { - /// Syncing mode. - #[allow(missing_docs)] - #[derive(Debug, Clone, Copy)] - pub enum SyncMode { - // Full sync. Donwnload end verify all blocks. - Full, - // Download blocks without executing them. Download latest state with proofs. - Fast, - // Download blocks without executing them. Download latest state without proofs. - FastUnsafe, - // Prove finality and download the latest state. - Warp, - } +/// Syncing mode. +#[derive(Debug, Clone, Copy, ArgEnum, PartialEq)] +#[clap(rename_all = "PascalCase")] +pub enum SyncMode { + /// Full sync. Download end verify all blocks. + Full, + /// Download blocks without executing them. Download latest state with proofs. + Fast, + /// Download blocks without executing them. Download latest state without proofs. + FastUnsafe, + /// Prove finality and download the latest state. + Warp, } impl Into for SyncMode { @@ -266,14 +281,14 @@ impl Into for SyncMode { } /// Default value for the `--execution-syncing` parameter. -pub const DEFAULT_EXECUTION_SYNCING: ExecutionStrategy = ExecutionStrategy::NativeElseWasm; +pub const DEFAULT_EXECUTION_SYNCING: ExecutionStrategy = ExecutionStrategy::Wasm; /// Default value for the `--execution-import-block` parameter. -pub const DEFAULT_EXECUTION_IMPORT_BLOCK: ExecutionStrategy = ExecutionStrategy::NativeElseWasm; +pub const DEFAULT_EXECUTION_IMPORT_BLOCK: ExecutionStrategy = ExecutionStrategy::Wasm; /// Default value for the `--execution-import-block` parameter when the node is a validator. pub const DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR: ExecutionStrategy = ExecutionStrategy::Wasm; /// Default value for the `--execution-block-construction` parameter. pub const DEFAULT_EXECUTION_BLOCK_CONSTRUCTION: ExecutionStrategy = ExecutionStrategy::Wasm; /// Default value for the `--execution-offchain-worker` parameter. -pub const DEFAULT_EXECUTION_OFFCHAIN_WORKER: ExecutionStrategy = ExecutionStrategy::Native; +pub const DEFAULT_EXECUTION_OFFCHAIN_WORKER: ExecutionStrategy = ExecutionStrategy::Wasm; /// Default value for the `--execution-other` parameter. -pub const DEFAULT_EXECUTION_OTHER: ExecutionStrategy = ExecutionStrategy::Native; +pub const DEFAULT_EXECUTION_OTHER: ExecutionStrategy = ExecutionStrategy::Wasm; diff --git a/client/cli/src/commands/build_spec_cmd.rs b/client/cli/src/commands/build_spec_cmd.rs index 75fdf07643ee..3196a3e7b915 100644 --- a/client/cli/src/commands/build_spec_cmd.rs +++ b/client/cli/src/commands/build_spec_cmd.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,6 +21,7 @@ use crate::{ params::{NodeKeyParams, SharedParams}, CliConfiguration, }; +use clap::Parser; use log::info; use sc_network::config::build_multiaddr; use sc_service::{ @@ -28,28 +29,27 @@ use sc_service::{ ChainSpec, }; use std::io::Write; -use structopt::StructOpt; /// The `build-spec` command used to build a specification. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Parser)] pub struct BuildSpecCmd { /// Force raw genesis storage output. - #[structopt(long = "raw")] + #[clap(long)] pub raw: bool, /// Disable adding the default bootnode to the specification. /// /// By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the /// specification when no bootnode exists. - #[structopt(long = "disable-default-bootnode")] + #[clap(long)] pub disable_default_bootnode: bool, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub shared_params: SharedParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub node_key_params: NodeKeyParams, } @@ -65,7 +65,7 @@ impl BuildSpecCmd { if spec.boot_nodes().is_empty() && !self.disable_default_bootnode { let keys = network_config.node_key.into_keypair()?; - let peer_id = keys.public().into_peer_id(); + let peer_id = keys.public().to_peer_id(); let addr = MultiaddrWithPeerId { multiaddr: build_multiaddr![Ip4([127, 0, 0, 1]), Tcp(30333u16)], peer_id, diff --git a/client/cli/src/commands/check_block_cmd.rs b/client/cli/src/commands/check_block_cmd.rs index 07a76319dca3..b7e69b1360a0 100644 --- a/client/cli/src/commands/check_block_cmd.rs +++ b/client/cli/src/commands/check_block_cmd.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,30 +21,30 @@ use crate::{ params::{BlockNumberOrHash, ImportParams, SharedParams}, CliConfiguration, }; -use sc_client_api::{BlockBackend, UsageProvider}; +use clap::Parser; +use sc_client_api::{BlockBackend, HeaderBackend}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use std::{fmt::Debug, str::FromStr, sync::Arc}; -use structopt::StructOpt; /// The `check-block` command used to validate blocks. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Parser)] pub struct CheckBlockCmd { /// Block hash or number - #[structopt(value_name = "HASH or NUMBER")] + #[clap(value_name = "HASH or NUMBER")] pub input: BlockNumberOrHash, /// The default number of 64KB pages to ever allocate for Wasm execution. /// /// Don't alter this unless you know what you're doing. - #[structopt(long = "default-heap-pages", value_name = "COUNT")] + #[clap(long, value_name = "COUNT")] pub default_heap_pages: Option, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub shared_params: SharedParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub import_params: ImportParams, } @@ -53,7 +53,7 @@ impl CheckBlockCmd { pub async fn run(&self, client: Arc, import_queue: IQ) -> error::Result<()> where B: BlockT + for<'de> serde::Deserialize<'de>, - C: BlockBackend + UsageProvider + Send + Sync + 'static, + C: BlockBackend + HeaderBackend + Send + Sync + 'static, IQ: sc_service::ImportQueue + 'static, B::Hash: FromStr, ::Err: Debug, diff --git a/client/cli/src/commands/export_blocks_cmd.rs b/client/cli/src/commands/export_blocks_cmd.rs index ca3069442a1d..ff35b5a51fca 100644 --- a/client/cli/src/commands/export_blocks_cmd.rs +++ b/client/cli/src/commands/export_blocks_cmd.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,46 +21,46 @@ use crate::{ params::{DatabaseParams, GenericNumber, PruningParams, SharedParams}, CliConfiguration, }; +use clap::Parser; use log::info; use sc_client_api::{BlockBackend, UsageProvider}; use sc_service::{chain_ops::export_blocks, config::DatabaseSource}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use std::{fmt::Debug, fs, io, path::PathBuf, str::FromStr, sync::Arc}; -use structopt::StructOpt; /// The `export-blocks` command used to export blocks. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Parser)] pub struct ExportBlocksCmd { /// Output file name or stdout if unspecified. - #[structopt(parse(from_os_str))] + #[clap(parse(from_os_str))] pub output: Option, /// Specify starting block number. /// /// Default is 1. - #[structopt(long = "from", value_name = "BLOCK")] + #[clap(long, value_name = "BLOCK")] pub from: Option, /// Specify last block number. /// /// Default is best block. - #[structopt(long = "to", value_name = "BLOCK")] + #[clap(long, value_name = "BLOCK")] pub to: Option, /// Use binary output rather than JSON. - #[structopt(long)] + #[clap(long)] pub binary: bool, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub shared_params: SharedParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub pruning_params: PruningParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub database_params: DatabaseParams, } @@ -76,7 +76,7 @@ impl ExportBlocksCmd { C: BlockBackend + UsageProvider + 'static, <::Number as FromStr>::Err: Debug, { - if let DatabaseSource::RocksDb { ref path, .. } = database_config { + if let Some(path) = database_config.path() { info!("DB path: {}", path.display()); } diff --git a/client/cli/src/commands/export_state_cmd.rs b/client/cli/src/commands/export_state_cmd.rs index 36eabd2c24f5..b76724caf0fe 100644 --- a/client/cli/src/commands/export_state_cmd.rs +++ b/client/cli/src/commands/export_state_cmd.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,30 +18,34 @@ use crate::{ error, - params::{BlockNumberOrHash, PruningParams, SharedParams}, + params::{BlockNumberOrHash, DatabaseParams, PruningParams, SharedParams}, CliConfiguration, }; +use clap::Parser; use log::info; use sc_client_api::{StorageProvider, UsageProvider}; use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use std::{fmt::Debug, io::Write, str::FromStr, sync::Arc}; -use structopt::StructOpt; /// The `export-state` command used to export the state of a given block into /// a chain spec. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Parser)] pub struct ExportStateCmd { /// Block hash or number. - #[structopt(value_name = "HASH or NUMBER")] + #[clap(value_name = "HASH or NUMBER")] pub input: Option, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub shared_params: SharedParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub pruning_params: PruningParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub database_params: DatabaseParams, } impl ExportStateCmd { @@ -81,4 +85,8 @@ impl CliConfiguration for ExportStateCmd { fn pruning_params(&self) -> Option<&PruningParams> { Some(&self.pruning_params) } + + fn database_params(&self) -> Option<&DatabaseParams> { + Some(&self.database_params) + } } diff --git a/client/cli/src/commands/generate.rs b/client/cli/src/commands/generate.rs index 7032ebd72e0c..9c1e5b689584 100644 --- a/client/cli/src/commands/generate.rs +++ b/client/cli/src/commands/generate.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,30 +21,30 @@ use crate::{ NetworkSchemeFlag, OutputTypeFlag, }; use bip39::{Language, Mnemonic, MnemonicType}; -use structopt::StructOpt; +use clap::Parser; /// The `generate` command -#[derive(Debug, StructOpt, Clone)] -#[structopt(name = "generate", about = "Generate a random account")] +#[derive(Debug, Clone, Parser)] +#[clap(name = "generate", about = "Generate a random account")] pub struct GenerateCmd { /// The number of words in the phrase to generate. One of 12 (default), 15, 18, 21 and 24. - #[structopt(long, short = "w", value_name = "WORDS")] + #[clap(short = 'w', long, value_name = "WORDS")] words: Option, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub keystore_params: KeystoreParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub network_scheme: NetworkSchemeFlag, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub output_scheme: OutputTypeFlag, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub crypto_scheme: CryptoSchemeFlag, } @@ -78,12 +78,11 @@ impl GenerateCmd { #[cfg(test)] mod tests { - use super::GenerateCmd; - use structopt::StructOpt; + use super::*; #[test] fn generate() { - let generate = GenerateCmd::from_iter(&["generate", "--password", "12345"]); + let generate = GenerateCmd::parse_from(&["generate", "--password", "12345"]); assert!(generate.run().is_ok()) } } diff --git a/client/cli/src/commands/generate_node_key.rs b/client/cli/src/commands/generate_node_key.rs index 74a4197f3662..8c634dca9acf 100644 --- a/client/cli/src/commands/generate_node_key.rs +++ b/client/cli/src/commands/generate_node_key.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,13 +18,13 @@ //! Implementation of the `generate-node-key` subcommand use crate::Error; +use clap::Parser; use libp2p::identity::{ed25519 as libp2p_ed25519, PublicKey}; use std::{fs, path::PathBuf}; -use structopt::StructOpt; /// The `generate-node-key` command -#[derive(Debug, StructOpt)] -#[structopt( +#[derive(Debug, Parser)] +#[clap( name = "generate-node-key", about = "Generate a random node libp2p key, save it to \ file or print it to stdout and print its peer ID to stderr" @@ -33,7 +33,7 @@ pub struct GenerateNodeKeyCmd { /// Name of file to save secret key to. /// /// If not given, the secret key is printed to stdout. - #[structopt(long)] + #[clap(long)] file: Option, } @@ -42,7 +42,7 @@ impl GenerateNodeKeyCmd { pub fn run(&self) -> Result<(), Error> { let keypair = libp2p_ed25519::Keypair::generate(); let secret = keypair.secret(); - let peer_id = PublicKey::Ed25519(keypair.public()).into_peer_id(); + let peer_id = PublicKey::Ed25519(keypair.public()).to_peer_id(); let secret_hex = hex::encode(secret.as_ref()); match &self.file { @@ -66,7 +66,7 @@ mod tests { fn generate_node_key() { let mut file = Builder::new().prefix("keyfile").tempfile().unwrap(); let file_path = file.path().display().to_string(); - let generate = GenerateNodeKeyCmd::from_iter(&["generate-node-key", "--file", &file_path]); + let generate = GenerateNodeKeyCmd::parse_from(&["generate-node-key", "--file", &file_path]); assert!(generate.run().is_ok()); let mut buf = String::new(); assert!(file.read_to_string(&mut buf).is_ok()); diff --git a/client/cli/src/commands/import_blocks_cmd.rs b/client/cli/src/commands/import_blocks_cmd.rs index 9b211b88d556..749824834bf7 100644 --- a/client/cli/src/commands/import_blocks_cmd.rs +++ b/client/cli/src/commands/import_blocks_cmd.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,7 +21,8 @@ use crate::{ params::{ImportParams, SharedParams}, CliConfiguration, }; -use sc_client_api::UsageProvider; +use clap::Parser; +use sc_client_api::HeaderBackend; use sc_service::chain_ops::import_blocks; use sp_runtime::traits::Block as BlockT; use std::{ @@ -31,31 +32,30 @@ use std::{ path::PathBuf, sync::Arc, }; -use structopt::StructOpt; /// The `import-blocks` command used to import blocks. -#[derive(Debug, StructOpt)] +#[derive(Debug, Parser)] pub struct ImportBlocksCmd { /// Input file or stdin if unspecified. - #[structopt(parse(from_os_str))] + #[clap(parse(from_os_str))] pub input: Option, /// The default number of 64KB pages to ever allocate for Wasm execution. /// /// Don't alter this unless you know what you're doing. - #[structopt(long = "default-heap-pages", value_name = "COUNT")] + #[clap(long, value_name = "COUNT")] pub default_heap_pages: Option, /// Try importing blocks from binary format rather than JSON. - #[structopt(long)] + #[clap(long)] pub binary: bool, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub shared_params: SharedParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub import_params: ImportParams, } @@ -68,17 +68,13 @@ impl ImportBlocksCmd { /// Run the import-blocks command pub async fn run(&self, client: Arc, import_queue: IQ) -> error::Result<()> where - C: UsageProvider + Send + Sync + 'static, + C: HeaderBackend + Send + Sync + 'static, B: BlockT + for<'de> serde::Deserialize<'de>, IQ: sc_service::ImportQueue + 'static, { - let file: Box = match &self.input { + let file: Box = match &self.input { Some(filename) => Box::new(fs::File::open(filename)?), - None => { - let mut buffer = Vec::new(); - io::stdin().read_to_end(&mut buffer)?; - Box::new(io::Cursor::new(buffer)) - }, + None => Box::new(io::stdin()), }; import_blocks(client, import_queue, file, false, self.binary) diff --git a/client/cli/src/commands/insert_key.rs b/client/cli/src/commands/insert_key.rs index 05055dc53c1e..68201d7b4bff 100644 --- a/client/cli/src/commands/insert_key.rs +++ b/client/cli/src/commands/insert_key.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,40 +18,40 @@ //! Implementation of the `insert` subcommand use crate::{ - utils, with_crypto_scheme, CryptoSchemeFlag, Error, KeystoreParams, SharedParams, SubstrateCli, + utils, with_crypto_scheme, CryptoScheme, Error, KeystoreParams, SharedParams, SubstrateCli, }; +use clap::Parser; use sc_keystore::LocalKeystore; use sc_service::config::{BasePath, KeystoreConfig}; use sp_core::crypto::{KeyTypeId, SecretString}; use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; -use std::{convert::TryFrom, sync::Arc}; -use structopt::StructOpt; +use std::sync::Arc; /// The `insert` command -#[derive(Debug, StructOpt, Clone)] -#[structopt(name = "insert", about = "Insert a key to the keystore of a node.")] +#[derive(Debug, Clone, Parser)] +#[clap(name = "insert", about = "Insert a key to the keystore of a node.")] pub struct InsertKeyCmd { /// The secret key URI. /// If the value is a file, the file content is used as URI. /// If not given, you will be prompted for the URI. - #[structopt(long)] + #[clap(long)] suri: Option, /// Key type, examples: "gran", or "imon" - #[structopt(long)] + #[clap(long)] key_type: String, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub shared_params: SharedParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub keystore_params: KeystoreParams, - #[allow(missing_docs)] - #[structopt(flatten)] - pub crypto_scheme: CryptoSchemeFlag, + /// The cryptography scheme that should be used to generate the key out of the given URI. + #[clap(long, value_name = "SCHEME", arg_enum, ignore_case = true)] + pub scheme: CryptoScheme, } impl InsertKeyCmd { @@ -68,10 +68,7 @@ impl InsertKeyCmd { let (keystore, public) = match self.keystore_params.keystore_config(&config_dir)? { (_, KeystoreConfig::Path { path, password }) => { - let public = with_crypto_scheme!( - self.crypto_scheme.scheme, - to_vec(&suri, password.clone()) - )?; + let public = with_crypto_scheme!(self.scheme, to_vec(&suri, password.clone()))?; let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::open(path, password)?); (keystore, public) }, @@ -97,8 +94,7 @@ fn to_vec(uri: &str, pass: Option) -> Result, /// Is the given `uri` a hex encoded public key? - #[structopt(long)] + #[clap(long)] public: bool, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub keystore_params: KeystoreParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub network_scheme: NetworkSchemeFlag, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub output_scheme: OutputTypeFlag, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub crypto_scheme: CryptoSchemeFlag, + + /// Expect that `--uri` has the given public key/account-id. + /// + /// If `--uri` has any derivations, the public key is checked against the base `uri`, i.e. the + /// `uri` without any derivation applied. However, if `uri` has a password or there is one + /// given by `--password`, it will be used to decrypt `uri` before comparing the public + /// key/account-id. + /// + /// If there is no derivation in `--uri`, the public key will be checked against the public key + /// of `--uri` directly. + #[clap(long, conflicts_with = "public")] + pub expect_public: Option, } impl InspectKeyCmd { @@ -77,6 +92,13 @@ impl InspectKeyCmd { ) )?; } else { + if let Some(ref expect_public) = self.expect_public { + with_crypto_scheme!( + self.crypto_scheme.scheme, + expect_public_from_phrase(&&expect_public, &uri, password.as_ref(),) + )?; + } + with_crypto_scheme!( self.crypto_scheme.scheme, print_from_uri( @@ -92,10 +114,50 @@ impl InspectKeyCmd { } } +/// Checks that `expect_public` is the public key of `suri`. +/// +/// If `suri` has any derivations, `expect_public` is checked against the public key of the "bare" +/// `suri`, i.e. without any derivations. +/// +/// Returns an error if the public key does not match. +fn expect_public_from_phrase( + expect_public: &str, + suri: &str, + password: Option<&SecretString>, +) -> Result<(), Error> { + let secret_uri = SecretUri::from_str(suri).map_err(|e| format!("{:?}", e))?; + let expected_public = if let Some(public) = expect_public.strip_prefix("0x") { + let hex_public = hex::decode(&public) + .map_err(|_| format!("Invalid expected public key hex: `{}`", expect_public))?; + Pair::Public::try_from(&hex_public) + .map_err(|_| format!("Invalid expected public key: `{}`", expect_public))? + } else { + Pair::Public::from_string_with_version(expect_public) + .map_err(|_| format!("Invalid expected account id: `{}`", expect_public))? + .0 + }; + + let pair = Pair::from_string_with_seed( + secret_uri.phrase.expose_secret().as_str(), + password + .or_else(|| secret_uri.password.as_ref()) + .map(|p| p.expose_secret().as_str()), + ) + .map_err(|_| format!("Invalid secret uri: {}", suri))? + .0; + + if pair.public() == expected_public { + Ok(()) + } else { + Err(format!("Expected public ({}) key does not match.", expect_public).into()) + } +} + #[cfg(test)] mod tests { use super::*; - use structopt::StructOpt; + use sp_core::crypto::{ByteArray, Pair}; + use sp_runtime::traits::IdentifyAccount; #[test] fn inspect() { @@ -103,10 +165,10 @@ mod tests { "remember fiber forum demise paper uniform squirrel feel access exclude casual effort"; let seed = "0xad1fb77243b536b90cfe5f0d351ab1b1ac40e3890b41dc64f766ee56340cfca5"; - let inspect = InspectKeyCmd::from_iter(&["inspect-key", words, "--password", "12345"]); + let inspect = InspectKeyCmd::parse_from(&["inspect-key", words, "--password", "12345"]); assert!(inspect.run().is_ok()); - let inspect = InspectKeyCmd::from_iter(&["inspect-key", seed]); + let inspect = InspectKeyCmd::parse_from(&["inspect-key", seed]); assert!(inspect.run().is_ok()); } @@ -114,7 +176,93 @@ mod tests { fn inspect_public_key() { let public = "0x12e76e0ae8ce41b6516cce52b3f23a08dcb4cfeed53c6ee8f5eb9f7367341069"; - let inspect = InspectKeyCmd::from_iter(&["inspect-key", "--public", public]); + let inspect = InspectKeyCmd::parse_from(&["inspect-key", "--public", public]); assert!(inspect.run().is_ok()); } + + #[test] + fn inspect_with_expected_public_key() { + let check_cmd = |seed, expected_public, success| { + let inspect = InspectKeyCmd::parse_from(&[ + "inspect-key", + "--expect-public", + expected_public, + seed, + ]); + let res = inspect.run(); + + if success { + assert!(res.is_ok()); + } else { + assert!(res.unwrap_err().to_string().contains(&format!( + "Expected public ({}) key does not match.", + expected_public + ))); + } + }; + + let seed = + "remember fiber forum demise paper uniform squirrel feel access exclude casual effort"; + let invalid_public = "0x12e76e0ae8ce41b6516cce52b3f23a08dcb4cfeed53c6ee8f5eb9f7367341069"; + let valid_public = sp_core::sr25519::Pair::from_string_with_seed(seed, None) + .expect("Valid") + .0 + .public(); + let valid_public_hex = format!("0x{}", hex::encode(valid_public.as_slice())); + let valid_accountid = format!("{}", valid_public.into_account()); + + // It should fail with the invalid public key + check_cmd(seed, invalid_public, false); + + // It should work with the valid public key & account id + check_cmd(seed, &valid_public_hex, true); + check_cmd(seed, &valid_accountid, true); + + let password = "test12245"; + let seed_with_password = format!("{}///{}", seed, password); + let valid_public_with_password = + sp_core::sr25519::Pair::from_string_with_seed(&seed_with_password, Some(password)) + .expect("Valid") + .0 + .public(); + let valid_public_hex_with_password = + format!("0x{}", hex::encode(&valid_public_with_password.as_slice())); + let valid_accountid_with_password = + format!("{}", &valid_public_with_password.into_account()); + + // Only the public key that corresponds to the seed with password should be accepted. + check_cmd(&seed_with_password, &valid_public_hex, false); + check_cmd(&seed_with_password, &valid_accountid, false); + + check_cmd(&seed_with_password, &valid_public_hex_with_password, true); + check_cmd(&seed_with_password, &valid_accountid_with_password, true); + + let seed_with_password_and_derivation = format!("{}//test//account///{}", seed, password); + + let valid_public_with_password_and_derivation = + sp_core::sr25519::Pair::from_string_with_seed( + &seed_with_password_and_derivation, + Some(password), + ) + .expect("Valid") + .0 + .public(); + let valid_public_hex_with_password_and_derivation = + format!("0x{}", hex::encode(&valid_public_with_password_and_derivation.as_slice())); + + // They should still be valid, because we check the base secret key. + check_cmd(&seed_with_password_and_derivation, &valid_public_hex_with_password, true); + check_cmd(&seed_with_password_and_derivation, &valid_accountid_with_password, true); + + // And these should be invalid. + check_cmd(&seed_with_password_and_derivation, &valid_public_hex, false); + check_cmd(&seed_with_password_and_derivation, &valid_accountid, false); + + // The public of the derived account should fail. + check_cmd( + &seed_with_password_and_derivation, + &valid_public_hex_with_password_and_derivation, + false, + ); + } } diff --git a/client/cli/src/commands/inspect_node_key.rs b/client/cli/src/commands/inspect_node_key.rs index 92a71f897505..4c0798cc0635 100644 --- a/client/cli/src/commands/inspect_node_key.rs +++ b/client/cli/src/commands/inspect_node_key.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,23 +18,23 @@ //! Implementation of the `inspect-node-key` subcommand use crate::{Error, NetworkSchemeFlag}; +use clap::Parser; use libp2p::identity::{ed25519, PublicKey}; use std::{fs, path::PathBuf}; -use structopt::StructOpt; /// The `inspect-node-key` command -#[derive(Debug, StructOpt)] -#[structopt( +#[derive(Debug, Parser)] +#[clap( name = "inspect-node-key", about = "Print the peer ID corresponding to the node key in the given file." )] pub struct InspectNodeKeyCmd { /// Name of file to read the secret key from. - #[structopt(long)] + #[clap(long)] file: PathBuf, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub network_scheme: NetworkSchemeFlag, } @@ -47,7 +47,7 @@ impl InspectNodeKeyCmd { ed25519::SecretKey::from_bytes(&mut file_content).map_err(|_| "Bad node key file")?; let keypair = ed25519::Keypair::from(secret); - let peer_id = PublicKey::Ed25519(keypair.public()).into_peer_id(); + let peer_id = PublicKey::Ed25519(keypair.public()).to_peer_id(); println!("{}", peer_id); @@ -63,11 +63,11 @@ mod tests { fn inspect_node_key() { let path = tempfile::tempdir().unwrap().into_path().join("node-id").into_os_string(); let path = path.to_str().unwrap(); - let cmd = GenerateNodeKeyCmd::from_iter(&["generate-node-key", "--file", path.clone()]); + let cmd = GenerateNodeKeyCmd::parse_from(&["generate-node-key", "--file", path.clone()]); assert!(cmd.run().is_ok()); - let cmd = InspectNodeKeyCmd::from_iter(&["inspect-node-key", "--file", path]); + let cmd = InspectNodeKeyCmd::parse_from(&["inspect-node-key", "--file", path]); assert!(cmd.run().is_ok()); } } diff --git a/client/cli/src/commands/key.rs b/client/cli/src/commands/key.rs index 8e1103a8ca51..e0f3524196e2 100644 --- a/client/cli/src/commands/key.rs +++ b/client/cli/src/commands/key.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,16 +17,14 @@ //! Key related CLI utilities -use crate::{Error, SubstrateCli}; -use structopt::StructOpt; - use super::{ generate::GenerateCmd, generate_node_key::GenerateNodeKeyCmd, insert_key::InsertKeyCmd, inspect_key::InspectKeyCmd, inspect_node_key::InspectNodeKeyCmd, }; +use crate::{Error, SubstrateCli}; /// Key utilities for the cli. -#[derive(Debug, StructOpt)] +#[derive(Debug, clap::Subcommand)] pub enum KeySubcommand { /// Generate a random node libp2p key, save it to file or print it to stdout /// and print its peer ID to stderr. diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs index 9e7c5689b49c..2b46e1d99caa 100644 --- a/client/cli/src/commands/mod.rs +++ b/client/cli/src/commands/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/cli/src/commands/purge_chain_cmd.rs b/client/cli/src/commands/purge_chain_cmd.rs index e1bdb3a03cc5..7dd7c1f5a5a5 100644 --- a/client/cli/src/commands/purge_chain_cmd.rs +++ b/client/cli/src/commands/purge_chain_cmd.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,27 +21,27 @@ use crate::{ params::{DatabaseParams, SharedParams}, CliConfiguration, }; +use clap::Parser; use sc_service::DatabaseSource; use std::{ fmt::Debug, fs, io::{self, Write}, }; -use structopt::StructOpt; /// The `purge-chain` command used to remove the whole chain. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Parser)] pub struct PurgeChainCmd { /// Skip interactive prompt by answering yes automatically. - #[structopt(short = "y")] + #[clap(short = 'y')] pub yes: bool, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub shared_params: SharedParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub database_params: DatabaseParams, } diff --git a/client/cli/src/commands/revert_cmd.rs b/client/cli/src/commands/revert_cmd.rs index 9ad49a03aa5f..f65e348b37b8 100644 --- a/client/cli/src/commands/revert_cmd.rs +++ b/client/cli/src/commands/revert_cmd.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,31 +21,40 @@ use crate::{ params::{GenericNumber, PruningParams, SharedParams}, CliConfiguration, }; +use clap::Parser; use sc_client_api::{Backend, UsageProvider}; use sc_service::chain_ops::revert_chain; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; use std::{fmt::Debug, str::FromStr, sync::Arc}; -use structopt::StructOpt; /// The `revert` command used revert the chain to a previous state. -#[derive(Debug, StructOpt)] +#[derive(Debug, Parser)] pub struct RevertCmd { /// Number of blocks to revert. - #[structopt(default_value = "256")] + #[clap(default_value = "256")] pub num: GenericNumber, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub shared_params: SharedParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub pruning_params: PruningParams, } +/// Revert handler for auxiliary data (e.g. consensus). +type AuxRevertHandler = + Box, Arc, NumberFor) -> error::Result<()>>; + impl RevertCmd { /// Run the revert command - pub async fn run(&self, client: Arc, backend: Arc) -> error::Result<()> + pub async fn run( + &self, + client: Arc, + backend: Arc, + aux_revert: Option>, + ) -> error::Result<()> where B: BlockT, BA: Backend, @@ -53,6 +62,9 @@ impl RevertCmd { <<::Header as HeaderT>::Number as FromStr>::Err: Debug, { let blocks = self.num.parse()?; + if let Some(aux_revert) = aux_revert { + aux_revert(client.clone(), backend.clone(), blocks)?; + } revert_chain(client, backend, blocks)?; Ok(()) diff --git a/client/cli/src/commands/run_cmd.rs b/client/cli/src/commands/run_cmd.rs index 98f2090c6f44..b9318813b048 100644 --- a/client/cli/src/commands/run_cmd.rs +++ b/client/cli/src/commands/run_cmd.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -25,6 +25,7 @@ use crate::{ }, CliConfiguration, }; +use clap::Parser; use regex::Regex; use sc_service::{ config::{BasePath, PrometheusConfig, TransactionPoolOptions}, @@ -32,41 +33,40 @@ use sc_service::{ }; use sc_telemetry::TelemetryEndpoints; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use structopt::StructOpt; /// The `run` command used to run a node. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Parser)] pub struct RunCmd { /// Enable validator mode. /// /// The node will be started with the authority role and actively /// participate in any consensus task that it can (e.g. depending on /// availability of local keys). - #[structopt(long)] + #[clap(long)] pub validator: bool, /// Disable GRANDPA voter when running in validator mode, otherwise disable the GRANDPA /// observer. - #[structopt(long)] + #[clap(long)] pub no_grandpa: bool, /// Experimental: Run in light client mode. - #[structopt(long = "light")] + #[clap(long)] pub light: bool, /// Listen to all RPC interfaces. /// /// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use an RPC /// proxy server to filter out dangerous methods. More details: - /// . + /// . /// Use `--unsafe-rpc-external` to suppress the warning if you understand the risks. - #[structopt(long = "rpc-external")] + #[clap(long)] pub rpc_external: bool, /// Listen to all RPC interfaces. /// /// Same as `--rpc-external`. - #[structopt(long)] + #[clap(long)] pub unsafe_rpc_external: bool, /// RPC methods to expose. @@ -75,11 +75,11 @@ pub struct RunCmd { /// - `Safe`: Exposes only a safe subset of RPC methods, denying unsafe RPC methods. /// - `Auto`: Acts as `Safe` if RPC is served externally, e.g. when `--{rpc,ws}-external` is /// passed, otherwise acts as `Unsafe`. - #[structopt( + #[clap( long, value_name = "METHOD SET", - possible_values = &RpcMethods::variants(), - case_insensitive = true, + arg_enum, + ignore_case = true, default_value = "Auto", verbatim_doc_comment )] @@ -89,73 +89,77 @@ pub struct RunCmd { /// /// Default is local. Note: not all RPC methods are safe to be exposed publicly. Use an RPC /// proxy server to filter out dangerous methods. More details: - /// . + /// . /// Use `--unsafe-ws-external` to suppress the warning if you understand the risks. - #[structopt(long = "ws-external")] + #[clap(long)] pub ws_external: bool, /// Listen to all Websocket interfaces. /// /// Same as `--ws-external` but doesn't warn you about it. - #[structopt(long = "unsafe-ws-external")] + #[clap(long)] pub unsafe_ws_external: bool, /// Set the the maximum RPC payload size for both requests and responses (both http and ws), in /// megabytes. Default is 15MiB. - #[structopt(long = "rpc-max-payload")] + #[clap(long)] pub rpc_max_payload: Option, /// Expose Prometheus exporter on all interfaces. /// /// Default is local. - #[structopt(long = "prometheus-external")] + #[clap(long)] pub prometheus_external: bool, /// Specify IPC RPC server path - #[structopt(long = "ipc-path", value_name = "PATH")] + #[clap(long, value_name = "PATH")] pub ipc_path: Option, /// Specify HTTP RPC server TCP port. - #[structopt(long = "rpc-port", value_name = "PORT")] + #[clap(long, value_name = "PORT")] pub rpc_port: Option, /// Specify WebSockets RPC server TCP port. - #[structopt(long = "ws-port", value_name = "PORT")] + #[clap(long, value_name = "PORT")] pub ws_port: Option, /// Maximum number of WS RPC server connections. - #[structopt(long = "ws-max-connections", value_name = "COUNT")] + #[clap(long, value_name = "COUNT")] pub ws_max_connections: Option, + /// Set the the maximum WebSocket output buffer size in MiB. Default is 16. + #[clap(long)] + pub ws_max_out_buffer_capacity: Option, + /// Specify browser Origins allowed to access the HTTP & WS RPC servers. /// /// A comma-separated list of origins (protocol://domain or special `null` /// value). Value of `all` will disable origin validation. Default is to /// allow localhost and origins. When running in /// --dev mode the default is to allow all origins. - #[structopt(long = "rpc-cors", value_name = "ORIGINS", parse(try_from_str = parse_cors))] + #[clap(long, value_name = "ORIGINS", parse(from_str = parse_cors))] pub rpc_cors: Option, /// Specify Prometheus exporter TCP Port. - #[structopt(long = "prometheus-port", value_name = "PORT")] + #[clap(long, value_name = "PORT")] pub prometheus_port: Option, /// Do not expose a Prometheus exporter endpoint. /// /// Prometheus metric endpoint is enabled by default. - #[structopt(long = "no-prometheus")] + #[clap(long)] pub no_prometheus: bool, /// The human-readable name for this node. /// /// The node name will be reported to the telemetry server, if enabled. - #[structopt(long = "name", value_name = "NAME")] + #[clap(long, value_name = "NAME")] pub name: Option, /// Disable connecting to the Substrate telemetry server. /// /// Telemetry is on by default on global chains. - #[structopt(long = "no-telemetry")] + #[clap(long)] pub no_telemetry: bool, /// The URL of the telemetry server to connect to. @@ -164,76 +168,80 @@ pub struct RunCmd { /// telemetry endpoints. Verbosity levels range from 0-9, with 0 denoting /// the least verbosity. /// Expected format is 'URL VERBOSITY', e.g. `--telemetry-url 'wss://foo/bar 0'`. - #[structopt(long = "telemetry-url", value_name = "URL VERBOSITY", parse(try_from_str = parse_telemetry_endpoints))] + #[clap(long = "telemetry-url", value_name = "URL VERBOSITY", parse(try_from_str = parse_telemetry_endpoints))] pub telemetry_endpoints: Vec<(String, u8)>, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub offchain_worker_params: OffchainWorkerParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub shared_params: SharedParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub import_params: ImportParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub network_params: NetworkParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub pool_config: TransactionPoolParams, /// Shortcut for `--name Alice --validator` with session keys for `Alice` added to keystore. - #[structopt(long, conflicts_with_all = &["bob", "charlie", "dave", "eve", "ferdie", "one", "two"])] + #[clap(long, conflicts_with_all = &["bob", "charlie", "dave", "eve", "ferdie", "one", "two"])] pub alice: bool, /// Shortcut for `--name Bob --validator` with session keys for `Bob` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "charlie", "dave", "eve", "ferdie", "one", "two"])] + #[clap(long, conflicts_with_all = &["alice", "charlie", "dave", "eve", "ferdie", "one", "two"])] pub bob: bool, /// Shortcut for `--name Charlie --validator` with session keys for `Charlie` added to /// keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "dave", "eve", "ferdie", "one", "two"])] + #[clap(long, conflicts_with_all = &["alice", "bob", "dave", "eve", "ferdie", "one", "two"])] pub charlie: bool, /// Shortcut for `--name Dave --validator` with session keys for `Dave` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "eve", "ferdie", "one", "two"])] + #[clap(long, conflicts_with_all = &["alice", "bob", "charlie", "eve", "ferdie", "one", "two"])] pub dave: bool, /// Shortcut for `--name Eve --validator` with session keys for `Eve` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "ferdie", "one", "two"])] + #[clap(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "ferdie", "one", "two"])] pub eve: bool, /// Shortcut for `--name Ferdie --validator` with session keys for `Ferdie` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "one", "two"])] + #[clap(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "one", "two"])] pub ferdie: bool, /// Shortcut for `--name One --validator` with session keys for `One` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "ferdie", "two"])] + #[clap(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "ferdie", "two"])] pub one: bool, /// Shortcut for `--name Two --validator` with session keys for `Two` added to keystore. - #[structopt(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "ferdie", "one"])] + #[clap(long, conflicts_with_all = &["alice", "bob", "charlie", "dave", "eve", "ferdie", "one"])] pub two: bool, /// Enable authoring even when offline. - #[structopt(long = "force-authoring")] + #[clap(long)] pub force_authoring: bool, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub keystore_params: KeystoreParams, /// The size of the instances cache for each runtime. /// /// The default value is 8 and the values higher than 256 are ignored. - #[structopt(long)] + #[clap(long)] pub max_runtime_instances: Option, + /// Maximum number of different runtimes that can be cached. + #[clap(long, default_value = "2")] + pub runtime_cache_size: u8, + /// Run a temporary node. /// /// A temporary directory will be created to store the configuration and will be deleted @@ -241,7 +249,9 @@ pub struct RunCmd { /// /// Note: the directory is random per process execution. This directory is used as base path /// which includes: database, node key and keystore. - #[structopt(long, conflicts_with = "base-path")] + /// + /// When `--dev` is given and no explicit `--base-path`, this option is implied. + #[clap(long, conflicts_with = "base-path")] pub tmp: bool, } @@ -355,17 +365,24 @@ impl CliConfiguration for RunCmd { Ok(self.shared_params.dev || self.force_authoring) } - fn prometheus_config(&self, default_listen_port: u16) -> Result> { + fn prometheus_config( + &self, + default_listen_port: u16, + chain_spec: &Box, + ) -> Result> { Ok(if self.no_prometheus { None } else { let interface = if self.prometheus_external { Ipv4Addr::UNSPECIFIED } else { Ipv4Addr::LOCALHOST }; - Some(PrometheusConfig::new_with_default_registry(SocketAddr::new( - interface.into(), - self.prometheus_port.unwrap_or(default_listen_port), - ))) + Some(PrometheusConfig::new_with_default_registry( + SocketAddr::new( + interface.into(), + self.prometheus_port.unwrap_or(default_listen_port), + ), + chain_spec.id().into(), + )) }) } @@ -432,6 +449,10 @@ impl CliConfiguration for RunCmd { Ok(self.rpc_max_payload) } + fn ws_max_out_buffer_capacity(&self) -> Result> { + Ok(self.ws_max_out_buffer_capacity) + } + fn transaction_pool(&self) -> Result { Ok(self.pool_config.transaction_pool()) } @@ -440,11 +461,20 @@ impl CliConfiguration for RunCmd { Ok(self.max_runtime_instances.map(|x| x.min(256))) } + fn runtime_cache_size(&self) -> Result { + Ok(self.runtime_cache_size) + } + fn base_path(&self) -> Result> { Ok(if self.tmp { Some(BasePath::new_temp_dir()?) } else { - self.shared_params().base_path() + match self.shared_params().base_path() { + Some(r) => Some(r), + // If `dev` is enabled, we use the temp base path. + None if self.shared_params().is_dev() => Some(BasePath::new_temp_dir()?), + None => None, + } }) } } @@ -532,8 +562,7 @@ fn parse_telemetry_endpoints(s: &str) -> std::result::Result<(String, u8), Telem /// CORS setting /// -/// The type is introduced to overcome `Option>` -/// handling of `structopt`. +/// The type is introduced to overcome `Option>` handling of `clap`. #[derive(Clone, Debug)] pub enum Cors { /// All hosts allowed. @@ -552,7 +581,7 @@ impl From for Option> { } /// Parse cors origins. -fn parse_cors(s: &str) -> std::result::Result> { +fn parse_cors(s: &str) -> Cors { let mut is_all = false; let mut origins = Vec::new(); for part in s.split(',') { @@ -565,7 +594,11 @@ fn parse_cors(s: &str) -> std::result::Result> } } - Ok(if is_all { Cors::All } else { Cors::List(origins) }) + if is_all { + Cors::All + } else { + Cors::List(origins) + } } #[cfg(test)] diff --git a/client/cli/src/commands/sign.rs b/client/cli/src/commands/sign.rs index 20aacd9bf002..e0a5fce353ef 100644 --- a/client/cli/src/commands/sign.rs +++ b/client/cli/src/commands/sign.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,34 +18,34 @@ //! Implementation of the `sign` subcommand use crate::{error, utils, with_crypto_scheme, CryptoSchemeFlag, KeystoreParams}; +use clap::Parser; use sp_core::crypto::SecretString; -use structopt::StructOpt; /// The `sign` command -#[derive(Debug, StructOpt, Clone)] -#[structopt(name = "sign", about = "Sign a message, with a given (secret) key")] +#[derive(Debug, Clone, Parser)] +#[clap(name = "sign", about = "Sign a message, with a given (secret) key")] pub struct SignCmd { /// The secret key URI. /// If the value is a file, the file content is used as URI. /// If not given, you will be prompted for the URI. - #[structopt(long)] + #[clap(long)] suri: Option, /// Message to sign, if not provided you will be prompted to /// pass the message via STDIN - #[structopt(long)] + #[clap(long)] message: Option, /// The message on STDIN is hex-encoded data - #[structopt(long)] + #[clap(long)] hex: bool, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub keystore_params: KeystoreParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub crypto_scheme: CryptoSchemeFlag, } @@ -70,19 +70,18 @@ fn sign( message: Vec, ) -> error::Result { let pair = utils::pair_from_suri::

(suri, password)?; - Ok(format!("{}", hex::encode(pair.sign(&message)))) + Ok(hex::encode(pair.sign(&message))) } #[cfg(test)] mod test { - use super::SignCmd; - use structopt::StructOpt; + use super::*; #[test] fn sign() { let seed = "0xad1fb77243b536b90cfe5f0d351ab1b1ac40e3890b41dc64f766ee56340cfca5"; - let sign = SignCmd::from_iter(&[ + let sign = SignCmd::parse_from(&[ "sign", "--suri", seed, diff --git a/client/cli/src/commands/utils.rs b/client/cli/src/commands/utils.rs index 864d7e920f81..fa776c25a2ed 100644 --- a/client/cli/src/commands/utils.rs +++ b/client/cli/src/commands/utils.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,12 +23,15 @@ use crate::{ }; use serde_json::json; use sp_core::{ - crypto::{ExposeSecret, SecretString, Ss58AddressFormat, Ss58Codec, Zeroize}, + crypto::{ + unwrap_or_default_ss58_version, ExposeSecret, SecretString, Ss58AddressFormat, Ss58Codec, + Zeroize, + }, hexdisplay::HexDisplay, Pair, }; use sp_runtime::{traits::IdentifyAccount, MultiSigner}; -use std::{convert::TryFrom, io::Read, path::PathBuf}; +use std::{io::Read, path::PathBuf}; /// Public key type for Runtime pub type PublicFor

=

::Public; @@ -70,14 +73,16 @@ pub fn print_from_uri( Pair::Public: Into, { let password = password.as_ref().map(|s| s.expose_secret().as_str()); + let network_id = String::from(unwrap_or_default_ss58_version(network_override)); if let Ok((pair, seed)) = Pair::from_phrase(uri, password.clone()) { let public_key = pair.public(); - let network_override = network_override.unwrap_or_default(); + let network_override = unwrap_or_default_ss58_version(network_override); match output { OutputType::Json => { let json = json!({ "secretPhrase": uri, + "networkId": network_id, "secretSeed": format_seed::(seed), "publicKey": format_public_key::(public_key.clone()), "ss58PublicKey": public_key.to_ss58check_with_version(network_override), @@ -92,12 +97,14 @@ pub fn print_from_uri( OutputType::Text => { println!( "Secret phrase: {}\n \ + Network ID: {}\n \ Secret seed: {}\n \ Public key (hex): {}\n \ Account ID: {}\n \ Public key (SS58): {}\n \ SS58 Address: {}", uri, + network_id, format_seed::(seed), format_public_key::(public_key.clone()), format_account_id::(public_key.clone()), @@ -108,12 +115,13 @@ pub fn print_from_uri( } } else if let Ok((pair, seed)) = Pair::from_string_with_seed(uri, password.clone()) { let public_key = pair.public(); - let network_override = network_override.unwrap_or_default(); + let network_override = unwrap_or_default_ss58_version(network_override); match output { OutputType::Json => { let json = json!({ "secretKeyUri": uri, + "networkId": network_id, "secretSeed": if let Some(seed) = seed { format_seed::(seed) } else { "n/a".into() }, "publicKey": format_public_key::(public_key.clone()), "ss58PublicKey": public_key.to_ss58check_with_version(network_override), @@ -128,12 +136,14 @@ pub fn print_from_uri( OutputType::Text => { println!( "Secret Key URI `{}` is account:\n \ + Network ID: {} \n \ Secret seed: {}\n \ Public key (hex): {}\n \ Account ID: {}\n \ Public key (SS58): {}\n \ SS58 Address: {}", uri, + network_id, if let Some(seed) = seed { format_seed::(seed) } else { "n/a".into() }, format_public_key::(public_key.clone()), format_account_id::(public_key.clone()), @@ -164,7 +174,7 @@ pub fn print_from_uri( OutputType::Text => { println!( "Public Key URI `{}` is account:\n \ - Network ID/version: {}\n \ + Network ID/Version: {}\n \ Public key (hex): {}\n \ Account ID: {}\n \ Public key (SS58): {}\n \ @@ -198,7 +208,7 @@ where let public_key = Pair::Public::try_from(&public) .map_err(|_| "Failed to construct public key from given hex")?; - let network_override = network_override.unwrap_or_default(); + let network_override = unwrap_or_default_ss58_version(network_override); match output { OutputType::Json => { @@ -214,7 +224,7 @@ where }, OutputType::Text => { println!( - "Network ID/version: {}\n \ + "Network ID/Version: {}\n \ Public key (hex): {}\n \ Account ID: {}\n \ Public key (SS58): {}\n \ diff --git a/client/cli/src/commands/vanity.rs b/client/cli/src/commands/vanity.rs index daeb81e86a1a..834b220df638 100644 --- a/client/cli/src/commands/vanity.rs +++ b/client/cli/src/commands/vanity.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,30 +21,30 @@ use crate::{ error, utils, with_crypto_scheme, CryptoSchemeFlag, NetworkSchemeFlag, OutputTypeFlag, }; +use clap::Parser; use rand::{rngs::OsRng, RngCore}; -use sp_core::crypto::{Ss58AddressFormat, Ss58Codec}; +use sp_core::crypto::{unwrap_or_default_ss58_version, Ss58AddressFormat, Ss58Codec}; use sp_runtime::traits::IdentifyAccount; -use structopt::StructOpt; use utils::print_from_uri; /// The `vanity` command -#[derive(Debug, StructOpt, Clone)] -#[structopt(name = "vanity", about = "Generate a seed that provides a vanity address")] +#[derive(Debug, Clone, Parser)] +#[clap(name = "vanity", about = "Generate a seed that provides a vanity address")] pub struct VanityCmd { /// Desired pattern - #[structopt(long, parse(try_from_str = assert_non_empty_string))] + #[clap(long, parse(try_from_str = assert_non_empty_string))] pattern: String, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] network_scheme: NetworkSchemeFlag, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] output_scheme: OutputTypeFlag, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] crypto_scheme: CryptoSchemeFlag, } @@ -53,7 +53,10 @@ impl VanityCmd { pub fn run(&self) -> error::Result<()> { let formated_seed = with_crypto_scheme!( self.crypto_scheme.scheme, - generate_key(&self.pattern, self.network_scheme.network.clone().unwrap_or_default()), + generate_key( + &self.pattern, + unwrap_or_default_ss58_version(self.network_scheme.network) + ), )?; with_crypto_scheme!( @@ -159,20 +162,22 @@ fn assert_non_empty_string(pattern: &str) -> Result { #[cfg(test)] mod tests { use super::*; - use sp_core::{crypto::Ss58Codec, sr25519, Pair}; - use structopt::StructOpt; + use sp_core::{ + crypto::{default_ss58_version, Ss58AddressFormatRegistry, Ss58Codec}, + sr25519, Pair, + }; #[cfg(feature = "bench")] use test::Bencher; #[test] fn vanity() { - let vanity = VanityCmd::from_iter(&["vanity", "--pattern", "j"]); + let vanity = VanityCmd::parse_from(&["vanity", "--pattern", "j"]); assert!(vanity.run().is_ok()); } #[test] fn test_generation_with_single_char() { - let seed = generate_key::("ab", Default::default()).unwrap(); + let seed = generate_key::("ab", default_ss58_version()).unwrap(); assert!(sr25519::Pair::from_seed_slice(&hex::decode(&seed[2..]).unwrap()) .unwrap() .public() @@ -182,11 +187,13 @@ mod tests { #[test] fn generate_key_respects_network_override() { - let seed = generate_key::("ab", Ss58AddressFormat::PolkadotAccount).unwrap(); + let seed = + generate_key::("ab", Ss58AddressFormatRegistry::PolkadotAccount.into()) + .unwrap(); assert!(sr25519::Pair::from_seed_slice(&hex::decode(&seed[2..]).unwrap()) .unwrap() .public() - .to_ss58check_with_version(Ss58AddressFormat::PolkadotAccount) + .to_ss58check_with_version(Ss58AddressFormatRegistry::PolkadotAccount.into()) .contains("ab")); } diff --git a/client/cli/src/commands/verify.rs b/client/cli/src/commands/verify.rs index 760793374242..b004a948a7a4 100644 --- a/client/cli/src/commands/verify.rs +++ b/client/cli/src/commands/verify.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,12 +19,12 @@ //! implementation of the `verify` subcommand use crate::{error, utils, with_crypto_scheme, CryptoSchemeFlag}; -use sp_core::{crypto::Ss58Codec, Public}; -use structopt::StructOpt; +use clap::Parser; +use sp_core::crypto::{ByteArray, Ss58Codec}; /// The `verify` command -#[derive(Debug, StructOpt, Clone)] -#[structopt( +#[derive(Debug, Clone, Parser)] +#[clap( name = "verify", about = "Verify a signature for a message, provided on STDIN, with a given (public or secret) key" )] @@ -39,15 +39,15 @@ pub struct VerifyCmd { /// Message to verify, if not provided you will be prompted to /// pass the message via STDIN - #[structopt(long)] + #[clap(long)] message: Option, /// The message on STDIN is hex-encoded data - #[structopt(long)] + #[clap(long)] hex: bool, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub crypto_scheme: CryptoSchemeFlag, } @@ -57,7 +57,7 @@ impl VerifyCmd { let message = utils::read_message(self.message.as_ref(), self.hex)?; let sig_data = utils::decode_hex(&self.sig)?; let uri = utils::read_uri(self.uri.as_ref())?; - let uri = if uri.starts_with("0x") { &uri[2..] } else { &uri }; + let uri = if let Some(uri) = uri.strip_prefix("0x") { uri } else { &uri }; with_crypto_scheme!(self.crypto_scheme.scheme, verify(sig_data, message, uri)) } @@ -66,19 +66,14 @@ impl VerifyCmd { fn verify(sig_data: Vec, message: Vec, uri: &str) -> error::Result<()> where Pair: sp_core::Pair, - Pair::Signature: Default + AsMut<[u8]>, + Pair::Signature: for<'a> TryFrom<&'a [u8]>, { - let mut signature = Pair::Signature::default(); - if sig_data.len() != signature.as_ref().len() { - return Err(error::Error::SignatureInvalidLength { - read: sig_data.len(), - expected: signature.as_ref().len(), - }) - } - signature.as_mut().copy_from_slice(&sig_data); + let signature = + Pair::Signature::try_from(&sig_data).map_err(|_| error::Error::SignatureFormatInvalid)?; let pubkey = if let Ok(pubkey_vec) = hex::decode(uri) { Pair::Public::from_slice(pubkey_vec.as_slice()) + .map_err(|_| error::Error::KeyFormatInvalid)? } else { Pair::Public::from_string(uri)? }; diff --git a/client/cli/src/config.rs b/client/cli/src/config.rs index 59fc6bd438a1..d0f10dc9f6f3 100644 --- a/client/cli/src/config.rs +++ b/client/cli/src/config.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -31,7 +31,7 @@ use sc_service::{ NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode, Role, RpcMethods, TelemetryEndpoints, TransactionPoolOptions, WasmExecutionMethod, }, - ChainSpec, KeepBlocks, TracingReceiver, TransactionStorageMode, + ChainSpec, KeepBlocks, TracingReceiver, }; use sc_tracing::logging::LoggerBuilder; use std::{net::SocketAddr, path::PathBuf}; @@ -198,14 +198,6 @@ pub trait CliConfiguration: Sized { Ok(self.database_params().map(|x| x.database_cache_size()).unwrap_or_default()) } - /// Get the database transaction storage scheme. - fn database_transaction_storage(&self) -> Result { - Ok(self - .database_params() - .map(|x| x.transaction_storage()) - .unwrap_or(TransactionStorageMode::BlockBody)) - } - /// Get the database backend variant. /// /// By default this is retrieved from `DatabaseParams` if it is available. Otherwise its `None`. @@ -229,7 +221,14 @@ pub trait CliConfiguration: Sized { let paritydb_path = base_path.join("paritydb").join(role_dir); Ok(match database { Database::RocksDb => DatabaseSource::RocksDb { path: rocksdb_path, cache_size }, - Database::ParityDb => DatabaseSource::ParityDb { path: rocksdb_path }, + Database::ParityDb => DatabaseSource::ParityDb { path: paritydb_path }, + Database::ParityDbDeprecated => { + eprintln!( + "WARNING: \"paritydb-experimental\" database setting is deprecated and will be removed in future releases. \ + Please update your setup to use the new value: \"paritydb\"." + ); + DatabaseSource::ParityDb { path: paritydb_path } + }, Database::Auto => DatabaseSource::Auto { paritydb_path, rocksdb_path, cache_size }, }) } @@ -360,10 +359,19 @@ pub trait CliConfiguration: Sized { Ok(None) } + /// Get maximum WS output buffer capacity. + fn ws_max_out_buffer_capacity(&self) -> Result> { + Ok(None) + } + /// Get the prometheus configuration (`None` if disabled) /// /// By default this is `None`. - fn prometheus_config(&self, _default_listen_port: u16) -> Result> { + fn prometheus_config( + &self, + _default_listen_port: u16, + _chain_spec: &Box, + ) -> Result> { Ok(None) } @@ -447,6 +455,13 @@ pub trait CliConfiguration: Sized { Ok(Default::default()) } + /// Get maximum different runtimes in cache + /// + /// By default this is `2`. + fn runtime_cache_size(&self) -> Result { + Ok(2) + } + /// Activate or not the automatic announcing of blocks after import /// /// By default this is `false`. @@ -469,7 +484,7 @@ pub trait CliConfiguration: Sized { let config_dir = base_path.config_dir(chain_spec.id()); let net_config_dir = config_dir.join(DEFAULT_NETWORK_CONFIG_PATH); let client_id = C::client_id(); - let database_cache_size = self.database_cache_size()?.unwrap_or(128); + let database_cache_size = self.database_cache_size()?.unwrap_or(1024); let database = self.database()?.unwrap_or(Database::RocksDb); let node_key = self.node_key(&net_config_dir)?; let role = self.role(is_dev)?; @@ -477,6 +492,7 @@ pub trait CliConfiguration: Sized { let is_validator = role.is_authority(); let (keystore_remote, keystore) = self.keystore_config(&config_dir)?; let telemetry_endpoints = self.telemetry_endpoints(&chain_spec)?; + let runtime_cache_size = self.runtime_cache_size()?; let unsafe_pruning = self.import_params().map(|p| p.unsafe_pruning).unwrap_or(false); @@ -502,7 +518,6 @@ pub trait CliConfiguration: Sized { state_cache_child_ratio: self.state_cache_child_ratio()?, state_pruning: self.state_pruning(unsafe_pruning, &role)?, keep_blocks: self.keep_blocks()?, - transaction_storage: self.database_transaction_storage()?, wasm_method: self.wasm_method()?, wasm_runtime_overrides: self.wasm_runtime_overrides(), execution_strategies: self.execution_strategies(is_dev, is_validator)?, @@ -513,7 +528,9 @@ pub trait CliConfiguration: Sized { rpc_ws_max_connections: self.rpc_ws_max_connections()?, rpc_cors: self.rpc_cors(is_dev)?, rpc_max_payload: self.rpc_max_payload()?, - prometheus_config: self.prometheus_config(DCV::prometheus_listen_port())?, + ws_max_out_buffer_capacity: self.ws_max_out_buffer_capacity()?, + prometheus_config: self + .prometheus_config(DCV::prometheus_listen_port(), &chain_spec)?, telemetry_endpoints, default_heap_pages: self.default_heap_pages()?, offchain_worker: self.offchain_worker(&role)?, @@ -522,13 +539,13 @@ pub trait CliConfiguration: Sized { dev_key_seed: self.dev_key_seed(is_dev)?, tracing_targets: self.tracing_targets()?, tracing_receiver: self.tracing_receiver()?, - disable_log_reloading: self.is_log_filter_reloading_disabled()?, chain_spec, max_runtime_instances, announce_block: self.announce_block()?, role, base_path: Some(base_path), informant_output_format: Default::default(), + runtime_cache_size, }) } @@ -542,9 +559,14 @@ pub trait CliConfiguration: Sized { Ok(self.shared_params().log_filters().join(",")) } - /// Is log reloading disabled (enabled by default) - fn is_log_filter_reloading_disabled(&self) -> Result { - Ok(self.shared_params().is_log_filter_reloading_disabled()) + /// Should the detailed log output be enabled. + fn detailed_log_output(&self) -> Result { + Ok(self.shared_params().detailed_log_output()) + } + + /// Is log reloading enabled? + fn enable_log_reloading(&self) -> Result { + Ok(self.shared_params().enable_log_reloading()) } /// Should the log color output be disabled? @@ -557,13 +579,45 @@ pub trait CliConfiguration: Sized { /// This method: /// /// 1. Sets the panic handler + /// 2. Optionally customize logger/profiling /// 2. Initializes the logger /// 3. Raises the FD limit - fn init(&self) -> Result<()> { - sp_panic_handler::set(&C::support_url(), &C::impl_version()); + /// + /// The `logger_hook` closure is executed before the logger is constructed + /// and initialized. It is useful for setting up a custom profiler. + /// + /// Example: + /// ``` + /// use sc_tracing::{SpanDatum, TraceEvent}; + /// struct TestProfiler; + /// + /// impl sc_tracing::TraceHandler for TestProfiler { + /// fn handle_span(&self, sd: &SpanDatum) {} + /// fn handle_event(&self, _event: &TraceEvent) {} + /// }; + /// + /// fn logger_hook() -> impl FnOnce(&mut sc_cli::LoggerBuilder, &sc_service::Configuration) -> () { + /// |logger_builder, config| { + /// logger_builder.with_custom_profiling(Box::new(TestProfiler{})); + /// } + /// } + /// ``` + fn init( + &self, + support_url: &String, + impl_version: &String, + logger_hook: F, + config: &Configuration, + ) -> Result<()> + where + F: FnOnce(&mut LoggerBuilder, &Configuration), + { + sp_panic_handler::set(support_url, impl_version); let mut logger = LoggerBuilder::new(self.log_filters()?); - logger.with_log_reloading(!self.is_log_filter_reloading_disabled()?); + logger + .with_log_reloading(self.enable_log_reloading()?) + .with_detailed_output(self.detailed_log_output()?); if let Some(tracing_targets) = self.tracing_targets()? { let tracing_receiver = self.tracing_receiver()?; @@ -574,6 +628,9 @@ pub trait CliConfiguration: Sized { logger.with_colors(false); } + // Call hook for custom profiling setup. + logger_hook(&mut logger, &config); + logger.init()?; if let Some(new_limit) = fdlimit::raise_fd_limit() { diff --git a/client/cli/src/error.rs b/client/cli/src/error.rs index c5784b201817..f38a95e0115f 100644 --- a/client/cli/src/error.rs +++ b/client/cli/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -31,7 +31,7 @@ pub enum Error { Io(#[from] std::io::Error), #[error(transparent)] - Cli(#[from] structopt::clap::Error), + Cli(#[from] clap::Error), #[error(transparent)] Service(#[from] sc_service::Error), @@ -51,13 +51,11 @@ pub enum Error { #[error("Invalid URI; expecting either a secret URI or a public URI.")] InvalidUri(crypto::PublicError), - #[error("Signature has an invalid length. Read {read} bytes, expected {expected} bytes")] - SignatureInvalidLength { - /// Amount of signature bytes read. - read: usize, - /// Expected number of signature bytes. - expected: usize, - }, + #[error("Signature is an invalid format.")] + SignatureFormatInvalid, + + #[error("Key is an invalid format.")] + KeyFormatInvalid, #[error("Unknown key type, must be a known 4-character sequence")] KeyTypeInvalid, @@ -82,19 +80,19 @@ pub enum Error { GlobalLoggerError(#[from] sc_tracing::logging::Error), } -impl std::convert::From<&str> for Error { +impl From<&str> for Error { fn from(s: &str) -> Error { Error::Input(s.to_string()) } } -impl std::convert::From for Error { +impl From for Error { fn from(s: String) -> Error { Error::Input(s) } } -impl std::convert::From for Error { +impl From for Error { fn from(e: crypto::PublicError) -> Error { Error::InvalidUri(e) } diff --git a/client/cli/src/lib.rs b/client/cli/src/lib.rs index bb1bff94145f..c920c6cd52cf 100644 --- a/client/cli/src/lib.rs +++ b/client/cli/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -22,6 +22,9 @@ #![warn(unused_extern_crates)] #![warn(unused_imports)] +use clap::{CommandFactory, FromArgMatches, Parser}; +use sc_service::Configuration; + pub mod arg_enums; mod commands; mod config; @@ -30,38 +33,32 @@ mod params; mod runner; pub use arg_enums::*; +pub use clap; pub use commands::*; pub use config::*; pub use error::*; pub use params::*; pub use runner::*; -use sc_service::Configuration; pub use sc_service::{ChainSpec, Role}; pub use sc_tracing::logging::LoggerBuilder; pub use sp_version::RuntimeVersion; -use std::io::Write; -pub use structopt; -use structopt::{ - clap::{self, AppSettings}, - StructOpt, -}; /// Substrate client CLI /// -/// This trait needs to be defined on the root structopt of the application. It will provide the -/// implementation name, version, executable name, description, author, support_url, copyright start -/// year and most importantly: how to load the chain spec. -/// -/// StructOpt must not be in scope to use from_args (or the similar methods). This trait provides -/// its own implementation that will fill the necessary field based on the trait's functions. +/// This trait needs to be implemented on the root CLI struct of the application. It will provide +/// the implementation `name`, `version`, `executable name`, `description`, `author`, `support_url`, +/// `copyright start year` and most importantly: how to load the chain spec. pub trait SubstrateCli: Sized { /// Implementation name. fn impl_name() -> String; /// Implementation version. /// - /// By default this will look like this: 2.0.0-b950f731c-x86_64-linux-gnu where the hash is the - /// short commit hash of the commit of in the Git repository. + /// By default this will look like this: + /// + /// `2.0.0-b950f731c-x86_64-linux-gnu` + /// + /// Where the hash is the short commit hash of the commit of in the Git repository. fn impl_version() -> String; /// Executable file name. @@ -92,39 +89,37 @@ pub trait SubstrateCli: Sized { fn load_spec(&self, id: &str) -> std::result::Result, String>; /// Helper function used to parse the command line arguments. This is the equivalent of - /// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the - /// name of the application, author, "about" and version. It will also set - /// `AppSettings::GlobalVersion`. + /// [`clap::Parser::parse()`]. /// - /// To allow running the node without subcommand, tt also sets a few more settings: - /// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. + /// To allow running the node without subcommand, it also sets a few more settings: + /// [`clap::Command::propagate_version`], [`clap::Command::args_conflicts_with_subcommands`], + /// [`clap::Command::subcommand_negates_reqs`]. /// - /// Gets the struct from the command line arguments. Print the + /// Creates `Self` from the command line arguments. Print the /// error message and quit the program in case of failure. fn from_args() -> Self where - Self: StructOpt + Sized, + Self: Parser + Sized, { ::from_iter(&mut std::env::args_os()) } /// Helper function used to parse the command line arguments. This is the equivalent of - /// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the - /// name of the application, author, "about" and version. It will also set - /// `AppSettings::GlobalVersion`. + /// [`clap::Parser::parse_from`]. /// /// To allow running the node without subcommand, it also sets a few more settings: - /// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. + /// [`clap::Command::propagate_version`], [`clap::Command::args_conflicts_with_subcommands`], + /// [`clap::Command::subcommand_negates_reqs`]. /// - /// Gets the struct from any iterator such as a `Vec` of your making. + /// Creates `Self` from any iterator over arguments. /// Print the error message and quit the program in case of failure. fn from_iter(iter: I) -> Self where - Self: StructOpt + Sized, + Self: Parser + Sized, I: IntoIterator, I::Item: Into + Clone, { - let app = ::clap(); + let app = ::command(); let mut full_version = Self::impl_version(); full_version.push_str("\n"); @@ -137,58 +132,36 @@ pub trait SubstrateCli: Sized { .author(author.as_str()) .about(about.as_str()) .version(full_version.as_str()) - .settings(&[ - AppSettings::GlobalVersion, - AppSettings::ArgsNegateSubcommands, - AppSettings::SubcommandsNegateReqs, - AppSettings::ColoredHelp, - ]); - - let matches = match app.get_matches_from_safe(iter) { - Ok(matches) => matches, - Err(mut e) => { - // To support pipes, we can not use `writeln!` as any error - // results in a "broken pipe" error. - // - // Instead we write directly to `stdout` and ignore any error - // as we exit afterwards anyway. - e.message.extend("\n".chars()); - - if e.use_stderr() { - let _ = std::io::stderr().write_all(e.message.as_bytes()); - std::process::exit(1); - } else { - let _ = std::io::stdout().write_all(e.message.as_bytes()); - std::process::exit(0); - } - }, - }; - - ::from_clap(&matches) + .propagate_version(true) + .args_conflicts_with_subcommands(true) + .subcommand_negates_reqs(true); + + let matches = app.try_get_matches_from(iter).unwrap_or_else(|e| e.exit()); + + ::from_arg_matches(&matches).unwrap_or_else(|e| e.exit()) } /// Helper function used to parse the command line arguments. This is the equivalent of - /// `structopt`'s `from_iter()` except that it takes a `VersionInfo` argument to provide the - /// name of the application, author, "about" and version. It will also set - /// `AppSettings::GlobalVersion`. + /// [`clap::Parser::try_parse_from`] /// /// To allow running the node without subcommand, it also sets a few more settings: - /// `AppSettings::ArgsNegateSubcommands` and `AppSettings::SubcommandsNegateReqs`. + /// [`clap::Command::propagate_version`], [`clap::Command::args_conflicts_with_subcommands`], + /// [`clap::Command::subcommand_negates_reqs`]. /// - /// Gets the struct from any iterator such as a `Vec` of your making. + /// Creates `Self` from any iterator over arguments. /// Print the error message and quit the program in case of failure. /// /// **NOTE:** This method WILL NOT exit when `--help` or `--version` (or short versions) are /// used. It will return a [`clap::Error`], where the [`clap::Error::kind`] is a - /// [`clap::ErrorKind::HelpDisplayed`] or [`clap::ErrorKind::VersionDisplayed`] respectively. + /// [`clap::ErrorKind::DisplayHelp`] or [`clap::ErrorKind::DisplayVersion`] respectively. /// You must call [`clap::Error::exit`] or perform a [`std::process::exit`]. fn try_from_iter(iter: I) -> clap::Result where - Self: StructOpt + Sized, + Self: Parser + Sized, I: IntoIterator, I::Item: Into + Clone, { - let app = ::clap(); + let app = ::command(); let mut full_version = Self::impl_version(); full_version.push_str("\n"); @@ -202,9 +175,9 @@ pub trait SubstrateCli: Sized { .about(about.as_str()) .version(full_version.as_str()); - let matches = app.get_matches_from_safe(iter)?; + let matches = app.try_get_matches_from(iter)?; - Ok(::from_clap(&matches)) + ::from_arg_matches(&matches) } /// Returns the client ID: `{impl_name}/v{impl_version}` @@ -224,10 +197,46 @@ pub trait SubstrateCli: Sized { /// Create a runner for the command provided in argument. This will create a Configuration and /// a tokio runtime fn create_runner(&self, command: &T) -> error::Result> { - command.init::()?; - Runner::new(self, command) + let tokio_runtime = build_runtime()?; + let config = command.create_configuration(self, tokio_runtime.handle().clone())?; + + command.init(&Self::support_url(), &Self::impl_version(), |_, _| {}, &config)?; + Runner::new(config, tokio_runtime) } + /// Create a runner for the command provided in argument. The `logger_hook` can be used to setup + /// a custom profiler or update the logger configuration before it is initialized. + /// + /// Example: + /// ``` + /// use sc_tracing::{SpanDatum, TraceEvent}; + /// struct TestProfiler; + /// + /// impl sc_tracing::TraceHandler for TestProfiler { + /// fn handle_span(&self, sd: &SpanDatum) {} + /// fn handle_event(&self, _event: &TraceEvent) {} + /// }; + /// + /// fn logger_hook() -> impl FnOnce(&mut sc_cli::LoggerBuilder, &sc_service::Configuration) -> () { + /// |logger_builder, config| { + /// logger_builder.with_custom_profiling(Box::new(TestProfiler{})); + /// } + /// } + /// ``` + fn create_runner_with_logger_hook( + &self, + command: &T, + logger_hook: F, + ) -> error::Result> + where + F: FnOnce(&mut LoggerBuilder, &Configuration), + { + let tokio_runtime = build_runtime()?; + let config = command.create_configuration(self, tokio_runtime.handle().clone())?; + + command.init(&Self::support_url(), &Self::impl_version(), logger_hook, &config)?; + Runner::new(config, tokio_runtime) + } /// Native runtime version. fn native_runtime_version(chain_spec: &Box) -> &'static RuntimeVersion; } diff --git a/client/cli/src/params/database_params.rs b/client/cli/src/params/database_params.rs index 4d6cf5f1d367..e954b8cc3bc2 100644 --- a/client/cli/src/params/database_params.rs +++ b/client/cli/src/params/database_params.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -17,34 +17,24 @@ // along with this program. If not, see . use crate::arg_enums::Database; -use sc_service::TransactionStorageMode; -use structopt::StructOpt; +use clap::Args; /// Parameters for block import. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, PartialEq, Args)] pub struct DatabaseParams { /// Select database backend to use. - #[structopt( + #[clap( long, alias = "db", value_name = "DB", - case_insensitive = true, - possible_values = &Database::variants(), + ignore_case = true, + possible_values = Database::variants(), )] pub database: Option, /// Limit the memory the database cache can use. - #[structopt(long = "db-cache", value_name = "MiB")] + #[clap(long = "db-cache", value_name = "MiB")] pub database_cache_size: Option, - - /// Enable storage chain mode - /// - /// This changes the storage format for blocks bodies. - /// If this is enabled, each transaction is stored separately in the - /// transaction database column and is only referenced by hash - /// in the block body column. - #[structopt(long)] - pub storage_chain: bool, } impl DatabaseParams { @@ -57,13 +47,4 @@ impl DatabaseParams { pub fn database_cache_size(&self) -> Option { self.database_cache_size } - - /// Transaction storage scheme. - pub fn transaction_storage(&self) -> TransactionStorageMode { - if self.storage_chain { - TransactionStorageMode::StorageChain - } else { - TransactionStorageMode::BlockBody - } - } } diff --git a/client/cli/src/params/import_params.rs b/client/cli/src/params/import_params.rs index 3c87e91c220f..1df11cff8d79 100644 --- a/client/cli/src/params/import_params.rs +++ b/client/cli/src/params/import_params.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,28 +21,23 @@ use crate::{ ExecutionStrategy, WasmExecutionMethod, DEFAULT_EXECUTION_BLOCK_CONSTRUCTION, DEFAULT_EXECUTION_IMPORT_BLOCK, DEFAULT_EXECUTION_IMPORT_BLOCK_VALIDATOR, DEFAULT_EXECUTION_OFFCHAIN_WORKER, DEFAULT_EXECUTION_OTHER, DEFAULT_EXECUTION_SYNCING, + DEFAULT_WASM_EXECUTION_METHOD, }, params::{DatabaseParams, PruningParams}, }; +use clap::Args; use sc_client_api::execution_extensions::ExecutionStrategies; use std::path::PathBuf; -use structopt::StructOpt; - -#[cfg(feature = "wasmtime")] -const WASM_METHOD_DEFAULT: &str = "Compiled"; - -#[cfg(not(feature = "wasmtime"))] -const WASM_METHOD_DEFAULT: &str = "interpreted-i-know-what-i-do"; /// Parameters for block import. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Args)] pub struct ImportParams { #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub pruning_params: PruningParams, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub database_params: DatabaseParams, /// Force start with unsafe pruning settings. @@ -50,31 +45,31 @@ pub struct ImportParams { /// When running as a validator it is highly recommended to disable state /// pruning (i.e. 'archive') which is the default. The node will refuse to /// start as a validator if pruning is enabled unless this option is set. - #[structopt(long = "unsafe-pruning")] + #[clap(long)] pub unsafe_pruning: bool, /// Method for executing Wasm runtime code. - #[structopt( + #[clap( long = "wasm-execution", value_name = "METHOD", - possible_values = &WasmExecutionMethod::variants(), - case_insensitive = true, - default_value = WASM_METHOD_DEFAULT + possible_values = WasmExecutionMethod::variants(), + ignore_case = true, + default_value = DEFAULT_WASM_EXECUTION_METHOD, )] pub wasm_method: WasmExecutionMethod, /// Specify the path where local WASM runtimes are stored. /// /// These runtimes will override on-chain runtimes when the version matches. - #[structopt(long, value_name = "PATH", parse(from_os_str))] + #[clap(long, value_name = "PATH", parse(from_os_str))] pub wasm_runtime_overrides: Option, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub execution_strategies: ExecutionStrategiesParams, /// Specify the state cache size. - #[structopt(long = "state-cache-size", value_name = "Bytes", default_value = "67108864")] + #[clap(long, value_name = "Bytes", default_value = "67108864")] pub state_cache_size: usize, } @@ -127,62 +122,37 @@ impl ImportParams { } /// Execution strategies parameters. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Args)] pub struct ExecutionStrategiesParams { /// The means of execution used when calling into the runtime for importing blocks as /// part of an initial sync. - #[structopt( - long = "execution-syncing", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - )] + #[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)] pub execution_syncing: Option, /// The means of execution used when calling into the runtime for general block import /// (including locally authored blocks). - #[structopt( - long = "execution-import-block", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - )] + #[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)] pub execution_import_block: Option, /// The means of execution used when calling into the runtime while constructing blocks. - #[structopt( - long = "execution-block-construction", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - )] + #[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)] pub execution_block_construction: Option, /// The means of execution used when calling into the runtime while using an off-chain worker. - #[structopt( - long = "execution-offchain-worker", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - )] + #[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)] pub execution_offchain_worker: Option, /// The means of execution used when calling into the runtime while not syncing, importing or /// constructing blocks. - #[structopt( - long = "execution-other", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - )] + #[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)] pub execution_other: Option, /// The execution strategy that should be used by all execution contexts. - #[structopt( - long = "execution", + #[clap( + long, value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, + arg_enum, + ignore_case = true, conflicts_with_all = &[ "execution-other", "execution-offchain-worker", diff --git a/client/cli/src/params/keystore_params.rs b/client/cli/src/params/keystore_params.rs index 951f61bd1bc5..72b09134f57a 100644 --- a/client/cli/src/params/keystore_params.rs +++ b/client/cli/src/params/keystore_params.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -17,50 +17,47 @@ // along with this program. If not, see . use crate::{error, error::Result}; +use clap::Args; use sc_service::config::KeystoreConfig; use sp_core::crypto::SecretString; use std::{ fs, path::{Path, PathBuf}, }; -use structopt::StructOpt; /// default sub directory for the key store const DEFAULT_KEYSTORE_CONFIG_PATH: &'static str = "keystore"; /// Parameters of the keystore -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Args)] pub struct KeystoreParams { /// Specify custom URIs to connect to for keystore-services - #[structopt(long = "keystore-uri")] + #[clap(long)] pub keystore_uri: Option, /// Specify custom keystore path. - #[structopt(long = "keystore-path", value_name = "PATH", parse(from_os_str))] + #[clap(long, value_name = "PATH", parse(from_os_str))] pub keystore_path: Option, /// Use interactive shell for entering the password used by the keystore. - #[structopt( - long = "password-interactive", - conflicts_with_all = &[ "password", "password-filename" ] - )] + #[clap(long, conflicts_with_all = &["password", "password-filename"])] pub password_interactive: bool, /// Password used by the keystore. This allows appending an extra user-defined secret to the /// seed. - #[structopt( - long = "password", + #[clap( + long, parse(try_from_str = secret_string_from_str), - conflicts_with_all = &[ "password-interactive", "password-filename" ] + conflicts_with_all = &["password-interactive", "password-filename"] )] pub password: Option, /// File that contains the password used by the keystore. - #[structopt( - long = "password-filename", + #[clap( + long, value_name = "PATH", parse(from_os_str), - conflicts_with_all = &[ "password-interactive", "password" ] + conflicts_with_all = &["password-interactive", "password"] )] pub password_filename: Option, } diff --git a/client/cli/src/params/mod.rs b/client/cli/src/params/mod.rs index dac832a1f897..33e714ec138b 100644 --- a/client/cli/src/params/mod.rs +++ b/client/cli/src/params/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -26,13 +26,13 @@ mod shared_params; mod transaction_pool_params; use crate::arg_enums::{CryptoScheme, OutputType}; +use clap::Args; use sp_core::crypto::Ss58AddressFormat; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, NumberFor}, }; -use std::{convert::TryFrom, fmt::Debug, str::FromStr}; -use structopt::StructOpt; +use std::{fmt::Debug, str::FromStr}; pub use crate::params::{ database_params::*, import_params::*, keystore_params::*, network_params::*, @@ -79,8 +79,8 @@ impl FromStr for BlockNumberOrHash { type Err = String; fn from_str(block_number: &str) -> Result { - if block_number.starts_with("0x") { - if let Some(pos) = &block_number[2..].chars().position(|c| !c.is_ascii_hexdigit()) { + if let Some(rest) = block_number.strip_prefix("0x") { + if let Some(pos) = rest.chars().position(|c| !c.is_ascii_hexdigit()) { Err(format!( "Expected block hash, found illegal hex character at position: {}", 2 + pos, @@ -115,44 +115,32 @@ impl BlockNumberOrHash { } /// Optional flag for specifying crypto algorithm -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Args)] pub struct CryptoSchemeFlag { /// cryptography scheme - #[structopt( - long, - value_name = "SCHEME", - possible_values = &CryptoScheme::variants(), - case_insensitive = true, - default_value = "Sr25519" - )] + #[clap(long, value_name = "SCHEME", arg_enum, ignore_case = true, default_value = "Sr25519")] pub scheme: CryptoScheme, } /// Optional flag for specifying output type -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Args)] pub struct OutputTypeFlag { /// output format - #[structopt( - long, - value_name = "FORMAT", - possible_values = &OutputType::variants(), - case_insensitive = true, - default_value = "Text" - )] + #[clap(long, value_name = "FORMAT", arg_enum, ignore_case = true, default_value = "Text")] pub output_type: OutputType, } /// Optional flag for specifying network scheme -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Args)] pub struct NetworkSchemeFlag { /// network address format - #[structopt( + #[clap( + short = 'n', long, value_name = "NETWORK", - short = "n", possible_values = &Ss58AddressFormat::all_names()[..], + ignore_case = true, parse(try_from_str = Ss58AddressFormat::try_from), - case_insensitive = true, )] pub network: Option, } diff --git a/client/cli/src/params/network_params.rs b/client/cli/src/params/network_params.rs index 6eaf068fdaec..7a265c5e2760 100644 --- a/client/cli/src/params/network_params.rs +++ b/client/cli/src/params/network_params.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -17,6 +17,7 @@ // along with this program. If not, see . use crate::{arg_enums::SyncMode, params::node_key_params::NodeKeyParams}; +use clap::Args; use sc_network::{ config::{ NetworkConfiguration, NodeKeyConfig, NonReservedPeerMode, SetConfig, TransportConfig, @@ -28,17 +29,16 @@ use sc_service::{ ChainSpec, ChainType, }; use std::{borrow::Cow, path::PathBuf}; -use structopt::StructOpt; /// Parameters used to create the network configuration. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Args)] pub struct NetworkParams { /// Specify a list of bootnodes. - #[structopt(long = "bootnodes", value_name = "ADDR")] + #[clap(long, value_name = "ADDR", multiple_values(true))] pub bootnodes: Vec, /// Specify a list of reserved node addresses. - #[structopt(long = "reserved-nodes", value_name = "ADDR")] + #[clap(long, value_name = "ADDR", multiple_values(true))] pub reserved_nodes: Vec, /// Whether to only synchronize the chain with reserved nodes. @@ -49,12 +49,12 @@ pub struct NetworkParams { /// In particular, if you are a validator your node might still connect to other /// validator nodes and collator nodes regardless of whether they are defined as /// reserved nodes. - #[structopt(long = "reserved-only")] + #[clap(long)] pub reserved_only: bool, /// The public address that other nodes will use to connect to it. /// This can be used if there's a proxy in front of this node. - #[structopt(long, value_name = "PUBLIC_ADDR")] + #[clap(long, value_name = "PUBLIC_ADDR", multiple_values(true))] pub public_addr: Vec, /// Listen on this multiaddress. @@ -62,57 +62,60 @@ pub struct NetworkParams { /// By default: /// If `--validator` is passed: `/ip4/0.0.0.0/tcp/` and `/ip6/[::]/tcp/`. /// Otherwise: `/ip4/0.0.0.0/tcp//ws` and `/ip6/[::]/tcp//ws`. - #[structopt(long = "listen-addr", value_name = "LISTEN_ADDR")] + #[clap(long, value_name = "LISTEN_ADDR", multiple_values(true))] pub listen_addr: Vec, /// Specify p2p protocol TCP port. - #[structopt(long = "port", value_name = "PORT", conflicts_with_all = &[ "listen-addr" ])] + #[clap(long, value_name = "PORT", conflicts_with_all = &[ "listen-addr" ])] pub port: Option, /// Always forbid connecting to private IPv4 addresses (as specified in /// [RFC1918](https://tools.ietf.org/html/rfc1918)), unless the address was passed with /// `--reserved-nodes` or `--bootnodes`. Enabled by default for chains marked as "live" in /// their chain specifications. - #[structopt(long = "no-private-ipv4", conflicts_with_all = &["allow-private-ipv4"])] + #[clap(long, conflicts_with_all = &["allow-private-ipv4"])] pub no_private_ipv4: bool, /// Always accept connecting to private IPv4 addresses (as specified in /// [RFC1918](https://tools.ietf.org/html/rfc1918)). Enabled by default for chains marked as /// "local" in their chain specifications, or when `--dev` is passed. - #[structopt(long = "allow-private-ipv4", conflicts_with_all = &["no-private-ipv4"])] + #[clap(long, conflicts_with_all = &["no-private-ipv4"])] pub allow_private_ipv4: bool, /// Specify the number of outgoing connections we're trying to maintain. - #[structopt(long = "out-peers", value_name = "COUNT", default_value = "25")] + #[clap(long, value_name = "COUNT", default_value = "25")] pub out_peers: u32, - /// Specify the maximum number of incoming connections we're accepting. - #[structopt(long = "in-peers", value_name = "COUNT", default_value = "25")] + /// Maximum number of inbound full nodes peers. + #[clap(long, value_name = "COUNT", default_value = "25")] pub in_peers: u32, + /// Maximum number of inbound light nodes peers. + #[clap(long, value_name = "COUNT", default_value = "100")] + pub in_peers_light: u32, /// Disable mDNS discovery. /// /// By default, the network will use mDNS to discover other nodes on the /// local network. This disables it. Automatically implied when using --dev. - #[structopt(long = "no-mdns")] + #[clap(long)] pub no_mdns: bool, /// Maximum number of peers from which to ask for the same blocks in parallel. /// /// This allows downloading announced blocks from multiple peers. Decrease to save /// traffic and risk increased latency. - #[structopt(long = "max-parallel-downloads", value_name = "COUNT", default_value = "5")] + #[clap(long, value_name = "COUNT", default_value = "5")] pub max_parallel_downloads: u32, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub node_key_params: NodeKeyParams, /// Enable peer discovery on local networks. /// /// By default this option is `true` for `--dev` or when the chain type is /// `Local`/`Development` and false otherwise. - #[structopt(long)] + #[clap(long)] pub discover_local: bool, /// Require iterative Kademlia DHT queries to use disjoint paths for increased resiliency in @@ -120,11 +123,11 @@ pub struct NetworkParams { /// /// See the S/Kademlia paper for more information on the high level design as well as its /// security improvements. - #[structopt(long)] + #[clap(long)] pub kademlia_disjoint_query_paths: bool, /// Join the IPFS network and serve transactions over bitswap protocol. - #[structopt(long)] + #[clap(long)] pub ipfs_server: bool, /// Blockchain syncing mode. @@ -134,7 +137,7 @@ pub struct NetworkParams { /// - `Fast`: Download blocks and the latest state only. /// /// - `FastUnsafe`: Same as `Fast`, but skip downloading state proofs. - #[structopt(long, value_name = "SYNC_MODE", default_value = "Full")] + #[clap(long, arg_enum, value_name = "SYNC_MODE", default_value = "Full", ignore_case(true))] pub sync: SyncMode, } @@ -203,7 +206,7 @@ impl NetworkParams { boot_nodes, net_config_path, default_peers_set: SetConfig { - in_peers: self.in_peers, + in_peers: self.in_peers + self.in_peers_light, out_peers: self.out_peers, reserved_nodes: self.reserved_nodes.clone(), non_reserved_mode: if self.reserved_only { @@ -212,6 +215,7 @@ impl NetworkParams { NonReservedPeerMode::Accept }, }, + default_peers_set_num_full: self.in_peers + self.out_peers, listen_addresses, public_addresses, extra_sets: Vec::new(), @@ -233,3 +237,55 @@ impl NetworkParams { } } } + +#[cfg(test)] +mod tests { + use super::*; + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[clap(flatten)] + network_params: NetworkParams, + } + + #[test] + fn reserved_nodes_multiple_values_and_occurrences() { + let params = Cli::try_parse_from([ + "", + "--reserved-nodes", + "/ip4/0.0.0.0/tcp/501/p2p/12D3KooWEBo1HUPQJwiBmM5kSeg4XgiVxEArArQdDarYEsGxMfbS", + "/ip4/0.0.0.0/tcp/502/p2p/12D3KooWEBo1HUPQJwiBmM5kSeg4XgiVxEArArQdDarYEsGxMfbS", + "--reserved-nodes", + "/ip4/0.0.0.0/tcp/503/p2p/12D3KooWEBo1HUPQJwiBmM5kSeg4XgiVxEArArQdDarYEsGxMfbS", + ]) + .expect("Parses network params"); + + let expected = vec![ + MultiaddrWithPeerId::try_from( + "/ip4/0.0.0.0/tcp/501/p2p/12D3KooWEBo1HUPQJwiBmM5kSeg4XgiVxEArArQdDarYEsGxMfbS" + .to_string(), + ) + .unwrap(), + MultiaddrWithPeerId::try_from( + "/ip4/0.0.0.0/tcp/502/p2p/12D3KooWEBo1HUPQJwiBmM5kSeg4XgiVxEArArQdDarYEsGxMfbS" + .to_string(), + ) + .unwrap(), + MultiaddrWithPeerId::try_from( + "/ip4/0.0.0.0/tcp/503/p2p/12D3KooWEBo1HUPQJwiBmM5kSeg4XgiVxEArArQdDarYEsGxMfbS" + .to_string(), + ) + .unwrap(), + ]; + + assert_eq!(expected, params.network_params.reserved_nodes); + } + + #[test] + fn sync_ingores_case() { + let params = Cli::try_parse_from(["", "--sync", "wArP"]).expect("Parses network params"); + + assert_eq!(SyncMode::Warp, params.network_params.sync); + } +} diff --git a/client/cli/src/params/node_key_params.rs b/client/cli/src/params/node_key_params.rs index 41f9033d282d..f31fd854cbb5 100644 --- a/client/cli/src/params/node_key_params.rs +++ b/client/cli/src/params/node_key_params.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,10 +16,10 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +use clap::Args; use sc_network::config::{identity::ed25519, NodeKeyConfig}; use sp_core::H256; use std::{path::PathBuf, str::FromStr}; -use structopt::StructOpt; use crate::{arg_enums::NodeKeyType, error}; @@ -30,7 +30,7 @@ const NODE_KEY_ED25519_FILE: &str = "secret_ed25519"; /// Parameters used to create the `NodeKeyConfig`, which determines the keypair /// used for libp2p networking. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Args)] pub struct NodeKeyParams { /// The secret key to use for libp2p networking. /// @@ -46,7 +46,7 @@ pub struct NodeKeyParams { /// WARNING: Secrets provided as command-line arguments are easily exposed. /// Use of this option should be limited to development and testing. To use /// an externally managed secret key, use `--node-key-file` instead. - #[structopt(long = "node-key", value_name = "KEY")] + #[clap(long, value_name = "KEY")] pub node_key: Option, /// The type of secret key to use for libp2p networking. @@ -66,13 +66,7 @@ pub struct NodeKeyParams { /// /// The node's secret key determines the corresponding public key and hence the /// node's peer ID in the context of libp2p. - #[structopt( - long = "node-key-type", - value_name = "TYPE", - possible_values = &NodeKeyType::variants(), - case_insensitive = true, - default_value = "Ed25519" - )] + #[clap(long, value_name = "TYPE", arg_enum, ignore_case = true, default_value = "Ed25519")] pub node_key_type: NodeKeyType, /// The file from which to read the node's secret key to use for libp2p networking. @@ -85,7 +79,7 @@ pub struct NodeKeyParams { /// /// If the file does not exist, it is created with a newly generated secret key of /// the chosen type. - #[structopt(long = "node-key-file", value_name = "FILE")] + #[clap(long, value_name = "FILE")] pub node_key_file: Option, } @@ -128,14 +122,15 @@ fn parse_ed25519_secret(hex: &str) -> error::Result error::Result<()> { - NodeKeyType::variants().iter().try_for_each(|t| { - let node_key_type = NodeKeyType::from_str(t).unwrap(); + NodeKeyType::value_variants().iter().try_for_each(|t| { + let node_key_type = *t; let sk = match node_key_type { NodeKeyType::Ed25519 => ed25519::SecretKey::generate().as_ref().to_vec(), }; @@ -194,8 +189,8 @@ mod tests { where F: Fn(NodeKeyParams) -> error::Result<()>, { - NodeKeyType::variants().iter().try_for_each(|t| { - let node_key_type = NodeKeyType::from_str(t).unwrap(); + NodeKeyType::value_variants().iter().try_for_each(|t| { + let node_key_type = *t; f(NodeKeyParams { node_key_type, node_key: None, node_key_file: None }) }) } diff --git a/client/cli/src/params/offchain_worker_params.rs b/client/cli/src/params/offchain_worker_params.rs index 685328ef1779..3ab507f10859 100644 --- a/client/cli/src/params/offchain_worker_params.rs +++ b/client/cli/src/params/offchain_worker_params.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,23 +23,23 @@ //! targeted at handling input parameter parsing providing //! a reasonable abstraction. +use clap::Args; use sc_network::config::Role; use sc_service::config::OffchainWorkerConfig; -use structopt::StructOpt; use crate::{error, OffchainWorkerEnabled}; /// Offchain worker related parameters. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Args)] pub struct OffchainWorkerParams { /// Should execute offchain workers on every block. /// /// By default it's only enabled for nodes that are authoring new blocks. - #[structopt( + #[clap( long = "offchain-worker", value_name = "ENABLED", - possible_values = &OffchainWorkerEnabled::variants(), - case_insensitive = true, + arg_enum, + ignore_case = true, default_value = "WhenValidating" )] pub enabled: OffchainWorkerEnabled, @@ -48,7 +48,7 @@ pub struct OffchainWorkerParams { /// /// Enables a runtime to write directly to a offchain workers /// DB during block import. - #[structopt(long = "enable-offchain-indexing", value_name = "ENABLE_OFFCHAIN_INDEXING")] + #[clap(long = "enable-offchain-indexing", value_name = "ENABLE_OFFCHAIN_INDEXING")] pub indexing_enabled: bool, } diff --git a/client/cli/src/params/pruning_params.rs b/client/cli/src/params/pruning_params.rs index 28c7fa301cc6..de9628ecf7ad 100644 --- a/client/cli/src/params/pruning_params.rs +++ b/client/cli/src/params/pruning_params.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -17,23 +17,23 @@ // along with this program. If not, see . use crate::error; +use clap::Args; use sc_service::{KeepBlocks, PruningMode, Role}; -use structopt::StructOpt; /// Parameters to define the pruning mode -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, PartialEq, Args)] pub struct PruningParams { /// Specify the state pruning mode, a number of blocks to keep or 'archive'. /// /// Default is to keep all block states if the node is running as a /// validator (i.e. 'archive'), otherwise state is only kept for the last /// 256 blocks. - #[structopt(long = "pruning", value_name = "PRUNING_MODE")] + #[clap(long, value_name = "PRUNING_MODE")] pub pruning: Option, /// Specify the number of finalized blocks to keep in the database. /// /// Default is to keep all blocks. - #[structopt(long, value_name = "COUNT")] + #[clap(long, value_name = "COUNT")] pub keep_blocks: Option, } diff --git a/client/cli/src/params/shared_params.rs b/client/cli/src/params/shared_params.rs index 41472387d263..99eddf4e7c4b 100644 --- a/client/cli/src/params/shared_params.rs +++ b/client/cli/src/params/shared_params.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -17,59 +17,66 @@ // along with this program. If not, see . use crate::arg_enums::TracingReceiver; +use clap::Args; use sc_service::config::BasePath; use std::path::PathBuf; -use structopt::StructOpt; /// Shared parameters used by all `CoreParams`. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, PartialEq, Args)] pub struct SharedParams { /// Specify the chain specification. /// /// It can be one of the predefined ones (dev, local, or staging) or it can be a path to a file /// with the chainspec (such as one exported by the `build-spec` subcommand). - #[structopt(long, value_name = "CHAIN_SPEC")] + #[clap(long, value_name = "CHAIN_SPEC")] pub chain: Option, /// Specify the development chain. - #[structopt(long, conflicts_with_all = &["chain"])] + /// + /// This flag sets `--chain=dev`, `--force-authoring`, `--rpc-cors=all`, + /// `--alice`, and `--tmp` flags, unless explicitly overridden. + #[clap(long, conflicts_with_all = &["chain"])] pub dev: bool, /// Specify custom base path. - #[structopt(long, short = "d", value_name = "PATH", parse(from_os_str))] + #[clap(long, short = 'd', value_name = "PATH", parse(from_os_str))] pub base_path: Option, /// Sets a custom logging filter. Syntax is =, e.g. -lsync=debug. /// /// Log levels (least to most verbose) are error, warn, info, debug, and trace. /// By default, all targets log `info`. The global log level can be set with -l. - #[structopt(short = "l", long, value_name = "LOG_PATTERN")] + #[clap(short = 'l', long, value_name = "LOG_PATTERN", multiple_values(true))] pub log: Vec, + /// Enable detailed log output. + /// + /// This includes displaying the log target, log level and thread name. + /// + /// This is automatically enabled when something is logged with any higher level than `info`. + #[clap(long)] + pub detailed_log_output: bool, + /// Disable log color output. - #[structopt(long)] + #[clap(long)] pub disable_log_color: bool, - /// Disable feature to dynamically update and reload the log filter. + /// Enable feature to dynamically update and reload the log filter. + /// + /// Be aware that enabling this feature can lead to a performance decrease up to factor six or + /// more. Depending on the global logging level the performance decrease changes. /// - /// By default this feature is enabled, however it leads to a small performance decrease. /// The `system_addLogFilter` and `system_resetLogFilter` RPCs will have no effect with this - /// option set. - #[structopt(long = "disable-log-reloading")] - pub disable_log_reloading: bool, + /// option not being set. + #[clap(long)] + pub enable_log_reloading: bool, /// Sets a custom profiling filter. Syntax is the same as for logging: = - #[structopt(long = "tracing-targets", value_name = "TARGETS")] + #[clap(long, value_name = "TARGETS")] pub tracing_targets: Option, /// Receiver to process tracing messages. - #[structopt( - long = "tracing-receiver", - value_name = "RECEIVER", - possible_values = &TracingReceiver::variants(), - case_insensitive = true, - default_value = "Log" - )] + #[clap(long, value_name = "RECEIVER", arg_enum, ignore_case = true, default_value = "Log")] pub tracing_receiver: TracingReceiver, } @@ -102,14 +109,19 @@ impl SharedParams { &self.log } + /// Should the detailed log output be enabled. + pub fn detailed_log_output(&self) -> bool { + self.detailed_log_output + } + /// Should the log color output be disabled? pub fn disable_log_color(&self) -> bool { self.disable_log_color } - /// Is log reloading disabled - pub fn is_log_filter_reloading_disabled(&self) -> bool { - self.disable_log_reloading + /// Is log reloading enabled + pub fn enable_log_reloading(&self) -> bool { + self.enable_log_reloading } /// Receiver to process tracing messages. diff --git a/client/cli/src/params/transaction_pool_params.rs b/client/cli/src/params/transaction_pool_params.rs index feea19c97c2d..efb78430ced5 100644 --- a/client/cli/src/params/transaction_pool_params.rs +++ b/client/cli/src/params/transaction_pool_params.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,18 +16,18 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +use clap::Args; use sc_service::config::TransactionPoolOptions; -use structopt::StructOpt; /// Parameters used to create the pool configuration. -#[derive(Debug, StructOpt, Clone)] +#[derive(Debug, Clone, Args)] pub struct TransactionPoolParams { /// Maximum number of transactions in the transaction pool. - #[structopt(long = "pool-limit", value_name = "COUNT", default_value = "8192")] + #[clap(long, value_name = "COUNT", default_value = "8192")] pub pool_limit: usize, /// Maximum number of kilobytes of all transactions stored in the pool. - #[structopt(long = "pool-kbytes", value_name = "COUNT", default_value = "20480")] + #[clap(long, value_name = "COUNT", default_value = "20480")] pub pool_kbytes: usize, } diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index 6f03e02a12d0..f6edd8444735 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,7 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::{error::Error as CliError, CliConfiguration, Result, SubstrateCli}; +use crate::{error::Error as CliError, Result, SubstrateCli}; use chrono::prelude::*; use futures::{future, future::FutureExt, pin_mut, select, Future}; use log::info; @@ -98,7 +98,7 @@ where pin_mut!(f); tokio_runtime.block_on(main(f))?; - tokio_runtime.block_on(task_manager.clean_shutdown()); + drop(task_manager); Ok(()) } @@ -112,15 +112,8 @@ pub struct Runner { impl Runner { /// Create a new runtime with the command provided in argument - pub fn new(cli: &C, command: &T) -> Result> { - let tokio_runtime = build_runtime()?; - let runtime_handle = tokio_runtime.handle().clone(); - - Ok(Runner { - config: command.create_configuration(cli, runtime_handle)?, - tokio_runtime, - phantom: PhantomData, - }) + pub fn new(config: Configuration, tokio_runtime: tokio::runtime::Runtime) -> Result> { + Ok(Runner { config, tokio_runtime, phantom: PhantomData }) } /// Log information about the node itself. @@ -132,7 +125,7 @@ impl Runner { /// 2020-06-03 16:14:21 ✌️ version 2.0.0-rc3-f4940588c-x86_64-linux-gnu /// 2020-06-03 16:14:21 ❤️ by Parity Technologies , 2017-2020 /// 2020-06-03 16:14:21 📋 Chain specification: Flaming Fir - /// 2020-06-03 16:14:21 🏷 Node name: jolly-rod-7462 + /// 2020-06-03 16:14:21 🏷 Node name: jolly-rod-7462 /// 2020-06-03 16:14:21 👤 Role: FULL /// 2020-06-03 16:14:21 💾 Database: RocksDb at /tmp/c/chains/flamingfir7/db /// 2020-06-03 16:14:21 ⛓ Native runtime: node-251 (substrate-node-1.tx1.au10) @@ -154,7 +147,6 @@ impl Runner { self.print_node_infos(); let mut task_manager = self.tokio_runtime.block_on(initialize(self.config))?; let res = self.tokio_runtime.block_on(main(task_manager.future().fuse())); - self.tokio_runtime.block_on(task_manager.clean_shutdown()); Ok(res?) } @@ -200,7 +192,7 @@ pub fn print_node_infos(config: &Configuration) { info!("✌️ version {}", C::impl_version()); info!("❤️ by {}, {}-{}", C::author(), C::copyright_start_year(), Local::today().year()); info!("📋 Chain specification: {}", config.chain_spec.name()); - info!("🏷 Node name: {}", config.network.node_name); + info!("🏷 Node name: {}", config.network.node_name); info!("👤 Role: {}", config.display_role()); info!( "💾 Database: {} at {}", diff --git a/client/consensus/aura/Cargo.toml b/client/consensus/aura/Cargo.toml index 75595779427b..ac90d35dbce3 100644 --- a/client/consensus/aura/Cargo.toml +++ b/client/consensus/aura/Cargo.toml @@ -3,9 +3,9 @@ name = "sc-consensus-aura" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "Aura consensus algorithm for substrate" -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,39 +13,36 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "4.0.0-dev", path = "../../../primitives/application-crypto" } +sp-application-crypto = { version = "6.0.0", path = "../../../primitives/application-crypto" } sp-consensus-aura = { version = "0.10.0-dev", path = "../../../primitives/consensus/aura" } sp-block-builder = { version = "4.0.0-dev", path = "../../../primitives/block-builder" } sc-block-builder = { version = "0.10.0-dev", path = "../../block-builder" } sc-client-api = { version = "4.0.0-dev", path = "../../api" } -codec = { package = "parity-scale-codec", version = "2.0.0" } +codec = { package = "parity-scale-codec", version = "3.0.0" } sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } sc-consensus = { version = "0.10.0-dev", path = "../../../client/consensus/common" } sp-consensus-slots = { version = "0.10.0-dev", path = "../../../primitives/consensus/slots" } -derive_more = "0.99.2" -futures = "0.3.9" +thiserror = "1.0" +futures = "0.3.21" sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } log = "0.4.8" -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } sc-consensus-slots = { version = "0.10.0-dev", path = "../slots" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -sp-keystore = { version = "0.10.0-dev", path = "../../../primitives/keystore" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } +sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" } sc-telemetry = { version = "4.0.0-dev", path = "../../telemetry" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.9.0" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.10.0-dev" } async-trait = "0.1.50" -# We enable it only for web-wasm check -# See https://docs.rs/getrandom/0.2.1/getrandom/#webassembly-support -getrandom = { version = "0.2", features = ["js"], optional = true } [dev-dependencies] sp-timestamp = { version = "4.0.0-dev", path = "../../../primitives/timestamp" } -sp-keyring = { version = "4.0.0-dev", path = "../../../primitives/keyring" } -sp-tracing = { version = "4.0.0-dev", path = "../../../primitives/tracing" } +sp-keyring = { version = "6.0.0", path = "../../../primitives/keyring" } +sp-tracing = { version = "5.0.0", path = "../../../primitives/tracing" } sc-keystore = { version = "4.0.0-dev", path = "../../keystore" } sc-network = { version = "0.10.0-dev", path = "../../network" } sc-network-test = { version = "0.8.0", path = "../../network/test" } substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } tempfile = "3.1.0" -parking_lot = "0.11.1" +parking_lot = "0.12.0" diff --git a/client/consensus/aura/src/import_queue.rs b/client/consensus/aura/src/import_queue.rs index a4dbe5012ea1..56eb45c621a1 100644 --- a/client/consensus/aura/src/import_queue.rs +++ b/client/consensus/aura/src/import_queue.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -33,7 +33,7 @@ use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_block_builder::BlockBuilder as BlockBuilderApi; use sp_blockchain::{ well_known_cache_keys::{self, Id as CacheKeyId}, - HeaderBackend, ProvideCache, + HeaderBackend, }; use sp_consensus::{CanAuthorWith, Error as ConsensusError}; use sp_consensus_aura::{ @@ -45,7 +45,8 @@ use sp_core::{crypto::Pair, ExecutionContext}; use sp_inherents::{CreateInherentDataProviders, InherentDataProvider as _}; use sp_runtime::{ generic::{BlockId, OpaqueDigestItemId}, - traits::{Block as BlockT, DigestItemFor, Header}, + traits::{Block as BlockT, Header}, + DigestItem, }; use std::{fmt::Debug, hash::Hash, marker::PhantomData, sync::Arc}; @@ -61,9 +62,8 @@ fn check_header( hash: B::Hash, authorities: &[AuthorityId

], check_for_equivocation: CheckForEquivocation, -) -> Result)>, Error> +) -> Result, Error> where - DigestItemFor: CompatibleDigestItem, P::Signature: Codec, C: sc_client_api::backend::AuxStore, P::Public: Encode + Decode + PartialEq + Clone, @@ -189,14 +189,8 @@ where #[async_trait::async_trait] impl Verifier for AuraVerifier where - C: ProvideRuntimeApi - + Send - + Sync - + sc_client_api::backend::AuxStore - + ProvideCache - + BlockOf, + C: ProvideRuntimeApi + Send + Sync + sc_client_api::backend::AuxStore + BlockOf, C::Api: BlockBuilderApi + AuraApi> + ApiExt, - DigestItemFor: CompatibleDigestItem, P: Pair + Send + Sync + 'static, P::Public: Send + Sync + Hash + Eq + Clone + Decode + Encode + Debug + 'static, P::Signature: Encode + Decode, @@ -211,7 +205,7 @@ where let hash = block.header.hash(); let parent_hash = *block.header.parent_hash(); let authorities = authorities(self.client.as_ref(), &BlockId::Hash(parent_hash)) - .map_err(|e| format!("Could not fetch authorities at {:?}: {:?}", parent_hash, e))?; + .map_err(|e| format!("Could not fetch authorities at {:?}: {}", parent_hash, e))?; let create_inherent_data_providers = self .create_inherent_data_providers @@ -255,7 +249,7 @@ where &BlockId::Hash(parent_hash), |v| v >= 2, ) - .map_err(|e| format!("{:?}", e))? + .map_err(|e| e.to_string())? { self.check_inherents( new_block.clone(), @@ -385,7 +379,6 @@ where C: 'static + ProvideRuntimeApi + BlockOf - + ProvideCache + Send + Sync + AuxStore @@ -395,7 +388,6 @@ where + Send + Sync + 'static, - DigestItemFor: CompatibleDigestItem, P: Pair + Send + Sync + 'static, P::Public: Clone + Eq + Send + Sync + Hash + Debug + Encode + Decode, P::Signature: Encode + Decode, diff --git a/client/consensus/aura/src/lib.rs b/client/consensus/aura/src/lib.rs index 946e0b90c4dd..f6ea2e08e1e8 100644 --- a/client/consensus/aura/src/lib.rs +++ b/client/consensus/aura/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -30,14 +30,7 @@ //! //! NOTE: Aura itself is designed to be generic over the crypto used. #![forbid(missing_docs, unsafe_code)] -use std::{ - convert::{TryFrom, TryInto}, - fmt::Debug, - hash::Hash, - marker::PhantomData, - pin::Pin, - sync::Arc, -}; +use std::{fmt::Debug, hash::Hash, marker::PhantomData, pin::Pin, sync::Arc}; use futures::prelude::*; use log::{debug, trace}; @@ -52,17 +45,18 @@ use sc_consensus_slots::{ use sc_telemetry::TelemetryHandle; use sp_api::ProvideRuntimeApi; use sp_application_crypto::{AppKey, AppPublic}; -use sp_blockchain::{HeaderBackend, ProvideCache, Result as CResult}; +use sp_blockchain::{HeaderBackend, Result as CResult}; use sp_consensus::{ BlockOrigin, CanAuthorWith, Environment, Error as ConsensusError, Proposer, SelectChain, }; use sp_consensus_slots::Slot; -use sp_core::crypto::{Pair, Public}; +use sp_core::crypto::{ByteArray, Pair, Public}; use sp_inherents::CreateInherentDataProviders; use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, DigestItemFor, Header, Member, NumberFor, Zero}, + traits::{Block as BlockT, Header, Member, NumberFor, Zero}, + DigestItem, }; mod import_queue; @@ -76,15 +70,12 @@ pub use sp_consensus::SyncOracle; pub use sp_consensus_aura::{ digests::CompatibleDigestItem, inherents::{InherentDataProvider, InherentType as AuraInherent, INHERENT_IDENTIFIER}, - AuraApi, ConsensusLog, AURA_ENGINE_ID, + AuraApi, ConsensusLog, SlotDuration, AURA_ENGINE_ID, }; type AuthorityId

=

::Public; -/// Slot duration type for Aura. -pub type SlotDuration = sc_consensus_slots::SlotDuration; - -/// Get type of `SlotDuration` for Aura. +/// Get the slot duration for Aura. pub fn slot_duration(client: &C) -> CResult where A: Codec, @@ -92,7 +83,8 @@ where C: AuxStore + ProvideRuntimeApi + UsageProvider, C::Api: AuraApi, { - SlotDuration::get_or_compute(client, |a, b| a.slot_duration(b).map_err(Into::into)) + let best_block_id = BlockId::Hash(client.usage_info().chain.best_hash); + client.runtime_api().slot_duration(&best_block_id).map_err(|err| err.into()) } /// Get slot author for given block along with authorities. @@ -178,7 +170,7 @@ where P::Public: AppPublic + Hash + Member + Encode + Decode, P::Signature: TryFrom> + Hash + Member + Encode + Decode, B: BlockT, - C: ProvideRuntimeApi + BlockOf + ProvideCache + AuxStore + HeaderBackend + Send + Sync, + C: ProvideRuntimeApi + BlockOf + AuxStore + HeaderBackend + Send + Sync, C::Api: AuraApi>, SC: SelectChain, I: BlockImport> + Send + Sync + 'static, @@ -267,7 +259,7 @@ pub fn build_aura_worker( ) -> impl sc_consensus_slots::SlotWorker>::Proof> where B: BlockT, - C: ProvideRuntimeApi + BlockOf + ProvideCache + AuxStore + HeaderBackend + Send + Sync, + C: ProvideRuntimeApi + BlockOf + AuxStore + HeaderBackend + Send + Sync, C::Api: AuraApi>, PF: Environment + Send + Sync + 'static, PF::Proposer: Proposer>, @@ -280,7 +272,7 @@ where L: sc_consensus::JustificationSyncLink, BS: BackoffAuthoringBlocksStrategy> + Send + Sync + 'static, { - AuraWorker { + sc_consensus_slots::SimpleSlotWorkerToSlotWorker(AuraWorker { client, block_import, env: proposer_factory, @@ -293,7 +285,7 @@ where block_proposal_slot_portion, max_block_proposal_slot_portion, _key_type: PhantomData::

, - } + }) } struct AuraWorker { @@ -316,7 +308,7 @@ impl sc_consensus_slots::SimpleSlotWorker for AuraWorker where B: BlockT, - C: ProvideRuntimeApi + BlockOf + ProvideCache + HeaderBackend + Sync, + C: ProvideRuntimeApi + BlockOf + HeaderBackend + Sync, C::Api: AuraApi>, E: Environment + Send + Sync, E::Proposer: Proposer>, @@ -377,66 +369,55 @@ where }) } - fn pre_digest_data( - &self, - slot: Slot, - _claim: &Self::Claim, - ) -> Vec> { - vec![ as CompatibleDigestItem>::aura_pre_digest(slot)] + fn pre_digest_data(&self, slot: Slot, _claim: &Self::Claim) -> Vec { + vec![>::aura_pre_digest(slot)] } - fn block_import_params( + async fn block_import_params( &self, - ) -> Box< - dyn Fn( - B::Header, - &B::Hash, - Vec, - StorageChanges, B>, - Self::Claim, - Self::EpochData, - ) -> Result< - sc_consensus::BlockImportParams>, - sp_consensus::Error, - > + Send - + 'static, + header: B::Header, + header_hash: &B::Hash, + body: Vec, + storage_changes: StorageChanges<>::Transaction, B>, + public: Self::Claim, + _epoch: Self::EpochData, + ) -> Result< + sc_consensus::BlockImportParams>::Transaction>, + sp_consensus::Error, > { - let keystore = self.keystore.clone(); - Box::new(move |header, header_hash, body, storage_changes, public, _epoch| { - // sign the pre-sealed hash of the block and then - // add it to a digest item. - let public_type_pair = public.to_public_crypto_pair(); - let public = public.to_raw_vec(); - let signature = SyncCryptoStore::sign_with( - &*keystore, - as AppKey>::ID, - &public_type_pair, - header_hash.as_ref(), + // sign the pre-sealed hash of the block and then + // add it to a digest item. + let public_type_pair = public.to_public_crypto_pair(); + let public = public.to_raw_vec(); + let signature = SyncCryptoStore::sign_with( + &*self.keystore, + as AppKey>::ID, + &public_type_pair, + header_hash.as_ref(), + ) + .map_err(|e| sp_consensus::Error::CannotSign(public.clone(), e.to_string()))? + .ok_or_else(|| { + sp_consensus::Error::CannotSign( + public.clone(), + "Could not find key in keystore.".into(), ) - .map_err(|e| sp_consensus::Error::CannotSign(public.clone(), e.to_string()))? - .ok_or_else(|| { - sp_consensus::Error::CannotSign( - public.clone(), - "Could not find key in keystore.".into(), - ) - })?; - let signature = signature - .clone() - .try_into() - .map_err(|_| sp_consensus::Error::InvalidSignature(signature, public))?; - - let signature_digest_item = - as CompatibleDigestItem>::aura_seal(signature); - - let mut import_block = BlockImportParams::new(BlockOrigin::Own, header); - import_block.post_digests.push(signature_digest_item); - import_block.body = Some(body); - import_block.state_action = - StateAction::ApplyChanges(sc_consensus::StorageChanges::Changes(storage_changes)); - import_block.fork_choice = Some(ForkChoiceStrategy::LongestChain); - - Ok(import_block) - }) + })?; + let signature = signature + .clone() + .try_into() + .map_err(|_| sp_consensus::Error::InvalidSignature(signature, public))?; + + let signature_digest_item = + >::aura_seal(signature); + + let mut import_block = BlockImportParams::new(BlockOrigin::Own, header); + import_block.post_digests.push(signature_digest_item); + import_block.body = Some(body); + import_block.state_action = + StateAction::ApplyChanges(sc_consensus::StorageChanges::Changes(storage_changes)); + import_block.fork_choice = Some(ForkChoiceStrategy::LongestChain); + + Ok(import_block) } fn force_authoring(&self) -> bool { @@ -497,34 +478,46 @@ fn aura_err(error: Error) -> Error { error } -#[derive(derive_more::Display, Debug)] -enum Error { - #[display(fmt = "Multiple Aura pre-runtime headers")] +/// Aura Errors +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Multiple Aura pre-runtime headers + #[error("Multiple Aura pre-runtime headers")] MultipleHeaders, - #[display(fmt = "No Aura pre-runtime digest found")] + /// No Aura pre-runtime digest found + #[error("No Aura pre-runtime digest found")] NoDigestFound, - #[display(fmt = "Header {:?} is unsealed", _0)] + /// Header is unsealed + #[error("Header {0:?} is unsealed")] HeaderUnsealed(B::Hash), - #[display(fmt = "Header {:?} has a bad seal", _0)] + /// Header has a bad seal + #[error("Header {0:?} has a bad seal")] HeaderBadSeal(B::Hash), - #[display(fmt = "Slot Author not found")] + /// Slot Author not found + #[error("Slot Author not found")] SlotAuthorNotFound, - #[display(fmt = "Bad signature on {:?}", _0)] + /// Bad signature + #[error("Bad signature on {0:?}")] BadSignature(B::Hash), + /// Client Error + #[error(transparent)] Client(sp_blockchain::Error), - #[display(fmt = "Unknown inherent error for identifier: {}", "String::from_utf8_lossy(_0)")] + /// Unknown inherent error for identifier + #[error("Unknown inherent error for identifier: {}", String::from_utf8_lossy(.0))] UnknownInherentError(sp_inherents::InherentIdentifier), - #[display(fmt = "Inherent error: {}", _0)] + /// Inherents Error + #[error("Inherent error: {0}")] Inherent(sp_inherents::Error), } -impl std::convert::From> for String { +impl From> for String { fn from(error: Error) -> String { error.to_string() } } -fn find_pre_digest(header: &B::Header) -> Result> { +/// Get pre-digests from the header +pub fn find_pre_digest(header: &B::Header) -> Result> { if header.number().is_zero() { return Ok(0.into()) } @@ -545,7 +538,7 @@ fn authorities(client: &C, at: &BlockId) -> Result, Consensus where A: Codec + Debug, B: BlockT, - C: ProvideRuntimeApi + BlockOf + ProvideCache, + C: ProvideRuntimeApi + BlockOf, C::Api: AuraApi, { client @@ -569,12 +562,15 @@ mod tests { use sc_network_test::{Block as TestBlock, *}; use sp_application_crypto::key_types::AURA; use sp_consensus::{ - AlwaysCanAuthor, DisableProofRecording, NoNetwork as DummyOracle, Proposal, SlotData, + AlwaysCanAuthor, DisableProofRecording, NoNetwork as DummyOracle, Proposal, }; use sp_consensus_aura::sr25519::AuthorityPair; use sp_inherents::InherentData; use sp_keyring::sr25519::Keyring; - use sp_runtime::traits::{Block as BlockT, DigestFor, Header as _}; + use sp_runtime::{ + traits::{Block as BlockT, Header as _}, + Digest, + }; use sp_timestamp::InherentDataProvider as TimestampInherentDataProvider; use std::{ task::Poll, @@ -611,7 +607,7 @@ mod tests { fn propose( self, _: InherentData, - digests: DigestFor, + digests: Digest, _: Duration, _: Option, ) -> Self::Proposal { @@ -661,29 +657,25 @@ mod tests { _cfg: &ProtocolConfig, _peer_data: &(), ) -> Self::Verifier { - match client { - PeersClient::Full(client, _) => { - let slot_duration = slot_duration(&*client).expect("slot duration available"); - - assert_eq!(slot_duration.slot_duration().as_millis() as u64, SLOT_DURATION); - import_queue::AuraVerifier::new( - client, - Box::new(|_, _| async { - let timestamp = TimestampInherentDataProvider::from_system_time(); - let slot = InherentDataProvider::from_timestamp_and_duration( - *timestamp, - Duration::from_secs(6), - ); - - Ok((timestamp, slot)) - }), - AlwaysCanAuthor, - CheckForEquivocation::Yes, - None, - ) - }, - PeersClient::Light(_, _) => unreachable!("No (yet) tests for light client + Aura"), - } + let client = client.as_client(); + let slot_duration = slot_duration(&*client).expect("slot duration available"); + + assert_eq!(slot_duration.as_millis() as u64, SLOT_DURATION); + import_queue::AuraVerifier::new( + client, + Box::new(|_, _| async { + let timestamp = TimestampInherentDataProvider::from_system_time(); + let slot = InherentDataProvider::from_timestamp_and_slot_duration( + *timestamp, + SlotDuration::from_millis(6000), + ); + + Ok((timestamp, slot)) + }), + AlwaysCanAuthor, + CheckForEquivocation::Yes, + None, + ) } fn make_block_import( @@ -724,7 +716,7 @@ mod tests { for (peer_id, key) in peers { let mut net = net.lock(); let peer = net.peer(*peer_id); - let client = peer.client().as_full().expect("full clients are created").clone(); + let client = peer.client().as_client(); let select_chain = peer.select_chain().expect("full client has a select chain"); let keystore_path = tempfile::tempdir().expect("Creates keystore path"); let keystore = Arc::new( @@ -758,9 +750,9 @@ mod tests { justification_sync_link: (), create_inherent_data_providers: |_, _| async { let timestamp = TimestampInherentDataProvider::from_system_time(); - let slot = InherentDataProvider::from_timestamp_and_duration( + let slot = InherentDataProvider::from_timestamp_and_slot_duration( *timestamp, - Duration::from_secs(6), + SlotDuration::from_millis(6000), ); Ok((timestamp, slot)) @@ -823,7 +815,7 @@ mod tests { let mut net = net.lock(); let peer = net.peer(3); - let client = peer.client().as_full().expect("full clients are created").clone(); + let client = peer.client().as_client(); let environ = DummyFactory(client.clone()); let worker = AuraWorker { @@ -875,7 +867,7 @@ mod tests { let mut net = net.lock(); let peer = net.peer(3); - let client = peer.client().as_full().expect("full clients are created").clone(); + let client = peer.client().as_client(); let environ = DummyFactory(client.clone()); let mut worker = AuraWorker { diff --git a/client/consensus/babe/Cargo.toml b/client/consensus/babe/Cargo.toml index 65dfc5713320..20cc359849f5 100644 --- a/client/consensus/babe/Cargo.toml +++ b/client/consensus/babe/Cargo.toml @@ -3,9 +3,9 @@ name = "sc-consensus-babe" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "BABE consensus algorithm for substrate" -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sc-consensus-babe" readme = "README.md" @@ -14,20 +14,20 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", features = [ "derive", ] } sp-consensus-babe = { version = "0.10.0-dev", path = "../../../primitives/consensus/babe" } sc-consensus = { version = "0.10.0-dev", path = "../../../client/consensus/common" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-application-crypto = { version = "4.0.0-dev", path = "../../../primitives/application-crypto" } -sp-keystore = { version = "0.10.0-dev", path = "../../../primitives/keystore" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-application-crypto = { version = "6.0.0", path = "../../../primitives/application-crypto" } +sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" } num-bigint = "0.2.3" num-rational = "0.2.2" num-traits = "0.2.8" -serde = { version = "1.0.126", features = ["derive"] } -sp-version = { version = "4.0.0-dev", path = "../../../primitives/version" } -sp-io = { version = "4.0.0-dev", path = "../../../primitives/io" } +serde = { version = "1.0.136", features = ["derive"] } +sp-version = { version = "5.0.0", path = "../../../primitives/version" } +sp-io = { version = "6.0.0", path = "../../../primitives/io" } sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } sc-telemetry = { version = "4.0.0-dev", path = "../../telemetry" } sc-keystore = { version = "4.0.0-dev", path = "../../keystore" } @@ -40,22 +40,22 @@ sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/c sp-consensus-slots = { version = "0.10.0-dev", path = "../../../primitives/consensus/slots" } sp-consensus-vrf = { version = "0.10.0-dev", path = "../../../primitives/consensus/vrf" } sc-consensus-slots = { version = "0.10.0-dev", path = "../slots" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } fork-tree = { version = "3.0.0", path = "../../../utils/fork-tree" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.9.0" } -futures = "0.3.9" -parking_lot = "0.11.1" +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.10.0-dev" } +futures = "0.3.21" +parking_lot = "0.12.0" log = "0.4.8" schnorrkel = { version = "0.9.1", features = ["preaudit_deprecated"] } rand = "0.7.2" merlin = "2.0" -derive_more = "0.99.2" -retain_mut = "0.1.3" +thiserror = "1.0" +retain_mut = "0.1.4" async-trait = "0.1.50" [dev-dependencies] sp-timestamp = { version = "4.0.0-dev", path = "../../../primitives/timestamp" } -sp-tracing = { version = "4.0.0-dev", path = "../../../primitives/tracing" } +sp-tracing = { version = "5.0.0", path = "../../../primitives/tracing" } sc-network = { version = "0.10.0-dev", path = "../../network" } sc-network-test = { version = "0.8.0", path = "../../network/test" } substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } diff --git a/client/consensus/babe/rpc/Cargo.toml b/client/consensus/babe/rpc/Cargo.toml index 8d5625705a48..cce4544f0970 100644 --- a/client/consensus/babe/rpc/Cargo.toml +++ b/client/consensus/babe/rpc/Cargo.toml @@ -3,9 +3,9 @@ name = "sc-consensus-babe-rpc" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "RPC extensions for the BABE consensus algorithm" -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -19,22 +19,22 @@ jsonrpc-core = "18.0.0" jsonrpc-core-client = "18.0.0" jsonrpc-derive = "18.0.0" sp-consensus-babe = { version = "0.10.0-dev", path = "../../../../primitives/consensus/babe" } -serde = { version = "1.0.126", features=["derive"] } +serde = { version = "1.0.136", features = ["derive"] } sp-blockchain = { version = "4.0.0-dev", path = "../../../../primitives/blockchain" } -sp-runtime = { version = "4.0.0-dev", path = "../../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", path = "../../../../primitives/runtime" } sc-consensus-epochs = { version = "0.10.0-dev", path = "../../epochs" } -futures = "0.3.16" -derive_more = "0.99.2" +futures = "0.3.21" +thiserror = "1.0" sp-api = { version = "4.0.0-dev", path = "../../../../primitives/api" } sp-consensus = { version = "0.10.0-dev", path = "../../../../primitives/consensus/common" } -sp-core = { version = "4.0.0-dev", path = "../../../../primitives/core" } -sp-application-crypto = { version = "4.0.0-dev", path = "../../../../primitives/application-crypto" } -sp-keystore = { version = "0.10.0-dev", path = "../../../../primitives/keystore" } +sp-core = { version = "6.0.0", path = "../../../../primitives/core" } +sp-application-crypto = { version = "6.0.0", path = "../../../../primitives/application-crypto" } +sp-keystore = { version = "0.12.0", path = "../../../../primitives/keystore" } [dev-dependencies] sc-consensus = { version = "0.10.0-dev", path = "../../../consensus/common" } -serde_json = "1.0.68" -sp-keyring = { version = "4.0.0-dev", path = "../../../../primitives/keyring" } +serde_json = "1.0.79" +sp-keyring = { version = "6.0.0", path = "../../../../primitives/keyring" } sc-keystore = { version = "4.0.0-dev", path = "../../../keystore" } substrate-test-runtime-client = { version = "2.0.0", path = "../../../../test-utils/runtime/client" } tempfile = "3.1.0" diff --git a/client/consensus/babe/rpc/src/lib.rs b/client/consensus/babe/rpc/src/lib.rs index 285cfe543cee..9dd6424a43a9 100644 --- a/client/consensus/babe/rpc/src/lib.rs +++ b/client/consensus/babe/rpc/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -30,7 +30,7 @@ use sp_application_crypto::AppKey; use sp_blockchain::{Error as BlockChainError, HeaderBackend, HeaderMetadata}; use sp_consensus::{Error as ConsensusError, SelectChain}; use sp_consensus_babe::{digests::PreDigest, AuthorityId, BabeApi as BabeRuntimeApi}; -use sp_core::crypto::Public; +use sp_core::crypto::ByteArray; use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; use sp_runtime::traits::{Block as BlockT, Header as _}; use std::{collections::HashMap, sync::Arc}; @@ -104,7 +104,7 @@ where let epoch_start = client .runtime_api() .current_epoch_start(&BlockId::Hash(header.hash())) - .map_err(|err| Error::StringError(format!("{:?}", err)))?; + .map_err(|err| Error::StringError(err.to_string()))?; let epoch = epoch_data(&shared_epoch, &client, &babe_config, *epoch_start, &select_chain) .await?; @@ -166,11 +166,13 @@ pub struct EpochAuthorship { } /// Errors encountered by the RPC -#[derive(Debug, derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] pub enum Error { /// Consensus error - Consensus(ConsensusError), + #[error(transparent)] + Consensus(#[from] ConsensusError), /// Errors that can be formatted as a String + #[error("{0}")] StringError(String), } @@ -205,9 +207,9 @@ where &parent.hash(), parent.number().clone(), slot.into(), - |slot| Epoch::genesis(&babe_config, slot), + |slot| Epoch::genesis(babe_config.genesis_config(), slot), ) - .map_err(|e| Error::Consensus(ConsensusError::ChainLookup(format!("{:?}", e))))? + .map_err(|e| Error::Consensus(ConsensusError::ChainLookup(e.to_string())))? .ok_or(Error::Consensus(ConsensusError::InvalidAuthoritiesSet)) } @@ -247,7 +249,7 @@ mod tests { let builder = TestClientBuilder::new(); let (client, longest_chain) = builder.build_with_longest_chain(); let client = Arc::new(client); - let config = Config::get_or_compute(&*client).expect("config available"); + let config = Config::get(&*client).expect("config available"); let (_, link) = block_import(config.clone(), client.clone(), client.clone()) .expect("can initialize block-import"); diff --git a/client/consensus/babe/src/authorship.rs b/client/consensus/babe/src/authorship.rs index 609f96c83c19..1f74afb0e78b 100644 --- a/client/consensus/babe/src/authorship.rs +++ b/client/consensus/babe/src/authorship.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -27,7 +27,7 @@ use sp_consensus_babe::{ make_transcript, make_transcript_data, AuthorityId, BabeAuthorityWeight, Slot, BABE_VRF_PREFIX, }; use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof}; -use sp_core::{blake2_256, crypto::Public, U256}; +use sp_core::{blake2_256, crypto::ByteArray, U256}; use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; /// Calculates the primary selection threshold for a given authority, taking diff --git a/client/consensus/babe/src/aux_schema.rs b/client/consensus/babe/src/aux_schema.rs index b18220c3e360..2ab84b9b132c 100644 --- a/client/consensus/babe/src/aux_schema.rs +++ b/client/consensus/babe/src/aux_schema.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,14 +23,17 @@ use log::info; use crate::{migration::EpochV0, Epoch}; use sc_client_api::backend::AuxStore; -use sc_consensus_epochs::{migration::EpochChangesForV0, EpochChangesFor, SharedEpochChanges}; +use sc_consensus_epochs::{ + migration::{EpochChangesV0For, EpochChangesV1For}, + EpochChangesFor, SharedEpochChanges, +}; use sp_blockchain::{Error as ClientError, Result as ClientResult}; use sp_consensus_babe::{BabeBlockWeight, BabeGenesisConfiguration}; use sp_runtime::traits::Block as BlockT; const BABE_EPOCH_CHANGES_VERSION: &[u8] = b"babe_epoch_changes_version"; const BABE_EPOCH_CHANGES_KEY: &[u8] = b"babe_epoch_changes"; -const BABE_EPOCH_CHANGES_CURRENT_VERSION: u32 = 2; +const BABE_EPOCH_CHANGES_CURRENT_VERSION: u32 = 3; /// The aux storage key used to store the block weight of the given block hash. pub fn block_weight_key(block_hash: H) -> Vec { @@ -60,11 +63,16 @@ pub fn load_epoch_changes( let maybe_epoch_changes = match version { None => - load_decode::<_, EpochChangesForV0>(backend, BABE_EPOCH_CHANGES_KEY)? + load_decode::<_, EpochChangesV0For>(backend, BABE_EPOCH_CHANGES_KEY)? .map(|v0| v0.migrate().map(|_, _, epoch| epoch.migrate(config))), Some(1) => - load_decode::<_, EpochChangesFor>(backend, BABE_EPOCH_CHANGES_KEY)? - .map(|v1| v1.map(|_, _, epoch| epoch.migrate(config))), + load_decode::<_, EpochChangesV1For>(backend, BABE_EPOCH_CHANGES_KEY)? + .map(|v1| v1.migrate().map(|_, _, epoch| epoch.migrate(config))), + Some(2) => { + // v2 still uses `EpochChanges` v1 format but with a different `Epoch` type. + load_decode::<_, EpochChangesV1For>(backend, BABE_EPOCH_CHANGES_KEY)? + .map(|v2| v2.migrate()) + }, Some(BABE_EPOCH_CHANGES_CURRENT_VERSION) => load_decode::<_, EpochChangesFor>(backend, BABE_EPOCH_CHANGES_KEY)?, Some(other) => @@ -164,7 +172,7 @@ mod test { .insert_aux( &[( BABE_EPOCH_CHANGES_KEY, - &EpochChangesForV0::::from_raw(v0_tree).encode()[..], + &EpochChangesV0For::::from_raw(v0_tree).encode()[..], )], &[], ) @@ -202,6 +210,6 @@ mod test { client.insert_aux(values, &[]).unwrap(); }); - assert_eq!(load_decode::<_, u32>(&client, BABE_EPOCH_CHANGES_VERSION).unwrap(), Some(2)); + assert_eq!(load_decode::<_, u32>(&client, BABE_EPOCH_CHANGES_VERSION).unwrap(), Some(3)); } } diff --git a/client/consensus/babe/src/lib.rs b/client/consensus/babe/src/lib.rs index a0b6bde025b3..3d3a7f24df81 100644 --- a/client/consensus/babe/src/lib.rs +++ b/client/consensus/babe/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -67,7 +67,13 @@ #![warn(missing_docs)] use std::{ - borrow::Cow, collections::HashMap, convert::TryInto, pin::Pin, sync::Arc, time::Duration, u64, + borrow::Cow, + collections::{HashMap, HashSet}, + future::Future, + pin::Pin, + sync::Arc, + task::{Context, Poll}, + time::Duration, }; use codec::{Decode, Encode}; @@ -84,7 +90,10 @@ use prometheus_endpoint::Registry; use retain_mut::RetainMut; use schnorrkel::SignatureError; -use sc_client_api::{backend::AuxStore, BlockchainEvents, ProvideUncles, UsageProvider}; +use sc_client_api::{ + backend::AuxStore, AuxDataOperations, Backend as BackendT, BlockchainEvents, + FinalityNotification, PreCommitActions, ProvideUncles, UsageProvider, +}; use sc_consensus::{ block_import::{ BlockCheckParams, BlockImport, BlockImportParams, ForkChoiceStrategy, ImportResult, @@ -100,24 +109,25 @@ use sc_consensus_slots::{ SlotInfo, StorageChanges, }; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_TRACE}; -use sp_api::{ApiExt, NumberFor, ProvideRuntimeApi}; +use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_application_crypto::AppKey; use sp_block_builder::BlockBuilder as BlockBuilderApi; use sp_blockchain::{ - Error as ClientError, HeaderBackend, HeaderMetadata, ProvideCache, Result as ClientResult, + Backend as _, Error as ClientError, HeaderBackend, HeaderMetadata, Result as ClientResult, }; use sp_consensus::{ BlockOrigin, CacheKeyId, CanAuthorWith, Environment, Error as ConsensusError, Proposer, - SelectChain, SlotData, + SelectChain, }; use sp_consensus_babe::inherents::BabeInherentData; -use sp_consensus_slots::Slot; -use sp_core::{crypto::Public, ExecutionContext}; +use sp_consensus_slots::{Slot, SlotDuration}; +use sp_core::{crypto::ByteArray, ExecutionContext}; use sp_inherents::{CreateInherentDataProviders, InherentData, InherentDataProvider}; use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; use sp_runtime::{ generic::{BlockId, OpaqueDigestItemId}, - traits::{Block as BlockT, DigestItemFor, Header, Zero}, + traits::{Block as BlockT, Header, NumberFor, One, SaturatedConversion, Saturating, Zero}, + DigestItem, }; pub use sc_consensus_slots::SlotProportion; @@ -218,99 +228,98 @@ impl Epoch { } /// Errors encountered by the babe authorship task. -#[derive(derive_more::Display, Debug)] +#[derive(Debug, thiserror::Error)] pub enum Error { /// Multiple BABE pre-runtime digests - #[display(fmt = "Multiple BABE pre-runtime digests, rejecting!")] + #[error("Multiple BABE pre-runtime digests, rejecting!")] MultiplePreRuntimeDigests, /// No BABE pre-runtime digest found - #[display(fmt = "No BABE pre-runtime digest found")] + #[error("No BABE pre-runtime digest found")] NoPreRuntimeDigest, /// Multiple BABE epoch change digests - #[display(fmt = "Multiple BABE epoch change digests, rejecting!")] + #[error("Multiple BABE epoch change digests, rejecting!")] MultipleEpochChangeDigests, /// Multiple BABE config change digests - #[display(fmt = "Multiple BABE config change digests, rejecting!")] + #[error("Multiple BABE config change digests, rejecting!")] MultipleConfigChangeDigests, /// Could not extract timestamp and slot - #[display(fmt = "Could not extract timestamp and slot: {:?}", _0)] + #[error("Could not extract timestamp and slot: {0}")] Extraction(sp_consensus::Error), /// Could not fetch epoch - #[display(fmt = "Could not fetch epoch at {:?}", _0)] + #[error("Could not fetch epoch at {0:?}")] FetchEpoch(B::Hash), /// Header rejected: too far in the future - #[display(fmt = "Header {:?} rejected: too far in the future", _0)] + #[error("Header {0:?} rejected: too far in the future")] TooFarInFuture(B::Hash), /// Parent unavailable. Cannot import - #[display(fmt = "Parent ({}) of {} unavailable. Cannot import", _0, _1)] + #[error("Parent ({0}) of {1} unavailable. Cannot import")] ParentUnavailable(B::Hash, B::Hash), /// Slot number must increase - #[display(fmt = "Slot number must increase: parent slot: {}, this slot: {}", _0, _1)] + #[error("Slot number must increase: parent slot: {0}, this slot: {1}")] SlotMustIncrease(Slot, Slot), /// Header has a bad seal - #[display(fmt = "Header {:?} has a bad seal", _0)] + #[error("Header {0:?} has a bad seal")] HeaderBadSeal(B::Hash), /// Header is unsealed - #[display(fmt = "Header {:?} is unsealed", _0)] + #[error("Header {0:?} is unsealed")] HeaderUnsealed(B::Hash), /// Slot author not found - #[display(fmt = "Slot author not found")] + #[error("Slot author not found")] SlotAuthorNotFound, /// Secondary slot assignments are disabled for the current epoch. - #[display(fmt = "Secondary slot assignments are disabled for the current epoch.")] + #[error("Secondary slot assignments are disabled for the current epoch.")] SecondarySlotAssignmentsDisabled, /// Bad signature - #[display(fmt = "Bad signature on {:?}", _0)] + #[error("Bad signature on {0:?}")] BadSignature(B::Hash), /// Invalid author: Expected secondary author - #[display(fmt = "Invalid author: Expected secondary author: {:?}, got: {:?}.", _0, _1)] + #[error("Invalid author: Expected secondary author: {0:?}, got: {1:?}.")] InvalidAuthor(AuthorityId, AuthorityId), /// No secondary author expected. - #[display(fmt = "No secondary author expected.")] + #[error("No secondary author expected.")] NoSecondaryAuthorExpected, /// VRF verification of block by author failed - #[display( - fmt = "VRF verification of block by author {:?} failed: threshold {} exceeded", - _0, - _1 - )] + #[error("VRF verification of block by author {0:?} failed: threshold {1} exceeded")] VRFVerificationOfBlockFailed(AuthorityId, u128), /// VRF verification failed - #[display(fmt = "VRF verification failed: {:?}", _0)] + #[error("VRF verification failed: {0:?}")] VRFVerificationFailed(SignatureError), /// Could not fetch parent header - #[display(fmt = "Could not fetch parent header: {:?}", _0)] + #[error("Could not fetch parent header: {0}")] FetchParentHeader(sp_blockchain::Error), /// Expected epoch change to happen. - #[display(fmt = "Expected epoch change to happen at {:?}, s{}", _0, _1)] + #[error("Expected epoch change to happen at {0:?}, s{1}")] ExpectedEpochChange(B::Hash, Slot), /// Unexpected config change. - #[display(fmt = "Unexpected config change")] + #[error("Unexpected config change")] UnexpectedConfigChange, /// Unexpected epoch change - #[display(fmt = "Unexpected epoch change")] + #[error("Unexpected epoch change")] UnexpectedEpochChange, /// Parent block has no associated weight - #[display(fmt = "Parent block of {} has no associated weight", _0)] + #[error("Parent block of {0} has no associated weight")] ParentBlockNoAssociatedWeight(B::Hash), /// Check inherents error - #[display(fmt = "Checking inherents failed: {}", _0)] + #[error("Checking inherents failed: {0}")] CheckInherents(sp_inherents::Error), /// Unhandled check inherents error - #[display(fmt = "Checking inherents unhandled error: {}", "String::from_utf8_lossy(_0)")] + #[error("Checking inherents unhandled error: {}", String::from_utf8_lossy(.0))] CheckInherentsUnhandled(sp_inherents::InherentIdentifier), /// Create inherents error. - #[display(fmt = "Creating inherents failed: {}", _0)] + #[error("Creating inherents failed: {0}")] CreateInherents(sp_inherents::Error), /// Client error + #[error(transparent)] Client(sp_blockchain::Error), /// Runtime Api error. + #[error(transparent)] RuntimeApi(sp_api::ApiError), /// Fork tree error + #[error(transparent)] ForkTree(Box>), } -impl std::convert::From> for String { +impl From> for String { fn from(error: Error) -> String { error.to_string() } @@ -330,60 +339,55 @@ pub struct BabeIntermediate { /// Intermediate key for Babe engine. pub static INTERMEDIATE_KEY: &[u8] = b"babe1"; -/// A slot duration. Create with `get_or_compute`. -// FIXME: Once Rust has higher-kinded types, the duplication between this -// and `super::babe::Config` can be eliminated. -// https://github.com/paritytech/substrate/issues/2434 +/// Configuration for BABE used for defining block verification parameters as +/// well as authoring (e.g. the slot duration). #[derive(Clone)] -pub struct Config(sc_consensus_slots::SlotDuration); +pub struct Config { + genesis_config: BabeGenesisConfiguration, +} impl Config { - /// Either fetch the slot duration from disk or compute it from the genesis - /// state. - pub fn get_or_compute(client: &C) -> ClientResult + /// Create a new config by reading the genesis configuration from the runtime. + pub fn get(client: &C) -> ClientResult where C: AuxStore + ProvideRuntimeApi + UsageProvider, C::Api: BabeApi, { trace!(target: "babe", "Getting slot duration"); - match sc_consensus_slots::SlotDuration::get_or_compute(client, |a, b| { - let has_api_v1 = a.has_api_with::, _>(&b, |v| v == 1)?; - let has_api_v2 = a.has_api_with::, _>(&b, |v| v == 2)?; - if has_api_v1 { - #[allow(deprecated)] - { - Ok(a.configuration_before_version_2(b)?.into()) - } - } else if has_api_v2 { - a.configuration(b).map_err(Into::into) - } else { - Err(sp_blockchain::Error::VersionInvalid( - "Unsupported or invalid BabeApi version".to_string(), - )) - } - }) - .map(Self) - { - Ok(s) => Ok(s), - Err(s) => { - warn!(target: "babe", "Failed to get slot duration"); - Err(s) - }, + let mut best_block_id = BlockId::Hash(client.usage_info().chain.best_hash); + if client.usage_info().chain.finalized_state.is_none() { + debug!(target: "babe", "No finalized state is available. Reading config from genesis"); + best_block_id = BlockId::Hash(client.usage_info().chain.genesis_hash); } - } + let runtime_api = client.runtime_api(); + + let version = runtime_api.api_version::>(&best_block_id)?; + + let genesis_config = if version == Some(1) { + #[allow(deprecated)] + { + runtime_api.configuration_before_version_2(&best_block_id)?.into() + } + } else if version == Some(2) { + runtime_api.configuration(&best_block_id)? + } else { + return Err(sp_blockchain::Error::VersionInvalid( + "Unsupported or invalid BabeApi version".to_string(), + )) + }; - /// Get the inner slot duration - pub fn slot_duration(&self) -> Duration { - self.0.slot_duration() + Ok(Config { genesis_config }) } -} -impl std::ops::Deref for Config { - type Target = BabeGenesisConfiguration; + /// Get the genesis configuration. + pub fn genesis_config(&self) -> &BabeGenesisConfiguration { + &self.genesis_config + } - fn deref(&self) -> &BabeGenesisConfiguration { - &*self.0 + /// Get the slot duration defined in the genesis configuration. + pub fn slot_duration(&self) -> SlotDuration { + SlotDuration::from_millis(self.genesis_config.slot_duration) } } @@ -465,9 +469,9 @@ pub fn start_babe( where B: BlockT, C: ProvideRuntimeApi - + ProvideCache + ProvideUncles + BlockchainEvents + + PreCommitActions + HeaderBackend + HeaderMetadata + Send @@ -491,7 +495,6 @@ where { const HANDLE_BUFFER_SIZE: usize = 1024; - let config = babe_link.config; let slot_notification_sinks = Arc::new(Mutex::new(Vec::new())); let worker = BabeSlotWorker { @@ -505,17 +508,18 @@ where keystore, epoch_changes: babe_link.epoch_changes.clone(), slot_notification_sinks: slot_notification_sinks.clone(), - config: config.clone(), + config: babe_link.config.clone(), block_proposal_slot_portion, max_block_proposal_slot_portion, telemetry, }; info!(target: "babe", "👶 Starting BABE Authorship worker"); - let inner = sc_consensus_slots::start_slot_worker( - config.0.clone(), + + let slot_worker = sc_consensus_slots::start_slot_worker( + babe_link.config.slot_duration(), select_chain, - worker, + sc_consensus_slots::SimpleSlotWorkerToSlotWorker(worker), sync_oracle, create_inherent_data_providers, can_author_with, @@ -524,22 +528,78 @@ where let (worker_tx, worker_rx) = channel(HANDLE_BUFFER_SIZE); let answer_requests = - answer_requests(worker_rx, config.0, client, babe_link.epoch_changes.clone()); + answer_requests(worker_rx, babe_link.config, client, babe_link.epoch_changes.clone()); + + let inner = future::select(Box::pin(slot_worker), Box::pin(answer_requests)); Ok(BabeWorker { - inner: Box::pin(future::join(inner, answer_requests).map(|_| ())), + inner: Box::pin(inner.map(|_| ())), slot_notification_sinks, handle: BabeWorkerHandle(worker_tx), }) } +// Remove obsolete block's weight data by leveraging finality notifications. +// This includes data for all finalized blocks (excluding the most recent one) +// and all stale branches. +fn aux_storage_cleanup, Block: BlockT>( + client: &C, + notification: &FinalityNotification, +) -> AuxDataOperations { + let mut aux_keys = HashSet::new(); + + // Cleans data for finalized block's ancestors down to, and including, the previously + // finalized one. + + let first_new_finalized = notification.tree_route.get(0).unwrap_or(¬ification.hash); + match client.header_metadata(*first_new_finalized) { + Ok(meta) => { + aux_keys.insert(aux_schema::block_weight_key(meta.parent)); + }, + Err(err) => { + warn!(target: "babe", "header lookup fail while cleaning data for block {}: {}", first_new_finalized.to_string(), err.to_string()); + }, + } + + aux_keys.extend(notification.tree_route.iter().map(aux_schema::block_weight_key)); + + // Cleans data for stale branches. + + // A safenet in case of malformed notification. + let height_limit = notification.header.number().saturating_sub( + notification.tree_route.len().saturated_into::>() + One::one(), + ); + for head in notification.stale_heads.iter() { + let mut hash = *head; + // Insert stale blocks hashes until canonical chain is not reached. + // Soon or late we should hit an element already present within the `aux_keys` set. + while aux_keys.insert(aux_schema::block_weight_key(hash)) { + match client.header_metadata(hash) { + Ok(meta) => { + // This should never happen and must be considered a bug. + if meta.number <= height_limit { + warn!(target: "babe", "unexpected canonical chain state or malformed finality notification"); + break + } + hash = meta.parent; + }, + Err(err) => { + warn!(target: "babe", "header lookup fail while cleaning data for block {}: {}", head.to_string(), err.to_string()); + break + }, + } + } + } + + aux_keys.into_iter().map(|val| (val, None)).collect() +} + async fn answer_requests( mut request_rx: Receiver>, - genesis_config: sc_consensus_slots::SlotDuration, + config: Config, client: Arc, epoch_changes: SharedEpochChanges, ) where C: ProvideRuntimeApi - + ProvideCache + ProvideUncles + BlockchainEvents + HeaderBackend @@ -565,7 +625,7 @@ async fn answer_requests( let viable_epoch = epoch_changes .viable_epoch(&epoch_descriptor, |slot| { - Epoch::genesis(&genesis_config, slot) + Epoch::genesis(&config.genesis_config, slot) }) .ok_or_else(|| Error::::FetchEpoch(parent_hash))?; @@ -615,7 +675,7 @@ impl BabeWorkerHandle { /// Worker for Babe which implements `Future`. This must be polled. #[must_use] pub struct BabeWorker { - inner: Pin + Send + 'static>>, + inner: Pin + Send + 'static>>, slot_notification_sinks: SlotNotificationSinks, handle: BabeWorkerHandle, } @@ -639,13 +699,10 @@ impl BabeWorker { } } -impl futures::Future for BabeWorker { +impl Future for BabeWorker { type Output = (); - fn poll( - mut self: Pin<&mut Self>, - cx: &mut futures::task::Context, - ) -> futures::task::Poll { + fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { self.inner.as_mut().poll(cx) } } @@ -677,10 +734,7 @@ impl sc_consensus_slots::SimpleSlotWorker for BabeSlotWorker where B: BlockT, - C: ProvideRuntimeApi - + ProvideCache - + HeaderBackend - + HeaderMetadata, + C: ProvideRuntimeApi + HeaderBackend + HeaderMetadata, C::Api: BabeApi, E: Environment + Sync, E::Proposer: Proposer>, @@ -720,14 +774,16 @@ where parent.number().clone(), slot, ) - .map_err(|e| ConsensusError::ChainLookup(format!("{:?}", e)))? + .map_err(|e| ConsensusError::ChainLookup(e.to_string()))? .ok_or(sp_consensus::Error::InvalidAuthoritiesSet) } fn authorities_len(&self, epoch_descriptor: &Self::EpochData) -> Option { self.epoch_changes .shared_data() - .viable_epoch(&epoch_descriptor, |slot| Epoch::genesis(&self.config, slot)) + .viable_epoch(&epoch_descriptor, |slot| { + Epoch::genesis(&self.config.genesis_config, slot) + }) .map(|epoch| epoch.as_ref().authorities.len()) } @@ -742,7 +798,9 @@ where slot, self.epoch_changes .shared_data() - .viable_epoch(&epoch_descriptor, |slot| Epoch::genesis(&self.config, slot))? + .viable_epoch(&epoch_descriptor, |slot| { + Epoch::genesis(&self.config.genesis_config, slot) + })? .as_ref(), &self.keystore, ); @@ -760,7 +818,7 @@ where slot: Slot, epoch_descriptor: &ViableEpochDescriptor, Epoch>, ) { - self.slot_notification_sinks.lock().retain_mut(|sink| { + RetainMut::retain_mut(&mut *self.slot_notification_sinks.lock(), |sink| { match sink.try_send((slot, epoch_descriptor.clone())) { Ok(()) => true, Err(e) => @@ -774,69 +832,56 @@ where }); } - fn pre_digest_data( - &self, - _slot: Slot, - claim: &Self::Claim, - ) -> Vec> { - vec![ as CompatibleDigestItem>::babe_pre_digest(claim.0.clone())] + fn pre_digest_data(&self, _slot: Slot, claim: &Self::Claim) -> Vec { + vec![::babe_pre_digest(claim.0.clone())] } - fn block_import_params( + async fn block_import_params( &self, - ) -> Box< - dyn Fn( - B::Header, - &B::Hash, - Vec, - StorageChanges, - Self::Claim, - Self::EpochData, - ) -> Result, sp_consensus::Error> - + Send - + 'static, + header: B::Header, + header_hash: &B::Hash, + body: Vec, + storage_changes: StorageChanges<>::Transaction, B>, + (_, public): Self::Claim, + epoch_descriptor: Self::EpochData, + ) -> Result< + sc_consensus::BlockImportParams>::Transaction>, + sp_consensus::Error, > { - let keystore = self.keystore.clone(); - Box::new( - move |header, header_hash, body, storage_changes, (_, public), epoch_descriptor| { - // sign the pre-sealed hash of the block and then - // add it to a digest item. - let public_type_pair = public.clone().into(); - let public = public.to_raw_vec(); - let signature = SyncCryptoStore::sign_with( - &*keystore, - ::ID, - &public_type_pair, - header_hash.as_ref(), - ) - .map_err(|e| sp_consensus::Error::CannotSign(public.clone(), e.to_string()))? - .ok_or_else(|| { - sp_consensus::Error::CannotSign( - public.clone(), - "Could not find key in keystore.".into(), - ) - })?; - let signature: AuthoritySignature = signature - .clone() - .try_into() - .map_err(|_| sp_consensus::Error::InvalidSignature(signature, public))?; - let digest_item = - as CompatibleDigestItem>::babe_seal(signature.into()); - - let mut import_block = BlockImportParams::new(BlockOrigin::Own, header); - import_block.post_digests.push(digest_item); - import_block.body = Some(body); - import_block.state_action = StateAction::ApplyChanges( - sc_consensus::StorageChanges::Changes(storage_changes), - ); - import_block.intermediates.insert( - Cow::from(INTERMEDIATE_KEY), - Box::new(BabeIntermediate:: { epoch_descriptor }) as Box<_>, - ); - - Ok(import_block) - }, + // sign the pre-sealed hash of the block and then + // add it to a digest item. + let public_type_pair = public.clone().into(); + let public = public.to_raw_vec(); + let signature = SyncCryptoStore::sign_with( + &*self.keystore, + ::ID, + &public_type_pair, + header_hash.as_ref(), ) + .map_err(|e| sp_consensus::Error::CannotSign(public.clone(), e.to_string()))? + .ok_or_else(|| { + sp_consensus::Error::CannotSign( + public.clone(), + "Could not find key in keystore.".into(), + ) + })?; + let signature: AuthoritySignature = signature + .clone() + .try_into() + .map_err(|_| sp_consensus::Error::InvalidSignature(signature, public))?; + let digest_item = ::babe_seal(signature.into()); + + let mut import_block = BlockImportParams::new(BlockOrigin::Own, header); + import_block.post_digests.push(digest_item); + import_block.body = Some(body); + import_block.state_action = + StateAction::ApplyChanges(sc_consensus::StorageChanges::Changes(storage_changes)); + import_block.intermediates.insert( + Cow::from(INTERMEDIATE_KEY), + Box::new(BabeIntermediate:: { epoch_descriptor }) as Box<_>, + ); + + Ok(import_block) } fn force_authoring(&self) -> bool { @@ -880,7 +925,7 @@ where self.telemetry.clone() } - fn proposing_remaining_duration(&self, slot_info: &SlotInfo) -> std::time::Duration { + fn proposing_remaining_duration(&self, slot_info: &SlotInfo) -> Duration { let parent_slot = find_pre_digest::(&slot_info.chain_head).ok().map(|d| d.slot()); sc_consensus_slots::proposing_remaining_duration( @@ -921,10 +966,7 @@ pub fn find_pre_digest(header: &B::Header) -> Result( header: &B::Header, -) -> Result, Error> -where - DigestItemFor: CompatibleDigestItem, -{ +) -> Result, Error> { let mut epoch_digest: Option<_> = None; for log in header.digest().logs() { trace!(target: "babe", "Checking log {:?}, looking for epoch change digest.", log); @@ -943,10 +985,7 @@ where /// Extract the BABE config change digest from the given header, if it exists. fn find_next_config_digest( header: &B::Header, -) -> Result, Error> -where - DigestItemFor: CompatibleDigestItem, -{ +) -> Result, Error> { let mut config_digest: Option<_> = None; for log in header.digest().logs() { trace!(target: "babe", "Checking log {:?}, looking for epoch change digest.", log); @@ -1132,8 +1171,7 @@ where + ProvideRuntimeApi + Send + Sync - + AuxStore - + ProvideCache, + + AuxStore, Client::Api: BlockBuilderApi + BabeApi, SelectChain: sp_consensus::SelectChain, CAW: CanAuthorWith + Send + Sync, @@ -1192,7 +1230,9 @@ where .map_err(|e| Error::::ForkTree(Box::new(e)))? .ok_or_else(|| Error::::FetchEpoch(parent_hash))?; let viable_epoch = epoch_changes - .viable_epoch(&epoch_descriptor, |slot| Epoch::genesis(&self.config, slot)) + .viable_epoch(&epoch_descriptor, |slot| { + Epoch::genesis(&self.config.genesis_config, slot) + }) .ok_or_else(|| Error::::FetchEpoch(parent_hash))?; // We add one to the current slot to allow for some small drift. @@ -1228,7 +1268,7 @@ where ) .await { - warn!(target: "babe", "Error checking/reporting BABE equivocation: {:?}", err); + warn!(target: "babe", "Error checking/reporting BABE equivocation: {}", err); } // if the body is passed through, we need to use the runtime @@ -1332,7 +1372,6 @@ where + HeaderMetadata + AuxStore + ProvideRuntimeApi - + ProvideCache + Send + Sync, Client::Api: BabeApi + ApiExt, @@ -1399,7 +1438,6 @@ where + HeaderMetadata + AuxStore + ProvideRuntimeApi - + ProvideCache + Send + Sync, Client::Api: BabeApi + ApiExt, @@ -1527,7 +1565,9 @@ where old_epoch_changes = Some((*epoch_changes).clone()); let viable_epoch = epoch_changes - .viable_epoch(&epoch_descriptor, |slot| Epoch::genesis(&self.config, slot)) + .viable_epoch(&epoch_descriptor, |slot| { + Epoch::genesis(&self.config.genesis_config, slot) + }) .ok_or_else(|| { ConsensusError::ClientImport(Error::::FetchEpoch(parent_hash).into()) })?; @@ -1578,13 +1618,17 @@ where *block.header.parent_hash(), next_epoch, ) - .map_err(|e| ConsensusError::ClientImport(format!("{:?}", e)))?; - + .map_err(|e| { + ConsensusError::ClientImport(format!( + "Error importing epoch changes: {}", + e + )) + })?; Ok(()) }; if let Err(e) = prune_and_import() { - debug!(target: "babe", "Failed to launch next epoch: {:?}", e); + debug!(target: "babe", "Failed to launch next epoch: {}", e); *epoch_changes = old_epoch_changes.expect("set `Some` above and not taken; qed"); return Err(e) @@ -1615,7 +1659,7 @@ where parent_weight } else { aux_schema::load_block_weight(&*self.client, last_best) - .map_err(|e| ConsensusError::ChainLookup(format!("{:?}", e)))? + .map_err(|e| ConsensusError::ChainLookup(e.to_string()))? .ok_or_else(|| { ConsensusError::ChainLookup( "No block weight for parent header.".to_string(), @@ -1667,11 +1711,14 @@ where Client: HeaderBackend + HeaderMetadata, { let info = client.info(); + if info.block_gap.is_none() { + epoch_changes.clear_gap(); + } let finalized_slot = { let finalized_header = client .header(BlockId::Hash(info.finalized_hash)) - .map_err(|e| ConsensusError::ClientImport(format!("{:?}", e)))? + .map_err(|e| ConsensusError::ClientImport(e.to_string()))? .expect( "best finalized hash was given by client; finalized headers must exist in db; qed", ); @@ -1688,7 +1735,7 @@ where info.finalized_number, finalized_slot, ) - .map_err(|e| ConsensusError::ClientImport(format!("{:?}", e)))?; + .map_err(|e| ConsensusError::ClientImport(e.to_string()))?; Ok(()) } @@ -1704,9 +1751,14 @@ pub fn block_import( client: Arc, ) -> ClientResult<(BabeBlockImport, BabeLink)> where - Client: AuxStore + HeaderBackend + HeaderMetadata, + Client: AuxStore + + HeaderBackend + + HeaderMetadata + + PreCommitActions + + 'static, { - let epoch_changes = aux_schema::load_epoch_changes::(&*client, &config)?; + let epoch_changes = + aux_schema::load_epoch_changes::(&*client, &config.genesis_config)?; let link = BabeLink { epoch_changes: epoch_changes.clone(), config: config.clone() }; // NOTE: this isn't entirely necessary, but since we didn't use to prune the @@ -1714,6 +1766,12 @@ where // startup rather than waiting until importing the next epoch change block. prune_finalized(client.clone(), &mut epoch_changes.shared_data())?; + let client_clone = client.clone(); + let on_finality = move |summary: &FinalityNotification| { + aux_storage_cleanup(client_clone.as_ref(), summary) + }; + client.register_finality_action(Box::new(on_finality)); + let import = BabeBlockImport::new(client, epoch_changes, wrapped_block_import, config); Ok((import, link)) @@ -1749,7 +1807,6 @@ where + Sync + 'static, Client: ProvideRuntimeApi - + ProvideCache + HeaderBackend + HeaderMetadata + AuxStore @@ -1774,3 +1831,76 @@ where Ok(BasicQueue::new(verifier, Box::new(block_import), justification_import, spawner, registry)) } + +/// Reverts protocol aux data to at most the last finalized block. +/// In particular, epoch-changes and block weights announced after the revert +/// point are removed. +pub fn revert( + client: Arc, + backend: Arc, + blocks: NumberFor, +) -> ClientResult<()> +where + Block: BlockT, + Client: AuxStore + + HeaderMetadata + + HeaderBackend + + ProvideRuntimeApi + + UsageProvider, + Client::Api: BabeApi, + Backend: BackendT, +{ + let best_number = client.info().best_number; + let finalized = client.info().finalized_number; + let revertible = blocks.min(best_number - finalized); + + let number = best_number - revertible; + let hash = client + .block_hash_from_id(&BlockId::Number(number))? + .ok_or(ClientError::Backend(format!( + "Unexpected hash lookup failure for block number: {}", + number + )))?; + + // Revert epoch changes tree. + + let config = Config::get(&*client)?; + let epoch_changes = + aux_schema::load_epoch_changes::(&*client, config.genesis_config())?; + let mut epoch_changes = epoch_changes.shared_data(); + + if number == Zero::zero() { + // Special case, no epoch changes data were present on genesis. + *epoch_changes = EpochChangesFor::::default(); + } else { + epoch_changes.revert(descendent_query(&*client), hash, number); + } + + // Remove block weights added after the revert point. + + let mut weight_keys = HashSet::with_capacity(revertible.saturated_into()); + let leaves = backend.blockchain().leaves()?.into_iter().filter(|&leaf| { + sp_blockchain::tree_route(&*client, hash, leaf) + .map(|route| route.retracted().is_empty()) + .unwrap_or_default() + }); + for leaf in leaves { + let mut hash = leaf; + // Insert parent after parent until we don't hit an already processed + // branch or we reach a direct child of the rollback point. + while weight_keys.insert(aux_schema::block_weight_key(hash)) { + let meta = client.header_metadata(hash)?; + if meta.number <= number + One::one() { + // We've reached a child of the revert point, stop here. + break + } + hash = client.header_metadata(hash)?.parent; + } + } + let weight_keys: Vec<_> = weight_keys.iter().map(|val| val.as_slice()).collect(); + + // Write epoch changes and remove weights in one shot. + aux_schema::write_epoch_changes::(&epoch_changes, |values| { + client.insert_aux(values, weight_keys.iter()) + }) +} diff --git a/client/consensus/babe/src/migration.rs b/client/consensus/babe/src/migration.rs index a248c9da24db..a8c3772bbefb 100644 --- a/client/consensus/babe/src/migration.rs +++ b/client/consensus/babe/src/migration.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/consensus/babe/src/tests.rs b/client/consensus/babe/src/tests.rs index c033f4535be0..080387c88655 100644 --- a/client/consensus/babe/src/tests.rs +++ b/client/consensus/babe/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -28,7 +28,7 @@ use log::debug; use rand::RngCore; use rand_chacha::{rand_core::SeedableRng, ChaChaRng}; use sc_block_builder::{BlockBuilder, BlockBuilderProvider}; -use sc_client_api::{backend::TransactionFor, BlockchainEvents}; +use sc_client_api::{backend::TransactionFor, BlockchainEvents, Finalizer}; use sc_consensus::{BoxBlockImport, BoxJustificationImport}; use sc_consensus_slots::BackoffAuthoringOnFinalizedHeadLagging; use sc_keystore::LocalKeystore; @@ -43,13 +43,13 @@ use sp_consensus_babe::{ use sp_core::crypto::Pair; use sp_keystore::{vrf::make_transcript as transcript_from_data, SyncCryptoStore}; use sp_runtime::{ - generic::DigestItem, - traits::{Block as BlockT, DigestFor}, + generic::{Digest, DigestItem}, + traits::Block as BlockT, }; use sp_timestamp::InherentDataProvider as TimestampInherentDataProvider; use std::{cell::RefCell, task::Poll, time::Duration}; -type Item = DigestItem; +type Item = DigestItem; type Error = sp_blockchain::Error; @@ -108,7 +108,7 @@ impl Environment for DummyFactory { impl DummyProposer { fn propose_with( &mut self, - pre_digests: DigestFor, + pre_digests: Digest, ) -> future::Ready< Result< Proposal< @@ -143,7 +143,7 @@ impl DummyProposer { &self.parent_hash, self.parent_number, this_slot, - |slot| Epoch::genesis(&self.factory.config, slot), + |slot| Epoch::genesis(self.factory.config.genesis_config(), slot), ) .expect("client has data to find epoch") .expect("can compute epoch for baked block"); @@ -181,7 +181,7 @@ impl Proposer for DummyProposer { fn propose( mut self, _: InherentData, - pre_digests: DigestFor, + pre_digests: Digest, _: Duration, _: Option, ) -> Self::Proposal { @@ -295,9 +295,9 @@ impl TestNetFactory for BabeTestNet { Option>, Option, ) { - let client = client.as_full().expect("only full clients are tested"); + let client = client.as_client(); - let config = Config::get_or_compute(&*client).expect("config available"); + let config = Config::get(&*client).expect("config available"); let (block_import, link) = crate::block_import(config, client.clone(), client.clone()) .expect("can initialize block-import"); @@ -320,7 +320,7 @@ impl TestNetFactory for BabeTestNet { ) -> Self::Verifier { use substrate_test_runtime_client::DefaultTestClientBuilderExt; - let client = client.as_full().expect("only full clients are used in test"); + let client = client.as_client(); trace!(target: "babe", "Creating a verifier"); // ensure block import and verifier are linked correctly. @@ -336,9 +336,9 @@ impl TestNetFactory for BabeTestNet { select_chain: longest_chain, create_inherent_data_providers: Box::new(|_, _| async { let timestamp = TimestampInherentDataProvider::from_system_time(); - let slot = InherentDataProvider::from_timestamp_and_duration( + let slot = InherentDataProvider::from_timestamp_and_slot_duration( *timestamp, - Duration::from_secs(6), + SlotDuration::from_millis(6000), ); Ok((timestamp, slot)) @@ -395,7 +395,7 @@ fn run_one_test(mutator: impl Fn(&mut TestHeader, Stage) + Send + Sync + 'static for (peer_id, seed) in peers { let mut net = net.lock(); let peer = net.peer(*peer_id); - let client = peer.client().as_full().expect("Only full clients are used in tests").clone(); + let client = peer.client().as_client(); let select_chain = peer.select_chain().expect("Full client has select_chain"); let keystore_path = tempfile::tempdir().expect("Creates keystore path"); @@ -449,9 +449,9 @@ fn run_one_test(mutator: impl Fn(&mut TestHeader, Stage) + Send + Sync + 'static sync_oracle: DummyOracle, create_inherent_data_providers: Box::new(|_, _| async { let timestamp = TimestampInherentDataProvider::from_system_time(); - let slot = InherentDataProvider::from_timestamp_and_duration( + let slot = InherentDataProvider::from_timestamp_and_slot_duration( *timestamp, - Duration::from_secs(6), + SlotDuration::from_millis(6000), ); Ok((timestamp, slot)) @@ -608,8 +608,8 @@ fn propose_and_import_block( slot: Option, proposer_factory: &mut DummyFactory, block_import: &mut BoxBlockImport, -) -> sp_core::H256 { - let mut proposer = futures::executor::block_on(proposer_factory.init(parent)).unwrap(); +) -> Hash { + let mut proposer = block_on(proposer_factory.init(parent)).unwrap(); let slot = slot.unwrap_or_else(|| { let parent_pre_digest = find_pre_digest::(parent).unwrap(); @@ -625,7 +625,7 @@ fn propose_and_import_block( let parent_hash = parent.hash(); - let mut block = futures::executor::block_on(proposer.propose_with(pre_digest)).unwrap().block; + let mut block = block_on(proposer.propose_with(pre_digest)).unwrap().block; let epoch_descriptor = proposer_factory .epoch_changes @@ -673,13 +673,36 @@ fn propose_and_import_block( post_hash } +// Propose and import n valid BABE blocks that are built on top of the given parent. +// The proposer takes care of producing epoch change digests according to the epoch +// duration (which is set to 6 slots in the test runtime). +fn propose_and_import_blocks( + client: &PeersFullClient, + proposer_factory: &mut DummyFactory, + block_import: &mut BoxBlockImport, + parent_id: BlockId, + n: usize, +) -> Vec { + let mut hashes = Vec::with_capacity(n); + let mut parent_header = client.header(&parent_id).unwrap().unwrap(); + + for _ in 0..n { + let block_hash = + propose_and_import_block(&parent_header, None, proposer_factory, block_import); + hashes.push(block_hash); + parent_header = client.header(&BlockId::Hash(block_hash)).unwrap().unwrap(); + } + + hashes +} + #[test] fn importing_block_one_sets_genesis_epoch() { let mut net = BabeTestNet::new(1); let peer = net.peer(0); let data = peer.data.as_ref().expect("babe link set up during initialization"); - let client = peer.client().as_full().expect("Only full clients are used in tests").clone(); + let client = peer.client().as_client(); let mut proposer_factory = DummyFactory { client: client.clone(), @@ -699,12 +722,12 @@ fn importing_block_one_sets_genesis_epoch() { &mut block_import, ); - let genesis_epoch = Epoch::genesis(&data.link.config, 999.into()); + let genesis_epoch = Epoch::genesis(data.link.config.genesis_config(), 999.into()); let epoch_changes = data.link.epoch_changes.shared_data(); let epoch_for_second_block = epoch_changes .epoch_data_for_child_of(descendent_query(&*client), &block_hash, 1, 1000.into(), |slot| { - Epoch::genesis(&data.link.config, slot) + Epoch::genesis(data.link.config.genesis_config(), slot) }) .unwrap() .unwrap(); @@ -713,15 +736,14 @@ fn importing_block_one_sets_genesis_epoch() { } #[test] -fn importing_epoch_change_block_prunes_tree() { - use sc_client_api::Finalizer; - +fn revert_prunes_epoch_changes_and_removes_weights() { let mut net = BabeTestNet::new(1); let peer = net.peer(0); let data = peer.data.as_ref().expect("babe link set up during initialization"); - let client = peer.client().as_full().expect("Only full clients are used in tests").clone(); + let client = peer.client().as_client(); + let backend = peer.client().as_backend(); let mut block_import = data.block_import.lock().take().expect("import set up during init"); let epoch_changes = data.link.epoch_changes.clone(); @@ -732,26 +754,89 @@ fn importing_epoch_change_block_prunes_tree() { mutator: Arc::new(|_, _| ()), }; - // This is just boilerplate code for proposing and importing n valid BABE - // blocks that are built on top of the given parent. The proposer takes care - // of producing epoch change digests according to the epoch duration (which - // is set to 6 slots in the test runtime). - let mut propose_and_import_blocks = |parent_id, n| { - let mut hashes = Vec::new(); - let mut parent_header = client.header(&parent_id).unwrap().unwrap(); - - for _ in 0..n { - let block_hash = propose_and_import_block( - &parent_header, - None, - &mut proposer_factory, - &mut block_import, - ); - hashes.push(block_hash); - parent_header = client.header(&BlockId::Hash(block_hash)).unwrap().unwrap(); - } + let mut propose_and_import_blocks_wrap = |parent_id, n| { + propose_and_import_blocks(&client, &mut proposer_factory, &mut block_import, parent_id, n) + }; + + // Test scenario. + // Information for epoch 19 is produced on three different forks at block #13. + // One branch starts before the revert point (epoch data should be maintained). + // One branch starts after the revert point (epoch data should be removed). + // + // *----------------- F(#13) --#18 < fork #2 + // / + // A(#1) ---- B(#7) ----#8----+-----#12----- C(#13) ---- D(#19) ------#21 < canon + // \ ^ \ + // \ revert *---- G(#13) ---- H(#19) ---#20 < fork #3 + // \ to #10 + // *-----E(#7)---#11 < fork #1 + let canon = propose_and_import_blocks_wrap(BlockId::Number(0), 21); + let fork1 = propose_and_import_blocks_wrap(BlockId::Hash(canon[0]), 10); + let fork2 = propose_and_import_blocks_wrap(BlockId::Hash(canon[7]), 10); + let fork3 = propose_and_import_blocks_wrap(BlockId::Hash(canon[11]), 8); + + // We should be tracking a total of 9 epochs in the fork tree + assert_eq!(epoch_changes.shared_data().tree().iter().count(), 8); + // And only one root + assert_eq!(epoch_changes.shared_data().tree().roots().count(), 1); + + // Revert canon chain to block #10 (best(21) - 11) + revert(client.clone(), backend, 11).expect("revert should work for baked test scenario"); + + // Load and check epoch changes. + + let actual_nodes = aux_schema::load_epoch_changes::( + &*client, + data.link.config.genesis_config(), + ) + .expect("load epoch changes") + .shared_data() + .tree() + .iter() + .map(|(h, _, _)| *h) + .collect::>(); + + let expected_nodes = vec![ + canon[0], // A + canon[6], // B + fork2[4], // F + fork1[5], // E + ]; + + assert_eq!(actual_nodes, expected_nodes); + + let weight_data_check = |hashes: &[Hash], expected: bool| { + hashes.iter().all(|hash| { + aux_schema::load_block_weight(&*client, hash).unwrap().is_some() == expected + }) + }; + assert!(weight_data_check(&canon[..10], true)); + assert!(weight_data_check(&canon[10..], false)); + assert!(weight_data_check(&fork1, true)); + assert!(weight_data_check(&fork2, true)); + assert!(weight_data_check(&fork3, false)); +} + +#[test] +fn importing_epoch_change_block_prunes_tree() { + let mut net = BabeTestNet::new(1); + + let peer = net.peer(0); + let data = peer.data.as_ref().expect("babe link set up during initialization"); + + let client = peer.client().as_client(); + let mut block_import = data.block_import.lock().take().expect("import set up during init"); + let epoch_changes = data.link.epoch_changes.clone(); + + let mut proposer_factory = DummyFactory { + client: client.clone(), + config: data.link.config.clone(), + epoch_changes: data.link.epoch_changes.clone(), + mutator: Arc::new(|_, _| ()), + }; - hashes + let mut propose_and_import_blocks_wrap = |parent_id, n| { + propose_and_import_blocks(&client, &mut proposer_factory, &mut block_import, parent_id, n) }; // This is the block tree that we're going to use in this test. Each node @@ -766,12 +851,12 @@ fn importing_epoch_change_block_prunes_tree() { // Create and import the canon chain and keep track of fork blocks (A, C, D) // from the diagram above. - let canon_hashes = propose_and_import_blocks(BlockId::Number(0), 30); + let canon_hashes = propose_and_import_blocks_wrap(BlockId::Number(0), 30); // Create the forks - let fork_1 = propose_and_import_blocks(BlockId::Hash(canon_hashes[0]), 10); - let fork_2 = propose_and_import_blocks(BlockId::Hash(canon_hashes[12]), 15); - let fork_3 = propose_and_import_blocks(BlockId::Hash(canon_hashes[18]), 10); + let fork_1 = propose_and_import_blocks_wrap(BlockId::Hash(canon_hashes[0]), 10); + let fork_2 = propose_and_import_blocks_wrap(BlockId::Hash(canon_hashes[12]), 15); + let fork_3 = propose_and_import_blocks_wrap(BlockId::Hash(canon_hashes[18]), 10); // We should be tracking a total of 9 epochs in the fork tree assert_eq!(epoch_changes.shared_data().tree().iter().count(), 9); @@ -782,7 +867,7 @@ fn importing_epoch_change_block_prunes_tree() { // We finalize block #13 from the canon chain, so on the next epoch // change the tree should be pruned, to not contain F (#7). client.finalize_block(BlockId::Hash(canon_hashes[12]), None, false).unwrap(); - propose_and_import_blocks(BlockId::Hash(client.chain_info().best_hash), 7); + propose_and_import_blocks_wrap(BlockId::Hash(client.chain_info().best_hash), 7); // at this point no hashes from the first fork must exist on the tree assert!(!epoch_changes @@ -809,7 +894,7 @@ fn importing_epoch_change_block_prunes_tree() { // finalizing block #25 from the canon chain should prune out the second fork client.finalize_block(BlockId::Hash(canon_hashes[24]), None, false).unwrap(); - propose_and_import_blocks(BlockId::Hash(client.chain_info().best_hash), 8); + propose_and_import_blocks_wrap(BlockId::Hash(client.chain_info().best_hash), 8); // at this point no hashes from the second fork must exist on the tree assert!(!epoch_changes @@ -836,7 +921,7 @@ fn verify_slots_are_strictly_increasing() { let peer = net.peer(0); let data = peer.data.as_ref().expect("babe link set up during initialization"); - let client = peer.client().as_full().expect("Only full clients are used in tests").clone(); + let client = peer.client().as_client(); let mut block_import = data.block_import.lock().take().expect("import set up during init"); let mut proposer_factory = DummyFactory { @@ -894,3 +979,68 @@ fn babe_transcript_generation_match() { }; debug_assert!(test(orig_transcript) == test(transcript_from_data(new_transcript))); } + +#[test] +fn obsolete_blocks_aux_data_cleanup() { + let mut net = BabeTestNet::new(1); + + let peer = net.peer(0); + let data = peer.data.as_ref().expect("babe link set up during initialization"); + let client = peer.client().as_client(); + + // Register the handler (as done by `babe_start`) + let client_clone = client.clone(); + let on_finality = move |summary: &FinalityNotification| { + aux_storage_cleanup(client_clone.as_ref(), summary) + }; + client.register_finality_action(Box::new(on_finality)); + + let mut proposer_factory = DummyFactory { + client: client.clone(), + config: data.link.config.clone(), + epoch_changes: data.link.epoch_changes.clone(), + mutator: Arc::new(|_, _| ()), + }; + + let mut block_import = data.block_import.lock().take().expect("import set up during init"); + + let mut propose_and_import_blocks_wrap = |parent_id, n| { + propose_and_import_blocks(&client, &mut proposer_factory, &mut block_import, parent_id, n) + }; + + let aux_data_check = |hashes: &[Hash], expected: bool| { + hashes.iter().all(|hash| { + aux_schema::load_block_weight(&*peer.client().as_backend(), hash) + .unwrap() + .is_some() == expected + }) + }; + + // Create the following test scenario: + // + // /-----B3 --- B4 ( < fork2 ) + // G --- A1 --- A2 --- A3 --- A4 ( < fork1 ) + // \-----C4 --- C5 ( < fork3 ) + + let fork1_hashes = propose_and_import_blocks_wrap(BlockId::Number(0), 4); + let fork2_hashes = propose_and_import_blocks_wrap(BlockId::Number(2), 2); + let fork3_hashes = propose_and_import_blocks_wrap(BlockId::Number(3), 2); + + // Check that aux data is present for all but the genesis block. + assert!(aux_data_check(&[client.chain_info().genesis_hash], false)); + assert!(aux_data_check(&fork1_hashes, true)); + assert!(aux_data_check(&fork2_hashes, true)); + assert!(aux_data_check(&fork3_hashes, true)); + + // Finalize A3 + client.finalize_block(BlockId::Number(3), None, true).unwrap(); + + // Wiped: A1, A2 + assert!(aux_data_check(&fork1_hashes[..2], false)); + // Present: A3, A4 + assert!(aux_data_check(&fork1_hashes[2..], true)); + // Wiped: B3, B4 + assert!(aux_data_check(&fork2_hashes, false)); + // Present C4, C5 + assert!(aux_data_check(&fork3_hashes, true)); +} diff --git a/client/consensus/babe/src/verification.rs b/client/consensus/babe/src/verification.rs index af118312dd07..41d1e1bfa5d3 100644 --- a/client/consensus/babe/src/verification.rs +++ b/client/consensus/babe/src/verification.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -31,8 +31,8 @@ use sp_consensus_babe::{ make_transcript, AuthorityId, AuthorityPair, AuthoritySignature, }; use sp_consensus_slots::Slot; -use sp_core::{Pair, Public}; -use sp_runtime::traits::{DigestItemFor, Header}; +use sp_core::{ByteArray, Pair}; +use sp_runtime::{traits::Header, DigestItem}; /// BABE verification parameters pub(super) struct VerificationParams<'a, B: 'a + BlockT> { @@ -61,10 +61,7 @@ pub(super) struct VerificationParams<'a, B: 'a + BlockT> { /// with each having different validation logic. pub(super) fn check_header( params: VerificationParams, -) -> Result>, Error> -where - DigestItemFor: CompatibleDigestItem, -{ +) -> Result, Error> { let VerificationParams { mut header, pre_digest, slot_now, epoch } = params; let authorities = &epoch.authorities; @@ -114,7 +111,7 @@ where ); check_secondary_plain_header::(pre_hash, secondary, sig, &epoch)?; - } + }, PreDigest::SecondaryVRF(secondary) if epoch.config.allowed_slots.is_secondary_vrf_slots_allowed() => { @@ -125,7 +122,7 @@ where ); check_secondary_vrf_header::(pre_hash, secondary, sig, &epoch)?; - } + }, _ => return Err(babe_err(Error::SecondarySlotAssignmentsDisabled)), } @@ -137,9 +134,9 @@ where Ok(CheckedHeader::Checked(header, info)) } -pub(super) struct VerifiedHeaderInfo { - pub(super) pre_digest: DigestItemFor, - pub(super) seal: DigestItemFor, +pub(super) struct VerifiedHeaderInfo { + pub(super) pre_digest: DigestItem, + pub(super) seal: DigestItem, pub(super) author: AuthorityId, } diff --git a/client/consensus/common/Cargo.toml b/client/consensus/common/Cargo.toml index 6829bd2c6d8b..4bd5cff457f5 100644 --- a/client/consensus/common/Cargo.toml +++ b/client/consensus/common/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-consensus" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Collection of common consensus specific imlementations for Substrate (client)" readme = "README.md" @@ -13,22 +13,22 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -thiserror = "1.0.21" -libp2p = { version = "0.39.1", default-features = false } +thiserror = "1.0.30" +libp2p = { version = "0.40.0", default-features = false } log = "0.4.8" -futures = { version = "0.3.1", features = ["thread-pool"] } +futures = { version = "0.3.21", features = ["thread-pool"] } futures-timer = "3.0.1" sc-client-api = { version = "4.0.0-dev", path = "../../api" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sp-core = { path = "../../../primitives/core", version = "4.0.0-dev" } +sp-core = { path = "../../../primitives/core", version = "6.0.0"} sp-consensus = { path = "../../../primitives/consensus/common", version = "0.10.0-dev" } -sp-state-machine = { version = "0.10.0-dev", path = "../../../primitives/state-machine" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-state-machine = { version = "0.12.0", path = "../../../primitives/state-machine" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } sc-utils = { version = "4.0.0-dev", path = "../../utils" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } -parking_lot = "0.11.1" +parking_lot = "0.12.0" serde = { version = "1.0", features = ["derive"] } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.9.0" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.10.0-dev" } async-trait = "0.1.42" [dev-dependencies] diff --git a/client/consensus/common/src/block_import.rs b/client/consensus/common/src/block_import.rs index 6d411dd9afbf..24fec9b974a4 100644 --- a/client/consensus/common/src/block_import.rs +++ b/client/consensus/common/src/block_import.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,8 +20,8 @@ use serde::{Deserialize, Serialize}; use sp_runtime::{ - traits::{Block as BlockT, DigestItemFor, HashFor, Header as HeaderT, NumberFor}, - Justification, Justifications, + traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor}, + DigestItem, Justification, Justifications, }; use std::{any::Any, borrow::Cow, collections::HashMap, sync::Arc}; @@ -74,7 +74,7 @@ impl ImportResult { &self, hash: &B::Hash, number: NumberFor, - justification_sync_link: &mut dyn JustificationSyncLink, + justification_sync_link: &dyn JustificationSyncLink, ) where B: BlockT, { @@ -122,7 +122,7 @@ pub struct BlockCheckParams { /// Precomputed storage. pub enum StorageChanges { /// Changes coming from block execution. - Changes(sp_state_machine::StorageChanges, NumberFor>), + Changes(sp_state_machine::StorageChanges>), /// Whole new state. Import(ImportedState), } @@ -133,7 +133,7 @@ pub struct ImportedState { /// Target block hash. pub block: B::Hash, /// State keys and values. - pub state: Vec<(Vec, Vec)>, + pub state: sp_state_machine::KeyValueStates, } impl std::fmt::Debug for ImportedState { @@ -175,7 +175,7 @@ pub struct BlockImportParams { pub justifications: Option, /// Digest items that have been added after the runtime for external /// work, like a consensus signature. - pub post_digests: Vec>, + pub post_digests: Vec, /// The body of the block. pub body: Option>, /// Indexed transaction body of the block. @@ -190,8 +190,9 @@ pub struct BlockImportParams { /// rejects block import if there are still intermediate values that remain unhandled. pub intermediates: HashMap, Box>, /// Auxiliary consensus data produced by the block. - /// Contains a list of key-value pairs. If values are `None`, the keys - /// will be deleted. + /// Contains a list of key-value pairs. If values are `None`, the keys will be deleted. These + /// changes will be applied to `AuxStore` database all as one batch, which is more efficient + /// than updating `AuxStore` directly. pub auxiliary: Vec<(Vec, Option>)>, /// Fork choice strategy of this import. This should only be set by a /// synchronous import, otherwise it may race against other imports. @@ -286,9 +287,9 @@ impl BlockImportParams { pub fn take_intermediate(&mut self, key: &[u8]) -> Result, Error> { let (k, v) = self.intermediates.remove_entry(key).ok_or(Error::NoIntermediate)?; - v.downcast::().or_else(|v| { + v.downcast::().map_err(|v| { self.intermediates.insert(k, v); - Err(Error::InvalidIntermediate) + Error::InvalidIntermediate }) } diff --git a/client/consensus/common/src/import_queue.rs b/client/consensus/common/src/import_queue.rs index 3f2126ccadf6..8b560d044741 100644 --- a/client/consensus/common/src/import_queue.rs +++ b/client/consensus/common/src/import_queue.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -161,21 +161,34 @@ pub enum BlockImportStatus { } /// Block import error. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum BlockImportError { /// Block missed header, can't be imported + #[error("block is missing a header (origin = {0:?})")] IncompleteHeader(Option), + /// Block verification failed, can't be imported + #[error("block verification failed (origin = {0:?}): {1}")] VerificationFailed(Option, String), + /// Block is known to be Bad + #[error("bad block (origin = {0:?})")] BadBlock(Option), + /// Parent state is missing. + #[error("block is missing parent state")] MissingState, + /// Block has an unknown parent + #[error("block has an unknown parent")] UnknownParent, + /// Block import has been cancelled. This can happen if the parent block fails to be imported. + #[error("import has been cancelled")] Cancelled, + /// Other error. + #[error("consensus error: {0}")] Other(ConsensusError), } @@ -245,7 +258,7 @@ pub(crate) async fn import_single_block_metered< Err(BlockImportError::BadBlock(peer.clone())) }, Err(e) => { - debug!(target: "sync", "Error importing block {}: {:?}: {:?}", number, hash, e); + debug!(target: "sync", "Error importing block {}: {:?}: {}", number, hash, e); Err(BlockImportError::Other(e)) }, }; diff --git a/client/consensus/common/src/import_queue/basic_queue.rs b/client/consensus/common/src/import_queue/basic_queue.rs index 9042c8798be4..5134dc041c26 100644 --- a/client/consensus/common/src/import_queue/basic_queue.rs +++ b/client/consensus/common/src/import_queue/basic_queue.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -89,7 +89,11 @@ impl BasicQueue { metrics, ); - spawner.spawn_essential_blocking("basic-block-import-worker", future.boxed()); + spawner.spawn_essential_blocking( + "basic-block-import-worker", + Some("block-import"), + future.boxed(), + ); Self { justification_sender, block_import_sender, result_port, _phantom: PhantomData } } @@ -299,11 +303,11 @@ impl BlockImportWorker { .map_err(|e| { debug!( target: "sync", - "Justification import failed with {:?} for hash: {:?} number: {:?} coming from node: {:?}", - e, + "Justification import failed for hash = {:?} with number = {:?} coming from node = {:?} with error: {}", hash, number, who, + e, ); e }) diff --git a/client/consensus/common/src/import_queue/buffered_link.rs b/client/consensus/common/src/import_queue/buffered_link.rs index 87ea6dde5c47..8fb5689075ab 100644 --- a/client/consensus/common/src/import_queue/buffered_link.rs +++ b/client/consensus/common/src/import_queue/buffered_link.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/consensus/common/src/lib.rs b/client/consensus/common/src/lib.rs index 640bad237e88..f291cc270481 100644 --- a/client/consensus/common/src/lib.rs +++ b/client/consensus/common/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/consensus/common/src/longest_chain.rs b/client/consensus/common/src/longest_chain.rs index 7ec91a5ad87e..b38183b8ac11 100644 --- a/client/consensus/common/src/longest_chain.rs +++ b/client/consensus/common/src/longest_chain.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/consensus/common/src/metrics.rs b/client/consensus/common/src/metrics.rs index ade45e3ffb68..8314049238b8 100644 --- a/client/consensus/common/src/metrics.rs +++ b/client/consensus/common/src/metrics.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -41,28 +41,34 @@ impl Metrics { Ok(Self { import_queue_processed: register( CounterVec::new( - Opts::new("import_queue_processed_total", "Blocks processed by import queue"), + Opts::new( + "substrate_import_queue_processed_total", + "Blocks processed by import queue", + ), &["result"], // 'success or failure )?, registry, )?, block_verification_time: register( HistogramVec::new( - HistogramOpts::new("block_verification_time", "Time taken to verify blocks"), + HistogramOpts::new( + "substrate_block_verification_time", + "Time taken to verify blocks", + ), &["result"], )?, registry, )?, block_verification_and_import_time: register( Histogram::with_opts(HistogramOpts::new( - "block_verification_and_import_time", + "substrate_block_verification_and_import_time", "Time taken to verify and import blocks", ))?, registry, )?, justification_import_time: register( Histogram::with_opts(HistogramOpts::new( - "justification_import_time", + "substrate_justification_import_time", "Time taken to import justifications", ))?, registry, diff --git a/client/consensus/common/src/shared_data.rs b/client/consensus/common/src/shared_data.rs index 7a25660e08aa..fb04966b5db6 100644 --- a/client/consensus/common/src/shared_data.rs +++ b/client/consensus/common/src/shared_data.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -167,6 +167,13 @@ struct SharedDataInner { /// // As we don't know the order of the threads, we need to check for both combinations /// assert!(*data == "hello world321" || *data == "hello world312"); /// ``` +/// +/// # Deadlock +/// +/// Be aware that this data structure doesn't give you any guarantees that you can not create a +/// deadlock. If you use [`release_mutex`](SharedDataLocked::release_mutex) followed by a call +/// to [`shared_data`](Self::shared_data) in the same thread will make your program dead lock. +/// The same applies when you are using a single threaded executor. pub struct SharedData { inner: Arc>>, cond_var: Arc, diff --git a/client/consensus/epochs/Cargo.toml b/client/consensus/epochs/Cargo.toml index 78e5cc31ea07..2caf60547cce 100644 --- a/client/consensus/epochs/Cargo.toml +++ b/client/consensus/epochs/Cargo.toml @@ -3,9 +3,9 @@ name = "sc-consensus-epochs" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "Generic epochs-based utilities for consensus" -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,9 +13,9 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", features = ["derive"] } fork-tree = { version = "3.0.0", path = "../../../utils/fork-tree" } -sp-runtime = { path = "../../../primitives/runtime" , version = "4.0.0-dev"} +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } sc-client-api = { path = "../../api" , version = "4.0.0-dev"} sc-consensus = { path = "../common" , version = "0.10.0-dev"} diff --git a/client/consensus/epochs/src/lib.rs b/client/consensus/epochs/src/lib.rs index f3cfc55bae69..90081bf9af44 100644 --- a/client/consensus/epochs/src/lib.rs +++ b/client/consensus/epochs/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,7 +21,7 @@ pub mod migration; use codec::{Decode, Encode}; -use fork_tree::ForkTree; +use fork_tree::{FilterAction, ForkTree}; use sc_client_api::utils::is_descendent_of; use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata}; use sp_runtime::traits::{Block as BlockT, NumberFor, One, Zero}; @@ -78,11 +78,11 @@ where /// /// Once an epoch is created, it must have a known `start_slot` and `end_slot`, which cannot be /// changed. Consensus engine may modify any other data in the epoch, if needed. -pub trait Epoch { +pub trait Epoch: std::fmt::Debug { /// Descriptor for the next epoch. type NextEpochDescriptor; /// Type of the slot number. - type Slot: Ord + Copy; + type Slot: Ord + Copy + std::fmt::Debug; /// The starting slot of the epoch. fn start_slot(&self) -> Self::Slot; @@ -228,14 +228,21 @@ impl ViableEpochDescriptor { } /// Persisted epoch stored in EpochChanges. -#[derive(Clone, Encode, Decode)] -pub enum PersistedEpoch { +#[derive(Clone, Encode, Decode, Debug)] +pub enum PersistedEpoch { /// Genesis persisted epoch data. epoch_0, epoch_1. Genesis(E, E), /// Regular persisted epoch data. epoch_n. Regular(E), } +impl PersistedEpoch { + /// Returns if this is a genesis epoch. + pub fn is_genesis(&self) -> bool { + matches!(self, Self::Genesis(_, _)) + } +} + impl<'a, E: Epoch> From<&'a PersistedEpoch> for PersistedEpochHeader { fn from(epoch: &'a PersistedEpoch) -> Self { match epoch { @@ -246,8 +253,23 @@ impl<'a, E: Epoch> From<&'a PersistedEpoch> for PersistedEpochHeader { } } +impl PersistedEpoch { + /// Map the epoch to a different type using a conversion function. + pub fn map(self, h: &Hash, n: &Number, f: &mut F) -> PersistedEpoch + where + B: Epoch, + F: FnMut(&Hash, &Number, E) -> B, + { + match self { + PersistedEpoch::Genesis(epoch_0, epoch_1) => + PersistedEpoch::Genesis(f(h, n, epoch_0), f(h, n, epoch_1)), + PersistedEpoch::Regular(epoch_n) => PersistedEpoch::Regular(f(h, n, epoch_n)), + } + } +} + /// Persisted epoch header stored in ForkTree. -#[derive(Encode, Decode, PartialEq, Eq)] +#[derive(Encode, Decode, PartialEq, Eq, Debug)] pub enum PersistedEpochHeader { /// Genesis persisted epoch header. epoch_0, epoch_1. Genesis(EpochHeader, EpochHeader), @@ -264,6 +286,25 @@ impl Clone for PersistedEpochHeader { } } +impl PersistedEpochHeader { + /// Map the epoch header to a different type. + pub fn map(self) -> PersistedEpochHeader + where + B: Epoch, + { + match self { + PersistedEpochHeader::Genesis(epoch_0, epoch_1) => PersistedEpochHeader::Genesis( + EpochHeader { start_slot: epoch_0.start_slot, end_slot: epoch_0.end_slot }, + EpochHeader { start_slot: epoch_1.start_slot, end_slot: epoch_1.end_slot }, + ), + PersistedEpochHeader::Regular(epoch_n) => PersistedEpochHeader::Regular(EpochHeader { + start_slot: epoch_n.start_slot, + end_slot: epoch_n.end_slot, + }), + } + } +} + /// A fresh, incremented epoch to import into the underlying fork-tree. /// /// Create this with `ViableEpoch::increment`. @@ -279,6 +320,106 @@ impl AsRef for IncrementedEpoch { } } +/// A pair of epochs for the gap block download validation. +/// Block gap is created after the warp sync is complete. Blocks +/// are imported both at the tip of the chain and at the start of the gap. +/// This holds a pair of epochs that are required to validate headers +/// at the start of the gap. Since gap download does not allow forks we don't +/// need to keep a tree of epochs. +#[derive(Clone, Encode, Decode, Debug)] +pub struct GapEpochs { + current: (Hash, Number, PersistedEpoch), + next: Option<(Hash, Number, E)>, +} + +impl GapEpochs +where + Hash: Copy + PartialEq + std::fmt::Debug, + Number: Copy + PartialEq + std::fmt::Debug, + E: Epoch, +{ + /// Check if given slot matches one of the gap epochs. + /// Returns epoch identifier if it does. + fn matches( + &self, + slot: E::Slot, + ) -> Option<(Hash, Number, EpochHeader, EpochIdentifierPosition)> { + match &self.current { + (_, _, PersistedEpoch::Genesis(epoch_0, _)) + if slot >= epoch_0.start_slot() && slot < epoch_0.end_slot() => + return Some(( + self.current.0, + self.current.1, + epoch_0.into(), + EpochIdentifierPosition::Genesis0, + )), + (_, _, PersistedEpoch::Genesis(_, epoch_1)) + if slot >= epoch_1.start_slot() && slot < epoch_1.end_slot() => + return Some(( + self.current.0, + self.current.1, + epoch_1.into(), + EpochIdentifierPosition::Genesis1, + )), + (_, _, PersistedEpoch::Regular(epoch_n)) + if slot >= epoch_n.start_slot() && slot < epoch_n.end_slot() => + return Some(( + self.current.0, + self.current.1, + epoch_n.into(), + EpochIdentifierPosition::Regular, + )), + _ => {}, + }; + match &self.next { + Some((h, n, epoch_n)) if slot >= epoch_n.start_slot() && slot < epoch_n.end_slot() => + Some((*h, *n, epoch_n.into(), EpochIdentifierPosition::Regular)), + _ => None, + } + } + + /// Returns epoch data if it matches given identifier. + pub fn epoch(&self, id: &EpochIdentifier) -> Option<&E> { + match (&self.current, &self.next) { + ((h, n, e), _) if h == &id.hash && n == &id.number => match e { + PersistedEpoch::Genesis(ref epoch_0, _) + if id.position == EpochIdentifierPosition::Genesis0 => + Some(epoch_0), + PersistedEpoch::Genesis(_, ref epoch_1) + if id.position == EpochIdentifierPosition::Genesis1 => + Some(epoch_1), + PersistedEpoch::Regular(ref epoch_n) + if id.position == EpochIdentifierPosition::Regular => + Some(epoch_n), + _ => None, + }, + (_, Some((h, n, e))) + if h == &id.hash && + n == &id.number && id.position == EpochIdentifierPosition::Regular => + Some(e), + _ => None, + } + } + + /// Import a new gap epoch, potentially replacing an old epoch. + fn import(&mut self, slot: E::Slot, hash: Hash, number: Number, epoch: E) -> Result<(), E> { + match (&mut self.current, &mut self.next) { + ((_, _, PersistedEpoch::Genesis(_, epoch_1)), _) if slot == epoch_1.end_slot() => { + self.next = Some((hash, number, epoch)); + Ok(()) + }, + (_, Some((_, _, epoch_n))) if slot == epoch_n.end_slot() => { + let (cur_h, cur_n, cur_epoch) = + self.next.take().expect("Already matched as `Some`"); + self.current = (cur_h, cur_n, PersistedEpoch::Regular(cur_epoch)); + self.next = Some((hash, number, epoch)); + Ok(()) + }, + _ => Err(epoch), + } + } +} + /// Tree of all epoch changes across all *seen* forks. Data stored in tree is /// the hash and block number of the block signaling the epoch change, and the /// epoch that was signalled at that block. @@ -294,10 +435,14 @@ impl AsRef for IncrementedEpoch { /// same DAG entry, pinned to a specific block #1. /// /// Further epochs (epoch_2, ..., epoch_n) each get their own entry. -#[derive(Clone, Encode, Decode)] +/// +/// Also maintains a pair of epochs for the start of the gap, +/// as long as there's an active gap download after a warp sync. +#[derive(Clone, Encode, Decode, Debug)] pub struct EpochChanges { inner: ForkTree>, epochs: BTreeMap<(Hash, Number), PersistedEpoch>, + gap: Option>, } // create a fake header hash which hasn't been included in the chain. @@ -315,14 +460,14 @@ where Number: Ord, { fn default() -> Self { - EpochChanges { inner: ForkTree::new(), epochs: BTreeMap::new() } + EpochChanges { inner: ForkTree::new(), epochs: BTreeMap::new(), gap: None } } } impl EpochChanges where - Hash: PartialEq + Ord + AsRef<[u8]> + AsMut<[u8]> + Copy, - Number: Ord + One + Zero + Add + Sub + Copy, + Hash: PartialEq + Ord + AsRef<[u8]> + AsMut<[u8]> + Copy + std::fmt::Debug, + Number: Ord + One + Zero + Add + Sub + Copy + std::fmt::Debug, { /// Create a new epoch change. pub fn new() -> Self { @@ -335,6 +480,11 @@ where self.inner.rebalance() } + /// Clear gap epochs if any. + pub fn clear_gap(&mut self) { + self.gap = None; + } + /// Map the epoch changes from one storing data to a different one. pub fn map(self, mut f: F) -> EpochChanges where @@ -342,31 +492,15 @@ where F: FnMut(&Hash, &Number, E) -> B, { EpochChanges { - inner: self.inner.map(&mut |_, _, header| match header { - PersistedEpochHeader::Genesis(epoch_0, epoch_1) => PersistedEpochHeader::Genesis( - EpochHeader { start_slot: epoch_0.start_slot, end_slot: epoch_0.end_slot }, - EpochHeader { start_slot: epoch_1.start_slot, end_slot: epoch_1.end_slot }, - ), - PersistedEpochHeader::Regular(epoch_n) => - PersistedEpochHeader::Regular(EpochHeader { - start_slot: epoch_n.start_slot, - end_slot: epoch_n.end_slot, - }), + inner: self.inner.map(&mut |_, _, header: PersistedEpochHeader| header.map()), + gap: self.gap.map(|GapEpochs { current: (h, n, header), next }| GapEpochs { + current: (h, n, header.map(&h, &n, &mut f)), + next: next.map(|(h, n, e)| (h, n, f(&h, &n, e))), }), epochs: self .epochs .into_iter() - .map(|((hash, number), epoch)| { - let bepoch = match epoch { - PersistedEpoch::Genesis(epoch_0, epoch_1) => PersistedEpoch::Genesis( - f(&hash, &number, epoch_0), - f(&hash, &number, epoch_1), - ), - PersistedEpoch::Regular(epoch_n) => - PersistedEpoch::Regular(f(&hash, &number, epoch_n)), - }; - ((hash, number), bepoch) - }) + .map(|((hash, number), epoch)| ((hash, number), epoch.map(&hash, &number, &mut f))) .collect(), } } @@ -402,6 +536,9 @@ where /// Get a reference to an epoch with given identifier. pub fn epoch(&self, id: &EpochIdentifier) -> Option<&E> { + if let Some(e) = &self.gap.as_ref().and_then(|gap| gap.epoch(id)) { + return Some(e) + } self.epochs.get(&(id.hash, id.number)).and_then(|v| match v { PersistedEpoch::Genesis(ref epoch_0, _) if id.position == EpochIdentifierPosition::Genesis0 => @@ -523,6 +660,20 @@ where parent_number: Number, slot: E::Slot, ) -> Result>, fork_tree::Error> { + if parent_number == Zero::zero() { + // need to insert the genesis epoch. + return Ok(Some(ViableEpochDescriptor::UnimportedGenesis(slot))) + } + + if let Some(gap) = &self.gap { + if let Some((hash, number, hdr, position)) = gap.matches(slot) { + return Ok(Some(ViableEpochDescriptor::Signaled( + EpochIdentifier { position, hash, number }, + hdr, + ))) + } + } + // find_node_where will give you the node in the fork-tree which is an ancestor // of the `parent_hash` by default. if the last epoch was signalled at the parent_hash, // then it won't be returned. we need to create a new fake chain head hash which @@ -532,11 +683,6 @@ where let is_descendent_of = descendent_of_builder.build_is_descendent_of(Some((fake_head_hash, *parent_hash))); - if parent_number == Zero::zero() { - // need to insert the genesis epoch. - return Ok(Some(ViableEpochDescriptor::UnimportedGenesis(slot))) - } - // We want to find the deepest node in the tree which is an ancestor // of our block and where the start slot of the epoch was before the // slot of our block. The genesis special-case doesn't need to look @@ -598,13 +744,30 @@ where ) -> Result<(), fork_tree::Error> { let is_descendent_of = descendent_of_builder.build_is_descendent_of(Some((hash, parent_hash))); - let header = PersistedEpochHeader::::from(&epoch.0); + let slot = epoch.as_ref().start_slot(); + let IncrementedEpoch(mut epoch) = epoch; + let header = PersistedEpochHeader::::from(&epoch); + + if let Some(gap) = &mut self.gap { + if let PersistedEpoch::Regular(e) = epoch { + epoch = match gap.import(slot, hash.clone(), number.clone(), e) { + Ok(()) => return Ok(()), + Err(e) => PersistedEpoch::Regular(e), + } + } + } else if epoch.is_genesis() && !self.epochs.values().all(|e| e.is_genesis()) { + // There's a genesis epoch imported when we already have an active epoch. + // This happens after the warp sync as the ancient blocks download start. + // We need to start tracking gap epochs here. + self.gap = Some(GapEpochs { current: (hash, number, epoch), next: None }); + return Ok(()) + } let res = self.inner.import(hash, number, header, &is_descendent_of); match res { Ok(_) | Err(fork_tree::Error::Duplicate) => { - self.epochs.insert((hash, number), epoch.0); + self.epochs.insert((hash, number), epoch); Ok(()) }, Err(e) => Err(e), @@ -635,6 +798,37 @@ where }); self.epochs.insert((hash, number), persisted); } + + /// Revert to a specified block given its `hash` and `number`. + /// This removes all the epoch changes information that were announced by + /// all the given block descendents. + pub fn revert>( + &mut self, + descendent_of_builder: D, + hash: Hash, + number: Number, + ) { + let is_descendent_of = descendent_of_builder.build_is_descendent_of(None); + + let filter = |node_hash: &Hash, node_num: &Number, _: &PersistedEpochHeader| { + if number >= *node_num && + (is_descendent_of(node_hash, &hash).unwrap_or_default() || *node_hash == hash) + { + // Continue the search in this subtree. + FilterAction::KeepNode + } else if number < *node_num && is_descendent_of(&hash, node_hash).unwrap_or_default() { + // Found a node to be removed. + FilterAction::Remove + } else { + // Not a parent or child of the one we're looking for, stop processing this branch. + FilterAction::KeepTree + } + }; + + self.inner.drain_filter(filter).for_each(|(h, n, _)| { + self.epochs.remove(&(h, n)); + }); + } } /// Type alias to produce the epoch-changes tree from a block type. @@ -916,4 +1110,182 @@ mod tests { assert!(epoch_for_x_child_before_genesis.is_none()); } } + + /// Test that ensures that the gap is not enabled when we import multiple genesis blocks. + #[test] + fn gap_is_not_enabled_when_multiple_genesis_epochs_are_imported() { + // X + // / + // 0 - A + // + let is_descendent_of = |base: &Hash, block: &Hash| -> Result { + match (base, *block) { + (b"0", _) => Ok(true), + _ => Ok(false), + } + }; + + let duration = 100; + + let make_genesis = |slot| Epoch { start_slot: slot, duration }; + + let mut epoch_changes = EpochChanges::new(); + let next_descriptor = (); + + // insert genesis epoch for A + { + let genesis_epoch_a_descriptor = epoch_changes + .epoch_descriptor_for_child_of(&is_descendent_of, b"0", 0, 100) + .unwrap() + .unwrap(); + + let incremented_epoch = epoch_changes + .viable_epoch(&genesis_epoch_a_descriptor, &make_genesis) + .unwrap() + .increment(next_descriptor.clone()); + + epoch_changes + .import(&is_descendent_of, *b"A", 1, *b"0", incremented_epoch) + .unwrap(); + } + + // insert genesis epoch for X + { + let genesis_epoch_x_descriptor = epoch_changes + .epoch_descriptor_for_child_of(&is_descendent_of, b"0", 0, 1000) + .unwrap() + .unwrap(); + + let incremented_epoch = epoch_changes + .viable_epoch(&genesis_epoch_x_descriptor, &make_genesis) + .unwrap() + .increment(next_descriptor.clone()); + + epoch_changes + .import(&is_descendent_of, *b"X", 1, *b"0", incremented_epoch) + .unwrap(); + } + + // Clearing the gap should be a no-op. + epoch_changes.clear_gap(); + + // Check that both epochs are available. + epoch_changes + .epoch_data_for_child_of(&is_descendent_of, b"A", 1, 101, &make_genesis) + .unwrap() + .unwrap(); + + epoch_changes + .epoch_data_for_child_of(&is_descendent_of, b"X", 1, 1001, &make_genesis) + .unwrap() + .unwrap(); + } + + #[test] + fn gap_epochs_advance() { + // 0 - 1 - 2 - 3 - .... 42 - 43 + let is_descendent_of = |base: &Hash, block: &Hash| -> Result { + match (base, *block) { + (b"0", _) => Ok(true), + (b"1", b) => Ok(b == *b"0"), + (b"2", b) => Ok(b == *b"1"), + (b"3", b) => Ok(b == *b"2"), + _ => Ok(false), + } + }; + + let duration = 100; + + let make_genesis = |slot| Epoch { start_slot: slot, duration }; + + let mut epoch_changes = EpochChanges::new(); + let next_descriptor = (); + + let epoch42 = Epoch { start_slot: 42, duration: 100 }; + let epoch43 = Epoch { start_slot: 43, duration: 100 }; + epoch_changes.reset(*b"0", *b"1", 4200, epoch42, epoch43); + assert!(epoch_changes.gap.is_none()); + + // Import a new genesis epoch, this should crate the gap. + let genesis_epoch_a_descriptor = epoch_changes + .epoch_descriptor_for_child_of(&is_descendent_of, b"0", 0, 100) + .unwrap() + .unwrap(); + + let incremented_epoch = epoch_changes + .viable_epoch(&genesis_epoch_a_descriptor, &make_genesis) + .unwrap() + .increment(next_descriptor.clone()); + + epoch_changes + .import(&is_descendent_of, *b"1", 1, *b"0", incremented_epoch) + .unwrap(); + assert!(epoch_changes.gap.is_some()); + + let genesis_epoch = epoch_changes + .epoch_descriptor_for_child_of(&is_descendent_of, b"0", 0, 100) + .unwrap() + .unwrap(); + + assert_eq!(genesis_epoch, ViableEpochDescriptor::UnimportedGenesis(100)); + + // Import more epochs and check that gap advances. + let import_epoch_1 = + epoch_changes.viable_epoch(&genesis_epoch, &make_genesis).unwrap().increment(()); + + let epoch_1 = import_epoch_1.as_ref().clone(); + epoch_changes + .import(&is_descendent_of, *b"1", 1, *b"0", import_epoch_1) + .unwrap(); + let genesis_epoch_data = epoch_changes.epoch_data(&genesis_epoch, &make_genesis).unwrap(); + let end_slot = genesis_epoch_data.end_slot(); + let x = epoch_changes + .epoch_data_for_child_of(&is_descendent_of, b"1", 1, end_slot, &make_genesis) + .unwrap() + .unwrap(); + + assert_eq!(x, epoch_1); + assert_eq!(epoch_changes.gap.as_ref().unwrap().current.0, *b"1"); + assert!(epoch_changes.gap.as_ref().unwrap().next.is_none()); + + let epoch_1_desriptor = epoch_changes + .epoch_descriptor_for_child_of(&is_descendent_of, b"1", 1, end_slot) + .unwrap() + .unwrap(); + let epoch_1 = epoch_changes.epoch_data(&epoch_1_desriptor, &make_genesis).unwrap(); + let import_epoch_2 = epoch_changes + .viable_epoch(&epoch_1_desriptor, &make_genesis) + .unwrap() + .increment(()); + let epoch_2 = import_epoch_2.as_ref().clone(); + epoch_changes + .import(&is_descendent_of, *b"2", 2, *b"1", import_epoch_2) + .unwrap(); + + let end_slot = epoch_1.end_slot(); + let x = epoch_changes + .epoch_data_for_child_of(&is_descendent_of, b"2", 2, end_slot, &make_genesis) + .unwrap() + .unwrap(); + assert_eq!(epoch_changes.gap.as_ref().unwrap().current.0, *b"1"); + assert_eq!(epoch_changes.gap.as_ref().unwrap().next.as_ref().unwrap().0, *b"2"); + assert_eq!(x, epoch_2); + + let epoch_2_desriptor = epoch_changes + .epoch_descriptor_for_child_of(&is_descendent_of, b"2", 2, end_slot) + .unwrap() + .unwrap(); + let import_epoch_3 = epoch_changes + .viable_epoch(&epoch_2_desriptor, &make_genesis) + .unwrap() + .increment(()); + epoch_changes + .import(&is_descendent_of, *b"3", 3, *b"2", import_epoch_3) + .unwrap(); + + assert_eq!(epoch_changes.gap.as_ref().unwrap().current.0, *b"2"); + + epoch_changes.clear_gap(); + assert!(epoch_changes.gap.is_none()); + } } diff --git a/client/consensus/epochs/src/migration.rs b/client/consensus/epochs/src/migration.rs index 49e08240df8c..c4ed47e9c1c0 100644 --- a/client/consensus/epochs/src/migration.rs +++ b/client/consensus/epochs/src/migration.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -30,9 +30,19 @@ pub struct EpochChangesV0 { inner: ForkTree>, } -/// Type alias for legacy definition of epoch changes. -pub type EpochChangesForV0 = +/// Legacy definition of epoch changes. +#[derive(Clone, Encode, Decode)] +pub struct EpochChangesV1 { + inner: ForkTree>, + epochs: BTreeMap<(Hash, Number), PersistedEpoch>, +} + +/// Type alias for v0 definition of epoch changes. +pub type EpochChangesV0For = EpochChangesV0<::Hash, NumberFor, Epoch>; +/// Type alias for v1 and v2 definition of epoch changes. +pub type EpochChangesV1For = + EpochChangesV1<::Hash, NumberFor, Epoch>; impl EpochChangesV0 where @@ -54,6 +64,17 @@ where header }); - EpochChanges { inner, epochs } + EpochChanges { inner, epochs, gap: None } + } +} + +impl EpochChangesV1 +where + Hash: PartialEq + Ord + Copy, + Number: Ord + Copy, +{ + /// Migrate the type into current epoch changes definition. + pub fn migrate(self) -> EpochChanges { + EpochChanges { inner: self.inner, epochs: self.epochs, gap: None } } } diff --git a/client/consensus/manual-seal/Cargo.toml b/client/consensus/manual-seal/Cargo.toml index d9ae8521c12f..360cfe862f87 100644 --- a/client/consensus/manual-seal/Cargo.toml +++ b/client/consensus/manual-seal/Cargo.toml @@ -3,9 +3,9 @@ name = "sc-consensus-manual-seal" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "Manual sealing engine for Substrate" -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,13 +13,13 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -derive_more = "0.99.2" -futures = "0.3.9" +thiserror = "1.0" +futures = "0.3.21" jsonrpc-core = "18.0.0" jsonrpc-core-client = "18.0.0" jsonrpc-derive = "18.0.0" log = "0.4.8" -codec = { package = "parity-scale-codec", version = "2.0.0" } +codec = { package = "parity-scale-codec", version = "3.0.0" } serde = { version = "1.0", features = ["derive"] } assert_matches = "1.3.0" async-trait = "0.1.50" @@ -27,25 +27,27 @@ async-trait = "0.1.50" sc-client-api = { path = "../../api", version = "4.0.0-dev" } sc-consensus = { version = "0.10.0-dev", path = "../../consensus/common" } sc-consensus-babe = { path = "../../consensus/babe", version = "0.10.0-dev" } +sc-consensus-aura = { path = "../../consensus/aura", version = "0.10.0-dev" } sc-consensus-epochs = { path = "../../consensus/epochs", version = "0.10.0-dev" } sp-consensus-babe = { path = "../../../primitives/consensus/babe", version = "0.10.0-dev" } +sp-consensus-aura = { path = "../../../primitives/consensus/aura", version = "0.10.0-dev" } sc-transaction-pool = { path = "../../transaction-pool", version = "4.0.0-dev" } sp-blockchain = { path = "../../../primitives/blockchain", version = "4.0.0-dev" } sp-consensus = { path = "../../../primitives/consensus/common", version = "0.10.0-dev" } sp-consensus-slots = { path = "../../../primitives/consensus/slots", version = "0.10.0-dev" } sp-inherents = { path = "../../../primitives/inherents", version = "4.0.0-dev" } -sp-runtime = { path = "../../../primitives/runtime", version = "4.0.0-dev" } -sp-core = { path = "../../../primitives/core", version = "4.0.0-dev" } -sp-keystore = { path = "../../../primitives/keystore", version = "0.10.0-dev" } +sp-runtime = { path = "../../../primitives/runtime", version = "6.0.0"} +sp-core = { path = "../../../primitives/core", version = "6.0.0"} +sp-keystore = { path = "../../../primitives/keystore", version = "0.12.0"} sp-api = { path = "../../../primitives/api", version = "4.0.0-dev" } sc-transaction-pool-api = { path = "../../../client/transaction-pool/api", version = "4.0.0-dev" } sp-timestamp = { path = "../../../primitives/timestamp", version = "4.0.0-dev" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.9.0" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.10.0-dev" } [dev-dependencies] -tokio = { version = "1.10.0", features = ["rt-multi-thread", "macros"] } +tokio = { version = "1.17.0", features = ["rt-multi-thread", "macros"] } sc-basic-authorship = { path = "../../basic-authorship", version = "0.10.0-dev" } substrate-test-runtime-client = { path = "../../../test-utils/runtime/client", version = "2.0.0" } substrate-test-runtime-transaction-pool = { path = "../../../test-utils/runtime/transaction-pool", version = "2.0.0" } diff --git a/client/consensus/manual-seal/src/consensus.rs b/client/consensus/manual-seal/src/consensus.rs index 33a4c8616f6d..dfd3730fd342 100644 --- a/client/consensus/manual-seal/src/consensus.rs +++ b/client/consensus/manual-seal/src/consensus.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,9 +21,11 @@ use super::Error; use sc_consensus::BlockImportParams; use sp_inherents::InherentData; -use sp_runtime::traits::{Block as BlockT, DigestFor}; +use sp_runtime::{traits::Block as BlockT, Digest}; +pub mod aura; pub mod babe; +pub mod timestamp; /// Consensus data provider, manual seal uses this trait object for authoring blocks valid /// for any runtime. @@ -32,11 +34,7 @@ pub trait ConsensusDataProvider: Send + Sync { type Transaction; /// Attempt to create a consensus digest. - fn create_digest( - &self, - parent: &B::Header, - inherents: &InherentData, - ) -> Result, Error>; + fn create_digest(&self, parent: &B::Header, inherents: &InherentData) -> Result; /// set up the neccessary import params. fn append_block_import( diff --git a/client/consensus/manual-seal/src/consensus/aura.rs b/client/consensus/manual-seal/src/consensus/aura.rs new file mode 100644 index 000000000000..7b5d6720562b --- /dev/null +++ b/client/consensus/manual-seal/src/consensus/aura.rs @@ -0,0 +1,98 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Aura consensus data provider, This allows manual seal author blocks that are valid for +//! runtimes that expect the aura-specific digests. + +use crate::{ConsensusDataProvider, Error}; +use sc_client_api::{AuxStore, UsageProvider}; +use sc_consensus::BlockImportParams; +use sp_api::{ProvideRuntimeApi, TransactionFor}; +use sp_blockchain::{HeaderBackend, HeaderMetadata}; +use sp_consensus_aura::{ + digests::CompatibleDigestItem, + sr25519::{AuthorityId, AuthoritySignature}, + AuraApi, Slot, SlotDuration, +}; +use sp_inherents::InherentData; +use sp_runtime::{traits::Block as BlockT, Digest, DigestItem}; +use sp_timestamp::TimestampInherentData; +use std::{marker::PhantomData, sync::Arc}; + +/// Consensus data provider for Aura. +pub struct AuraConsensusDataProvider { + // slot duration + slot_duration: SlotDuration, + // phantom data for required generics + _phantom: PhantomData<(B, C)>, +} + +impl AuraConsensusDataProvider +where + B: BlockT, + C: AuxStore + ProvideRuntimeApi + UsageProvider, + C::Api: AuraApi, +{ + /// Creates a new instance of the [`AuraConsensusDataProvider`], requires that `client` + /// implements [`sp_consensus_aura::AuraApi`] + pub fn new(client: Arc) -> Self { + let slot_duration = sc_consensus_aura::slot_duration(&*client) + .expect("slot_duration is always present; qed."); + + Self { slot_duration, _phantom: PhantomData } + } +} + +impl ConsensusDataProvider for AuraConsensusDataProvider +where + B: BlockT, + C: AuxStore + + HeaderBackend + + HeaderMetadata + + UsageProvider + + ProvideRuntimeApi, + C::Api: AuraApi, +{ + type Transaction = TransactionFor; + + fn create_digest( + &self, + _parent: &B::Header, + inherents: &InherentData, + ) -> Result { + let timestamp = + inherents.timestamp_inherent_data()?.expect("Timestamp is always present; qed"); + + // we always calculate the new slot number based on the current time-stamp and the slot + // duration. + let digest_item = >::aura_pre_digest( + Slot::from_timestamp(timestamp, self.slot_duration), + ); + + Ok(Digest { logs: vec![digest_item] }) + } + + fn append_block_import( + &self, + _parent: &B::Header, + _params: &mut BlockImportParams, + _inherents: &InherentData, + ) -> Result<(), Error> { + Ok(()) + } +} diff --git a/client/consensus/manual-seal/src/consensus/babe.rs b/client/consensus/manual-seal/src/consensus/babe.rs index 1d3afe392d62..53cc58df30a3 100644 --- a/client/consensus/manual-seal/src/consensus/babe.rs +++ b/client/consensus/manual-seal/src/consensus/babe.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,7 +16,8 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! BABE consensus data provider +//! BABE consensus data provider, This allows manual seal author blocks that are valid for runtimes +//! that expect babe-specific digests. use super::ConsensusDataProvider; use crate::Error; @@ -30,11 +31,7 @@ use sc_consensus_epochs::{ descendent_query, EpochHeader, SharedEpochChanges, ViableEpochDescriptor, }; use sp_keystore::SyncCryptoStorePtr; -use std::{ - borrow::Cow, - sync::{atomic, Arc}, - time::SystemTime, -}; +use std::{borrow::Cow, sync::Arc}; use sc_consensus::{BlockImportParams, ForkChoiceStrategy, Verifier}; use sp_api::{ProvideRuntimeApi, TransactionFor}; @@ -46,12 +43,13 @@ use sp_consensus_babe::{ AuthorityId, BabeApi, BabeAuthorityWeight, ConsensusLog, BABE_ENGINE_ID, }; use sp_consensus_slots::Slot; -use sp_inherents::{InherentData, InherentDataProvider, InherentIdentifier}; +use sp_inherents::InherentData; use sp_runtime::{ generic::{BlockId, Digest}, - traits::{Block as BlockT, DigestFor, DigestItemFor, Header, Zero}, + traits::{Block as BlockT, Header}, + DigestItem, }; -use sp_timestamp::{InherentType, TimestampInherentData, INHERENT_IDENTIFIER}; +use sp_timestamp::TimestampInherentData; /// Provides BABE-compatible predigests and BlockImportParams. /// Intended for use with BABE runtimes. @@ -120,7 +118,7 @@ where pre_digest.slot(), ) .map_err(|e| format!("failed to fetch epoch_descriptor: {}", e))? - .ok_or_else(|| format!("{:?}", sp_consensus::Error::InvalidAuthoritiesSet))?; + .ok_or_else(|| format!("{}", sp_consensus::Error::InvalidAuthoritiesSet))?; // drop the lock drop(epoch_changes); @@ -153,7 +151,7 @@ where return Err(Error::StringError("Cannot supply empty authority set!".into())) } - let config = Config::get_or_compute(&*client)?; + let config = Config::get(&*client)?; Ok(Self { config, client, keystore, epoch_changes, authorities }) } @@ -171,7 +169,9 @@ where .ok_or_else(|| sp_consensus::Error::InvalidAuthoritiesSet)?; let epoch = epoch_changes - .viable_epoch(&epoch_descriptor, |slot| Epoch::genesis(&self.config, slot)) + .viable_epoch(&epoch_descriptor, |slot| { + Epoch::genesis(self.config.genesis_config(), slot) + }) .ok_or_else(|| { log::info!(target: "babe", "create_digest: no viable_epoch :("); sp_consensus::Error::InvalidAuthoritiesSet @@ -193,11 +193,7 @@ where { type Transaction = TransactionFor; - fn create_digest( - &self, - parent: &B::Header, - inherents: &InherentData, - ) -> Result, Error> { + fn create_digest(&self, parent: &B::Header, inherents: &InherentData) -> Result { let slot = inherents .babe_inherent_data()? .ok_or_else(|| Error::StringError("No babe inherent data".into()))?; @@ -207,7 +203,7 @@ where let logs = if let Some((predigest, _)) = authorship::claim_slot(slot, &epoch, &self.keystore) { - vec![ as CompatibleDigestItem>::babe_pre_digest(predigest)] + vec![::babe_pre_digest(predigest)] } else { // well we couldn't claim a slot because this is an existing chain and we're not in the // authorities. we need to tell BabeBlockImport that the epoch has changed, and we put @@ -244,13 +240,13 @@ where }); vec![ - DigestItemFor::::PreRuntime(BABE_ENGINE_ID, predigest.encode()), - DigestItemFor::::Consensus(BABE_ENGINE_ID, next_epoch.encode()), + DigestItem::PreRuntime(BABE_ENGINE_ID, predigest.encode()), + DigestItem::Consensus(BABE_ENGINE_ID, next_epoch.encode()), ] }, ViableEpochDescriptor::UnimportedGenesis(_) => { // since this is the genesis, secondary predigest works for now. - vec![DigestItemFor::::PreRuntime(BABE_ENGINE_ID, predigest.encode())] + vec![DigestItem::PreRuntime(BABE_ENGINE_ID, predigest.encode())] }, } }; @@ -282,22 +278,24 @@ where // a quick check to see if we're in the authorities let epoch = self.epoch(parent, slot)?; let (authority, _) = self.authorities.first().expect("authorities is non-emptyp; qed"); - let has_authority = epoch.authorities.iter().find(|(id, _)| *id == *authority).is_some(); + let has_authority = epoch.authorities.iter().any(|(id, _)| *id == *authority); if !has_authority { log::info!(target: "manual-seal", "authority not found"); let timestamp = inherents .timestamp_inherent_data()? .ok_or_else(|| Error::StringError("No timestamp inherent data".into()))?; - let slot = *timestamp / self.config.slot_duration; + + let slot = Slot::from_timestamp(timestamp, self.config.slot_duration()); + // manually hard code epoch descriptor epoch_descriptor = match epoch_descriptor { ViableEpochDescriptor::Signaled(identifier, _header) => ViableEpochDescriptor::Signaled( identifier, EpochHeader { - start_slot: slot.into(), - end_slot: (slot * self.config.epoch_length).into(), + start_slot: slot, + end_slot: (*slot * self.config.genesis_config().epoch_length).into(), }, ), _ => unreachable!( @@ -314,67 +312,3 @@ where Ok(()) } } - -/// Provide duration since unix epoch in millisecond for timestamp inherent. -/// Mocks the timestamp inherent to always produce the timestamp for the next babe slot. -pub struct SlotTimestampProvider { - time: atomic::AtomicU64, - slot_duration: u64, -} - -impl SlotTimestampProvider { - /// Create a new mocked time stamp provider. - pub fn new(client: Arc) -> Result - where - B: BlockT, - C: AuxStore + HeaderBackend + ProvideRuntimeApi + UsageProvider, - C::Api: BabeApi, - { - let slot_duration = Config::get_or_compute(&*client)?.slot_duration; - let info = client.info(); - - // looks like this isn't the first block, rehydrate the fake time. - // otherwise we'd be producing blocks for older slots. - let time = if info.best_number != Zero::zero() { - let header = client.header(BlockId::Hash(info.best_hash))?.unwrap(); - let slot = find_pre_digest::(&header).unwrap().slot(); - // add the slot duration so there's no collision of slots - (*slot * slot_duration) + slot_duration - } else { - // this is the first block, use the correct time. - let now = SystemTime::now(); - now.duration_since(SystemTime::UNIX_EPOCH) - .map_err(|err| Error::StringError(format!("{}", err)))? - .as_millis() as u64 - }; - - Ok(Self { time: atomic::AtomicU64::new(time), slot_duration }) - } - - /// Get the current slot number - pub fn slot(&self) -> u64 { - self.time.load(atomic::Ordering::SeqCst) / self.slot_duration - } -} - -#[async_trait::async_trait] -impl InherentDataProvider for SlotTimestampProvider { - fn provide_inherent_data( - &self, - inherent_data: &mut InherentData, - ) -> Result<(), sp_inherents::Error> { - // we update the time here. - let duration: InherentType = - self.time.fetch_add(self.slot_duration, atomic::Ordering::SeqCst).into(); - inherent_data.put_data(INHERENT_IDENTIFIER, &duration)?; - Ok(()) - } - - async fn try_handle_error( - &self, - _: &InherentIdentifier, - _: &[u8], - ) -> Option> { - None - } -} diff --git a/client/consensus/manual-seal/src/consensus/timestamp.rs b/client/consensus/manual-seal/src/consensus/timestamp.rs new file mode 100644 index 000000000000..e7f4e709ab99 --- /dev/null +++ b/client/consensus/manual-seal/src/consensus/timestamp.rs @@ -0,0 +1,164 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Mocked timestamp inherent, allows for manual seal to create blocks for runtimes +//! that expect this inherent. + +use crate::Error; +use sc_client_api::{AuxStore, UsageProvider}; +use sp_api::ProvideRuntimeApi; +use sp_blockchain::HeaderBackend; +use sp_consensus_aura::{ + sr25519::{AuthorityId, AuthoritySignature}, + AuraApi, +}; +use sp_consensus_babe::BabeApi; +use sp_consensus_slots::{Slot, SlotDuration}; +use sp_inherents::{InherentData, InherentDataProvider, InherentIdentifier}; +use sp_runtime::{ + generic::BlockId, + traits::{Block as BlockT, Zero}, +}; +use sp_timestamp::{InherentType, INHERENT_IDENTIFIER}; +use std::{ + sync::{atomic, Arc}, + time::SystemTime, +}; + +/// Provide duration since unix epoch in millisecond for timestamp inherent. +/// Mocks the timestamp inherent to always produce a valid timestamp for the next slot. +/// +/// This works by either fetching the `slot_number` from the most recent header and dividing +/// that value by `slot_duration` in order to fork chains that expect this inherent. +/// +/// It produces timestamp inherents that are increaed by `slot_duraation` whenever +/// `provide_inherent_data` is called. +pub struct SlotTimestampProvider { + // holds the unix millisecnd timestamp for the most recent block + unix_millis: atomic::AtomicU64, + // configured slot_duration in the runtime + slot_duration: SlotDuration, +} + +impl SlotTimestampProvider { + /// Create a new mocked time stamp provider, for babe. + pub fn new_babe(client: Arc) -> Result + where + B: BlockT, + C: AuxStore + HeaderBackend + ProvideRuntimeApi + UsageProvider, + C::Api: BabeApi, + { + let slot_duration = sc_consensus_babe::Config::get(&*client)?.slot_duration(); + + let time = Self::with_header(&client, slot_duration, |header| { + let slot_number = *sc_consensus_babe::find_pre_digest::(&header) + .map_err(|err| format!("{}", err))? + .slot(); + Ok(slot_number) + })?; + + Ok(Self { unix_millis: atomic::AtomicU64::new(time), slot_duration }) + } + + /// Create a new mocked time stamp provider, for aura + pub fn new_aura(client: Arc) -> Result + where + B: BlockT, + C: AuxStore + HeaderBackend + ProvideRuntimeApi + UsageProvider, + C::Api: AuraApi, + { + let slot_duration = sc_consensus_aura::slot_duration(&*client)?; + + let time = Self::with_header(&client, slot_duration, |header| { + let slot_number = *sc_consensus_aura::find_pre_digest::(&header) + .map_err(|err| format!("{}", err))?; + Ok(slot_number) + })?; + + Ok(Self { unix_millis: atomic::AtomicU64::new(time), slot_duration }) + } + + fn with_header( + client: &Arc, + slot_duration: SlotDuration, + func: F, + ) -> Result + where + B: BlockT, + C: AuxStore + HeaderBackend + UsageProvider, + F: Fn(B::Header) -> Result, + { + let info = client.info(); + + // looks like this isn't the first block, rehydrate the fake time. + // otherwise we'd be producing blocks for older slots. + let time = if info.best_number != Zero::zero() { + let header = client + .header(BlockId::Hash(info.best_hash))? + .ok_or_else(|| "best header not found in the db!".to_string())?; + let slot = func(header)?; + // add the slot duration so there's no collision of slots + (slot * slot_duration.as_millis() as u64) + slot_duration.as_millis() as u64 + } else { + // this is the first block, use the correct time. + let now = SystemTime::now(); + now.duration_since(SystemTime::UNIX_EPOCH) + .map_err(|err| Error::StringError(format!("{}", err)))? + .as_millis() as u64 + }; + + Ok(time) + } + + /// Get the current slot number + pub fn slot(&self) -> Slot { + Slot::from_timestamp( + self.unix_millis.load(atomic::Ordering::SeqCst).into(), + self.slot_duration, + ) + } + + /// Gets the current time stamp. + pub fn timestamp(&self) -> sp_timestamp::Timestamp { + sp_timestamp::Timestamp::new(self.unix_millis.load(atomic::Ordering::SeqCst)) + } +} + +#[async_trait::async_trait] +impl InherentDataProvider for SlotTimestampProvider { + fn provide_inherent_data( + &self, + inherent_data: &mut InherentData, + ) -> Result<(), sp_inherents::Error> { + // we update the time here. + let new_time: InherentType = self + .unix_millis + .fetch_add(self.slot_duration.as_millis() as u64, atomic::Ordering::SeqCst) + .into(); + inherent_data.put_data(INHERENT_IDENTIFIER, &new_time)?; + Ok(()) + } + + async fn try_handle_error( + &self, + _: &InherentIdentifier, + _: &[u8], + ) -> Option> { + None + } +} diff --git a/client/consensus/manual-seal/src/error.rs b/client/consensus/manual-seal/src/error.rs index 8585e6a70d64..7c3211203bf5 100644 --- a/client/consensus/manual-seal/src/error.rs +++ b/client/consensus/manual-seal/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -38,41 +38,52 @@ mod codes { } /// errors encountered by background block authorship task -#[derive(Debug, derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] pub enum Error { /// An error occurred while importing the block - #[display(fmt = "Block import failed: {:?}", _0)] + #[error("Block import failed: {0:?}")] BlockImportError(ImportResult), /// Transaction pool is empty, cannot create a block - #[display(fmt = "Transaction pool is empty, set create_empty to true,\ - if you want to create empty blocks")] + #[error( + "Transaction pool is empty, set create_empty to true, if you want to create empty blocks" + )] EmptyTransactionPool, /// encountered during creation of Proposer. - #[display(fmt = "Consensus Error: {}", _0)] - ConsensusError(ConsensusError), + #[error("Consensus Error: {0}")] + ConsensusError(#[from] ConsensusError), /// Failed to create Inherents data - #[display(fmt = "Inherents Error: {}", _0)] - InherentError(InherentsError), + #[error("Inherents Error: {0}")] + InherentError(#[from] InherentsError), /// error encountered during finalization - #[display(fmt = "Finalization Error: {}", _0)] - BlockchainError(BlockchainError), + #[error("Finalization Error: {0}")] + BlockchainError(#[from] BlockchainError), /// Supplied parent_hash doesn't exist in chain - #[display(fmt = "Supplied parent_hash: {} doesn't exist in chain", _0)] - #[from(ignore)] + #[error("Supplied parent_hash: {0} doesn't exist in chain")] BlockNotFound(String), /// Some string error - #[display(fmt = "{}", _0)] - #[from(ignore)] + #[error("{0}")] StringError(String), /// send error - #[display(fmt = "Consensus process is terminating")] - Canceled(oneshot::Canceled), + #[error("Consensus process is terminating")] + Canceled(#[from] oneshot::Canceled), /// send error - #[display(fmt = "Consensus process is terminating")] - SendError(SendError), + #[error("Consensus process is terminating")] + SendError(#[from] SendError), /// Some other error. - #[display(fmt = "Other error: {}", _0)] - Other(Box), + #[error("Other error: {0}")] + Other(#[from] Box), +} + +impl From for Error { + fn from(err: ImportResult) -> Self { + Error::BlockImportError(err) + } +} + +impl From for Error { + fn from(s: String) -> Self { + Error::StringError(s) + } } impl Error { @@ -91,7 +102,7 @@ impl Error { } } -impl std::convert::From for jsonrpc_core::Error { +impl From for jsonrpc_core::Error { fn from(error: Error) -> Self { jsonrpc_core::Error { code: jsonrpc_core::ErrorCode::ServerError(error.to_code()), diff --git a/client/consensus/manual-seal/src/finalize_block.rs b/client/consensus/manual-seal/src/finalize_block.rs index a5ddf1d162f7..d134ce773457 100644 --- a/client/consensus/manual-seal/src/finalize_block.rs +++ b/client/consensus/manual-seal/src/finalize_block.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -48,7 +48,7 @@ where match finalizer.finalize_block(BlockId::Hash(hash), justification, true) { Err(e) => { - log::warn!("Failed to finalize block {:?}", e); + log::warn!("Failed to finalize block {}", e); rpc::send_result(&mut sender, Err(e.into())) }, Ok(()) => { diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index 390c23fe032f..a8d2634ade56 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -173,7 +173,7 @@ pub async fn run_manual_seal( env: &mut env, select_chain: &select_chain, block_import: &mut block_import, - consensus_data_provider: consensus_data_provider.as_ref().map(|p| &**p), + consensus_data_provider: consensus_data_provider.as_deref(), pool: pool.clone(), client: client.clone(), create_inherent_data_providers: &create_inherent_data_providers, @@ -253,7 +253,8 @@ mod tests { use sc_consensus::ImportedAux; use sc_transaction_pool::{BasicPool, Options, RevalidationType}; use sc_transaction_pool_api::{MaintainedTransactionPool, TransactionPool, TransactionSource}; - use sp_runtime::generic::BlockId; + use sp_inherents::InherentData; + use sp_runtime::generic::{BlockId, Digest, DigestItem}; use substrate_test_runtime_client::{ AccountKeyring::*, DefaultTestClientBuilderExt, TestClientBuilder, TestClientBuilderExt, }; @@ -265,6 +266,35 @@ mod tests { const SOURCE: TransactionSource = TransactionSource::External; + struct TestDigestProvider { + _client: Arc, + } + impl ConsensusDataProvider for TestDigestProvider + where + B: BlockT, + C: ProvideRuntimeApi + Send + Sync, + { + type Transaction = TransactionFor; + + fn create_digest( + &self, + _parent: &B::Header, + _inherents: &InherentData, + ) -> Result { + Ok(Digest { logs: vec![] }) + } + + fn append_block_import( + &self, + _parent: &B::Header, + params: &mut BlockImportParams, + _inherents: &InherentData, + ) -> Result<(), Error> { + params.post_digests.push(DigestItem::Other(vec![1])); + Ok(()) + } + } + #[tokio::test] async fn instant_seal() { let builder = TestClientBuilder::new(); @@ -408,8 +438,8 @@ mod tests { }) .await .unwrap(); - // assert that the background task returns ok - assert_eq!(rx.await.unwrap().unwrap(), ()); + // check that the background task returns ok: + rx.await.unwrap().unwrap(); } #[tokio::test] @@ -519,4 +549,53 @@ mod tests { // assert that fork block is in the db assert!(client.header(&BlockId::Hash(imported.hash)).unwrap().is_some()) } + + #[tokio::test] + async fn manual_seal_post_hash() { + let builder = TestClientBuilder::new(); + let (client, select_chain) = builder.build_with_longest_chain(); + let client = Arc::new(client); + let spawner = sp_core::testing::TaskExecutor::new(); + let pool = Arc::new(BasicPool::with_revalidation_type( + Options::default(), + true.into(), + api(), + None, + RevalidationType::Full, + spawner.clone(), + 0, + )); + let env = ProposerFactory::new(spawner.clone(), client.clone(), pool.clone(), None, None); + + let (mut sink, commands_stream) = futures::channel::mpsc::channel(1024); + let future = run_manual_seal(ManualSealParams { + block_import: client.clone(), + env, + client: client.clone(), + pool: pool.clone(), + commands_stream, + select_chain, + // use a provider that pushes some post digest data + consensus_data_provider: Some(Box::new(TestDigestProvider { _client: client.clone() })), + create_inherent_data_providers: |_, _| async { Ok(()) }, + }); + std::thread::spawn(|| { + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(future); + }); + let (tx, rx) = futures::channel::oneshot::channel(); + sink.send(EngineCommand::SealNewBlock { + parent_hash: None, + sender: Some(tx), + create_empty: true, + finalize: false, + }) + .await + .unwrap(); + let created_block = rx.await.unwrap().unwrap(); + + // assert that the background task returned the actual header hash + let header = client.header(&BlockId::Number(1)).unwrap().unwrap(); + assert_eq!(header.hash(), created_block.hash); + } } diff --git a/client/consensus/manual-seal/src/rpc.rs b/client/consensus/manual-seal/src/rpc.rs index 6755879ceedd..4a8dcbc0cb76 100644 --- a/client/consensus/manual-seal/src/rpc.rs +++ b/client/consensus/manual-seal/src/rpc.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -155,7 +155,10 @@ pub fn send_result( ) { if let Some(sender) = sender.take() { if let Err(err) = sender.send(result) { - log::warn!("Server is shutting down: {:?}", err) + match err { + Ok(value) => log::warn!("Server is shutting down: {:?}", value), + Err(error) => log::warn!("Server is shutting down with error: {}", error), + } } } else { // instant seal doesn't report errors over rpc, simply log them. diff --git a/client/consensus/manual-seal/src/seal_block.rs b/client/consensus/manual-seal/src/seal_block.rs index 502705b41162..202b54fe5d0c 100644 --- a/client/consensus/manual-seal/src/seal_block.rs +++ b/client/consensus/manual-seal/src/seal_block.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -114,10 +114,7 @@ pub async fn seal_block( let inherent_data = inherent_data_providers.create_inherent_data()?; - let proposer = env - .init(&parent) - .map_err(|err| Error::StringError(format!("{:?}", err))) - .await?; + let proposer = env.init(&parent).map_err(|err| Error::StringError(err.to_string())).await?; let inherents_len = inherent_data.len(); let digest = if let Some(digest_provider) = digest_provider { @@ -133,7 +130,7 @@ pub async fn seal_block( Duration::from_secs(MAX_PROPOSAL_DURATION), None, ) - .map_err(|err| Error::StringError(format!("{:?}", err))) + .map_err(|err| Error::StringError(err.to_string())) .await?; if proposal.block.extrinsics().len() == inherents_len && !create_empty { @@ -153,9 +150,14 @@ pub async fn seal_block( digest_provider.append_block_import(&parent, &mut params, &inherent_data)?; } + // Make sure we return the same post-hash that will be calculated when importing the block + // This is important in case the digest_provider added any signature, seal, ect. + let mut post_header = header.clone(); + post_header.digest_mut().logs.extend(params.post_digests.iter().cloned()); + match block_import.import_block(params, HashMap::new()).await? { ImportResult::Imported(aux) => - Ok(CreatedBlock { hash: ::Header::hash(&header), aux }), + Ok(CreatedBlock { hash: ::Header::hash(&post_header), aux }), other => Err(other.into()), } }; diff --git a/client/consensus/pow/Cargo.toml b/client/consensus/pow/Cargo.toml index c71e11aef275..4ddbf0c0ead7 100644 --- a/client/consensus/pow/Cargo.toml +++ b/client/consensus/pow/Cargo.toml @@ -3,9 +3,9 @@ name = "sc-consensus-pow" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "PoW consensus algorithm for substrate" -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,10 +13,10 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", features = ["derive"] } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } +codec = { package = "parity-scale-codec", version = "3.0.0", features = ["derive"] } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sc-client-api = { version = "4.0.0-dev", path = "../../api" } sp-block-builder = { version = "4.0.0-dev", path = "../../../primitives/block-builder" } @@ -25,9 +25,9 @@ sp-consensus-pow = { version = "0.10.0-dev", path = "../../../primitives/consens sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } sc-consensus = { version = "0.10.0-dev", path = "../../../client/consensus/common" } log = "0.4.8" -futures = "0.3.16" +futures = "0.3.21" futures-timer = "3.0.1" -parking_lot = "0.11.1" -derive_more = "0.99.2" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.9.0"} +parking_lot = "0.12.0" +thiserror = "1.0" +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus", version = "0.10.0-dev"} async-trait = "0.1.50" diff --git a/client/consensus/pow/src/lib.rs b/client/consensus/pow/src/lib.rs index 1f5781434ef7..8885099ceb51 100644 --- a/client/consensus/pow/src/lib.rs +++ b/client/consensus/pow/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -27,9 +27,9 @@ //! started via the [`start_mining_worker`] function. It returns a worker //! handle together with a future. The future must be pulled. Through //! the worker handle, you can pull the metadata needed to start the -//! mining process via [`MiningWorker::metadata`], and then do the actual +//! mining process via [`MiningHandle::metadata`], and then do the actual //! mining on a standalone thread. Finally, when a seal is found, call -//! [`MiningWorker::submit`] to build the block. +//! [`MiningHandle::submit`] to build the block. //! //! The auxiliary storage for PoW engine only stores the total difficulty. //! For other storage requirements for particular PoW algorithm (such as @@ -41,13 +41,12 @@ mod worker; -pub use crate::worker::{MiningBuild, MiningMetadata, MiningWorker}; +pub use crate::worker::{MiningBuild, MiningHandle, MiningMetadata}; use crate::worker::UntilImportedOrTimeout; use codec::{Decode, Encode}; use futures::{Future, StreamExt}; use log::*; -use parking_lot::Mutex; use prometheus_endpoint::Registry; use sc_client_api::{self, backend::AuxStore, BlockOf, BlockchainEvents}; use sc_consensus::{ @@ -56,7 +55,7 @@ use sc_consensus::{ }; use sp_api::ProvideRuntimeApi; use sp_block_builder::BlockBuilder as BlockBuilderApi; -use sp_blockchain::{well_known_cache_keys::Id as CacheKeyId, HeaderBackend, ProvideCache}; +use sp_blockchain::{well_known_cache_keys::Id as CacheKeyId, HeaderBackend}; use sp_consensus::{ CanAuthorWith, Environment, Error as ConsensusError, Proposer, SelectChain, SyncOracle, }; @@ -73,55 +72,60 @@ use std::{ time::Duration, }; -#[derive(derive_more::Display, Debug)] +#[derive(Debug, thiserror::Error)] pub enum Error { - #[display(fmt = "Header uses the wrong engine {:?}", _0)] + #[error("Header uses the wrong engine {0:?}")] WrongEngine([u8; 4]), - #[display(fmt = "Header {:?} is unsealed", _0)] + #[error("Header {0:?} is unsealed")] HeaderUnsealed(B::Hash), - #[display(fmt = "PoW validation error: invalid seal")] + #[error("PoW validation error: invalid seal")] InvalidSeal, - #[display(fmt = "PoW validation error: preliminary verification failed")] + #[error("PoW validation error: preliminary verification failed")] FailedPreliminaryVerify, - #[display(fmt = "Rejecting block too far in future")] + #[error("Rejecting block too far in future")] TooFarInFuture, - #[display(fmt = "Fetching best header failed using select chain: {:?}", _0)] + #[error("Fetching best header failed using select chain: {0}")] BestHeaderSelectChain(ConsensusError), - #[display(fmt = "Fetching best header failed: {:?}", _0)] + #[error("Fetching best header failed: {0}")] BestHeader(sp_blockchain::Error), - #[display(fmt = "Best header does not exist")] + #[error("Best header does not exist")] NoBestHeader, - #[display(fmt = "Block proposing error: {:?}", _0)] + #[error("Block proposing error: {0}")] BlockProposingError(String), - #[display(fmt = "Fetch best hash failed via select chain: {:?}", _0)] + #[error("Fetch best hash failed via select chain: {0}")] BestHashSelectChain(ConsensusError), - #[display(fmt = "Error with block built on {:?}: {:?}", _0, _1)] + #[error("Error with block built on {0:?}: {1}")] BlockBuiltError(B::Hash, ConsensusError), - #[display(fmt = "Creating inherents failed: {}", _0)] + #[error("Creating inherents failed: {0}")] CreateInherents(sp_inherents::Error), - #[display(fmt = "Checking inherents failed: {}", _0)] + #[error("Checking inherents failed: {0}")] CheckInherents(sp_inherents::Error), - #[display( - fmt = "Checking inherents unknown error for identifier: {:?}", - "String::from_utf8_lossy(_0)" + #[error( + "Checking inherents unknown error for identifier: {}", + String::from_utf8_lossy(.0) )] CheckInherentsUnknownError(sp_inherents::InherentIdentifier), - #[display(fmt = "Multiple pre-runtime digests")] + #[error("Multiple pre-runtime digests")] MultiplePreRuntimeDigests, + #[error(transparent)] Client(sp_blockchain::Error), + #[error(transparent)] Codec(codec::Error), + #[error("{0}")] Environment(String), + #[error("{0}")] Runtime(RuntimeString), + #[error("{0}")] Other(String), } -impl std::convert::From> for String { +impl From> for String { fn from(error: Error) -> String { error.to_string() } } -impl std::convert::From> for ConsensusError { +impl From> for ConsensusError { fn from(error: Error) -> ConsensusError { ConsensusError::ClientImport(error.to_string()) } @@ -241,7 +245,7 @@ where B: BlockT, I: BlockImport> + Send + Sync, I::Error: Into, - C: ProvideRuntimeApi + Send + Sync + HeaderBackend + AuxStore + ProvideCache + BlockOf, + C: ProvideRuntimeApi + Send + Sync + HeaderBackend + AuxStore + BlockOf, C::Api: BlockBuilderApi, Algorithm: PowAlgorithm, CAW: CanAuthorWith, @@ -320,7 +324,7 @@ where I: BlockImport> + Send + Sync, I::Error: Into, S: SelectChain, - C: ProvideRuntimeApi + Send + Sync + HeaderBackend + AuxStore + ProvideCache + BlockOf, + C: ProvideRuntimeApi + Send + Sync + HeaderBackend + AuxStore + BlockOf, C::Api: BlockBuilderApi, Algorithm: PowAlgorithm + Send + Sync, Algorithm::Difficulty: 'static + Send, @@ -346,7 +350,7 @@ where .select_chain .best_chain() .await - .map_err(|e| format!("Fetch best chain failed via select chain: {:?}", e))?; + .map_err(|e| format!("Fetch best chain failed via select chain: {}", e))?; let best_hash = best_header.hash(); let parent_hash = *block.header.parent_hash(); @@ -426,10 +430,7 @@ impl PowVerifier { Self { algorithm, _marker: PhantomData } } - fn check_header( - &self, - mut header: B::Header, - ) -> Result<(B::Header, DigestItem), Error> + fn check_header(&self, mut header: B::Header) -> Result<(B::Header, DigestItem), Error> where Algorithm: PowAlgorithm, { @@ -525,7 +526,7 @@ pub fn start_mining_worker( build_time: Duration, can_author_with: CAW, ) -> ( - Arc>::Proof>>>, + MiningHandle>::Proof>, impl Future, ) where @@ -543,12 +544,7 @@ where CAW: CanAuthorWith + Clone + Send + 'static, { let mut timer = UntilImportedOrTimeout::new(client.import_notification_stream(), timeout); - let worker = Arc::new(Mutex::new(MiningWorker { - build: None, - algorithm: algorithm.clone(), - block_import, - justification_sync_link, - })); + let worker = MiningHandle::new(algorithm.clone(), block_import, justification_sync_link); let worker_ret = worker.clone(); let task = async move { @@ -559,7 +555,7 @@ where if sync_oracle.is_major_syncing() { debug!(target: "pow", "Skipping proposal due to sync."); - worker.lock().on_major_syncing(); + worker.on_major_syncing(); continue } @@ -569,7 +565,7 @@ where warn!( target: "pow", "Unable to pull new block for authoring. \ - Select best chain error: {:?}", + Select best chain error: {}", err ); continue @@ -587,7 +583,7 @@ where continue } - if worker.lock().best_hash() == Some(best_hash) { + if worker.best_hash() == Some(best_hash) { continue } @@ -600,7 +596,7 @@ where warn!( target: "pow", "Unable to propose new block for authoring. \ - Fetch difficulty failed: {:?}", + Fetch difficulty failed: {}", err, ); continue @@ -616,7 +612,7 @@ where warn!( target: "pow", "Unable to propose new block for authoring. \ - Creating inherent data providers failed: {:?}", + Creating inherent data providers failed: {}", err, ); continue @@ -629,14 +625,14 @@ where warn!( target: "pow", "Unable to propose new block for authoring. \ - Creating inherent data failed: {:?}", + Creating inherent data failed: {}", e, ); continue }, }; - let mut inherent_digest = Digest::::default(); + let mut inherent_digest = Digest::default(); if let Some(pre_runtime) = &pre_runtime { inherent_digest.push(DigestItem::PreRuntime(POW_ENGINE_ID, pre_runtime.to_vec())); } @@ -665,7 +661,7 @@ where warn!( target: "pow", "Unable to propose new block for authoring. \ - Creating proposal failed: {:?}", + Creating proposal failed: {}", err, ); continue @@ -682,7 +678,7 @@ where proposal, }; - worker.lock().on_build(build); + worker.on_build(build); } }; @@ -708,10 +704,7 @@ fn find_pre_digest(header: &B::Header) -> Result>, Err } /// Fetch PoW seal. -fn fetch_seal( - digest: Option<&DigestItem>, - hash: B::Hash, -) -> Result, Error> { +fn fetch_seal(digest: Option<&DigestItem>, hash: B::Hash) -> Result, Error> { match digest { Some(DigestItem::Seal(id, seal)) => if id == &POW_ENGINE_ID { diff --git a/client/consensus/pow/src/worker.rs b/client/consensus/pow/src/worker.rs index c0ca16ccad3a..42f82fb43ef7 100644 --- a/client/consensus/pow/src/worker.rs +++ b/client/consensus/pow/src/worker.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -22,6 +22,7 @@ use futures::{ }; use futures_timer::Delay; use log::*; +use parking_lot::Mutex; use sc_client_api::ImportNotifications; use sc_consensus::{BlockImportParams, BoxBlockImport, StateAction, StorageChanges}; use sp_consensus::{BlockOrigin, Proposal}; @@ -30,7 +31,16 @@ use sp_runtime::{ traits::{Block as BlockT, Header as HeaderT}, DigestItem, }; -use std::{borrow::Cow, collections::HashMap, pin::Pin, time::Duration}; +use std::{ + borrow::Cow, + collections::HashMap, + pin::Pin, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; use crate::{PowAlgorithm, PowIntermediate, Seal, INTERMEDIATE_KEY, POW_ENGINE_ID}; @@ -60,21 +70,26 @@ pub struct MiningBuild< pub proposal: Proposal, Proof>, } +/// Version of the mining worker. +#[derive(Eq, PartialEq, Clone, Copy)] +pub struct Version(usize); + /// Mining worker that exposes structs to query the current mining build and submit mined blocks. -pub struct MiningWorker< +pub struct MiningHandle< Block: BlockT, Algorithm: PowAlgorithm, C: sp_api::ProvideRuntimeApi, L: sc_consensus::JustificationSyncLink, Proof, > { - pub(crate) build: Option>, - pub(crate) algorithm: Algorithm, - pub(crate) block_import: BoxBlockImport>, - pub(crate) justification_sync_link: L, + version: Arc, + algorithm: Arc, + justification_sync_link: Arc, + build: Arc>>>, + block_import: Arc>>>, } -impl MiningWorker +impl MiningHandle where Block: BlockT, C: sp_api::ProvideRuntimeApi, @@ -83,35 +98,65 @@ where L: sc_consensus::JustificationSyncLink, sp_api::TransactionFor: Send + 'static, { - /// Get the current best hash. `None` if the worker has just started or the client is doing - /// major syncing. - pub fn best_hash(&self) -> Option { - self.build.as_ref().map(|b| b.metadata.best_hash) + fn increment_version(&self) { + self.version.fetch_add(1, Ordering::SeqCst); } - pub(crate) fn on_major_syncing(&mut self) { - self.build = None; + pub(crate) fn new( + algorithm: Algorithm, + block_import: BoxBlockImport>, + justification_sync_link: L, + ) -> Self { + Self { + version: Arc::new(AtomicUsize::new(0)), + algorithm: Arc::new(algorithm), + justification_sync_link: Arc::new(justification_sync_link), + build: Arc::new(Mutex::new(None)), + block_import: Arc::new(Mutex::new(block_import)), + } } - pub(crate) fn on_build(&mut self, build: MiningBuild) { - self.build = Some(build); + pub(crate) fn on_major_syncing(&self) { + let mut build = self.build.lock(); + *build = None; + self.increment_version(); + } + + pub(crate) fn on_build(&self, value: MiningBuild) { + let mut build = self.build.lock(); + *build = Some(value); + self.increment_version(); + } + + /// Get the version of the mining worker. + /// + /// This returns type `Version` which can only compare equality. If `Version` is unchanged, then + /// it can be certain that `best_hash` and `metadata` were not changed. + pub fn version(&self) -> Version { + Version(self.version.load(Ordering::SeqCst)) + } + + /// Get the current best hash. `None` if the worker has just started or the client is doing + /// major syncing. + pub fn best_hash(&self) -> Option { + self.build.lock().as_ref().map(|b| b.metadata.best_hash) } /// Get a copy of the current mining metadata, if available. pub fn metadata(&self) -> Option> { - self.build.as_ref().map(|b| b.metadata.clone()) + self.build.lock().as_ref().map(|b| b.metadata.clone()) } /// Submit a mined seal. The seal will be validated again. Returns true if the submission is /// successful. - pub async fn submit(&mut self, seal: Seal) -> bool { - if let Some(build) = self.build.take() { + pub async fn submit(&self, seal: Seal) -> bool { + if let Some(metadata) = self.metadata() { match self.algorithm.verify( - &BlockId::Hash(build.metadata.best_hash), - &build.metadata.pre_hash, - build.metadata.pre_runtime.as_ref().map(|v| &v[..]), + &BlockId::Hash(metadata.best_hash), + &metadata.pre_hash, + metadata.pre_runtime.as_ref().map(|v| &v[..]), &seal, - build.metadata.difficulty, + metadata.difficulty, ) { Ok(true) => (), Ok(false) => { @@ -124,61 +169,98 @@ where Err(err) => { warn!( target: "pow", - "Unable to import mined block: {:?}", + "Unable to import mined block: {}", err, ); return false }, } + } else { + warn!( + target: "pow", + "Unable to import mined block: metadata does not exist", + ); + return false + } - let seal = DigestItem::Seal(POW_ENGINE_ID, seal); - let (header, body) = build.proposal.block.deconstruct(); - - let mut import_block = BlockImportParams::new(BlockOrigin::Own, header); - import_block.post_digests.push(seal); - import_block.body = Some(body); - import_block.state_action = - StateAction::ApplyChanges(StorageChanges::Changes(build.proposal.storage_changes)); - - let intermediate = PowIntermediate:: { - difficulty: Some(build.metadata.difficulty), - }; - - import_block - .intermediates - .insert(Cow::from(INTERMEDIATE_KEY), Box::new(intermediate) as Box<_>); - - let header = import_block.post_header(); - match self.block_import.import_block(import_block, HashMap::default()).await { - Ok(res) => { - res.handle_justification( - &header.hash(), - *header.number(), - &mut self.justification_sync_link, - ); - - info!( - target: "pow", - "✅ Successfully mined block on top of: {}", - build.metadata.best_hash - ); - true - }, - Err(err) => { - warn!( - target: "pow", - "Unable to import mined block: {:?}", - err, - ); - false - }, + let build = if let Some(build) = { + let mut build = self.build.lock(); + let value = build.take(); + if value.is_some() { + self.increment_version(); } + value + } { + build } else { warn!( target: "pow", "Unable to import mined block: build does not exist", ); - false + return false + }; + + let seal = DigestItem::Seal(POW_ENGINE_ID, seal); + let (header, body) = build.proposal.block.deconstruct(); + + let mut import_block = BlockImportParams::new(BlockOrigin::Own, header); + import_block.post_digests.push(seal); + import_block.body = Some(body); + import_block.state_action = + StateAction::ApplyChanges(StorageChanges::Changes(build.proposal.storage_changes)); + + let intermediate = PowIntermediate:: { + difficulty: Some(build.metadata.difficulty), + }; + + import_block + .intermediates + .insert(Cow::from(INTERMEDIATE_KEY), Box::new(intermediate) as Box<_>); + + let header = import_block.post_header(); + let mut block_import = self.block_import.lock(); + + match block_import.import_block(import_block, HashMap::default()).await { + Ok(res) => { + res.handle_justification( + &header.hash(), + *header.number(), + &self.justification_sync_link, + ); + + info!( + target: "pow", + "✅ Successfully mined block on top of: {}", + build.metadata.best_hash + ); + true + }, + Err(err) => { + warn!( + target: "pow", + "Unable to import mined block: {}", + err, + ); + false + }, + } + } +} + +impl Clone for MiningHandle +where + Block: BlockT, + Algorithm: PowAlgorithm, + C: sp_api::ProvideRuntimeApi, + L: sc_consensus::JustificationSyncLink, +{ + fn clone(&self) -> Self { + Self { + version: self.version.clone(), + algorithm: self.algorithm.clone(), + justification_sync_link: self.justification_sync_link.clone(), + build: self.build.clone(), + block_import: self.block_import.clone(), } } } diff --git a/client/consensus/slots/Cargo.toml b/client/consensus/slots/Cargo.toml index 4c0142829bb5..2884f8443e30 100644 --- a/client/consensus/slots/Cargo.toml +++ b/client/consensus/slots/Cargo.toml @@ -3,10 +3,10 @@ name = "sc-consensus-slots" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "Generic slots-based utilities for consensus" -edition = "2018" +edition = "2021" build = "build.rs" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -14,24 +14,23 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0" } +codec = { package = "parity-scale-codec", version = "3.0.0" } sc-client-api = { version = "4.0.0-dev", path = "../../api" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-arithmetic = { version = "4.0.0-dev", path = "../../../primitives/arithmetic" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-arithmetic = { version = "5.0.0", path = "../../../primitives/arithmetic" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } sc-consensus = { version = "0.10.0-dev", path = "../../../client/consensus/common" } sp-consensus-slots = { version = "0.10.0-dev", path = "../../../primitives/consensus/slots" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -sp-state-machine = { version = "0.10.0-dev", path = "../../../primitives/state-machine" } -sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } +sp-state-machine = { version = "0.12.0", path = "../../../primitives/state-machine" } sc-telemetry = { version = "4.0.0-dev", path = "../../telemetry" } sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } sp-timestamp = { version = "4.0.0-dev", path = "../../../primitives/timestamp" } -futures = "0.3.9" +futures = "0.3.21" futures-timer = "3.0.1" log = "0.4.11" -thiserror = "1.0.21" +thiserror = "1.0.30" async-trait = "0.1.50" [dev-dependencies] diff --git a/client/consensus/slots/build.rs b/client/consensus/slots/build.rs index 57424f016f3e..b700b28e322c 100644 --- a/client/consensus/slots/build.rs +++ b/client/consensus/slots/build.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/consensus/slots/src/aux_schema.rs b/client/consensus/slots/src/aux_schema.rs index c2fe3f6f4e6b..275b12ff48f8 100644 --- a/client/consensus/slots/src/aux_schema.rs +++ b/client/consensus/slots/src/aux_schema.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/consensus/slots/src/lib.rs b/client/consensus/slots/src/lib.rs index bfaa388014ef..a97469fbcc30 100644 --- a/client/consensus/slots/src/lib.rs +++ b/client/consensus/slots/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -32,20 +32,18 @@ pub use aux_schema::{check_equivocation, MAX_SLOT_CAPACITY, PRUNING_BOUND}; pub use slots::SlotInfo; use slots::Slots; -use codec::{Decode, Encode}; use futures::{future::Either, Future, TryFutureExt}; use futures_timer::Delay; -use log::{debug, error, info, warn}; +use log::{debug, info, warn}; use sc_consensus::{BlockImport, JustificationSyncLink}; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO, CONSENSUS_WARN}; -use sp_api::{ApiRef, ProvideRuntimeApi}; use sp_arithmetic::traits::BaseArithmetic; -use sp_consensus::{CanAuthorWith, Proposer, SelectChain, SlotData, SyncOracle}; -use sp_consensus_slots::Slot; +use sp_consensus::{CanAuthorWith, Proposer, SelectChain, SyncOracle}; +use sp_consensus_slots::{Slot, SlotDuration}; use sp_inherents::CreateInherentDataProviders; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor}, + traits::{Block as BlockT, HashFor, Header as HeaderT}, }; use sp_timestamp::Timestamp; use std::{fmt::Debug, ops::Deref, time::Duration}; @@ -54,7 +52,7 @@ use std::{fmt::Debug, ops::Deref, time::Duration}; /// /// See [`sp_state_machine::StorageChanges`] for more information. pub type StorageChanges = - sp_state_machine::StorageChanges, NumberFor>; + sp_state_machine::StorageChanges>; /// The result of [`SlotWorker::on_slot`]. #[derive(Debug, Clone)] @@ -141,31 +139,20 @@ pub trait SimpleSlotWorker { fn notify_slot(&self, _header: &B::Header, _slot: Slot, _epoch_data: &Self::EpochData) {} /// Return the pre digest data to include in a block authored with the given claim. - fn pre_digest_data( - &self, - slot: Slot, - claim: &Self::Claim, - ) -> Vec>; + fn pre_digest_data(&self, slot: Slot, claim: &Self::Claim) -> Vec; /// Returns a function which produces a `BlockImportParams`. - fn block_import_params( + async fn block_import_params( &self, - ) -> Box< - dyn Fn( - B::Header, - &B::Hash, - Vec, - StorageChanges<>::Transaction, B>, - Self::Claim, - Self::EpochData, - ) -> Result< - sc_consensus::BlockImportParams< - B, - >::Transaction, - >, - sp_consensus::Error, - > + Send - + 'static, + header: B::Header, + header_hash: &B::Hash, + body: Vec, + storage_changes: StorageChanges<>::Transaction, B>, + public: Self::Claim, + epoch: Self::EpochData, + ) -> Result< + sc_consensus::BlockImportParams>::Transaction>, + sp_consensus::Error, >; /// Whether to force authoring if offline. @@ -226,7 +213,7 @@ pub trait SimpleSlotWorker { Err(err) => { warn!( target: logging_target, - "Unable to fetch epoch data at block {:?}: {:?}", + "Unable to fetch epoch data at block {:?}: {}", slot_info.chain_head.hash(), err, ); @@ -286,10 +273,7 @@ pub trait SimpleSlotWorker { let proposer = match self.proposer(&slot_info.chain_head).await { Ok(p) => p, Err(err) => { - warn!( - target: logging_target, - "Unable to author block in slot {:?}: {:?}", slot, err, - ); + warn!(target: logging_target, "Unable to author block in slot {:?}: {}", slot, err,); telemetry!( telemetry; @@ -315,12 +299,12 @@ pub trait SimpleSlotWorker { proposing_remaining_duration.mul_f32(0.98), None, ) - .map_err(|e| sp_consensus::Error::ClientImport(format!("{:?}", e))); + .map_err(|e| sp_consensus::Error::ClientImport(e.to_string())); let proposal = match futures::future::select(proposing, proposing_remaining).await { Either::Left((Ok(p), _)) => p, Either::Left((Err(err), _)) => { - warn!(target: logging_target, "Proposing failed: {:?}", err); + warn!(target: logging_target, "Proposing failed: {}", err); return None }, @@ -346,26 +330,26 @@ pub trait SimpleSlotWorker { }, }; - let block_import_params_maker = self.block_import_params(); - let block_import = self.block_import(); - let (block, storage_proof) = (proposal.block, proposal.proof); let (header, body) = block.deconstruct(); let header_num = *header.number(); let header_hash = header.hash(); let parent_hash = *header.parent_hash(); - let block_import_params = match block_import_params_maker( - header, - &header_hash, - body.clone(), - proposal.storage_changes, - claim, - epoch_data, - ) { + let block_import_params = match self + .block_import_params( + header, + &header_hash, + body.clone(), + proposal.storage_changes, + claim, + epoch_data, + ) + .await + { Ok(bi) => bi, Err(err) => { - warn!(target: logging_target, "Failed to create block import params: {:?}", err); + warn!(target: logging_target, "Failed to create block import params: {}", err); return None }, @@ -389,7 +373,7 @@ pub trait SimpleSlotWorker { ); let header = block_import_params.post_header(); - match block_import.import_block(block_import_params, Default::default()).await { + match self.block_import().import_block(block_import_params, Default::default()).await { Ok(res) => { res.handle_justification( &header.hash(), @@ -400,7 +384,7 @@ pub trait SimpleSlotWorker { Err(err) => { warn!( target: logging_target, - "Error with block built on {:?}: {:?}", parent_hash, err, + "Error with block built on {:?}: {}", parent_hash, err, ); telemetry!( @@ -417,15 +401,22 @@ pub trait SimpleSlotWorker { } } +/// A type that implements [`SlotWorker`] for a type that implements [`SimpleSlotWorker`]. +/// +/// This is basically a workaround for Rust not supporting specialization. Otherwise we could +/// implement [`SlotWorker`] for any `T` that implements [`SimpleSlotWorker`], but currently +/// that would prevent downstream users to implement [`SlotWorker`] for their own types. +pub struct SimpleSlotWorkerToSlotWorker(pub T); + #[async_trait::async_trait] -impl + Send + Sync> - SlotWorker>::Proof> for T +impl + Send + Sync, B: BlockT> + SlotWorker>::Proof> for SimpleSlotWorkerToSlotWorker { async fn on_slot( &mut self, slot_info: SlotInfo, ) -> Option>::Proof>> { - SimpleSlotWorker::on_slot(self, slot_info).await + self.0.on_slot(slot_info).await } } @@ -474,8 +465,8 @@ impl_inherent_data_provider_ext_tuple!(T, S, A, B, C, D, E, F, G, H, I, J); /// /// Every time a new slot is triggered, `worker.on_slot` is called and the future it returns is /// polled until completion, unless we are major syncing. -pub async fn start_slot_worker( - slot_duration: SlotDuration, +pub async fn start_slot_worker( + slot_duration: SlotDuration, client: C, mut worker: W, mut sync_oracle: SO, @@ -486,21 +477,17 @@ pub async fn start_slot_worker( C: SelectChain, W: SlotWorker, SO: SyncOracle + Send, - T: SlotData + Clone, CIDP: CreateInherentDataProviders + Send, CIDP::InherentDataProviders: InherentDataProviderExt + Send, CAW: CanAuthorWith + Send, { - let SlotDuration(slot_duration) = slot_duration; - - let mut slots = - Slots::new(slot_duration.slot_duration(), create_inherent_data_providers, client); + let mut slots = Slots::new(slot_duration.as_duration(), create_inherent_data_providers, client); loop { let slot_info = match slots.next_slot().await { Ok(r) => r, Err(e) => { - warn!(target: "slots", "Error while polling for next slot: {:?}", e); + warn!(target: "slots", "Error while polling for next slot: {}", e); return }, }; @@ -538,89 +525,6 @@ pub enum CheckedHeader { Checked(H, S), } -#[derive(Debug, thiserror::Error)] -#[allow(missing_docs)] -pub enum Error -where - T: Debug, -{ - #[error("Slot duration is invalid: {0:?}")] - SlotDurationInvalid(SlotDuration), -} - -/// A slot duration. Create with [`get_or_compute`](Self::get_or_compute). -// The internal member should stay private here to maintain invariants of -// `get_or_compute`. -#[derive(Clone, Copy, Debug, Encode, Decode, Hash, PartialOrd, Ord, PartialEq, Eq)] -pub struct SlotDuration(T); - -impl Deref for SlotDuration { - type Target = T; - fn deref(&self) -> &T { - &self.0 - } -} - -impl SlotData for SlotDuration { - fn slot_duration(&self) -> std::time::Duration { - self.0.slot_duration() - } - - const SLOT_KEY: &'static [u8] = T::SLOT_KEY; -} - -impl SlotDuration { - /// Either fetch the slot duration from disk or compute it from the - /// genesis state. - /// - /// `slot_key` is marked as `'static`, as it should really be a - /// compile-time constant. - pub fn get_or_compute(client: &C, cb: CB) -> sp_blockchain::Result - where - C: sc_client_api::backend::AuxStore + sc_client_api::UsageProvider, - C: ProvideRuntimeApi, - CB: FnOnce(ApiRef, &BlockId) -> sp_blockchain::Result, - T: SlotData + Encode + Decode + Debug, - { - let slot_duration = match client.get_aux(T::SLOT_KEY)? { - Some(v) => ::decode(&mut &v[..]).map(SlotDuration).map_err(|_| { - sp_blockchain::Error::Backend({ - error!(target: "slots", "slot duration kept in invalid format"); - "slot duration kept in invalid format".to_string() - }) - }), - None => { - let best_hash = client.usage_info().chain.best_hash; - let slot_duration = cb(client.runtime_api(), &BlockId::hash(best_hash))?; - - info!( - "⏱ Loaded block-time = {:?} from block {:?}", - slot_duration.slot_duration(), - best_hash, - ); - - slot_duration - .using_encoded(|s| client.insert_aux(&[(T::SLOT_KEY, &s[..])], &[]))?; - - Ok(SlotDuration(slot_duration)) - }, - }?; - - if slot_duration.slot_duration() == Default::default() { - return Err(sp_blockchain::Error::Application(Box::new(Error::SlotDurationInvalid( - slot_duration, - )))) - } - - Ok(slot_duration) - } - - /// Returns slot data value. - pub fn get(&self) -> T { - self.0.clone() - } -} - /// A unit type wrapper to express the proportion of a slot. pub struct SlotProportion(f32); @@ -713,10 +617,10 @@ pub fn proposing_remaining_duration( debug!( target: log_target, - "No block for {} slots. Applying {} lenience, total proposing duration: {}", + "No block for {} slots. Applying {} lenience, total proposing duration: {}ms", slot_info.slot.saturating_sub(parent_slot + 1), slot_lenience_type.as_str(), - lenient_proposing_duration.as_secs(), + lenient_proposing_duration.as_millis(), ); lenient_proposing_duration @@ -848,7 +752,9 @@ where return false } - let unfinalized_block_length = chain_head_number - finalized_number; + // There can be race between getting the finalized number and getting the best number. + // So, better be safe than sorry. + let unfinalized_block_length = chain_head_number.saturating_sub(finalized_number); let interval = unfinalized_block_length.saturating_sub(self.unfinalized_slack) / self.authoring_bias; let interval = interval.min(self.max_interval); @@ -886,7 +792,7 @@ impl BackoffAuthoringBlocksStrategy for () { #[cfg(test)] mod test { use super::*; - use sp_api::NumberFor; + use sp_runtime::traits::NumberFor; use std::time::{Duration, Instant}; use substrate_test_runtime_client::runtime::{Block, Header}; diff --git a/client/consensus/slots/src/slots.rs b/client/consensus/slots/src/slots.rs index c2ed986e1e7f..a7b9f3e3ff61 100644 --- a/client/consensus/slots/src/slots.rs +++ b/client/consensus/slots/src/slots.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -150,7 +150,7 @@ where Err(e) => { log::warn!( target: "slots", - "Unable to author block in slot. No best block header: {:?}", + "Unable to author block in slot. No best block header: {}", e, ); // Let's try at the next slot.. diff --git a/client/consensus/uncles/Cargo.toml b/client/consensus/uncles/Cargo.toml index 7e821db197b3..c512eb7e9f83 100644 --- a/client/consensus/uncles/Cargo.toml +++ b/client/consensus/uncles/Cargo.toml @@ -3,9 +3,9 @@ name = "sc-consensus-uncles" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "Generic uncle inclusion utilities for consensus" -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -14,6 +14,6 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sc-client-api = { version = "4.0.0-dev", path = "../../api" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } sp-authorship = { version = "4.0.0-dev", path = "../../../primitives/authorship" } -thiserror = "1.0.21" +thiserror = "1.0.30" diff --git a/client/consensus/uncles/src/lib.rs b/client/consensus/uncles/src/lib.rs index 368a994cfe52..d03c2f8aa6b4 100644 --- a/client/consensus/uncles/src/lib.rs +++ b/client/consensus/uncles/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index 1d3d76ee7a55..12bb29958cfe 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-client-db" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Client backend that uses RocksDB database as storage." readme = "README.md" @@ -13,37 +13,38 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -parking_lot = "0.11.1" +parking_lot = "0.12.0" log = "0.4.8" -kvdb = "0.10.0" -kvdb-rocksdb = { version = "0.14.0", optional = true } -kvdb-memorydb = "0.10.0" +kvdb = "0.11.0" +kvdb-rocksdb = { version = "0.15.2", optional = true } +kvdb-memorydb = "0.11.0" linked-hash-map = "0.5.4" hash-db = "0.15.2" -codec = { package = "parity-scale-codec", version = "2.0.0", features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", features = [ "derive", ] } sc-client-api = { version = "4.0.0-dev", path = "../api" } -sp-arithmetic = { version = "4.0.0-dev", path = "../../primitives/arithmetic" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sp-state-machine = { version = "0.10.0-dev", path = "../../primitives/state-machine" } +sp-arithmetic = { version = "5.0.0", path = "../../primitives/arithmetic" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } +sp-state-machine = { version = "0.12.0", path = "../../primitives/state-machine" } sc-state-db = { version = "0.10.0-dev", path = "../state-db" } -sp-trie = { version = "4.0.0-dev", path = "../../primitives/trie" } +sp-trie = { version = "6.0.0", path = "../../primitives/trie" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sp-database = { version = "4.0.0-dev", path = "../../primitives/database" } -parity-db = { version = "0.3.1", optional = true } +parity-db = { version = "0.3.9", optional = true } [dev-dependencies] -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } quickcheck = "1.0.3" -kvdb-rocksdb = "0.14.0" +kvdb-rocksdb = "0.15.1" tempfile = "3" [features] default = [] test-helpers = [] +runtime-benchmarks = [] with-kvdb-rocksdb = ["kvdb-rocksdb"] with-parity-db = ["parity-db"] diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index d46aca8e8ff7..fe31d31dfef9 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -34,7 +34,7 @@ use sp_core::{ }; use sp_runtime::{ traits::{Block as BlockT, HashFor}, - Storage, + StateVersion, Storage, }; use sp_state_machine::{ backend::Backend as StateBackend, ChildStorageCollection, DBValue, ProofRecorder, @@ -73,6 +73,7 @@ impl sp_state_machine::Storage> for StorageDb { root: Cell, @@ -105,9 +106,10 @@ impl BenchmarkingState { record_proof: bool, enable_tracking: bool, ) -> Result { + let state_version = sp_runtime::StateVersion::default(); let mut root = B::Hash::default(); let mut mdb = MemoryDB::>::default(); - sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); + sp_state_machine::TrieDBMutV1::>::new(&mut mdb, &mut root); let mut state = BenchmarkingState { state: RefCell::new(None), @@ -138,6 +140,7 @@ impl BenchmarkingState { state.state.borrow_mut().as_mut().unwrap().full_storage_root( genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, + state_version, ); state.genesis = transaction.clone().drain(); state.genesis_root = root.clone(); @@ -415,6 +418,7 @@ impl StateBackend> for BenchmarkingState { fn storage_root<'a>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord, @@ -422,13 +426,14 @@ impl StateBackend> for BenchmarkingState { self.state .borrow() .as_ref() - .map_or(Default::default(), |s| s.storage_root(delta)) + .map_or(Default::default(), |s| s.storage_root(delta, state_version)) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord, @@ -436,7 +441,7 @@ impl StateBackend> for BenchmarkingState { self.state .borrow() .as_ref() - .map_or(Default::default(), |s| s.child_storage_root(child_info, delta)) + .map_or(Default::default(), |s| s.child_storage_root(child_info, delta, state_version)) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -667,7 +672,6 @@ mod test { assert_eq!(rw_tracker.1, 0); assert_eq!(rw_tracker.2, 2); assert_eq!(rw_tracker.3, 0); - drop(rw_tracker); bench_state.wipe().unwrap(); } } diff --git a/client/db/src/cache/list_cache.rs b/client/db/src/cache/list_cache.rs deleted file mode 100644 index 795cb8f90118..000000000000 --- a/client/db/src/cache/list_cache.rs +++ /dev/null @@ -1,2351 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! List-based cache. -//! -//! Maintains several lists, containing nodes that are inserted whenever -//! cached value at new block differs from the value at previous block. -//! Example: -//! B1(a) <--- B2(b) <--- B3(b) <--- B4(c) -//! N1(b) <-------------- N2(c) -//! -//! There's single list for all finalized blocks and >= 0 lists for unfinalized -//! blocks. -//! When new non-final block is inserted (with value that differs from the value -//! at parent), it starts new unfinalized fork. -//! When new final block is inserted (with value that differs from the value at -//! parent), new entry is appended to the finalized fork. -//! When existing non-final block is finalized (with value that differs from the -//! value at parent), new entry is appended to the finalized fork AND unfinalized -//! fork is dropped. -//! -//! Entries from abandoned unfinalized forks (forks that are forking from block B -//! which is ascendant of the best finalized block) are deleted when block F with -//! number B.number (i.e. 'parallel' canon block) is finalized. -//! -//! Finalized entry E1 is pruned when block B is finalized so that: -//! EntryAt(B.number - prune_depth).points_to(E1) - -use std::collections::{BTreeMap, BTreeSet}; - -use log::warn; - -use sp_blockchain::{Error as ClientError, Result as ClientResult}; -use sp_runtime::traits::{Block as BlockT, Bounded, CheckedSub, NumberFor, Zero}; - -use crate::cache::{ - list_entry::{Entry, StorageEntry}, - list_storage::{Metadata, Storage, StorageTransaction}, - CacheItemT, ComplexBlockId, EntryType, -}; - -/// Pruning strategy. -#[derive(Debug, Clone, Copy)] -pub enum PruningStrategy { - /// Prune entries when they're too far behind best finalized block. - ByDepth(N), - /// Do not prune old entries at all. - NeverPrune, -} - -/// List-based cache. -pub struct ListCache> { - /// Cache storage. - storage: S, - /// Pruning strategy. - pruning_strategy: PruningStrategy>, - /// Best finalized block. - best_finalized_block: ComplexBlockId, - /// Best finalized entry (if exists). - best_finalized_entry: Option>, - /// All unfinalized 'forks'. - unfinalized: Vec>, -} - -/// All possible list cache operations that could be performed after transaction is committed. -#[derive(Debug)] -#[cfg_attr(test, derive(PartialEq))] -pub enum CommitOperation { - /// New block is appended to the fork without changing the cached value. - AppendNewBlock(usize, ComplexBlockId), - /// New block is appended to the fork with the different value. - AppendNewEntry(usize, Entry), - /// New fork is added with the given head entry. - AddNewFork(Entry), - /// New block is finalized and possibly: - /// - new entry is finalized AND/OR - /// - some forks are destroyed - BlockFinalized(ComplexBlockId, Option>, BTreeSet), - /// When best block is reverted - contains the forks that have to be updated - /// (they're either destroyed, or their best entry is updated to earlier block). - BlockReverted(BTreeMap>>), -} - -/// A set of commit operations. -#[derive(Debug)] -pub struct CommitOperations { - operations: Vec>, -} - -/// Single fork of list-based cache. -#[derive(Debug)] -#[cfg_attr(test, derive(PartialEq))] -pub struct Fork { - /// The best block of this fork. We do not save this field in the database to avoid - /// extra updates => it could be None after restart. It will be either filled when - /// the block is appended to this fork, or the whole fork will be abandoned when the - /// block from the other fork is finalized - best_block: Option>, - /// The head entry of this fork. - head: Entry, -} - -/// Outcome of Fork::try_append_or_fork. -#[derive(Debug)] -#[cfg_attr(test, derive(PartialEq))] -pub enum ForkAppendResult { - /// New entry should be appended to the end of the fork. - Append, - /// New entry should be forked from the fork, starting with entry at given block. - Fork(ComplexBlockId), -} - -impl> ListCache { - /// Create new db list cache entry. - pub fn new( - storage: S, - pruning_strategy: PruningStrategy>, - best_finalized_block: ComplexBlockId, - ) -> ClientResult { - let (best_finalized_entry, unfinalized) = - storage.read_meta().and_then(|meta| read_forks(&storage, meta))?; - - Ok(ListCache { - storage, - pruning_strategy, - best_finalized_block, - best_finalized_entry, - unfinalized, - }) - } - - /// Get reference to the storage. - pub fn storage(&self) -> &S { - &self.storage - } - - /// Get unfinalized forks reference. - #[cfg(test)] - pub fn unfinalized(&self) -> &[Fork] { - &self.unfinalized - } - - /// Get value valid at block. - pub fn value_at_block( - &self, - at: &ComplexBlockId, - ) -> ClientResult, Option>, T)>> { - let head = if at.number <= self.best_finalized_block.number { - // if the block is older than the best known finalized block - // => we're should search for the finalized value - - // BUT since we're not guaranteeing to provide correct values for forks - // behind the finalized block, check if the block is finalized first - if !chain::is_finalized_block(&self.storage, &at, Bounded::max_value())? { - return Err(ClientError::NotInFinalizedChain) - } - - self.best_finalized_entry.as_ref() - } else if self.unfinalized.is_empty() { - // there are no unfinalized entries - // => we should search for the finalized value - self.best_finalized_entry.as_ref() - } else { - // there are unfinalized entries - // => find the fork containing given block and read from this fork - // IF there's no matching fork, ensure that this isn't a block from a fork that has - // forked behind the best finalized block and search at finalized fork - - match self.find_unfinalized_fork(&at)? { - Some(fork) => Some(&fork.head), - None => match self.best_finalized_entry.as_ref() { - Some(best_finalized_entry) - if chain::is_connected_to_block( - &self.storage, - &at, - &best_finalized_entry.valid_from, - )? => - Some(best_finalized_entry), - _ => None, - }, - } - }; - - match head { - Some(head) => head - .search_best_before(&self.storage, at.number) - .map(|e| e.map(|e| (e.0.valid_from, e.1, e.0.value))), - None => Ok(None), - } - } - - /// When new block is inserted into database. - /// - /// None passed as value means that the value has not changed since previous block. - pub fn on_block_insert>( - &self, - tx: &mut Tx, - parent: ComplexBlockId, - block: ComplexBlockId, - value: Option, - entry_type: EntryType, - operations: &mut CommitOperations, - ) -> ClientResult<()> { - Ok(operations - .append(self.do_on_block_insert(tx, parent, block, value, entry_type, operations)?)) - } - - /// When previously inserted block is finalized. - pub fn on_block_finalize>( - &self, - tx: &mut Tx, - parent: ComplexBlockId, - block: ComplexBlockId, - operations: &mut CommitOperations, - ) -> ClientResult<()> { - Ok(operations.append(self.do_on_block_finalize(tx, parent, block, operations)?)) - } - - /// When block is reverted. - pub fn on_block_revert>( - &self, - tx: &mut Tx, - reverted_block: &ComplexBlockId, - operations: &mut CommitOperations, - ) -> ClientResult<()> { - Ok(operations.append(Some(self.do_on_block_revert(tx, reverted_block)?))) - } - - /// When transaction is committed. - pub fn on_transaction_commit(&mut self, ops: CommitOperations) { - for op in ops.operations { - match op { - CommitOperation::AppendNewBlock(index, best_block) => { - let mut fork = self.unfinalized.get_mut(index).expect( - "ListCache is a crate-private type; - internal clients of ListCache are committing transaction while cache is locked; - CommitOperation holds valid references while cache is locked; qed", - ); - fork.best_block = Some(best_block); - }, - CommitOperation::AppendNewEntry(index, entry) => { - let mut fork = self.unfinalized.get_mut(index).expect( - "ListCache is a crate-private type; - internal clients of ListCache are committing transaction while cache is locked; - CommitOperation holds valid references while cache is locked; qed", - ); - fork.best_block = Some(entry.valid_from.clone()); - fork.head = entry; - }, - CommitOperation::AddNewFork(entry) => { - self.unfinalized - .push(Fork { best_block: Some(entry.valid_from.clone()), head: entry }); - }, - CommitOperation::BlockFinalized(block, finalizing_entry, forks) => { - self.best_finalized_block = block; - if let Some(finalizing_entry) = finalizing_entry { - self.best_finalized_entry = Some(finalizing_entry); - } - for fork_index in forks.iter().rev() { - self.unfinalized.remove(*fork_index); - } - }, - CommitOperation::BlockReverted(forks) => { - for (fork_index, updated_fork) in forks.into_iter().rev() { - match updated_fork { - Some(updated_fork) => self.unfinalized[fork_index] = updated_fork, - None => { - self.unfinalized.remove(fork_index); - }, - } - } - }, - } - } - } - - fn do_on_block_insert>( - &self, - tx: &mut Tx, - parent: ComplexBlockId, - block: ComplexBlockId, - value: Option, - entry_type: EntryType, - operations: &CommitOperations, - ) -> ClientResult>> { - // this guarantee is currently provided by LightStorage && we're relying on it here - let prev_operation = operations.operations.last(); - debug_assert!( - entry_type != EntryType::Final || - self.unfinalized.is_empty() || - self.best_finalized_block.hash == parent.hash || - match prev_operation { - Some(&CommitOperation::BlockFinalized(ref best_finalized_block, _, _)) => - best_finalized_block.hash == parent.hash, - _ => false, - } - ); - - // we do not store any values behind finalized - if block.number != Zero::zero() && self.best_finalized_block.number >= block.number { - return Ok(None) - } - - // if the block is not final, it is possibly appended to/forking from existing unfinalized - // fork - let is_final = entry_type == EntryType::Final || entry_type == EntryType::Genesis; - if !is_final { - let mut fork_and_action = None; - - // when value hasn't changed and block isn't final, there's nothing we need to do - if value.is_none() { - return Ok(None) - } - - // first: try to find fork that is known to has the best block we're appending to - for (index, fork) in self.unfinalized.iter().enumerate() { - if fork.try_append(&parent) { - fork_and_action = Some((index, ForkAppendResult::Append)); - break - } - } - - // if not found, check cases: - // - we're appending to the fork for the first time after restart; - // - we're forking existing unfinalized fork from the middle; - if fork_and_action.is_none() { - let best_finalized_entry_block = - self.best_finalized_entry.as_ref().map(|f| f.valid_from.number); - for (index, fork) in self.unfinalized.iter().enumerate() { - if let Some(action) = - fork.try_append_or_fork(&self.storage, &parent, best_finalized_entry_block)? - { - fork_and_action = Some((index, action)); - break - } - } - } - - // if we have found matching unfinalized fork => early exit - match fork_and_action { - // append to unfinalized fork - Some((index, ForkAppendResult::Append)) => { - let new_storage_entry = match self.unfinalized[index].head.try_update(value) { - Some(new_storage_entry) => new_storage_entry, - None => return Ok(Some(CommitOperation::AppendNewBlock(index, block))), - }; - - tx.insert_storage_entry(&block, &new_storage_entry); - let operation = - CommitOperation::AppendNewEntry(index, new_storage_entry.into_entry(block)); - tx.update_meta( - self.best_finalized_entry.as_ref(), - &self.unfinalized, - &operation, - ); - return Ok(Some(operation)) - }, - // fork from the middle of unfinalized fork - Some((_, ForkAppendResult::Fork(prev_valid_from))) => { - // it is possible that we're inserting extra (but still required) fork here - let new_storage_entry = StorageEntry { - prev_valid_from: Some(prev_valid_from), - value: value.expect("checked above that !value.is_none(); qed"), - }; - - tx.insert_storage_entry(&block, &new_storage_entry); - let operation = - CommitOperation::AddNewFork(new_storage_entry.into_entry(block)); - tx.update_meta( - self.best_finalized_entry.as_ref(), - &self.unfinalized, - &operation, - ); - return Ok(Some(operation)) - }, - None => (), - } - } - - // if we're here, then one of following is true: - // - either we're inserting final block => all ancestors are already finalized AND the only - // thing we can do is to try to update last finalized entry - // - either we're inserting non-final blocks that has no ancestors in any known unfinalized - // forks - - let new_storage_entry = match self.best_finalized_entry.as_ref() { - Some(best_finalized_entry) => best_finalized_entry.try_update(value), - None if value.is_some() => Some(StorageEntry { - prev_valid_from: None, - value: value.expect("value.is_some(); qed"), - }), - None => None, - }; - - if !is_final { - return Ok(match new_storage_entry { - Some(new_storage_entry) => { - tx.insert_storage_entry(&block, &new_storage_entry); - let operation = - CommitOperation::AddNewFork(new_storage_entry.into_entry(block)); - tx.update_meta( - self.best_finalized_entry.as_ref(), - &self.unfinalized, - &operation, - ); - Some(operation) - }, - None => None, - }) - } - - // cleanup database from abandoned unfinalized forks and obsolete finalized entries - let abandoned_forks = self.destroy_abandoned_forks(tx, &block, prev_operation); - self.prune_finalized_entries(tx, &block); - - match new_storage_entry { - Some(new_storage_entry) => { - tx.insert_storage_entry(&block, &new_storage_entry); - let operation = CommitOperation::BlockFinalized( - block.clone(), - Some(new_storage_entry.into_entry(block)), - abandoned_forks, - ); - tx.update_meta(self.best_finalized_entry.as_ref(), &self.unfinalized, &operation); - Ok(Some(operation)) - }, - None => Ok(Some(CommitOperation::BlockFinalized(block, None, abandoned_forks))), - } - } - - fn do_on_block_finalize>( - &self, - tx: &mut Tx, - parent: ComplexBlockId, - block: ComplexBlockId, - operations: &CommitOperations, - ) -> ClientResult>> { - // this guarantee is currently provided by db backend && we're relying on it here - let prev_operation = operations.operations.last(); - debug_assert!( - self.best_finalized_block.hash == parent.hash || - match prev_operation { - Some(&CommitOperation::BlockFinalized(ref best_finalized_block, _, _)) => - best_finalized_block.hash == parent.hash, - _ => false, - } - ); - - // there could be at most one entry that is finalizing - let finalizing_entry = - self.storage.read_entry(&block)?.map(|entry| entry.into_entry(block.clone())); - - // cleanup database from abandoned unfinalized forks and obsolete finalized entries - let abandoned_forks = self.destroy_abandoned_forks(tx, &block, prev_operation); - self.prune_finalized_entries(tx, &block); - - let operation = CommitOperation::BlockFinalized(block, finalizing_entry, abandoned_forks); - tx.update_meta(self.best_finalized_entry.as_ref(), &self.unfinalized, &operation); - - Ok(Some(operation)) - } - - fn do_on_block_revert>( - &self, - tx: &mut Tx, - reverted_block: &ComplexBlockId, - ) -> ClientResult> { - // can't revert finalized blocks - debug_assert!(self.best_finalized_block.number < reverted_block.number); - - // iterate all unfinalized forks and truncate/destroy if required - let mut updated = BTreeMap::new(); - for (index, fork) in self.unfinalized.iter().enumerate() { - // we only need to truncate fork if its head is ancestor of truncated block - if fork.head.valid_from.number < reverted_block.number { - continue - } - - // we only need to truncate fork if its head is connected to truncated block - if !chain::is_connected_to_block(&self.storage, reverted_block, &fork.head.valid_from)? - { - continue - } - - let updated_fork = fork.truncate( - &self.storage, - tx, - reverted_block.number, - self.best_finalized_block.number, - )?; - updated.insert(index, updated_fork); - } - - // schedule commit operation and update meta - let operation = CommitOperation::BlockReverted(updated); - tx.update_meta(self.best_finalized_entry.as_ref(), &self.unfinalized, &operation); - - Ok(operation) - } - - /// Prune old finalized entries. - fn prune_finalized_entries>( - &self, - tx: &mut Tx, - block: &ComplexBlockId, - ) { - let prune_depth = match self.pruning_strategy { - PruningStrategy::ByDepth(prune_depth) => prune_depth, - PruningStrategy::NeverPrune => return, - }; - - let mut do_pruning = || -> ClientResult<()> { - // calculate last ancient block number - let ancient_block = match block.number.checked_sub(&prune_depth) { - Some(number) => match self.storage.read_id(number)? { - Some(hash) => ComplexBlockId::new(hash, number), - None => return Ok(()), - }, - None => return Ok(()), - }; - - // if there's an entry at this block: - // - remove reference from this entry to the previous entry - // - destroy fork starting with previous entry - let current_entry = match self.storage.read_entry(&ancient_block)? { - Some(current_entry) => current_entry, - None => return Ok(()), - }; - let first_entry_to_truncate = match current_entry.prev_valid_from { - Some(prev_valid_from) => prev_valid_from, - None => return Ok(()), - }; - - // truncate ancient entry - tx.insert_storage_entry( - &ancient_block, - &StorageEntry { prev_valid_from: None, value: current_entry.value }, - ); - - // destroy 'fork' ending with previous entry - destroy_fork(first_entry_to_truncate, &self.storage, tx, None) - }; - - if let Err(error) = do_pruning() { - warn!(target: "db", "Failed to prune ancient cache entries: {}", error); - } - } - - /// Try to destroy abandoned forks (forked before best finalized block) when block is finalized. - fn destroy_abandoned_forks>( - &self, - tx: &mut Tx, - block: &ComplexBlockId, - prev_operation: Option<&CommitOperation>, - ) -> BTreeSet { - // if some block has been finalized already => take it into account - let prev_abandoned_forks = match prev_operation { - Some(&CommitOperation::BlockFinalized(_, _, ref abandoned_forks)) => - Some(abandoned_forks), - _ => None, - }; - - let mut destroyed = prev_abandoned_forks.cloned().unwrap_or_else(|| BTreeSet::new()); - let live_unfinalized = self.unfinalized.iter().enumerate().filter(|(idx, _)| { - prev_abandoned_forks - .map(|prev_abandoned_forks| !prev_abandoned_forks.contains(idx)) - .unwrap_or(true) - }); - for (index, fork) in live_unfinalized { - if fork.head.valid_from.number == block.number { - destroyed.insert(index); - if fork.head.valid_from.hash != block.hash { - if let Err(error) = fork.destroy(&self.storage, tx, Some(block.number)) { - warn!(target: "db", "Failed to destroy abandoned unfinalized cache fork: {}", error); - } - } - } - } - - destroyed - } - - /// Search unfinalized fork where given block belongs. - fn find_unfinalized_fork( - &self, - block: &ComplexBlockId, - ) -> ClientResult>> { - for unfinalized in &self.unfinalized { - if unfinalized.matches(&self.storage, block)? { - return Ok(Some(&unfinalized)) - } - } - - Ok(None) - } -} - -impl Fork { - /// Get reference to the head entry of this fork. - pub fn head(&self) -> &Entry { - &self.head - } - - /// Check if the block is the part of the fork. - pub fn matches>( - &self, - storage: &S, - block: &ComplexBlockId, - ) -> ClientResult { - let range = self.head.search_best_range_before(storage, block.number)?; - match range { - None => Ok(false), - Some((begin, end)) => - chain::is_connected_to_range(storage, block, (&begin, end.as_ref())), - } - } - - /// Try to append NEW block to the fork. This method will only 'work' (return true) when block - /// is actually appended to the fork AND the best known block of the fork is known (i.e. some - /// block has been already appended to this fork after last restart). - pub fn try_append(&self, parent: &ComplexBlockId) -> bool { - // when the best block of the fork is known, the check is trivial - // - // most of calls will hopefully end here, because best_block is only unknown - // after restart and until new block is appended to the fork - self.best_block.as_ref() == Some(parent) - } - - /// Try to append new block to the fork OR fork it. - pub fn try_append_or_fork>( - &self, - storage: &S, - parent: &ComplexBlockId, - best_finalized_entry_block: Option>, - ) -> ClientResult>> { - // try to find entries that are (possibly) surrounding the parent block - let range = self.head.search_best_range_before(storage, parent.number)?; - let begin = match range { - Some((begin, _)) => begin, - None => return Ok(None), - }; - - // check if the parent is connected to the beginning of the range - if !chain::is_connected_to_block(storage, parent, &begin)? { - return Ok(None) - } - - // the block is connected to the begin-entry. If begin is the head entry - // => we need to append new block to the fork - if begin == self.head.valid_from { - return Ok(Some(ForkAppendResult::Append)) - } - - // the parent block belongs to this fork AND it is located after last finalized entry - // => we need to make a new fork - if best_finalized_entry_block.map(|f| begin.number > f).unwrap_or(true) { - return Ok(Some(ForkAppendResult::Fork(begin))) - } - - Ok(None) - } - - /// Destroy fork by deleting all unfinalized entries. - pub fn destroy, Tx: StorageTransaction>( - &self, - storage: &S, - tx: &mut Tx, - best_finalized_block: Option>, - ) -> ClientResult<()> { - destroy_fork(self.head.valid_from.clone(), storage, tx, best_finalized_block) - } - - /// Truncate fork by deleting all entries that are descendants of given block. - pub fn truncate, Tx: StorageTransaction>( - &self, - storage: &S, - tx: &mut Tx, - reverting_block: NumberFor, - best_finalized_block: NumberFor, - ) -> ClientResult>> { - let mut current = self.head.valid_from.clone(); - loop { - // read pointer to previous entry - let entry = storage.require_entry(¤t)?; - - // truncation stops when we have reached the ancestor of truncated block - if current.number < reverting_block { - // if we have reached finalized block => destroy fork - if chain::is_finalized_block(storage, ¤t, best_finalized_block)? { - return Ok(None) - } - - // else fork needs to be updated - return Ok(Some(Fork { best_block: None, head: entry.into_entry(current) })) - } - - tx.remove_storage_entry(¤t); - - // truncation also stops when there are no more entries in the list - current = match entry.prev_valid_from { - Some(prev_valid_from) => prev_valid_from, - None => return Ok(None), - }; - } - } -} - -impl Default for CommitOperations { - fn default() -> Self { - CommitOperations { operations: Vec::new() } - } -} - -// This should never be allowed for non-test code to avoid revealing its internals. -#[cfg(test)] -impl From>> - for CommitOperations -{ - fn from(operations: Vec>) -> Self { - CommitOperations { operations } - } -} - -impl CommitOperations { - /// Append operation to the set. - fn append(&mut self, new_operation: Option>) { - let new_operation = match new_operation { - Some(new_operation) => new_operation, - None => return, - }; - - let last_operation = match self.operations.pop() { - Some(last_operation) => last_operation, - None => { - self.operations.push(new_operation); - return - }, - }; - - // we are able (and obliged to) to merge two consequent block finalization operations - match last_operation { - CommitOperation::BlockFinalized( - old_finalized_block, - old_finalized_entry, - old_abandoned_forks, - ) => match new_operation { - CommitOperation::BlockFinalized( - new_finalized_block, - new_finalized_entry, - new_abandoned_forks, - ) => { - self.operations.push(CommitOperation::BlockFinalized( - new_finalized_block, - new_finalized_entry, - new_abandoned_forks, - )); - }, - _ => { - self.operations.push(CommitOperation::BlockFinalized( - old_finalized_block, - old_finalized_entry, - old_abandoned_forks, - )); - self.operations.push(new_operation); - }, - }, - _ => { - self.operations.push(last_operation); - self.operations.push(new_operation); - }, - } - } -} - -/// Destroy fork by deleting all unfinalized entries. -pub fn destroy_fork< - Block: BlockT, - T: CacheItemT, - S: Storage, - Tx: StorageTransaction, ->( - head_valid_from: ComplexBlockId, - storage: &S, - tx: &mut Tx, - best_finalized_block: Option>, -) -> ClientResult<()> { - let mut current = head_valid_from; - loop { - // optionally: deletion stops when we found entry at finalized block - if let Some(best_finalized_block) = best_finalized_block { - if chain::is_finalized_block(storage, ¤t, best_finalized_block)? { - return Ok(()) - } - } - - // read pointer to previous entry - let entry = storage.require_entry(¤t)?; - tx.remove_storage_entry(¤t); - - // deletion stops when there are no more entries in the list - current = match entry.prev_valid_from { - Some(prev_valid_from) => prev_valid_from, - None => return Ok(()), - }; - } -} - -/// Blockchain related functions. -mod chain { - use super::*; - use sp_runtime::traits::Header as HeaderT; - - /// Is the block1 connected both ends of the range. - pub fn is_connected_to_range>( - storage: &S, - block: &ComplexBlockId, - range: (&ComplexBlockId, Option<&ComplexBlockId>), - ) -> ClientResult { - let (begin, end) = range; - Ok(is_connected_to_block(storage, block, begin)? && - match end { - Some(end) => is_connected_to_block(storage, block, end)?, - None => true, - }) - } - - /// Is the block1 directly connected (i.e. part of the same fork) to block2? - pub fn is_connected_to_block>( - storage: &S, - block1: &ComplexBlockId, - block2: &ComplexBlockId, - ) -> ClientResult { - let (begin, end) = if *block1 > *block2 { (block2, block1) } else { (block1, block2) }; - let mut current = storage - .read_header(&end.hash)? - .ok_or_else(|| ClientError::UnknownBlock(format!("{}", end.hash)))?; - while *current.number() > begin.number { - current = storage - .read_header(current.parent_hash())? - .ok_or_else(|| ClientError::UnknownBlock(format!("{}", current.parent_hash())))?; - } - - Ok(begin.hash == current.hash()) - } - - /// Returns true if the given block is finalized. - pub fn is_finalized_block>( - storage: &S, - block: &ComplexBlockId, - best_finalized_block: NumberFor, - ) -> ClientResult { - if block.number > best_finalized_block { - return Ok(false) - } - - storage.read_id(block.number).map(|hash| hash.as_ref() == Some(&block.hash)) - } -} - -/// Read list cache forks at blocks IDs. -fn read_forks>( - storage: &S, - meta: Metadata, -) -> ClientResult<(Option>, Vec>)> { - let finalized = match meta.finalized { - Some(finalized) => Some(storage.require_entry(&finalized)?.into_entry(finalized)), - None => None, - }; - - let unfinalized = meta - .unfinalized - .into_iter() - .map(|unfinalized| { - storage.require_entry(&unfinalized).map(|storage_entry| Fork { - best_block: None, - head: storage_entry.into_entry(unfinalized), - }) - }) - .collect::>()?; - - Ok((finalized, unfinalized)) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::cache::list_storage::tests::{DummyStorage, DummyTransaction, FaultyStorage}; - use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper, Header}; - use substrate_test_runtime_client::runtime::H256; - - type Block = RawBlock>; - - fn test_id(number: u64) -> ComplexBlockId { - ComplexBlockId::new(H256::from_low_u64_be(number), number) - } - - fn correct_id(number: u64) -> ComplexBlockId { - ComplexBlockId::new(test_header(number).hash(), number) - } - - fn fork_id(fork_nonce: u64, fork_from: u64, number: u64) -> ComplexBlockId { - ComplexBlockId::new(fork_header(fork_nonce, fork_from, number).hash(), number) - } - - fn test_header(number: u64) -> Header { - Header { - parent_hash: if number == 0 { - Default::default() - } else { - test_header(number - 1).hash() - }, - number, - state_root: Default::default(), - extrinsics_root: Default::default(), - digest: Default::default(), - } - } - - fn fork_header(fork_nonce: u64, fork_from: u64, number: u64) -> Header { - if fork_from == number { - test_header(number) - } else { - Header { - parent_hash: fork_header(fork_nonce, fork_from, number - 1).hash(), - number, - state_root: H256::from_low_u64_be(1 + fork_nonce), - extrinsics_root: Default::default(), - digest: Default::default(), - } - } - } - - #[test] - fn list_value_at_block_works() { - // when block is earlier than best finalized block AND it is not finalized - // --- 50 --- - // ----------> [100] - assert!(ListCache::<_, u64, _>::new( - DummyStorage::new(), - PruningStrategy::ByDepth(1024), - test_id(100) - ) - .unwrap() - .value_at_block(&test_id(50)) - .is_err()); - // when block is earlier than best finalized block AND it is finalized AND value is some - // [30] ---- 50 ---> [100] - assert_eq!( - ListCache::new( - DummyStorage::new() - .with_meta(Some(test_id(100)), Vec::new()) - .with_id(50, H256::from_low_u64_be(50)) - .with_entry( - test_id(100), - StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 } - ) - .with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }), - PruningStrategy::ByDepth(1024), - test_id(100) - ) - .unwrap() - .value_at_block(&test_id(50)) - .unwrap(), - Some((test_id(30), Some(test_id(100)), 30)) - ); - // when block is the best finalized block AND value is some - // ---> [100] - assert_eq!( - ListCache::new( - DummyStorage::new() - .with_meta(Some(test_id(100)), Vec::new()) - .with_id(100, H256::from_low_u64_be(100)) - .with_entry( - test_id(100), - StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 } - ) - .with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }), - PruningStrategy::ByDepth(1024), - test_id(100) - ) - .unwrap() - .value_at_block(&test_id(100)) - .unwrap(), - Some((test_id(100), None, 100)) - ); - // when block is parallel to the best finalized block - // ---- 100 - // ---> [100] - assert!(ListCache::new( - DummyStorage::new() - .with_meta(Some(test_id(100)), Vec::new()) - .with_id(50, H256::from_low_u64_be(50)) - .with_entry( - test_id(100), - StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 } - ) - .with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }), - PruningStrategy::ByDepth(1024), - test_id(100) - ) - .unwrap() - .value_at_block(&ComplexBlockId::new(H256::from_low_u64_be(2), 100)) - .is_err()); - - // when block is later than last finalized block AND there are no forks AND finalized value - // is Some ---> [100] --- 200 - assert_eq!( - ListCache::new( - DummyStorage::new() - .with_meta(Some(test_id(100)), Vec::new()) - .with_id(50, H256::from_low_u64_be(50)) - .with_entry( - test_id(100), - StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 } - ), - PruningStrategy::ByDepth(1024), - test_id(100) - ) - .unwrap() - .value_at_block(&test_id(200)) - .unwrap(), - Some((test_id(100), None, 100)) - ); - - // when block is later than last finalized block AND there are no matching forks - // AND block is connected to finalized block AND finalized value is Some - // --- 3 - // ---> [2] /---------> [4] - assert_eq!( - ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![correct_id(4)]) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }) - .with_entry( - correct_id(4), - StorageEntry { prev_valid_from: Some(correct_id(2)), value: 4 } - ) - .with_header(test_header(2)) - .with_header(test_header(3)) - .with_header(test_header(4)) - .with_header(fork_header(0, 2, 3)), - PruningStrategy::ByDepth(1024), - test_id(2) - ) - .unwrap() - .value_at_block(&fork_id(0, 2, 3)) - .unwrap(), - Some((correct_id(2), None, 2)) - ); - // when block is later than last finalized block AND there are no matching forks - // AND block is not connected to finalized block - // --- 2 --- 3 - // 1 /---> [2] ---------> [4] - assert_eq!( - ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![correct_id(4)]) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }) - .with_entry( - correct_id(4), - StorageEntry { prev_valid_from: Some(correct_id(2)), value: 4 } - ) - .with_header(test_header(1)) - .with_header(test_header(2)) - .with_header(test_header(3)) - .with_header(test_header(4)) - .with_header(fork_header(0, 1, 3)) - .with_header(fork_header(0, 1, 2)), - PruningStrategy::ByDepth(1024), - test_id(2) - ) - .unwrap() - .value_at_block(&fork_id(0, 1, 3)) - .unwrap(), - None - ); - - // when block is later than last finalized block AND it appends to unfinalized fork from the - // end AND unfinalized value is Some - // ---> [2] ---> [4] ---> 5 - assert_eq!( - ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![correct_id(4)]) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }) - .with_entry( - correct_id(4), - StorageEntry { prev_valid_from: Some(correct_id(2)), value: 4 } - ) - .with_header(test_header(4)) - .with_header(test_header(5)), - PruningStrategy::ByDepth(1024), - test_id(2) - ) - .unwrap() - .value_at_block(&correct_id(5)) - .unwrap(), - Some((correct_id(4), None, 4)) - ); - // when block is later than last finalized block AND it does not fits unfinalized fork - // AND it is connected to the finalized block AND finalized value is Some - // ---> [2] ----------> [4] - // \--- 3 - assert_eq!( - ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![correct_id(4)]) - .with_entry( - correct_id(4), - StorageEntry { prev_valid_from: Some(correct_id(2)), value: 4 } - ) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }) - .with_header(test_header(2)) - .with_header(test_header(3)) - .with_header(test_header(4)) - .with_header(fork_header(0, 2, 3)), - PruningStrategy::ByDepth(1024), - test_id(2) - ) - .unwrap() - .value_at_block(&fork_id(0, 2, 3)) - .unwrap(), - Some((correct_id(2), None, 2)) - ); - } - - #[test] - fn list_on_block_insert_works() { - let nfin = EntryType::NonFinal; - let fin = EntryType::Final; - - // when trying to insert block < finalized number - let mut ops = Default::default(); - assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100)) - .unwrap() - .do_on_block_insert( - &mut DummyTransaction::new(), - test_id(49), - test_id(50), - Some(50), - nfin, - &mut ops, - ) - .unwrap() - .is_none()); - // when trying to insert block @ finalized number - assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100)) - .unwrap() - .do_on_block_insert( - &mut DummyTransaction::new(), - test_id(99), - test_id(100), - Some(100), - nfin, - &Default::default(), - ) - .unwrap() - .is_none()); - - // when trying to insert non-final block AND it appends to the best block of unfinalized - // fork AND new value is the same as in the fork' best block - let mut cache = ListCache::new( - DummyStorage::new() - .with_meta(None, vec![test_id(4)]) - .with_entry(test_id(4), StorageEntry { prev_valid_from: None, value: 4 }), - PruningStrategy::ByDepth(1024), - test_id(2), - ) - .unwrap(); - cache.unfinalized[0].best_block = Some(test_id(4)); - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_insert( - &mut tx, - test_id(4), - test_id(5), - Some(4), - nfin, - &Default::default() - ) - .unwrap(), - Some(CommitOperation::AppendNewBlock(0, test_id(5))), - ); - assert!(tx.inserted_entries().is_empty()); - assert!(tx.removed_entries().is_empty()); - assert!(tx.updated_meta().is_none()); - // when trying to insert non-final block AND it appends to the best block of unfinalized - // fork AND new value is the same as in the fork' best block - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_insert( - &mut tx, - test_id(4), - test_id(5), - Some(5), - nfin, - &Default::default() - ) - .unwrap(), - Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: test_id(5), value: 5 })), - ); - assert_eq!(*tx.inserted_entries(), vec![test_id(5).hash].into_iter().collect()); - assert!(tx.removed_entries().is_empty()); - assert_eq!( - *tx.updated_meta(), - Some(Metadata { finalized: None, unfinalized: vec![test_id(5)] }) - ); - - // when trying to insert non-final block AND it is the first block that appends to the best - // block of unfinalized fork AND new value is the same as in the fork' best block - let cache = ListCache::new( - DummyStorage::new() - .with_meta(None, vec![correct_id(4)]) - .with_entry(correct_id(4), StorageEntry { prev_valid_from: None, value: 4 }) - .with_header(test_header(4)), - PruningStrategy::ByDepth(1024), - test_id(2), - ) - .unwrap(); - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_insert( - &mut tx, - correct_id(4), - correct_id(5), - Some(4), - nfin, - &Default::default(), - ) - .unwrap(), - Some(CommitOperation::AppendNewBlock(0, correct_id(5))), - ); - assert!(tx.inserted_entries().is_empty()); - assert!(tx.removed_entries().is_empty()); - assert!(tx.updated_meta().is_none()); - // when trying to insert non-final block AND it is the first block that appends to the best - // block of unfinalized fork AND new value is the same as in the fork' best block - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_insert( - &mut tx, - correct_id(4), - correct_id(5), - Some(5), - nfin, - &Default::default(), - ) - .unwrap(), - Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(5), value: 5 })), - ); - assert_eq!(*tx.inserted_entries(), vec![correct_id(5).hash].into_iter().collect()); - assert!(tx.removed_entries().is_empty()); - assert_eq!( - *tx.updated_meta(), - Some(Metadata { finalized: None, unfinalized: vec![correct_id(5)] }) - ); - - // when trying to insert non-final block AND it forks unfinalized fork - let cache = ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![correct_id(4)]) - .with_entry( - correct_id(4), - StorageEntry { prev_valid_from: Some(correct_id(2)), value: 4 }, - ) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }) - .with_header(test_header(2)) - .with_header(test_header(3)) - .with_header(test_header(4)), - PruningStrategy::ByDepth(1024), - correct_id(2), - ) - .unwrap(); - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_insert( - &mut tx, - correct_id(3), - fork_id(0, 3, 4), - Some(14), - nfin, - &Default::default() - ) - .unwrap(), - Some(CommitOperation::AddNewFork(Entry { valid_from: fork_id(0, 3, 4), value: 14 })), - ); - assert_eq!(*tx.inserted_entries(), vec![fork_id(0, 3, 4).hash].into_iter().collect()); - assert!(tx.removed_entries().is_empty()); - assert_eq!( - *tx.updated_meta(), - Some(Metadata { - finalized: Some(correct_id(2)), - unfinalized: vec![correct_id(4), fork_id(0, 3, 4)] - }) - ); - - // when trying to insert non-final block AND there are no unfinalized forks - // AND value is the same as last finalized - let cache = ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![]) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }), - PruningStrategy::ByDepth(1024), - correct_id(2), - ) - .unwrap(); - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_insert( - &mut tx, - correct_id(2), - correct_id(3), - Some(2), - nfin, - &Default::default() - ) - .unwrap(), - None, - ); - assert!(tx.inserted_entries().is_empty()); - assert!(tx.removed_entries().is_empty()); - assert!(tx.updated_meta().is_none()); - // when trying to insert non-final block AND there are no unfinalized forks - // AND value differs from last finalized - let cache = ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![]) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }), - PruningStrategy::ByDepth(1024), - correct_id(2), - ) - .unwrap(); - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_insert( - &mut tx, - correct_id(2), - correct_id(3), - Some(3), - nfin, - &Default::default() - ) - .unwrap(), - Some(CommitOperation::AddNewFork(Entry { valid_from: correct_id(3), value: 3 })), - ); - assert_eq!(*tx.inserted_entries(), vec![correct_id(3).hash].into_iter().collect()); - assert!(tx.removed_entries().is_empty()); - assert_eq!( - *tx.updated_meta(), - Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(3)] }) - ); - - // when inserting finalized entry AND there are no previous finalized entries - let cache = - ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), correct_id(2)) - .unwrap(); - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_insert( - &mut tx, - correct_id(2), - correct_id(3), - Some(3), - fin, - &Default::default() - ) - .unwrap(), - Some(CommitOperation::BlockFinalized( - correct_id(3), - Some(Entry { valid_from: correct_id(3), value: 3 }), - Default::default(), - )), - ); - assert_eq!(*tx.inserted_entries(), vec![correct_id(3).hash].into_iter().collect()); - assert!(tx.removed_entries().is_empty()); - assert_eq!( - *tx.updated_meta(), - Some(Metadata { finalized: Some(correct_id(3)), unfinalized: vec![] }) - ); - // when inserting finalized entry AND value is the same as in previous finalized - let cache = ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![]) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }), - PruningStrategy::ByDepth(1024), - correct_id(2), - ) - .unwrap(); - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_insert( - &mut tx, - correct_id(2), - correct_id(3), - Some(2), - fin, - &Default::default() - ) - .unwrap(), - Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())), - ); - assert!(tx.inserted_entries().is_empty()); - assert!(tx.removed_entries().is_empty()); - assert!(tx.updated_meta().is_none()); - // when inserting finalized entry AND value differs from previous finalized - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_insert( - &mut tx, - correct_id(2), - correct_id(3), - Some(3), - fin, - &Default::default() - ) - .unwrap(), - Some(CommitOperation::BlockFinalized( - correct_id(3), - Some(Entry { valid_from: correct_id(3), value: 3 }), - Default::default(), - )), - ); - assert_eq!(*tx.inserted_entries(), vec![correct_id(3).hash].into_iter().collect()); - assert!(tx.removed_entries().is_empty()); - assert_eq!( - *tx.updated_meta(), - Some(Metadata { finalized: Some(correct_id(3)), unfinalized: vec![] }) - ); - - // inserting finalized entry removes abandoned fork EVEN if new entry is not inserted - let cache = ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![fork_id(0, 1, 3)]) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }) - .with_entry(fork_id(0, 1, 3), StorageEntry { prev_valid_from: None, value: 13 }), - PruningStrategy::ByDepth(1024), - correct_id(2), - ) - .unwrap(); - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_insert( - &mut tx, - correct_id(2), - correct_id(3), - Some(2), - fin, - &Default::default() - ) - .unwrap(), - Some(CommitOperation::BlockFinalized( - correct_id(3), - None, - vec![0].into_iter().collect() - )), - ); - } - - #[test] - fn list_on_block_finalized_works() { - // finalization does not finalizes entry if it does not exists - let cache = ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![correct_id(5)]) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }) - .with_entry( - correct_id(5), - StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }, - ), - PruningStrategy::ByDepth(1024), - correct_id(2), - ) - .unwrap(); - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_finalize(&mut tx, correct_id(2), correct_id(3), &Default::default()) - .unwrap(), - Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())), - ); - assert!(tx.inserted_entries().is_empty()); - assert!(tx.removed_entries().is_empty()); - assert_eq!( - *tx.updated_meta(), - Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(5)] }), - ); - // finalization finalizes entry - let cache = ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![correct_id(5)]) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }) - .with_entry( - correct_id(5), - StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }, - ), - PruningStrategy::ByDepth(1024), - correct_id(4), - ) - .unwrap(); - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_finalize(&mut tx, correct_id(4), correct_id(5), &Default::default()) - .unwrap(), - Some(CommitOperation::BlockFinalized( - correct_id(5), - Some(Entry { valid_from: correct_id(5), value: 5 }), - vec![0].into_iter().collect(), - )), - ); - assert!(tx.inserted_entries().is_empty()); - assert!(tx.removed_entries().is_empty()); - assert_eq!( - *tx.updated_meta(), - Some(Metadata { finalized: Some(correct_id(5)), unfinalized: vec![] }) - ); - // finalization removes abandoned forks - let cache = ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![fork_id(0, 1, 3)]) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }) - .with_entry(fork_id(0, 1, 3), StorageEntry { prev_valid_from: None, value: 13 }), - PruningStrategy::ByDepth(1024), - correct_id(2), - ) - .unwrap(); - let mut tx = DummyTransaction::new(); - assert_eq!( - cache - .do_on_block_finalize(&mut tx, correct_id(2), correct_id(3), &Default::default()) - .unwrap(), - Some(CommitOperation::BlockFinalized( - correct_id(3), - None, - vec![0].into_iter().collect() - )), - ); - } - - #[test] - fn list_transaction_commit_works() { - let mut cache = ListCache::new( - DummyStorage::new() - .with_meta(Some(correct_id(2)), vec![correct_id(5), correct_id(6)]) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }) - .with_entry( - correct_id(5), - StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }, - ) - .with_entry( - correct_id(6), - StorageEntry { prev_valid_from: Some(correct_id(5)), value: 6 }, - ), - PruningStrategy::ByDepth(1024), - correct_id(2), - ) - .unwrap(); - - // when new block is appended to unfinalized fork - cache.on_transaction_commit(vec![CommitOperation::AppendNewBlock(0, correct_id(6))].into()); - assert_eq!(cache.unfinalized[0].best_block, Some(correct_id(6))); - // when new entry is appended to unfinalized fork - cache.on_transaction_commit( - vec![CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(7), value: 7 })] - .into(), - ); - assert_eq!(cache.unfinalized[0].best_block, Some(correct_id(7))); - assert_eq!(cache.unfinalized[0].head, Entry { valid_from: correct_id(7), value: 7 }); - // when new fork is added - cache.on_transaction_commit( - vec![CommitOperation::AddNewFork(Entry { valid_from: correct_id(10), value: 10 })] - .into(), - ); - assert_eq!(cache.unfinalized[2].best_block, Some(correct_id(10))); - assert_eq!(cache.unfinalized[2].head, Entry { valid_from: correct_id(10), value: 10 }); - // when block is finalized + entry is finalized + unfinalized forks are deleted - cache.on_transaction_commit( - vec![CommitOperation::BlockFinalized( - correct_id(20), - Some(Entry { valid_from: correct_id(20), value: 20 }), - vec![0, 1, 2].into_iter().collect(), - )] - .into(), - ); - assert_eq!(cache.best_finalized_block, correct_id(20)); - assert_eq!( - cache.best_finalized_entry, - Some(Entry { valid_from: correct_id(20), value: 20 }) - ); - assert!(cache.unfinalized.is_empty()); - } - - #[test] - fn list_find_unfinalized_fork_works() { - // ----------> [3] - // --- [2] ---------> 4 ---> [5] - assert_eq!( - ListCache::new( - DummyStorage::new() - .with_meta(None, vec![fork_id(0, 1, 3), correct_id(5)]) - .with_entry( - fork_id(0, 1, 3), - StorageEntry { prev_valid_from: Some(correct_id(1)), value: 13 } - ) - .with_entry( - correct_id(5), - StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 } - ) - .with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }) - .with_header(test_header(2)) - .with_header(test_header(3)) - .with_header(test_header(4)) - .with_header(test_header(5)), - PruningStrategy::ByDepth(1024), - correct_id(0) - ) - .unwrap() - .find_unfinalized_fork((&correct_id(4)).into()) - .unwrap() - .unwrap() - .head - .valid_from, - correct_id(5) - ); - // --- [2] ---------------> [5] - // ----------> [3] ---> 4 - assert_eq!( - ListCache::new( - DummyStorage::new() - .with_meta(None, vec![correct_id(5), fork_id(0, 1, 3)]) - .with_entry( - fork_id(0, 1, 3), - StorageEntry { prev_valid_from: Some(correct_id(1)), value: 13 } - ) - .with_entry( - correct_id(5), - StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 } - ) - .with_entry( - correct_id(2), - StorageEntry { prev_valid_from: Some(correct_id(1)), value: 2 } - ) - .with_header(test_header(2)) - .with_header(test_header(3)) - .with_header(test_header(4)) - .with_header(test_header(5)) - .with_header(fork_header(0, 1, 2)) - .with_header(fork_header(0, 1, 3)) - .with_header(fork_header(0, 1, 4)), - PruningStrategy::ByDepth(1024), - correct_id(0) - ) - .unwrap() - .find_unfinalized_fork((&fork_id(0, 1, 4)).into()) - .unwrap() - .unwrap() - .head - .valid_from, - fork_id(0, 1, 3) - ); - // --- [2] ---------------> [5] - // ----------> [3] - // -----------------> 4 - assert!(ListCache::new( - DummyStorage::new() - .with_meta(None, vec![correct_id(5), fork_id(0, 1, 3)]) - .with_entry( - fork_id(0, 1, 3), - StorageEntry { prev_valid_from: Some(correct_id(1)), value: 13 } - ) - .with_entry( - correct_id(5), - StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 } - ) - .with_entry( - correct_id(2), - StorageEntry { prev_valid_from: Some(correct_id(1)), value: 2 } - ) - .with_header(test_header(2)) - .with_header(test_header(3)) - .with_header(test_header(4)) - .with_header(test_header(5)) - .with_header(fork_header(0, 1, 3)) - .with_header(fork_header(0, 1, 4)) - .with_header(fork_header(1, 1, 2)) - .with_header(fork_header(1, 1, 3)) - .with_header(fork_header(1, 1, 4)), - PruningStrategy::ByDepth(1024), - correct_id(0) - ) - .unwrap() - .find_unfinalized_fork((&fork_id(1, 1, 4)).into()) - .unwrap() - .is_none()); - } - - #[test] - fn fork_matches_works() { - // when block is not within list range - let storage = DummyStorage::new() - .with_entry( - test_id(100), - StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 }, - ) - .with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 }); - assert_eq!( - Fork::<_, u64> { best_block: None, head: Entry { valid_from: test_id(100), value: 0 } } - .matches(&storage, (&test_id(20)).into()) - .unwrap(), - false - ); - // when block is not connected to the begin block - let storage = DummyStorage::new() - .with_entry( - correct_id(5), - StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 }, - ) - .with_entry(correct_id(3), StorageEntry { prev_valid_from: None, value: 200 }) - .with_header(test_header(5)) - .with_header(test_header(4)) - .with_header(test_header(3)) - .with_header(fork_header(0, 2, 4)) - .with_header(fork_header(0, 2, 3)); - assert_eq!( - Fork::<_, u64> { - best_block: None, - head: Entry { valid_from: correct_id(5), value: 100 } - } - .matches(&storage, (&fork_id(0, 2, 4)).into()) - .unwrap(), - false - ); - // when block is not connected to the end block - let storage = DummyStorage::new() - .with_entry( - correct_id(5), - StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 }, - ) - .with_entry(correct_id(3), StorageEntry { prev_valid_from: None, value: 200 }) - .with_header(test_header(5)) - .with_header(test_header(4)) - .with_header(test_header(3)) - .with_header(fork_header(0, 3, 4)); - assert_eq!( - Fork::<_, u64> { - best_block: None, - head: Entry { valid_from: correct_id(5), value: 100 } - } - .matches(&storage, (&fork_id(0, 3, 4)).into()) - .unwrap(), - false - ); - // when block is connected to the begin block AND end is open - let storage = DummyStorage::new() - .with_entry(correct_id(5), StorageEntry { prev_valid_from: None, value: 100 }) - .with_header(test_header(5)) - .with_header(test_header(6)); - assert_eq!( - Fork::<_, u64> { - best_block: None, - head: Entry { valid_from: correct_id(5), value: 100 } - } - .matches(&storage, (&correct_id(6)).into()) - .unwrap(), - true - ); - // when block is connected to the begin block AND to the end block - let storage = DummyStorage::new() - .with_entry( - correct_id(5), - StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 }, - ) - .with_entry(correct_id(3), StorageEntry { prev_valid_from: None, value: 200 }) - .with_header(test_header(5)) - .with_header(test_header(4)) - .with_header(test_header(3)); - assert_eq!( - Fork::<_, u64> { - best_block: None, - head: Entry { valid_from: correct_id(5), value: 100 } - } - .matches(&storage, (&correct_id(4)).into()) - .unwrap(), - true - ); - } - - #[test] - fn fork_try_append_works() { - // when best block is unknown - assert_eq!( - Fork::<_, u64> { best_block: None, head: Entry { valid_from: test_id(100), value: 0 } } - .try_append(&test_id(100)), - false - ); - // when best block is known but different - assert_eq!( - Fork::<_, u64> { best_block: None, head: Entry { valid_from: test_id(100), value: 0 } } - .try_append(&test_id(101)), - false - ); - // when best block is known and the same - assert_eq!( - Fork::<_, u64> { - best_block: Some(test_id(100)), - head: Entry { valid_from: test_id(100), value: 0 } - } - .try_append(&test_id(100)), - true - ); - } - - #[test] - fn fork_try_append_or_fork_works() { - // when there's no entry before parent - let storage = DummyStorage::new() - .with_entry( - test_id(100), - StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 }, - ) - .with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 }); - assert_eq!( - Fork::<_, u64> { best_block: None, head: Entry { valid_from: test_id(100), value: 0 } } - .try_append_or_fork(&storage, &test_id(30), None) - .unwrap(), - None - ); - // when parent does not belong to the fork - let storage = DummyStorage::new() - .with_entry( - correct_id(5), - StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 }, - ) - .with_entry(correct_id(3), StorageEntry { prev_valid_from: None, value: 200 }) - .with_header(test_header(5)) - .with_header(test_header(4)) - .with_header(test_header(3)) - .with_header(fork_header(0, 2, 4)) - .with_header(fork_header(0, 2, 3)); - assert_eq!( - Fork::<_, u64> { - best_block: None, - head: Entry { valid_from: correct_id(5), value: 100 } - } - .try_append_or_fork(&storage, &fork_id(0, 2, 4), None) - .unwrap(), - None - ); - // when the entry before parent is the head entry - let storage = DummyStorage::new() - .with_entry( - ComplexBlockId::new(test_header(5).hash(), 5), - StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 }, - ) - .with_header(test_header(6)) - .with_header(test_header(5)); - assert_eq!( - Fork::<_, u64> { - best_block: None, - head: Entry { valid_from: correct_id(5), value: 100 } - } - .try_append_or_fork(&storage, &correct_id(6), None) - .unwrap(), - Some(ForkAppendResult::Append) - ); - // when the parent located after last finalized entry - let storage = DummyStorage::new() - .with_entry( - correct_id(6), - StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 }, - ) - .with_entry(correct_id(3), StorageEntry { prev_valid_from: None, value: 200 }) - .with_header(test_header(6)) - .with_header(test_header(5)) - .with_header(test_header(4)) - .with_header(test_header(3)) - .with_header(fork_header(0, 4, 5)); - assert_eq!( - Fork::<_, u64> { - best_block: None, - head: Entry { valid_from: correct_id(6), value: 100 } - } - .try_append_or_fork(&storage, &fork_id(0, 4, 5), None) - .unwrap(), - Some(ForkAppendResult::Fork(ComplexBlockId::new(test_header(3).hash(), 3))) - ); - // when the parent located before last finalized entry - let storage = DummyStorage::new() - .with_entry( - correct_id(6), - StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 }, - ) - .with_entry(correct_id(3), StorageEntry { prev_valid_from: None, value: 200 }) - .with_header(test_header(6)) - .with_header(test_header(5)) - .with_header(test_header(4)) - .with_header(test_header(3)) - .with_header(fork_header(0, 4, 5)); - assert_eq!( - Fork::<_, u64> { - best_block: None, - head: Entry { valid_from: correct_id(6), value: 100 } - } - .try_append_or_fork(&storage, &fork_id(0, 4, 5), Some(3)) - .unwrap(), - None - ); - } - - #[test] - fn fork_destroy_works() { - // when we reached finalized entry without iterations - let storage = DummyStorage::new().with_id(100, H256::from_low_u64_be(100)); - let mut tx = DummyTransaction::new(); - Fork::<_, u64> { best_block: None, head: Entry { valid_from: test_id(100), value: 0 } } - .destroy(&storage, &mut tx, Some(200)) - .unwrap(); - assert!(tx.removed_entries().is_empty()); - // when we reach finalized entry with iterations - let storage = DummyStorage::new() - .with_id(10, H256::from_low_u64_be(10)) - .with_entry( - test_id(100), - StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 }, - ) - .with_entry(test_id(50), StorageEntry { prev_valid_from: Some(test_id(20)), value: 50 }) - .with_entry(test_id(20), StorageEntry { prev_valid_from: Some(test_id(10)), value: 20 }) - .with_entry(test_id(10), StorageEntry { prev_valid_from: Some(test_id(5)), value: 10 }) - .with_entry(test_id(5), StorageEntry { prev_valid_from: Some(test_id(3)), value: 5 }) - .with_entry(test_id(3), StorageEntry { prev_valid_from: None, value: 0 }); - let mut tx = DummyTransaction::new(); - Fork::<_, u64> { best_block: None, head: Entry { valid_from: test_id(100), value: 0 } } - .destroy(&storage, &mut tx, Some(200)) - .unwrap(); - assert_eq!( - *tx.removed_entries(), - vec![test_id(100).hash, test_id(50).hash, test_id(20).hash] - .into_iter() - .collect() - ); - // when we reach beginning of fork before finalized block - let storage = DummyStorage::new() - .with_id(10, H256::from_low_u64_be(10)) - .with_entry( - test_id(100), - StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 }, - ) - .with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 }); - let mut tx = DummyTransaction::new(); - Fork::<_, u64> { best_block: None, head: Entry { valid_from: test_id(100), value: 0 } } - .destroy(&storage, &mut tx, Some(200)) - .unwrap(); - assert_eq!( - *tx.removed_entries(), - vec![test_id(100).hash, test_id(50).hash].into_iter().collect() - ); - } - - #[test] - fn is_connected_to_block_fails() { - // when storage returns error - assert!(chain::is_connected_to_block::<_, u64, _>( - &FaultyStorage, - (&test_id(1)).into(), - &test_id(100), - ) - .is_err(),); - // when there's no header in the storage - assert!(chain::is_connected_to_block::<_, u64, _>( - &DummyStorage::new(), - (&test_id(1)).into(), - &test_id(100), - ) - .is_err(),); - } - - #[test] - fn is_connected_to_block_works() { - // when without iterations we end up with different block - assert_eq!( - chain::is_connected_to_block::<_, u64, _>( - &DummyStorage::new().with_header(test_header(1)), - (&test_id(1)).into(), - &correct_id(1) - ) - .unwrap(), - false - ); - // when with ASC iterations we end up with different block - assert_eq!( - chain::is_connected_to_block::<_, u64, _>( - &DummyStorage::new() - .with_header(test_header(0)) - .with_header(test_header(1)) - .with_header(test_header(2)), - (&test_id(0)).into(), - &correct_id(2) - ) - .unwrap(), - false - ); - // when with DESC iterations we end up with different block - assert_eq!( - chain::is_connected_to_block::<_, u64, _>( - &DummyStorage::new() - .with_header(test_header(0)) - .with_header(test_header(1)) - .with_header(test_header(2)), - (&correct_id(2)).into(), - &test_id(0) - ) - .unwrap(), - false - ); - // when without iterations we end up with the same block - assert_eq!( - chain::is_connected_to_block::<_, u64, _>( - &DummyStorage::new().with_header(test_header(1)), - (&correct_id(1)).into(), - &correct_id(1) - ) - .unwrap(), - true - ); - // when with ASC iterations we end up with the same block - assert_eq!( - chain::is_connected_to_block::<_, u64, _>( - &DummyStorage::new() - .with_header(test_header(0)) - .with_header(test_header(1)) - .with_header(test_header(2)), - (&correct_id(0)).into(), - &correct_id(2) - ) - .unwrap(), - true - ); - // when with DESC iterations we end up with the same block - assert_eq!( - chain::is_connected_to_block::<_, u64, _>( - &DummyStorage::new() - .with_header(test_header(0)) - .with_header(test_header(1)) - .with_header(test_header(2)), - (&correct_id(2)).into(), - &correct_id(0) - ) - .unwrap(), - true - ); - } - - #[test] - fn is_finalized_block_fails() { - // when storage returns error - assert!(chain::is_finalized_block::<_, u64, _>(&FaultyStorage, &test_id(1), 100).is_err()); - } - - #[test] - fn is_finalized_block_works() { - // when number of block is larger than last finalized block - assert_eq!( - chain::is_finalized_block::<_, u64, _>(&DummyStorage::new(), &test_id(100), 1).unwrap(), - false - ); - // when there's no hash for this block number in the database - assert_eq!( - chain::is_finalized_block::<_, u64, _>(&DummyStorage::new(), &test_id(1), 100).unwrap(), - false - ); - // when there's different hash for this block number in the database - assert_eq!( - chain::is_finalized_block::<_, u64, _>( - &DummyStorage::new().with_id(1, H256::from_low_u64_be(2)), - &test_id(1), - 100 - ) - .unwrap(), - false - ); - // when there's the same hash for this block number in the database - assert_eq!( - chain::is_finalized_block::<_, u64, _>( - &DummyStorage::new().with_id(1, H256::from_low_u64_be(1)), - &test_id(1), - 100 - ) - .unwrap(), - true - ); - } - - #[test] - fn read_forks_fails() { - // when storage returns error during finalized entry read - assert!(read_forks::( - &FaultyStorage, - Metadata { finalized: Some(test_id(1)), unfinalized: vec![] } - ) - .is_err()); - // when storage returns error during unfinalized entry read - assert!(read_forks::( - &FaultyStorage, - Metadata { finalized: None, unfinalized: vec![test_id(1)] } - ) - .is_err()); - // when finalized entry is not found - assert!(read_forks::( - &DummyStorage::new(), - Metadata { finalized: Some(test_id(1)), unfinalized: vec![] } - ) - .is_err()); - // when unfinalized entry is not found - assert!(read_forks::( - &DummyStorage::new(), - Metadata { finalized: None, unfinalized: vec![test_id(1)] } - ) - .is_err()); - } - - #[test] - fn read_forks_works() { - let storage = DummyStorage::new() - .with_entry(test_id(10), StorageEntry { prev_valid_from: Some(test_id(1)), value: 11 }) - .with_entry(test_id(20), StorageEntry { prev_valid_from: Some(test_id(2)), value: 0 }) - .with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 33 }); - let expected = ( - Some(Entry { valid_from: test_id(10), value: 11 }), - vec![ - Fork { best_block: None, head: Entry { valid_from: test_id(20), value: 0 } }, - Fork { best_block: None, head: Entry { valid_from: test_id(30), value: 33 } }, - ], - ); - - assert_eq!( - expected, - read_forks( - &storage, - Metadata { - finalized: Some(test_id(10)), - unfinalized: vec![test_id(20), test_id(30)], - } - ) - .unwrap() - ); - } - - #[test] - fn ancient_entries_are_pruned_when_pruning_enabled() { - fn do_test(strategy: PruningStrategy) { - let cache = ListCache::new( - DummyStorage::new() - .with_id(10, H256::from_low_u64_be(10)) - .with_id(20, H256::from_low_u64_be(20)) - .with_id(30, H256::from_low_u64_be(30)) - .with_entry(test_id(10), StorageEntry { prev_valid_from: None, value: 10 }) - .with_entry( - test_id(20), - StorageEntry { prev_valid_from: Some(test_id(10)), value: 20 }, - ) - .with_entry( - test_id(30), - StorageEntry { prev_valid_from: Some(test_id(20)), value: 30 }, - ), - strategy, - test_id(9), - ) - .unwrap(); - let mut tx = DummyTransaction::new(); - - // when finalizing entry #10: no entries pruned - cache.prune_finalized_entries(&mut tx, &test_id(10)); - assert!(tx.removed_entries().is_empty()); - assert!(tx.inserted_entries().is_empty()); - // when finalizing entry #19: no entries pruned - cache.prune_finalized_entries(&mut tx, &test_id(19)); - assert!(tx.removed_entries().is_empty()); - assert!(tx.inserted_entries().is_empty()); - // when finalizing entry #20: no entries pruned - cache.prune_finalized_entries(&mut tx, &test_id(20)); - assert!(tx.removed_entries().is_empty()); - assert!(tx.inserted_entries().is_empty()); - // when finalizing entry #30: entry 10 pruned + entry 20 is truncated (if pruning is - // enabled) - cache.prune_finalized_entries(&mut tx, &test_id(30)); - match strategy { - PruningStrategy::NeverPrune => { - assert!(tx.removed_entries().is_empty()); - assert!(tx.inserted_entries().is_empty()); - }, - PruningStrategy::ByDepth(_) => { - assert_eq!(*tx.removed_entries(), vec![test_id(10).hash].into_iter().collect()); - assert_eq!( - *tx.inserted_entries(), - vec![test_id(20).hash].into_iter().collect() - ); - }, - } - } - - do_test(PruningStrategy::ByDepth(10)); - do_test(PruningStrategy::NeverPrune) - } - - #[test] - fn revert_block_works() { - // 1 -> (2) -> 3 -> 4 -> 5 - // \ - // -> 5'' - // \ - // -> (3') -> 4' -> 5' - let mut cache = ListCache::new( - DummyStorage::new() - .with_meta( - Some(correct_id(1)), - vec![correct_id(5), fork_id(1, 2, 5), fork_id(2, 4, 5)], - ) - .with_id(1, correct_id(1).hash) - .with_entry(correct_id(1), StorageEntry { prev_valid_from: None, value: 1 }) - .with_entry( - correct_id(3), - StorageEntry { prev_valid_from: Some(correct_id(1)), value: 3 }, - ) - .with_entry( - correct_id(4), - StorageEntry { prev_valid_from: Some(correct_id(3)), value: 4 }, - ) - .with_entry( - correct_id(5), - StorageEntry { prev_valid_from: Some(correct_id(4)), value: 5 }, - ) - .with_entry( - fork_id(1, 2, 4), - StorageEntry { prev_valid_from: Some(correct_id(1)), value: 14 }, - ) - .with_entry( - fork_id(1, 2, 5), - StorageEntry { prev_valid_from: Some(fork_id(1, 2, 4)), value: 15 }, - ) - .with_entry( - fork_id(2, 4, 5), - StorageEntry { prev_valid_from: Some(correct_id(4)), value: 25 }, - ) - .with_header(test_header(1)) - .with_header(test_header(2)) - .with_header(test_header(3)) - .with_header(test_header(4)) - .with_header(test_header(5)) - .with_header(fork_header(1, 2, 3)) - .with_header(fork_header(1, 2, 4)) - .with_header(fork_header(1, 2, 5)) - .with_header(fork_header(2, 4, 5)), - PruningStrategy::ByDepth(1024), - correct_id(1), - ) - .unwrap(); - - // when 5 is reverted: entry 5 is truncated - let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(5)).unwrap(); - assert_eq!( - op, - CommitOperation::BlockReverted( - vec![( - 0, - Some(Fork { - best_block: None, - head: Entry { valid_from: correct_id(4), value: 4 } - }) - ),] - .into_iter() - .collect() - ) - ); - cache.on_transaction_commit(vec![op].into()); - - // when 3 is reverted: entries 4+5' are truncated - let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(3)).unwrap(); - assert_eq!( - op, - CommitOperation::BlockReverted(vec![(0, None), (2, None),].into_iter().collect()) - ); - cache.on_transaction_commit(vec![op].into()); - - // when 2 is reverted: entries 4'+5' are truncated - let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(2)).unwrap(); - assert_eq!(op, CommitOperation::BlockReverted(vec![(0, None),].into_iter().collect())); - cache.on_transaction_commit(vec![op].into()); - } - - #[test] - fn append_commit_operation_works() { - let mut ops = CommitOperations::default(); - ops.append(None); - assert_eq!(ops.operations, Vec::new()); - - ops.append(Some(CommitOperation::BlockFinalized( - test_id(10), - Some(Entry { valid_from: test_id(10), value: 10 }), - vec![5].into_iter().collect(), - ))); - assert_eq!( - ops.operations, - vec![CommitOperation::BlockFinalized( - test_id(10), - Some(Entry { valid_from: test_id(10), value: 10 }), - vec![5].into_iter().collect(), - )], - ); - - ops.append(Some(CommitOperation::BlockFinalized( - test_id(20), - Some(Entry { valid_from: test_id(20), value: 20 }), - vec![5, 6].into_iter().collect(), - ))); - - assert_eq!( - ops.operations, - vec![CommitOperation::BlockFinalized( - test_id(20), - Some(Entry { valid_from: test_id(20), value: 20 }), - vec![5, 6].into_iter().collect(), - )], - ); - } -} diff --git a/client/db/src/cache/list_entry.rs b/client/db/src/cache/list_entry.rs deleted file mode 100644 index 7cee7a514626..000000000000 --- a/client/db/src/cache/list_entry.rs +++ /dev/null @@ -1,187 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! List-cache storage entries. - -use codec::{Decode, Encode}; -use sp_blockchain::Result as ClientResult; -use sp_runtime::traits::{Block as BlockT, NumberFor}; - -use crate::cache::{list_storage::Storage, CacheItemT, ComplexBlockId}; - -/// Single list-based cache entry. -#[derive(Debug)] -#[cfg_attr(test, derive(PartialEq))] -pub struct Entry { - /// first block, when this value became actual. - pub valid_from: ComplexBlockId, - /// Value stored at this entry. - pub value: T, -} - -/// Internal representation of the single list-based cache entry. The entry points to the -/// previous entry in the cache, allowing us to traverse back in time in list-style. -#[derive(Debug, Encode, Decode)] -#[cfg_attr(test, derive(Clone, PartialEq))] -pub struct StorageEntry { - /// None if valid from the beginning. - pub prev_valid_from: Option>, - /// Value stored at this entry. - pub value: T, -} - -impl Entry { - /// Returns Some if the entry should be updated with the new value. - pub fn try_update(&self, value: Option) -> Option> { - match value { - Some(value) => match self.value == value { - true => None, - false => - Some(StorageEntry { prev_valid_from: Some(self.valid_from.clone()), value }), - }, - None => None, - } - } - - /// Wrapper that calls search_before to get range where the given block fits. - pub fn search_best_range_before>( - &self, - storage: &S, - block: NumberFor, - ) -> ClientResult, Option>)>> { - Ok(self - .search_best_before(storage, block)? - .map(|(entry, next)| (entry.valid_from, next))) - } - - /// Searches the list, ending with THIS entry for the best entry preceding (or at) - /// given block number. - /// If the entry is found, result is the entry and the block id of next entry (if exists). - /// NOTE that this function does not check that the passed block is actually linked to - /// the blocks it found. - pub fn search_best_before>( - &self, - storage: &S, - block: NumberFor, - ) -> ClientResult, Option>)>> { - // we're looking for the best value - let mut next = None; - let mut current = self.valid_from.clone(); - if block >= self.valid_from.number { - let value = self.value.clone(); - return Ok(Some((Entry { valid_from: current, value }, next))) - } - - // else - travel back in time - loop { - let entry = storage.require_entry(¤t)?; - if block >= current.number { - return Ok(Some((Entry { valid_from: current, value: entry.value }, next))) - } - - next = Some(current); - current = match entry.prev_valid_from { - Some(prev_valid_from) => prev_valid_from, - None => return Ok(None), - }; - } - } -} - -impl StorageEntry { - /// Converts storage entry into an entry, valid from given block. - pub fn into_entry(self, valid_from: ComplexBlockId) -> Entry { - Entry { valid_from, value: self.value } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::cache::list_storage::tests::{DummyStorage, FaultyStorage}; - use substrate_test_runtime_client::runtime::{Block, H256}; - - fn test_id(number: u64) -> ComplexBlockId { - ComplexBlockId::new(H256::from_low_u64_be(number), number) - } - - #[test] - fn entry_try_update_works() { - // when trying to update with None value - assert_eq!(Entry::<_, u64> { valid_from: test_id(1), value: 42 }.try_update(None), None); - // when trying to update with the same Some value - assert_eq!(Entry { valid_from: test_id(1), value: 1 }.try_update(Some(1)), None); - // when trying to update with different Some value - assert_eq!( - Entry { valid_from: test_id(1), value: 1 }.try_update(Some(2)), - Some(StorageEntry { prev_valid_from: Some(test_id(1)), value: 2 }) - ); - } - - #[test] - fn entry_search_best_before_fails() { - // when storage returns error - assert!(Entry::<_, u64> { valid_from: test_id(100), value: 42 } - .search_best_before(&FaultyStorage, 50) - .is_err()); - } - - #[test] - fn entry_search_best_before_works() { - // when block is better than our best block - assert_eq!( - Entry::<_, u64> { valid_from: test_id(100), value: 100 } - .search_best_before(&DummyStorage::new(), 150) - .unwrap(), - Some((Entry::<_, u64> { valid_from: test_id(100), value: 100 }, None)) - ); - // when block is found between two entries - assert_eq!( - Entry::<_, u64> { valid_from: test_id(100), value: 100 } - .search_best_before( - &DummyStorage::new() - .with_entry( - test_id(100), - StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 } - ) - .with_entry( - test_id(50), - StorageEntry { prev_valid_from: Some(test_id(30)), value: 50 } - ), - 75 - ) - .unwrap(), - Some((Entry::<_, u64> { valid_from: test_id(50), value: 50 }, Some(test_id(100)))) - ); - // when block is not found - assert_eq!( - Entry::<_, u64> { valid_from: test_id(100), value: 100 } - .search_best_before( - &DummyStorage::new() - .with_entry( - test_id(100), - StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 } - ) - .with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 }), - 30 - ) - .unwrap(), - None - ); - } -} diff --git a/client/db/src/cache/list_storage.rs b/client/db/src/cache/list_storage.rs deleted file mode 100644 index bb47b8dab5a7..000000000000 --- a/client/db/src/cache/list_storage.rs +++ /dev/null @@ -1,441 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! List-cache storage definition and implementation. - -use std::sync::Arc; - -use crate::utils::{self, meta_keys}; -use codec::{Decode, Encode}; -use sp_blockchain::{Error as ClientError, Result as ClientResult}; -use sp_database::{Database, Transaction}; -use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, Header as HeaderT, NumberFor}, -}; - -use crate::{ - cache::{ - list_cache::{CommitOperation, Fork}, - list_entry::{Entry, StorageEntry}, - CacheItemT, ComplexBlockId, - }, - DbHash, -}; - -/// Single list-cache metadata. -#[derive(Debug)] -#[cfg_attr(test, derive(Clone, PartialEq))] -pub struct Metadata { - /// Block at which best finalized entry is stored. - pub finalized: Option>, - /// A set of blocks at which best unfinalized entries are stored. - pub unfinalized: Vec>, -} - -/// Readonly list-cache storage trait. -pub trait Storage { - /// Reads hash of the block at given number. - fn read_id(&self, at: NumberFor) -> ClientResult>; - - /// Reads header of the block with given hash. - fn read_header(&self, at: &Block::Hash) -> ClientResult>; - - /// Reads cache metadata: best finalized entry (if some) and the list. - fn read_meta(&self) -> ClientResult>; - - /// Reads cache entry from the storage. - fn read_entry( - &self, - at: &ComplexBlockId, - ) -> ClientResult>>; - - /// Reads referenced (and thus existing) cache entry from the storage. - fn require_entry(&self, at: &ComplexBlockId) -> ClientResult> { - self.read_entry(at).and_then(|entry| { - entry.ok_or_else(|| { - ClientError::from(ClientError::Backend(format!( - "Referenced cache entry at {:?} is not found", - at - ))) - }) - }) - } -} - -/// List-cache storage transaction. -pub trait StorageTransaction { - /// Insert storage entry at given block. - fn insert_storage_entry(&mut self, at: &ComplexBlockId, entry: &StorageEntry); - - /// Delete storage entry at given block. - fn remove_storage_entry(&mut self, at: &ComplexBlockId); - - /// Update metadata of the cache. - fn update_meta( - &mut self, - best_finalized_entry: Option<&Entry>, - unfinalized: &[Fork], - operation: &CommitOperation, - ); -} - -/// A set of columns used by the DbStorage. -#[derive(Debug)] -pub struct DbColumns { - /// Column holding cache meta. - pub meta: u32, - /// Column holding the mapping of { block number => block hash } for blocks of the best chain. - pub key_lookup: u32, - /// Column holding the mapping of { block hash => block header }. - pub header: u32, - /// Column holding cache entries. - pub cache: u32, -} - -/// Database-backed list cache storage. -pub struct DbStorage { - name: Vec, - meta_key: Vec, - db: Arc>, - columns: DbColumns, -} - -impl DbStorage { - /// Create new database-backed list cache storage. - pub fn new(name: Vec, db: Arc>, columns: DbColumns) -> Self { - let meta_key = meta::key(&name); - DbStorage { name, meta_key, db, columns } - } - - /// Get reference to the database. - pub fn db(&self) -> &Arc> { - &self.db - } - - /// Get reference to the database columns. - pub fn columns(&self) -> &DbColumns { - &self.columns - } - - /// Encode block id for storing as a key in cache column. - /// We append prefix to the actual encoding to allow several caches - /// store entries in the same column. - pub fn encode_block_id(&self, block: &ComplexBlockId) -> Vec { - let mut encoded = self.name.clone(); - encoded.extend(block.hash.as_ref()); - encoded - } -} - -impl Storage for DbStorage { - fn read_id(&self, at: NumberFor) -> ClientResult> { - utils::read_header::( - &*self.db, - self.columns.key_lookup, - self.columns.header, - BlockId::Number(at), - ) - .map(|maybe_header| maybe_header.map(|header| header.hash())) - } - - fn read_header(&self, at: &Block::Hash) -> ClientResult> { - utils::read_header::( - &*self.db, - self.columns.key_lookup, - self.columns.header, - BlockId::Hash(*at), - ) - } - - fn read_meta(&self) -> ClientResult> { - match self.db.get(self.columns.meta, &self.meta_key) { - Some(meta) => meta::decode(&*meta), - None => Ok(Metadata { finalized: None, unfinalized: Vec::new() }), - } - } - - fn read_entry( - &self, - at: &ComplexBlockId, - ) -> ClientResult>> { - match self.db.get(self.columns.cache, &self.encode_block_id(at)) { - Some(entry) => StorageEntry::::decode(&mut &entry[..]) - .map_err(|_| ClientError::Backend("Failed to decode cache entry".into())) - .map(Some), - None => Ok(None), - } - } -} - -/// Database-backed list cache storage transaction. -pub struct DbStorageTransaction<'a> { - storage: &'a DbStorage, - tx: &'a mut Transaction, -} - -impl<'a> DbStorageTransaction<'a> { - /// Create new database transaction. - pub fn new(storage: &'a DbStorage, tx: &'a mut Transaction) -> Self { - DbStorageTransaction { storage, tx } - } -} - -impl<'a, Block: BlockT, T: CacheItemT> StorageTransaction for DbStorageTransaction<'a> { - fn insert_storage_entry(&mut self, at: &ComplexBlockId, entry: &StorageEntry) { - self.tx.set_from_vec( - self.storage.columns.cache, - &self.storage.encode_block_id(at), - entry.encode(), - ); - } - - fn remove_storage_entry(&mut self, at: &ComplexBlockId) { - self.tx.remove(self.storage.columns.cache, &self.storage.encode_block_id(at)); - } - - fn update_meta( - &mut self, - best_finalized_entry: Option<&Entry>, - unfinalized: &[Fork], - operation: &CommitOperation, - ) { - self.tx.set_from_vec( - self.storage.columns.meta, - &self.storage.meta_key, - meta::encode(best_finalized_entry, unfinalized, operation), - ); - } -} - -/// Metadata related functions. -mod meta { - use super::*; - - /// Convert cache name into cache metadata key. - pub fn key(name: &[u8]) -> Vec { - let mut key_name = meta_keys::CACHE_META_PREFIX.to_vec(); - key_name.extend_from_slice(name); - key_name - } - - /// Encode cache metadata 'applying' commit operation before encoding. - pub fn encode( - best_finalized_entry: Option<&Entry>, - unfinalized: &[Fork], - op: &CommitOperation, - ) -> Vec { - let mut finalized = best_finalized_entry.as_ref().map(|entry| &entry.valid_from); - let mut unfinalized = - unfinalized.iter().map(|fork| &fork.head().valid_from).collect::>(); - - match op { - CommitOperation::AppendNewBlock(_, _) => (), - CommitOperation::AppendNewEntry(index, ref entry) => { - unfinalized[*index] = &entry.valid_from; - }, - CommitOperation::AddNewFork(ref entry) => { - unfinalized.push(&entry.valid_from); - }, - CommitOperation::BlockFinalized(_, ref finalizing_entry, ref forks) => { - if let Some(finalizing_entry) = finalizing_entry.as_ref() { - finalized = Some(&finalizing_entry.valid_from); - } - for fork_index in forks.iter().rev() { - unfinalized.remove(*fork_index); - } - }, - CommitOperation::BlockReverted(ref forks) => { - for (fork_index, updated_fork) in forks.iter().rev() { - match updated_fork { - Some(updated_fork) => - unfinalized[*fork_index] = &updated_fork.head().valid_from, - None => { - unfinalized.remove(*fork_index); - }, - } - } - }, - } - - (finalized, unfinalized).encode() - } - - /// Decode meta information. - pub fn decode(encoded: &[u8]) -> ClientResult> { - let input = &mut &*encoded; - let finalized: Option> = Decode::decode(input).map_err(|_| { - ClientError::from(ClientError::Backend("Error decoding cache meta".into())) - })?; - let unfinalized: Vec> = Decode::decode(input).map_err(|_| { - ClientError::from(ClientError::Backend("Error decoding cache meta".into())) - })?; - - Ok(Metadata { finalized, unfinalized }) - } -} - -#[cfg(test)] -pub mod tests { - use super::*; - use std::collections::{HashMap, HashSet}; - - pub struct FaultyStorage; - - impl Storage for FaultyStorage { - fn read_id(&self, _at: NumberFor) -> ClientResult> { - Err(ClientError::Backend("TestError".into())) - } - - fn read_header(&self, _at: &Block::Hash) -> ClientResult> { - Err(ClientError::Backend("TestError".into())) - } - - fn read_meta(&self) -> ClientResult> { - Err(ClientError::Backend("TestError".into())) - } - - fn read_entry( - &self, - _at: &ComplexBlockId, - ) -> ClientResult>> { - Err(ClientError::Backend("TestError".into())) - } - } - - pub struct DummyStorage { - meta: Metadata, - ids: HashMap, Block::Hash>, - headers: HashMap, - entries: HashMap>, - } - - impl DummyStorage { - pub fn new() -> Self { - DummyStorage { - meta: Metadata { finalized: None, unfinalized: Vec::new() }, - ids: HashMap::new(), - headers: HashMap::new(), - entries: HashMap::new(), - } - } - - pub fn with_meta( - mut self, - finalized: Option>, - unfinalized: Vec>, - ) -> Self { - self.meta.finalized = finalized; - self.meta.unfinalized = unfinalized; - self - } - - pub fn with_id(mut self, at: NumberFor, id: Block::Hash) -> Self { - self.ids.insert(at, id); - self - } - - pub fn with_header(mut self, header: Block::Header) -> Self { - self.headers.insert(header.hash(), header); - self - } - - pub fn with_entry( - mut self, - at: ComplexBlockId, - entry: StorageEntry, - ) -> Self { - self.entries.insert(at.hash, entry); - self - } - } - - impl Storage for DummyStorage { - fn read_id(&self, at: NumberFor) -> ClientResult> { - Ok(self.ids.get(&at).cloned()) - } - - fn read_header(&self, at: &Block::Hash) -> ClientResult> { - Ok(self.headers.get(&at).cloned()) - } - - fn read_meta(&self) -> ClientResult> { - Ok(self.meta.clone()) - } - - fn read_entry( - &self, - at: &ComplexBlockId, - ) -> ClientResult>> { - Ok(self.entries.get(&at.hash).cloned()) - } - } - - pub struct DummyTransaction { - updated_meta: Option>, - inserted_entries: HashSet, - removed_entries: HashSet, - } - - impl DummyTransaction { - pub fn new() -> Self { - DummyTransaction { - updated_meta: None, - inserted_entries: HashSet::new(), - removed_entries: HashSet::new(), - } - } - - pub fn inserted_entries(&self) -> &HashSet { - &self.inserted_entries - } - - pub fn removed_entries(&self) -> &HashSet { - &self.removed_entries - } - - pub fn updated_meta(&self) -> &Option> { - &self.updated_meta - } - } - - impl StorageTransaction for DummyTransaction { - fn insert_storage_entry( - &mut self, - at: &ComplexBlockId, - _entry: &StorageEntry, - ) { - self.inserted_entries.insert(at.hash); - } - - fn remove_storage_entry(&mut self, at: &ComplexBlockId) { - self.removed_entries.insert(at.hash); - } - - fn update_meta( - &mut self, - best_finalized_entry: Option<&Entry>, - unfinalized: &[Fork], - operation: &CommitOperation, - ) { - self.updated_meta = Some( - meta::decode(&meta::encode(best_finalized_entry, unfinalized, operation)).unwrap(), - ); - } - } -} diff --git a/client/db/src/cache/mod.rs b/client/db/src/cache/mod.rs deleted file mode 100644 index 5502896aced2..000000000000 --- a/client/db/src/cache/mod.rs +++ /dev/null @@ -1,413 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! DB-backed cache of blockchain data. - -use parking_lot::RwLock; -use std::{ - collections::{hash_map::Entry, HashMap}, - sync::Arc, -}; - -use crate::{ - utils::{self, COLUMN_META}, - DbHash, -}; -use codec::{Decode, Encode}; -use sc_client_api::blockchain::{ - well_known_cache_keys::{self, Id as CacheKeyId}, - Cache as BlockchainCache, -}; -use sp_blockchain::{HeaderMetadataCache, Result as ClientResult}; -use sp_database::{Database, Transaction}; -use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}, -}; - -use self::list_cache::{ListCache, PruningStrategy}; - -mod list_cache; -mod list_entry; -mod list_storage; - -/// Minimal post-finalization age of finalized blocks before they'll pruned. -const PRUNE_DEPTH: u32 = 1024; - -/// The type of entry that is inserted to the cache. -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum EntryType { - /// Non-final entry. - NonFinal, - /// Final entry. - Final, - /// Genesis entry (inserted during cache initialization). - Genesis, -} - -/// Block identifier that holds both hash and number. -#[derive(Clone, Debug, Encode, Decode, PartialEq)] -pub struct ComplexBlockId { - /// Hash of the block. - pub(crate) hash: Block::Hash, - /// Number of the block. - pub(crate) number: NumberFor, -} - -impl ComplexBlockId { - /// Create new complex block id. - pub fn new(hash: Block::Hash, number: NumberFor) -> Self { - ComplexBlockId { hash, number } - } -} - -impl ::std::cmp::PartialOrd for ComplexBlockId { - fn partial_cmp(&self, other: &ComplexBlockId) -> Option<::std::cmp::Ordering> { - self.number.partial_cmp(&other.number) - } -} - -/// All cache items must implement this trait. -pub trait CacheItemT: Clone + Decode + Encode + PartialEq {} - -impl CacheItemT for T where T: Clone + Decode + Encode + PartialEq {} - -/// Database-backed blockchain data cache. -pub struct DbCache { - cache_at: HashMap, self::list_storage::DbStorage>>, - header_metadata_cache: Arc>, - db: Arc>, - key_lookup_column: u32, - header_column: u32, - cache_column: u32, - genesis_hash: Block::Hash, - best_finalized_block: ComplexBlockId, -} - -impl DbCache { - /// Create new cache. - pub fn new( - db: Arc>, - header_metadata_cache: Arc>, - key_lookup_column: u32, - header_column: u32, - cache_column: u32, - genesis_hash: Block::Hash, - best_finalized_block: ComplexBlockId, - ) -> Self { - Self { - cache_at: HashMap::new(), - db, - header_metadata_cache, - key_lookup_column, - header_column, - cache_column, - genesis_hash, - best_finalized_block, - } - } - - /// Set genesis block hash. - pub fn set_genesis_hash(&mut self, genesis_hash: Block::Hash) { - self.genesis_hash = genesis_hash; - } - - /// Begin cache transaction. - pub fn transaction<'a>( - &'a mut self, - tx: &'a mut Transaction, - ) -> DbCacheTransaction<'a, Block> { - DbCacheTransaction { - cache: self, - tx, - cache_at_ops: HashMap::new(), - best_finalized_block: None, - } - } - - /// Begin cache transaction with given ops. - pub fn transaction_with_ops<'a>( - &'a mut self, - tx: &'a mut Transaction, - ops: DbCacheTransactionOps, - ) -> DbCacheTransaction<'a, Block> { - DbCacheTransaction { - cache: self, - tx, - cache_at_ops: ops.cache_at_ops, - best_finalized_block: ops.best_finalized_block, - } - } - - /// Run post-commit cache operations. - pub fn commit(&mut self, ops: DbCacheTransactionOps) -> ClientResult<()> { - for (name, ops) in ops.cache_at_ops.into_iter() { - self.get_cache(name)?.on_transaction_commit(ops); - } - if let Some(best_finalized_block) = ops.best_finalized_block { - self.best_finalized_block = best_finalized_block; - } - Ok(()) - } - - /// Creates `ListCache` with the given name or returns a reference to the existing. - pub(crate) fn get_cache( - &mut self, - name: CacheKeyId, - ) -> ClientResult<&mut ListCache, self::list_storage::DbStorage>> { - get_cache_helper( - &mut self.cache_at, - name, - &self.db, - self.key_lookup_column, - self.header_column, - self.cache_column, - &self.best_finalized_block, - ) - } -} - -// This helper is needed because otherwise the borrow checker will require to -// clone all parameters outside of the closure. -fn get_cache_helper<'a, Block: BlockT>( - cache_at: &'a mut HashMap, self::list_storage::DbStorage>>, - name: CacheKeyId, - db: &Arc>, - key_lookup: u32, - header: u32, - cache: u32, - best_finalized_block: &ComplexBlockId, -) -> ClientResult<&'a mut ListCache, self::list_storage::DbStorage>> { - match cache_at.entry(name) { - Entry::Occupied(entry) => Ok(entry.into_mut()), - Entry::Vacant(entry) => { - let cache = ListCache::new( - self::list_storage::DbStorage::new( - name.to_vec(), - db.clone(), - self::list_storage::DbColumns { meta: COLUMN_META, key_lookup, header, cache }, - ), - cache_pruning_strategy(name), - best_finalized_block.clone(), - )?; - Ok(entry.insert(cache)) - }, - } -} - -/// Cache operations that are to be committed after database transaction is committed. -#[derive(Default)] -pub struct DbCacheTransactionOps { - cache_at_ops: HashMap>>, - best_finalized_block: Option>, -} - -impl DbCacheTransactionOps { - /// Empty transaction ops. - pub fn empty() -> DbCacheTransactionOps { - DbCacheTransactionOps { cache_at_ops: HashMap::new(), best_finalized_block: None } - } -} - -/// Database-backed blockchain data cache transaction valid for single block import. -pub struct DbCacheTransaction<'a, Block: BlockT> { - cache: &'a mut DbCache, - tx: &'a mut Transaction, - cache_at_ops: HashMap>>, - best_finalized_block: Option>, -} - -impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> { - /// Convert transaction into post-commit operations set. - pub fn into_ops(self) -> DbCacheTransactionOps { - DbCacheTransactionOps { - cache_at_ops: self.cache_at_ops, - best_finalized_block: self.best_finalized_block, - } - } - - /// When new block is inserted into database. - pub fn on_block_insert( - mut self, - parent: ComplexBlockId, - block: ComplexBlockId, - data_at: HashMap>, - entry_type: EntryType, - ) -> ClientResult { - // prepare list of caches that are not update - // (we might still need to do some cache maintenance in this case) - let missed_caches = self - .cache - .cache_at - .keys() - .filter(|cache| !data_at.contains_key(*cache)) - .cloned() - .collect::>(); - - let mut insert_op = |name: CacheKeyId, - value: Option>| - -> Result<(), sp_blockchain::Error> { - let cache = self.cache.get_cache(name)?; - let cache_ops = self.cache_at_ops.entry(name).or_default(); - cache.on_block_insert( - &mut self::list_storage::DbStorageTransaction::new(cache.storage(), &mut self.tx), - parent.clone(), - block.clone(), - value, - entry_type, - cache_ops, - )?; - - Ok(()) - }; - - data_at.into_iter().try_for_each(|(name, data)| insert_op(name, Some(data)))?; - missed_caches.into_iter().try_for_each(|name| insert_op(name, None))?; - - match entry_type { - EntryType::Final | EntryType::Genesis => self.best_finalized_block = Some(block), - EntryType::NonFinal => (), - } - - Ok(self) - } - - /// When previously inserted block is finalized. - pub fn on_block_finalize( - mut self, - parent: ComplexBlockId, - block: ComplexBlockId, - ) -> ClientResult { - for (name, cache) in self.cache.cache_at.iter() { - let cache_ops = self.cache_at_ops.entry(*name).or_default(); - cache.on_block_finalize( - &mut self::list_storage::DbStorageTransaction::new(cache.storage(), &mut self.tx), - parent.clone(), - block.clone(), - cache_ops, - )?; - } - - self.best_finalized_block = Some(block); - - Ok(self) - } - - /// When block is reverted. - pub fn on_block_revert(mut self, reverted_block: &ComplexBlockId) -> ClientResult { - for (name, cache) in self.cache.cache_at.iter() { - let cache_ops = self.cache_at_ops.entry(*name).or_default(); - cache.on_block_revert( - &mut self::list_storage::DbStorageTransaction::new(cache.storage(), &mut self.tx), - reverted_block, - cache_ops, - )?; - } - - Ok(self) - } -} - -/// Synchronous implementation of database-backed blockchain data cache. -pub struct DbCacheSync(pub RwLock>); - -impl BlockchainCache for DbCacheSync { - fn initialize(&self, key: &CacheKeyId, data: Vec) -> ClientResult<()> { - let mut cache = self.0.write(); - let genesis_hash = cache.genesis_hash; - let cache_contents = vec![(*key, data)].into_iter().collect(); - let db = cache.db.clone(); - let mut dbtx = Transaction::new(); - let tx = cache.transaction(&mut dbtx); - let tx = tx.on_block_insert( - ComplexBlockId::new(Default::default(), Zero::zero()), - ComplexBlockId::new(genesis_hash, Zero::zero()), - cache_contents, - EntryType::Genesis, - )?; - let tx_ops = tx.into_ops(); - db.commit(dbtx)?; - cache.commit(tx_ops)?; - - Ok(()) - } - - fn get_at( - &self, - key: &CacheKeyId, - at: &BlockId, - ) -> ClientResult< - Option<((NumberFor, Block::Hash), Option<(NumberFor, Block::Hash)>, Vec)>, - > { - let mut cache = self.0.write(); - let header_metadata_cache = cache.header_metadata_cache.clone(); - let cache = cache.get_cache(*key)?; - let storage = cache.storage(); - let db = storage.db(); - let columns = storage.columns(); - let at = match *at { - BlockId::Hash(hash) => match header_metadata_cache.header_metadata(hash) { - Some(metadata) => ComplexBlockId::new(hash, metadata.number), - None => { - let header = utils::require_header::( - &**db, - columns.key_lookup, - columns.header, - BlockId::Hash(hash.clone()), - )?; - ComplexBlockId::new(hash, *header.number()) - }, - }, - BlockId::Number(number) => { - let hash = utils::require_header::( - &**db, - columns.key_lookup, - columns.header, - BlockId::Number(number.clone()), - )? - .hash(); - ComplexBlockId::new(hash, number) - }, - }; - - cache.value_at_block(&at).map(|block_and_value| { - block_and_value.map(|(begin_block, end_block, value)| { - ( - (begin_block.number, begin_block.hash), - end_block.map(|end_block| (end_block.number, end_block.hash)), - value, - ) - }) - }) - } -} - -/// Get pruning strategy for given cache. -fn cache_pruning_strategy>(cache: CacheKeyId) -> PruningStrategy { - // the cache is mostly used to store data from consensus engines - // this kind of data is only required for non-finalized blocks - // => by default we prune finalized cached entries - - match cache { - // we need to keep changes tries configurations forever (or at least until changes tries, - // that were built using this configuration, are pruned) to make it possible to refer - // to old changes tries - well_known_cache_keys::CHANGES_TRIE_CONFIG => PruningStrategy::NeverPrune, - _ => PruningStrategy::ByDepth(PRUNE_DEPTH.into()), - } -} diff --git a/client/db/src/changes_tries_storage.rs b/client/db/src/changes_tries_storage.rs deleted file mode 100644 index 3a3c5918535f..000000000000 --- a/client/db/src/changes_tries_storage.rs +++ /dev/null @@ -1,1168 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! DB-backed changes tries storage. - -use crate::{ - cache::{ - ComplexBlockId, DbCache, DbCacheSync, DbCacheTransactionOps, EntryType as CacheEntryType, - }, - utils::{self, meta_keys, Meta}, - Database, DbHash, -}; -use codec::{Decode, Encode}; -use hash_db::Prefix; -use parking_lot::RwLock; -use sc_client_api::backend::PrunableStateChangesTrieStorage; -use sp_blockchain::{ - well_known_cache_keys, Cache as BlockchainCache, Error as ClientError, HeaderMetadataCache, - Result as ClientResult, -}; -use sp_core::{ - convert_hash, storage::PrefixedStorageKey, ChangesTrieConfiguration, - ChangesTrieConfigurationRange, -}; -use sp_database::Transaction; -use sp_runtime::{ - generic::{BlockId, ChangesTrieSignal, DigestItem}, - traits::{Block as BlockT, CheckedSub, HashFor, Header as HeaderT, NumberFor, One, Zero}, -}; -use sp_state_machine::{ChangesTrieBuildCache, ChangesTrieCacheAction}; -use sp_trie::MemoryDB; -use std::{ - collections::{HashMap, HashSet}, - sync::Arc, -}; - -/// Extract new changes trie configuration (if available) from the header. -pub fn extract_new_configuration( - header: &Header, -) -> Option<&Option> { - header - .digest() - .log(DigestItem::as_changes_trie_signal) - .and_then(ChangesTrieSignal::as_new_configuration) -} - -/// Opaque configuration cache transaction. During its lifetime, no-one should modify cache. This is -/// currently guaranteed because import lock is held during block import/finalization. -pub struct DbChangesTrieStorageTransaction { - /// Cache operations that must be performed after db transaction is committed. - cache_ops: DbCacheTransactionOps, - /// New configuration (if changed at current block). - new_config: Option>, -} - -impl DbChangesTrieStorageTransaction { - /// Consume self and return transaction with given new configuration. - pub fn with_new_config(mut self, new_config: Option>) -> Self { - self.new_config = new_config; - self - } -} - -impl From> for DbChangesTrieStorageTransaction { - fn from(cache_ops: DbCacheTransactionOps) -> Self { - DbChangesTrieStorageTransaction { cache_ops, new_config: None } - } -} - -/// Changes tries storage. -/// -/// Stores all tries in separate DB column. -/// Lock order: meta, tries_meta, cache, build_cache. -pub struct DbChangesTrieStorage { - db: Arc>, - meta_column: u32, - changes_tries_column: u32, - key_lookup_column: u32, - header_column: u32, - meta: Arc, Block::Hash>>>, - tries_meta: RwLock>, - min_blocks_to_keep: Option, - /// The cache stores all ever existing changes tries configurations. - cache: DbCacheSync, - /// Build cache is a map of block => set of storage keys changed at this block. - /// They're used to build digest blocks - instead of reading+parsing tries from db - /// we just use keys sets from the cache. - build_cache: RwLock>>, -} - -/// Persistent struct that contains all the changes tries metadata. -#[derive(Decode, Encode, Debug)] -struct ChangesTriesMeta { - /// Oldest unpruned max-level (or skewed) digest trie blocks range. - /// The range is inclusive from both sides. - /// Is None only if: - /// 1) we haven't yet finalized any blocks (except genesis) - /// 2) if best_finalized_block - min_blocks_to_keep points to the range where changes tries are - /// disabled 3) changes tries pruning is disabled - pub oldest_digest_range: Option<(NumberFor, NumberFor)>, - /// End block (inclusive) of oldest pruned max-level (or skewed) digest trie blocks range. - /// It is guaranteed that we have no any changes tries before (and including) this block. - /// It is guaranteed that all existing changes tries after this block are not yet pruned (if - /// created). - pub oldest_pruned_digest_range_end: NumberFor, -} - -impl DbChangesTrieStorage { - /// Create new changes trie storage. - pub fn new( - db: Arc>, - header_metadata_cache: Arc>, - meta_column: u32, - changes_tries_column: u32, - key_lookup_column: u32, - header_column: u32, - cache_column: u32, - meta: Arc, Block::Hash>>>, - min_blocks_to_keep: Option, - ) -> ClientResult { - let (finalized_hash, finalized_number, genesis_hash) = { - let meta = meta.read(); - (meta.finalized_hash, meta.finalized_number, meta.genesis_hash) - }; - let tries_meta = read_tries_meta(&*db, meta_column)?; - Ok(Self { - db: db.clone(), - meta_column, - changes_tries_column, - key_lookup_column, - header_column, - meta, - min_blocks_to_keep, - cache: DbCacheSync(RwLock::new(DbCache::new( - db.clone(), - header_metadata_cache, - key_lookup_column, - header_column, - cache_column, - genesis_hash, - ComplexBlockId::new(finalized_hash, finalized_number), - ))), - build_cache: RwLock::new(ChangesTrieBuildCache::new()), - tries_meta: RwLock::new(tries_meta), - }) - } - - /// Commit new changes trie. - pub fn commit( - &self, - tx: &mut Transaction, - mut changes_trie: MemoryDB>, - parent_block: ComplexBlockId, - block: ComplexBlockId, - new_header: &Block::Header, - finalized: bool, - new_configuration: Option>, - cache_tx: Option>, - ) -> ClientResult> { - // insert changes trie, associated with block, into DB - for (key, (val, _)) in changes_trie.drain() { - tx.set(self.changes_tries_column, key.as_ref(), &val); - } - - // if configuration has not been changed AND block is not finalized => nothing to do here - let new_configuration = match new_configuration { - Some(new_configuration) => new_configuration, - None if !finalized => return Ok(DbCacheTransactionOps::empty().into()), - None => - return self.finalize( - tx, - parent_block.hash, - block.hash, - block.number, - Some(new_header), - cache_tx, - ), - }; - - // update configuration cache - let mut cache_at = HashMap::new(); - cache_at.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_configuration.encode()); - Ok(DbChangesTrieStorageTransaction::from(match cache_tx { - Some(cache_tx) => self - .cache - .0 - .write() - .transaction_with_ops(tx, cache_tx.cache_ops) - .on_block_insert( - parent_block, - block, - cache_at, - if finalized { CacheEntryType::Final } else { CacheEntryType::NonFinal }, - )? - .into_ops(), - None => self - .cache - .0 - .write() - .transaction(tx) - .on_block_insert( - parent_block, - block, - cache_at, - if finalized { CacheEntryType::Final } else { CacheEntryType::NonFinal }, - )? - .into_ops(), - }) - .with_new_config(Some(new_configuration))) - } - - /// Called when block is finalized. - pub fn finalize( - &self, - tx: &mut Transaction, - parent_block_hash: Block::Hash, - block_hash: Block::Hash, - block_num: NumberFor, - new_header: Option<&Block::Header>, - cache_tx: Option>, - ) -> ClientResult> { - // prune obsolete changes tries - self.prune(tx, block_hash, block_num, new_header.clone(), cache_tx.as_ref())?; - - // if we have inserted the block that we're finalizing in the same transaction - // => then we have already finalized it from the commit() call - if cache_tx.is_some() { - if let Some(new_header) = new_header { - if new_header.hash() == block_hash { - return Ok(cache_tx.expect("guarded by cache_tx.is_some(); qed")) - } - } - } - - // and finalize configuration cache entries - let block = ComplexBlockId::new(block_hash, block_num); - let parent_block_num = block_num.checked_sub(&One::one()).unwrap_or_else(|| Zero::zero()); - let parent_block = ComplexBlockId::new(parent_block_hash, parent_block_num); - Ok(match cache_tx { - Some(cache_tx) => DbChangesTrieStorageTransaction::from( - self.cache - .0 - .write() - .transaction_with_ops(tx, cache_tx.cache_ops) - .on_block_finalize(parent_block, block)? - .into_ops(), - ) - .with_new_config(cache_tx.new_config), - None => DbChangesTrieStorageTransaction::from( - self.cache - .0 - .write() - .transaction(tx) - .on_block_finalize(parent_block, block)? - .into_ops(), - ), - }) - } - - /// When block is reverted. - pub fn revert( - &self, - tx: &mut Transaction, - block: &ComplexBlockId, - ) -> ClientResult> { - Ok(self.cache.0.write().transaction(tx).on_block_revert(block)?.into_ops().into()) - } - - /// When transaction has been committed. - pub fn post_commit(&self, tx: Option>) { - if let Some(tx) = tx { - self.cache.0.write().commit(tx.cache_ops).expect( - "only fails if cache with given name isn't loaded yet; cache is already loaded \ - because there is tx; qed", - ); - } - } - - /// Commit changes into changes trie build cache. - pub fn commit_build_cache( - &self, - cache_update: ChangesTrieCacheAction>, - ) { - self.build_cache.write().perform(cache_update); - } - - /// Prune obsolete changes tries. - fn prune( - &self, - tx: &mut Transaction, - block_hash: Block::Hash, - block_num: NumberFor, - new_header: Option<&Block::Header>, - cache_tx: Option<&DbChangesTrieStorageTransaction>, - ) -> ClientResult<()> { - // never prune on archive nodes - let min_blocks_to_keep = match self.min_blocks_to_keep { - Some(min_blocks_to_keep) => min_blocks_to_keep, - None => return Ok(()), - }; - - let mut tries_meta = self.tries_meta.write(); - let mut next_digest_range_start = block_num; - loop { - // prune oldest digest if it is known - // it could be unknown if: - // 1) either we're finalizing block#1 - // 2) or we are (or were) in period where changes tries are disabled - if let Some((begin, end)) = tries_meta.oldest_digest_range { - if block_num <= end || block_num - end <= min_blocks_to_keep.into() { - break - } - - tries_meta.oldest_pruned_digest_range_end = end; - sp_state_machine::prune_changes_tries( - &*self, - begin, - end, - &sp_state_machine::ChangesTrieAnchorBlockId { - hash: convert_hash(&block_hash), - number: block_num, - }, - |node| tx.remove(self.changes_tries_column, node.as_ref()), - ); - - next_digest_range_start = end + One::one(); - } - - // proceed to the next configuration range - let next_digest_range_start_hash = match block_num == next_digest_range_start { - true => block_hash, - false => utils::require_header::( - &*self.db, - self.key_lookup_column, - self.header_column, - BlockId::Number(next_digest_range_start), - )? - .hash(), - }; - - let config_for_new_block = new_header - .map(|header| *header.number() == next_digest_range_start) - .unwrap_or(false); - let next_config = match cache_tx { - Some(cache_tx) if config_for_new_block && cache_tx.new_config.is_some() => { - let config = cache_tx.new_config.clone().expect("guarded by is_some(); qed"); - Ok(ChangesTrieConfigurationRange { - zero: (block_num, block_hash), - end: None, - config, - }) - }, - _ if config_for_new_block => self.configuration_at(&BlockId::Hash( - *new_header - .expect("config_for_new_block is only true when new_header is passed; qed") - .parent_hash(), - )), - _ => self.configuration_at(&BlockId::Hash(next_digest_range_start_hash)), - }; - let next_config = match next_config { - Ok(next_config) => next_config, - Err(ClientError::UnknownBlock(_)) => break, // No block means nothing to prune. - Err(e) => return Err(e), - }; - if let Some(config) = next_config.config { - let mut oldest_digest_range = config - .next_max_level_digest_range(next_config.zero.0, next_digest_range_start) - .unwrap_or_else(|| (next_digest_range_start, next_digest_range_start)); - - if let Some(end) = next_config.end { - if end.0 < oldest_digest_range.1 { - oldest_digest_range.1 = end.0; - } - } - - tries_meta.oldest_digest_range = Some(oldest_digest_range); - continue - } - - tries_meta.oldest_digest_range = None; - break - } - - write_tries_meta(tx, self.meta_column, &*tries_meta); - Ok(()) - } -} - -impl PrunableStateChangesTrieStorage for DbChangesTrieStorage { - fn storage( - &self, - ) -> &dyn sp_state_machine::ChangesTrieStorage, NumberFor> { - self - } - - fn configuration_at( - &self, - at: &BlockId, - ) -> ClientResult, Block::Hash>> { - self.cache - .get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, at)? - .and_then(|(zero, end, encoded)| { - Decode::decode(&mut &encoded[..]) - .ok() - .map(|config| ChangesTrieConfigurationRange { zero, end, config }) - }) - .ok_or_else(|| ClientError::ErrorReadingChangesTriesConfig) - } - - fn oldest_pruned_digest_range_end(&self) -> NumberFor { - self.tries_meta.read().oldest_pruned_digest_range_end - } -} - -impl sp_state_machine::ChangesTrieRootsStorage, NumberFor> - for DbChangesTrieStorage -{ - fn build_anchor( - &self, - hash: Block::Hash, - ) -> Result>, String> { - utils::read_header::( - &*self.db, - self.key_lookup_column, - self.header_column, - BlockId::Hash(hash), - ) - .map_err(|e| e.to_string()) - .and_then(|maybe_header| { - maybe_header - .map(|header| sp_state_machine::ChangesTrieAnchorBlockId { - hash, - number: *header.number(), - }) - .ok_or_else(|| format!("Unknown header: {}", hash)) - }) - } - - fn root( - &self, - anchor: &sp_state_machine::ChangesTrieAnchorBlockId>, - block: NumberFor, - ) -> Result, String> { - // check API requirement: we can't get NEXT block(s) based on anchor - if block > anchor.number { - return Err(format!( - "Can't get changes trie root at {} using anchor at {}", - block, anchor.number - )) - } - - // we need to get hash of the block to resolve changes trie root - let block_id = if block <= self.meta.read().finalized_number { - // if block is finalized, we could just read canonical hash - BlockId::Number(block) - } else { - // the block is not finalized - let mut current_num = anchor.number; - let mut current_hash: Block::Hash = convert_hash(&anchor.hash); - let maybe_anchor_header: Block::Header = utils::require_header::( - &*self.db, - self.key_lookup_column, - self.header_column, - BlockId::Number(current_num), - ) - .map_err(|e| e.to_string())?; - if maybe_anchor_header.hash() == current_hash { - // if anchor is canonicalized, then the block is also canonicalized - BlockId::Number(block) - } else { - // else (block is not finalized + anchor is not canonicalized): - // => we should find the required block hash by traversing - // back from the anchor to the block with given number - while current_num != block { - let current_header: Block::Header = utils::require_header::( - &*self.db, - self.key_lookup_column, - self.header_column, - BlockId::Hash(current_hash), - ) - .map_err(|e| e.to_string())?; - - current_hash = *current_header.parent_hash(); - current_num = current_num - One::one(); - } - - BlockId::Hash(current_hash) - } - }; - - Ok(utils::require_header::( - &*self.db, - self.key_lookup_column, - self.header_column, - block_id, - ) - .map_err(|e| e.to_string())? - .digest() - .log(DigestItem::as_changes_trie_root) - .cloned()) - } -} - -impl sp_state_machine::ChangesTrieStorage, NumberFor> - for DbChangesTrieStorage -where - Block: BlockT, -{ - fn as_roots_storage( - &self, - ) -> &dyn sp_state_machine::ChangesTrieRootsStorage, NumberFor> { - self - } - - fn with_cached_changed_keys( - &self, - root: &Block::Hash, - functor: &mut dyn FnMut(&HashMap, HashSet>>), - ) -> bool { - self.build_cache.read().with_changed_keys(root, functor) - } - - fn get(&self, key: &Block::Hash, _prefix: Prefix) -> Result>, String> { - Ok(self.db.get(self.changes_tries_column, key.as_ref())) - } -} - -/// Read changes tries metadata from database. -fn read_tries_meta( - db: &dyn Database, - meta_column: u32, -) -> ClientResult> { - match db.get(meta_column, meta_keys::CHANGES_TRIES_META) { - Some(h) => Decode::decode(&mut &h[..]).map_err(|err| { - ClientError::Backend(format!("Error decoding changes tries metadata: {}", err)) - }), - None => Ok(ChangesTriesMeta { - oldest_digest_range: None, - oldest_pruned_digest_range_end: Zero::zero(), - }), - } -} - -/// Write changes tries metadata from database. -fn write_tries_meta( - tx: &mut Transaction, - meta_column: u32, - meta: &ChangesTriesMeta, -) { - tx.set_from_vec(meta_column, meta_keys::CHANGES_TRIES_META, meta.encode()); -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - tests::{insert_header, prepare_changes, Block}, - Backend, - }; - use hash_db::EMPTY_PREFIX; - use sc_client_api::backend::{ - Backend as ClientBackend, BlockImportOperation, NewBlockState, - PrunableStateChangesTrieStorage, - }; - use sp_blockchain::HeaderBackend as BlockchainHeaderBackend; - use sp_core::H256; - use sp_runtime::{ - testing::{Digest, Header}, - traits::{BlakeTwo256, Hash}, - }; - use sp_state_machine::{ChangesTrieRootsStorage, ChangesTrieStorage}; - - fn changes(number: u64) -> Option, Vec)>> { - Some(vec![(number.to_le_bytes().to_vec(), number.to_le_bytes().to_vec())]) - } - - fn insert_header_with_configuration_change( - backend: &Backend, - number: u64, - parent_hash: H256, - changes: Option, Vec)>>, - new_configuration: Option, - ) -> H256 { - let mut digest = Digest::default(); - let mut changes_trie_update = Default::default(); - if let Some(changes) = changes { - let (root, update) = prepare_changes(changes); - digest.push(DigestItem::ChangesTrieRoot(root)); - changes_trie_update = update; - } - digest.push(DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration( - new_configuration, - ))); - - let header = Header { - number, - parent_hash, - state_root: BlakeTwo256::trie_root(Vec::new()), - digest, - extrinsics_root: Default::default(), - }; - let header_hash = header.hash(); - - let block_id = if number == 0 { - BlockId::Hash(Default::default()) - } else { - BlockId::Number(number - 1) - }; - let mut op = backend.begin_operation().unwrap(); - backend.begin_state_operation(&mut op, block_id).unwrap(); - op.set_block_data(header, None, None, None, NewBlockState::Best).unwrap(); - op.update_changes_trie((changes_trie_update, ChangesTrieCacheAction::Clear)) - .unwrap(); - backend.commit_operation(op).unwrap(); - - header_hash - } - - #[test] - fn changes_trie_storage_works() { - let backend = Backend::::new_test(1000, 100); - backend.changes_tries_storage.meta.write().finalized_number = 1000; - - let check_changes = |backend: &Backend, - block: u64, - changes: Vec<(Vec, Vec)>| { - let (changes_root, mut changes_trie_update) = prepare_changes(changes); - let anchor = sp_state_machine::ChangesTrieAnchorBlockId { - hash: backend.blockchain().header(BlockId::Number(block)).unwrap().unwrap().hash(), - number: block, - }; - assert_eq!(backend.changes_tries_storage.root(&anchor, block), Ok(Some(changes_root))); - - let storage = backend.changes_tries_storage.storage(); - for (key, (val, _)) in changes_trie_update.drain() { - assert_eq!(storage.get(&key, EMPTY_PREFIX), Ok(Some(val))); - } - }; - - let changes0 = vec![(b"key_at_0".to_vec(), b"val_at_0".to_vec())]; - let changes1 = vec![ - (b"key_at_1".to_vec(), b"val_at_1".to_vec()), - (b"another_key_at_1".to_vec(), b"another_val_at_1".to_vec()), - ]; - let changes2 = vec![(b"key_at_2".to_vec(), b"val_at_2".to_vec())]; - - let block0 = insert_header( - &backend, - 0, - Default::default(), - Some(changes0.clone()), - Default::default(), - ); - let block1 = insert_header(&backend, 1, block0, Some(changes1.clone()), Default::default()); - let _ = insert_header(&backend, 2, block1, Some(changes2.clone()), Default::default()); - - // check that the storage contains tries for all blocks - check_changes(&backend, 0, changes0); - check_changes(&backend, 1, changes1); - check_changes(&backend, 2, changes2); - } - - #[test] - fn changes_trie_storage_works_with_forks() { - let backend = Backend::::new_test(1000, 100); - - let changes0 = vec![(b"k0".to_vec(), b"v0".to_vec())]; - let changes1 = vec![(b"k1".to_vec(), b"v1".to_vec())]; - let changes2 = vec![(b"k2".to_vec(), b"v2".to_vec())]; - let block0 = insert_header( - &backend, - 0, - Default::default(), - Some(changes0.clone()), - Default::default(), - ); - let block1 = insert_header(&backend, 1, block0, Some(changes1.clone()), Default::default()); - let block2 = insert_header(&backend, 2, block1, Some(changes2.clone()), Default::default()); - - let changes2_1_0 = vec![(b"k3".to_vec(), b"v3".to_vec())]; - let changes2_1_1 = vec![(b"k4".to_vec(), b"v4".to_vec())]; - let block2_1_0 = - insert_header(&backend, 3, block2, Some(changes2_1_0.clone()), Default::default()); - let block2_1_1 = - insert_header(&backend, 4, block2_1_0, Some(changes2_1_1.clone()), Default::default()); - - let changes2_2_0 = vec![(b"k5".to_vec(), b"v5".to_vec())]; - let changes2_2_1 = vec![(b"k6".to_vec(), b"v6".to_vec())]; - let block2_2_0 = - insert_header(&backend, 3, block2, Some(changes2_2_0.clone()), Default::default()); - let block2_2_1 = - insert_header(&backend, 4, block2_2_0, Some(changes2_2_1.clone()), Default::default()); - - // finalize block1 - backend.changes_tries_storage.meta.write().finalized_number = 1; - - // branch1: when asking for finalized block hash - let (changes1_root, _) = prepare_changes(changes1); - let anchor = sp_state_machine::ChangesTrieAnchorBlockId { hash: block2_1_1, number: 4 }; - assert_eq!(backend.changes_tries_storage.root(&anchor, 1), Ok(Some(changes1_root))); - - // branch2: when asking for finalized block hash - let anchor = sp_state_machine::ChangesTrieAnchorBlockId { hash: block2_2_1, number: 4 }; - assert_eq!(backend.changes_tries_storage.root(&anchor, 1), Ok(Some(changes1_root))); - - // branch1: when asking for non-finalized block hash (search by traversal) - let (changes2_1_0_root, _) = prepare_changes(changes2_1_0); - let anchor = sp_state_machine::ChangesTrieAnchorBlockId { hash: block2_1_1, number: 4 }; - assert_eq!(backend.changes_tries_storage.root(&anchor, 3), Ok(Some(changes2_1_0_root))); - - // branch2: when asking for non-finalized block hash (search using canonicalized hint) - let (changes2_2_0_root, _) = prepare_changes(changes2_2_0); - let anchor = sp_state_machine::ChangesTrieAnchorBlockId { hash: block2_2_1, number: 4 }; - assert_eq!(backend.changes_tries_storage.root(&anchor, 3), Ok(Some(changes2_2_0_root))); - - // finalize first block of branch2 (block2_2_0) - backend.changes_tries_storage.meta.write().finalized_number = 3; - - // branch2: when asking for finalized block of this branch - assert_eq!(backend.changes_tries_storage.root(&anchor, 3), Ok(Some(changes2_2_0_root))); - - // branch1: when asking for finalized block of other branch - // => result is incorrect (returned for the block of branch1), but this is expected, - // because the other fork is abandoned (forked before finalized header) - let anchor = sp_state_machine::ChangesTrieAnchorBlockId { hash: block2_1_1, number: 4 }; - assert_eq!(backend.changes_tries_storage.root(&anchor, 3), Ok(Some(changes2_2_0_root))); - } - - #[test] - fn changes_tries_are_pruned_on_finalization() { - let mut backend = Backend::::new_test(1000, 100); - backend.changes_tries_storage.min_blocks_to_keep = Some(8); - - let parent_hash = |number| { - if number == 0 { - Default::default() - } else { - backend - .blockchain() - .header(BlockId::Number(number - 1)) - .unwrap() - .unwrap() - .hash() - } - }; - - let insert_regular_header = |with_changes, number| { - insert_header( - &backend, - number, - parent_hash(number), - if with_changes { changes(number) } else { None }, - Default::default(), - ); - }; - - let is_pruned = |number| { - let trie_root = backend - .blockchain() - .header(BlockId::Number(number)) - .unwrap() - .unwrap() - .digest() - .log(DigestItem::as_changes_trie_root) - .cloned(); - match trie_root { - Some(trie_root) => - backend.changes_tries_storage.get(&trie_root, EMPTY_PREFIX).unwrap().is_none(), - None => true, - } - }; - - let finalize_block = |number| { - let header = backend.blockchain().header(BlockId::Number(number)).unwrap().unwrap(); - let mut tx = Transaction::new(); - let cache_ops = backend - .changes_tries_storage - .finalize(&mut tx, *header.parent_hash(), header.hash(), number, None, None) - .unwrap(); - backend.storage.db.commit(tx).unwrap(); - backend.changes_tries_storage.post_commit(Some(cache_ops)); - }; - - // configuration ranges: - // (0; 6] - None - // [7; 17] - Some(2^2): D2 is built at #10, #14; SD is built at #17 - // [18; 21] - None - // [22; 32] - Some(8^1): D1 is built at #29; SD is built at #32 - // [33; ... - Some(1) - let config_at_6 = Some(ChangesTrieConfiguration::new(2, 2)); - let config_at_17 = None; - let config_at_21 = Some(ChangesTrieConfiguration::new(8, 1)); - let config_at_32 = Some(ChangesTrieConfiguration::new(1, 0)); - - (0..6).for_each(|number| insert_regular_header(false, number)); - insert_header_with_configuration_change(&backend, 6, parent_hash(6), None, config_at_6); - (7..17).for_each(|number| insert_regular_header(true, number)); - insert_header_with_configuration_change( - &backend, - 17, - parent_hash(17), - changes(17), - config_at_17, - ); - (18..21).for_each(|number| insert_regular_header(false, number)); - insert_header_with_configuration_change(&backend, 21, parent_hash(21), None, config_at_21); - (22..32).for_each(|number| insert_regular_header(true, number)); - insert_header_with_configuration_change( - &backend, - 32, - parent_hash(32), - changes(32), - config_at_32, - ); - (33..50).for_each(|number| insert_regular_header(true, number)); - - // when only genesis is finalized, nothing is pruned - (0..=6).for_each(|number| assert!(is_pruned(number))); - (7..=17).for_each(|number| assert!(!is_pruned(number))); - (18..=21).for_each(|number| assert!(is_pruned(number))); - (22..50).for_each(|number| assert!(!is_pruned(number))); - - // when blocks [1; 18] are finalized, nothing is pruned - (1..=18).for_each(|number| finalize_block(number)); - (0..=6).for_each(|number| assert!(is_pruned(number))); - (7..=17).for_each(|number| assert!(!is_pruned(number))); - (18..=21).for_each(|number| assert!(is_pruned(number))); - (22..50).for_each(|number| assert!(!is_pruned(number))); - - // when block 19 is finalized, changes tries for blocks [7; 10] are pruned - finalize_block(19); - (0..=10).for_each(|number| assert!(is_pruned(number))); - (11..=17).for_each(|number| assert!(!is_pruned(number))); - (18..=21).for_each(|number| assert!(is_pruned(number))); - (22..50).for_each(|number| assert!(!is_pruned(number))); - - // when blocks [20; 22] are finalized, nothing is pruned - (20..=22).for_each(|number| finalize_block(number)); - (0..=10).for_each(|number| assert!(is_pruned(number))); - (11..=17).for_each(|number| assert!(!is_pruned(number))); - (18..=21).for_each(|number| assert!(is_pruned(number))); - (22..50).for_each(|number| assert!(!is_pruned(number))); - - // when block 23 is finalized, changes tries for blocks [11; 14] are pruned - finalize_block(23); - (0..=14).for_each(|number| assert!(is_pruned(number))); - (15..=17).for_each(|number| assert!(!is_pruned(number))); - (18..=21).for_each(|number| assert!(is_pruned(number))); - (22..50).for_each(|number| assert!(!is_pruned(number))); - - // when blocks [24; 25] are finalized, nothing is pruned - (24..=25).for_each(|number| finalize_block(number)); - (0..=14).for_each(|number| assert!(is_pruned(number))); - (15..=17).for_each(|number| assert!(!is_pruned(number))); - (18..=21).for_each(|number| assert!(is_pruned(number))); - (22..50).for_each(|number| assert!(!is_pruned(number))); - - // when block 26 is finalized, changes tries for blocks [15; 17] are pruned - finalize_block(26); - (0..=21).for_each(|number| assert!(is_pruned(number))); - (22..50).for_each(|number| assert!(!is_pruned(number))); - - // when blocks [27; 37] are finalized, nothing is pruned - (27..=37).for_each(|number| finalize_block(number)); - (0..=21).for_each(|number| assert!(is_pruned(number))); - (22..50).for_each(|number| assert!(!is_pruned(number))); - - // when block 38 is finalized, changes tries for blocks [22; 29] are pruned - finalize_block(38); - (0..=29).for_each(|number| assert!(is_pruned(number))); - (30..50).for_each(|number| assert!(!is_pruned(number))); - - // when blocks [39; 40] are finalized, nothing is pruned - (39..=40).for_each(|number| finalize_block(number)); - (0..=29).for_each(|number| assert!(is_pruned(number))); - (30..50).for_each(|number| assert!(!is_pruned(number))); - - // when block 41 is finalized, changes tries for blocks [30; 32] are pruned - finalize_block(41); - (0..=32).for_each(|number| assert!(is_pruned(number))); - (33..50).for_each(|number| assert!(!is_pruned(number))); - - // when block 42 is finalized, changes trie for block 33 is pruned - finalize_block(42); - (0..=33).for_each(|number| assert!(is_pruned(number))); - (34..50).for_each(|number| assert!(!is_pruned(number))); - - // when block 43 is finalized, changes trie for block 34 is pruned - finalize_block(43); - (0..=34).for_each(|number| assert!(is_pruned(number))); - (35..50).for_each(|number| assert!(!is_pruned(number))); - } - - #[test] - fn changes_tries_configuration_is_updated_on_block_insert() { - let backend = Backend::::new_test(1000, 100); - - // configurations at blocks - let config_at_1 = Some(ChangesTrieConfiguration { digest_interval: 4, digest_levels: 2 }); - let config_at_3 = Some(ChangesTrieConfiguration { digest_interval: 8, digest_levels: 1 }); - let config_at_5 = None; - let config_at_7 = Some(ChangesTrieConfiguration { digest_interval: 8, digest_levels: 1 }); - - // insert some blocks - let block0 = insert_header(&backend, 0, Default::default(), None, Default::default()); - let block1 = - insert_header_with_configuration_change(&backend, 1, block0, None, config_at_1.clone()); - let block2 = insert_header(&backend, 2, block1, None, Default::default()); - let block3 = - insert_header_with_configuration_change(&backend, 3, block2, None, config_at_3.clone()); - let block4 = insert_header(&backend, 4, block3, None, Default::default()); - let block5 = - insert_header_with_configuration_change(&backend, 5, block4, None, config_at_5.clone()); - let block6 = insert_header(&backend, 6, block5, None, Default::default()); - let block7 = - insert_header_with_configuration_change(&backend, 7, block6, None, config_at_7.clone()); - - // test configuration cache - let storage = &backend.changes_tries_storage; - assert_eq!( - storage.configuration_at(&BlockId::Hash(block1)).unwrap().config, - config_at_1.clone(), - ); - assert_eq!( - storage.configuration_at(&BlockId::Hash(block2)).unwrap().config, - config_at_1.clone(), - ); - assert_eq!( - storage.configuration_at(&BlockId::Hash(block3)).unwrap().config, - config_at_3.clone(), - ); - assert_eq!( - storage.configuration_at(&BlockId::Hash(block4)).unwrap().config, - config_at_3.clone(), - ); - assert_eq!( - storage.configuration_at(&BlockId::Hash(block5)).unwrap().config, - config_at_5.clone(), - ); - assert_eq!( - storage.configuration_at(&BlockId::Hash(block6)).unwrap().config, - config_at_5.clone(), - ); - assert_eq!( - storage.configuration_at(&BlockId::Hash(block7)).unwrap().config, - config_at_7.clone(), - ); - } - - #[test] - fn test_finalize_several_configuration_change_blocks_in_single_operation() { - let mut backend = Backend::::new_test(10, 10); - backend.changes_tries_storage.min_blocks_to_keep = Some(8); - - let configs = - (0..=7).map(|i| Some(ChangesTrieConfiguration::new(2, i))).collect::>(); - - // insert unfinalized headers - let block0 = insert_header_with_configuration_change( - &backend, - 0, - Default::default(), - None, - configs[0].clone(), - ); - let block1 = insert_header_with_configuration_change( - &backend, - 1, - block0, - changes(1), - configs[1].clone(), - ); - let block2 = insert_header_with_configuration_change( - &backend, - 2, - block1, - changes(2), - configs[2].clone(), - ); - - let side_config2_1 = Some(ChangesTrieConfiguration::new(3, 2)); - let side_config2_2 = Some(ChangesTrieConfiguration::new(3, 3)); - let block2_1 = insert_header_with_configuration_change( - &backend, - 2, - block1, - changes(8), - side_config2_1.clone(), - ); - let _ = insert_header_with_configuration_change( - &backend, - 3, - block2_1, - changes(9), - side_config2_2.clone(), - ); - - // insert finalized header => 4 headers are finalized at once - let header3 = Header { - number: 3, - parent_hash: block2, - state_root: Default::default(), - digest: Digest { - logs: vec![DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration( - configs[3].clone(), - ))], - }, - extrinsics_root: Default::default(), - }; - let block3 = header3.hash(); - let mut op = backend.begin_operation().unwrap(); - backend.begin_state_operation(&mut op, BlockId::Hash(block2)).unwrap(); - op.mark_finalized(BlockId::Hash(block1), None).unwrap(); - op.mark_finalized(BlockId::Hash(block2), None).unwrap(); - op.set_block_data(header3, None, None, None, NewBlockState::Final).unwrap(); - backend.commit_operation(op).unwrap(); - - // insert more unfinalized headers - let block4 = insert_header_with_configuration_change( - &backend, - 4, - block3, - changes(4), - configs[4].clone(), - ); - let block5 = insert_header_with_configuration_change( - &backend, - 5, - block4, - changes(5), - configs[5].clone(), - ); - let block6 = insert_header_with_configuration_change( - &backend, - 6, - block5, - changes(6), - configs[6].clone(), - ); - - // insert finalized header => 4 headers are finalized at once - let header7 = Header { - number: 7, - parent_hash: block6, - state_root: Default::default(), - digest: Digest { - logs: vec![DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration( - configs[7].clone(), - ))], - }, - extrinsics_root: Default::default(), - }; - let mut op = backend.begin_operation().unwrap(); - backend.begin_state_operation(&mut op, BlockId::Hash(block6)).unwrap(); - op.mark_finalized(BlockId::Hash(block4), None).unwrap(); - op.mark_finalized(BlockId::Hash(block5), None).unwrap(); - op.mark_finalized(BlockId::Hash(block6), None).unwrap(); - op.set_block_data(header7, None, None, None, NewBlockState::Final).unwrap(); - backend.commit_operation(op).unwrap(); - } - - #[test] - fn changes_tries_configuration_is_reverted() { - let backend = Backend::::new_test(10, 10); - - let config0 = Some(ChangesTrieConfiguration::new(2, 5)); - let block0 = - insert_header_with_configuration_change(&backend, 0, Default::default(), None, config0); - let config1 = Some(ChangesTrieConfiguration::new(2, 6)); - let block1 = - insert_header_with_configuration_change(&backend, 1, block0, changes(0), config1); - let just1 = Some((*b"TEST", vec![42])); - backend.finalize_block(BlockId::Number(1), just1).unwrap(); - let config2 = Some(ChangesTrieConfiguration::new(2, 7)); - let block2 = - insert_header_with_configuration_change(&backend, 2, block1, changes(1), config2); - let config2_1 = Some(ChangesTrieConfiguration::new(2, 8)); - let _ = - insert_header_with_configuration_change(&backend, 3, block2, changes(10), config2_1); - let config2_2 = Some(ChangesTrieConfiguration::new(2, 9)); - let block2_2 = - insert_header_with_configuration_change(&backend, 3, block2, changes(20), config2_2); - let config2_3 = Some(ChangesTrieConfiguration::new(2, 10)); - let _ = - insert_header_with_configuration_change(&backend, 4, block2_2, changes(30), config2_3); - - // before truncate there are 2 unfinalized forks - block2_1+block2_3 - assert_eq!( - backend - .changes_tries_storage - .cache - .0 - .write() - .get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG) - .unwrap() - .unfinalized() - .iter() - .map(|fork| fork.head().valid_from.number) - .collect::>(), - vec![3, 4], - ); - - // after truncating block2_3 - there are 2 unfinalized forks - block2_1+block2_2 - backend.revert(1, false).unwrap(); - assert_eq!( - backend - .changes_tries_storage - .cache - .0 - .write() - .get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG) - .unwrap() - .unfinalized() - .iter() - .map(|fork| fork.head().valid_from.number) - .collect::>(), - vec![3, 3], - ); - - // after truncating block2_1 && block2_2 - there are still two unfinalized forks (cache impl - // specifics), the 1st one points to the block #3 because it isn't truncated - backend.revert(1, false).unwrap(); - assert_eq!( - backend - .changes_tries_storage - .cache - .0 - .write() - .get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG) - .unwrap() - .unfinalized() - .iter() - .map(|fork| fork.head().valid_from.number) - .collect::>(), - vec![3, 2], - ); - - // after truncating block2 - there are no unfinalized forks - backend.revert(1, false).unwrap(); - assert!(backend - .changes_tries_storage - .cache - .0 - .write() - .get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG) - .unwrap() - .unfinalized() - .iter() - .map(|fork| fork.head().valid_from.number) - .collect::>() - .is_empty(),); - } -} diff --git a/client/db/src/children.rs b/client/db/src/children.rs index c11e4204997d..538e51851f08 100644 --- a/client/db/src/children.rs +++ b/client/db/src/children.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 66adb64c0109..6a8a025f1f45 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -28,14 +28,11 @@ #![warn(missing_docs)] -pub mod light; pub mod offchain; #[cfg(any(feature = "with-kvdb-rocksdb", test))] pub mod bench; -mod cache; -mod changes_tries_storage; mod children; #[cfg(feature = "with-parity-db")] mod parity_db; @@ -56,7 +53,6 @@ use std::{ }; use crate::{ - changes_tries_storage::{DbChangesTrieStorage, DbChangesTrieStorageTransaction}, stats::StateUsageStats, storage_cache::{new_shared_cache, CachingState, SharedCache, SyncingCachingState}, utils::{meta_keys, read_db, read_meta, DatabaseType, Meta}, @@ -64,8 +60,7 @@ use crate::{ use codec::{Decode, Encode}; use hash_db::Prefix; use sc_client_api::{ - backend::{NewBlockState, ProvideChtRoots, PrunableStateChangesTrieStorage}, - cht, + backend::NewBlockState, leaves::{FinalizationDisplaced, LeafSet}, utils::is_descendent_of, IoInfo, MemoryInfo, MemorySize, UsageInfo, @@ -79,21 +74,19 @@ use sp_blockchain::{ use sp_core::{ offchain::OffchainOverlayedChange, storage::{well_known_keys, ChildInfo}, - ChangesTrieConfiguration, }; use sp_database::Transaction; use sp_runtime::{ - generic::{BlockId, DigestItem}, + generic::BlockId, traits::{ Block as BlockT, Hash, HashFor, Header as HeaderT, NumberFor, One, SaturatedConversion, Zero, }, - Justification, Justifications, Storage, + Justification, Justifications, StateVersion, Storage, }; use sp_state_machine::{ - backend::Backend as StateBackend, ChangesTrieCacheAction, ChangesTrieTransaction, - ChildStorageCollection, DBValue, IndexOperation, OffchainChangesCollection, StateMachineStats, - StorageCollection, UsageInfo as StateUsageInfo, + backend::Backend as StateBackend, ChildStorageCollection, DBValue, IndexOperation, + OffchainChangesCollection, StateMachineStats, StorageCollection, UsageInfo as StateUsageInfo, }; use sp_trie::{prefixed_key, MemoryDB, PrefixedMemoryDB}; @@ -104,7 +97,6 @@ pub use sp_database::Database; #[cfg(any(feature = "with-kvdb-rocksdb", test))] pub use bench::BenchmarkingState; -const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u32 = 32768; const CACHE_HEADERS: usize = 8; /// Default value for storage cache child ratio. @@ -114,17 +106,24 @@ const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10); pub type DbState = sp_state_machine::TrieBackend>>, HashFor>; +/// Length of a [`DbHash`]. const DB_HASH_LEN: usize = 32; + /// Hash type that this backend uses for the database. pub type DbHash = sp_core::H256; -/// This is used as block body when storage-chain mode is enabled. +/// An extrinsic entry in the database. #[derive(Debug, Encode, Decode)] -struct ExtrinsicHeader { - /// Hash of the indexed part - indexed_hash: DbHash, // Zero hash if there's no indexed data - /// The rest of the data. - data: Vec, +enum DbExtrinsic { + /// Extrinsic that contains indexed data. + Indexed { + /// Hash of the indexed part. + hash: DbHash, + /// Extrinsic header. + header: Vec, + }, + /// Complete extrinsic data. + Full(B::Extrinsic), } /// A reference tracking state. @@ -243,22 +242,24 @@ impl StateBackend> for RefTrackingState { fn storage_root<'a>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord, { - self.state.storage_root(delta) + self.state.storage_root(delta, state_version) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord, { - self.state.child_storage_root(child_info, delta) + self.state.child_storage_root(child_info, delta, state_version) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -300,8 +301,6 @@ pub struct DatabaseSettings { pub source: DatabaseSource, /// Block pruning mode. pub keep_blocks: KeepBlocks, - /// Block body/Transaction storage scheme. - pub transaction_storage: TransactionStorageMode, } /// Block pruning settings. @@ -313,16 +312,6 @@ pub enum KeepBlocks { Some(u32), } -/// Block body storage scheme. -#[derive(Debug, Clone, Copy)] -pub enum TransactionStorageMode { - /// Store block body as an encoded list of full transactions in the BODY column - BlockBody, - /// Store a list of hashes in the BODY column and each transaction individually - /// in the TRANSACTION column. - StorageChain, -} - /// Where to find the database.. #[derive(Debug, Clone)] pub enum DatabaseSource { @@ -406,13 +395,12 @@ pub(crate) mod columns { pub const HEADER: u32 = 4; pub const BODY: u32 = 5; pub const JUSTIFICATIONS: u32 = 6; - pub const CHANGES_TRIE: u32 = 7; pub const AUX: u32 = 8; /// Offchain workers local storage pub const OFFCHAIN: u32 = 9; - pub const CACHE: u32 = 10; /// Transactions pub const TRANSACTION: u32 = 11; + pub const BODY_INDEX: u32 = 12; } struct PendingBlock { @@ -460,14 +448,10 @@ pub struct BlockchainDb { leaves: RwLock>>, header_metadata_cache: Arc>, header_cache: Mutex>>, - transaction_storage: TransactionStorageMode, } impl BlockchainDb { - fn new( - db: Arc>, - transaction_storage: TransactionStorageMode, - ) -> ClientResult { + fn new(db: Arc>) -> ClientResult { let meta = read_meta::(&*db, columns::HEADER)?; let leaves = LeafSet::read_from_db(&*db, columns::META, meta_keys::LEAF_PREFIX)?; Ok(BlockchainDb { @@ -476,7 +460,6 @@ impl BlockchainDb { meta: Arc::new(RwLock::new(meta)), header_metadata_cache: Arc::new(HeaderMetadataCache::default()), header_cache: Default::default(), - transaction_storage, }) } @@ -485,7 +468,6 @@ impl BlockchainDb { let mut meta = self.meta.write(); if number.is_zero() { meta.genesis_hash = hash; - meta.finalized_hash = hash; } if is_best { @@ -502,11 +484,9 @@ impl BlockchainDb { } } - // Get block changes trie root, if available. - fn changes_trie_root(&self, block: BlockId) -> ClientResult> { - self.header(block).map(|header| { - header.and_then(|header| header.digest().log(DigestItem::as_changes_trie_root).cloned()) - }) + fn update_block_gap(&self, gap: Option<(NumberFor, NumberFor)>) { + let mut meta = self.meta.write(); + meta.block_gap = gap; } } @@ -538,6 +518,7 @@ impl sc_client_api::blockchain::HeaderBackend for Blockcha finalized_number: meta.finalized_number, finalized_state: meta.finalized_state.clone(), number_leaves: self.leaves.read().count(), + block_gap: meta.block_gap, } } @@ -567,59 +548,61 @@ impl sc_client_api::blockchain::HeaderBackend for Blockcha impl sc_client_api::blockchain::Backend for BlockchainDb { fn body(&self, id: BlockId) -> ClientResult>> { - let body = match read_db(&*self.db, columns::KEY_LOOKUP, columns::BODY, id)? { - Some(body) => body, - None => return Ok(None), - }; - match self.transaction_storage { - TransactionStorageMode::BlockBody => match Decode::decode(&mut &body[..]) { - Ok(body) => Ok(Some(body)), + if let Some(body) = read_db(&*self.db, columns::KEY_LOOKUP, columns::BODY, id)? { + // Plain body + match Decode::decode(&mut &body[..]) { + Ok(body) => return Ok(Some(body)), Err(err) => return Err(sp_blockchain::Error::Backend(format!( "Error decoding body: {}", err ))), - }, - TransactionStorageMode::StorageChain => { - match Vec::::decode(&mut &body[..]) { - Ok(index) => { - let extrinsics: ClientResult> = index - .into_iter() - .map(|ExtrinsicHeader { indexed_hash, data }| { - let decode_result = if indexed_hash != Default::default() { - match self.db.get(columns::TRANSACTION, indexed_hash.as_ref()) { - Some(t) => { - let mut input = - utils::join_input(data.as_ref(), t.as_ref()); - Block::Extrinsic::decode(&mut input) - }, - None => - return Err(sp_blockchain::Error::Backend(format!( - "Missing indexed transaction {:?}", - indexed_hash - ))), - } - } else { - Block::Extrinsic::decode(&mut data.as_ref()) + } + } + + if let Some(index) = read_db(&*self.db, columns::KEY_LOOKUP, columns::BODY_INDEX, id)? { + match Vec::>::decode(&mut &index[..]) { + Ok(index) => { + let mut body = Vec::new(); + for ex in index { + match ex { + DbExtrinsic::Indexed { hash, header } => { + match self.db.get(columns::TRANSACTION, hash.as_ref()) { + Some(t) => { + let mut input = + utils::join_input(header.as_ref(), t.as_ref()); + let ex = Block::Extrinsic::decode(&mut input).map_err( + |err| { + sp_blockchain::Error::Backend(format!( + "Error decoding indexed extrinsic: {}", + err + )) + }, + )?; + body.push(ex); + }, + None => + return Err(sp_blockchain::Error::Backend(format!( + "Missing indexed transaction {:?}", + hash + ))), }; - decode_result.map_err(|err| { - sp_blockchain::Error::Backend(format!( - "Error decoding extrinsic: {}", - err - )) - }) - }) - .collect(); - Ok(Some(extrinsics?)) - }, - Err(err) => - return Err(sp_blockchain::Error::Backend(format!( - "Error decoding body list: {}", - err - ))), - } - }, + }, + DbExtrinsic::Full(ex) => { + body.push(ex); + }, + } + } + return Ok(Some(body)) + }, + Err(err) => + return Err(sp_blockchain::Error::Backend(format!( + "Error decoding body list: {}", + err + ))), + } } + Ok(None) } fn justifications(&self, id: BlockId) -> ClientResult> { @@ -640,10 +623,6 @@ impl sc_client_api::blockchain::Backend for BlockchainDb Option>> { - None - } - fn leaves(&self) -> ClientResult> { Ok(self.leaves.read().hashes()) } @@ -661,47 +640,33 @@ impl sc_client_api::blockchain::Backend for BlockchainDb) -> ClientResult>>> { - match self.transaction_storage { - TransactionStorageMode::BlockBody => Ok(None), - TransactionStorageMode::StorageChain => { - let body = match read_db(&*self.db, columns::KEY_LOOKUP, columns::BODY, id)? { - Some(body) => body, - None => return Ok(None), - }; - match Vec::::decode(&mut &body[..]) { - Ok(index) => { - let mut transactions = Vec::new(); - for ExtrinsicHeader { indexed_hash, .. } in index.into_iter() { - if indexed_hash != Default::default() { - match self.db.get(columns::TRANSACTION, indexed_hash.as_ref()) { - Some(t) => transactions.push(t), - None => - return Err(sp_blockchain::Error::Backend(format!( - "Missing indexed transaction {:?}", - indexed_hash - ))), - } - } + let body = match read_db(&*self.db, columns::KEY_LOOKUP, columns::BODY_INDEX, id)? { + Some(body) => body, + None => return Ok(None), + }; + match Vec::>::decode(&mut &body[..]) { + Ok(index) => { + let mut transactions = Vec::new(); + for ex in index.into_iter() { + if let DbExtrinsic::Indexed { hash, .. } = ex { + match self.db.get(columns::TRANSACTION, hash.as_ref()) { + Some(t) => transactions.push(t), + None => + return Err(sp_blockchain::Error::Backend(format!( + "Missing indexed transaction {:?}", + hash + ))), } - Ok(Some(transactions)) - }, - Err(err) => - return Err(sp_blockchain::Error::Backend(format!( - "Error decoding body list: {}", - err - ))), + } } + Ok(Some(transactions)) }, + Err(err) => + Err(sp_blockchain::Error::Backend(format!("Error decoding body list: {}", err))), } } } -impl sc_client_api::blockchain::ProvideCache for BlockchainDb { - fn cache(&self) -> Option>> { - None - } -} - impl HeaderMetadata for BlockchainDb { type Error = sp_blockchain::Error; @@ -739,62 +704,6 @@ impl HeaderMetadata for BlockchainDb { } } -impl ProvideChtRoots for BlockchainDb { - fn header_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result> { - let cht_number = match cht::block_to_cht_number(cht_size, block) { - Some(number) => number, - None => return Ok(None), - }; - - let cht_start: NumberFor = cht::start_number(cht::size(), cht_number); - - let mut current_num = cht_start; - let cht_range = ::std::iter::from_fn(|| { - let old_current_num = current_num; - current_num = current_num + One::one(); - Some(old_current_num) - }); - - cht::compute_root::, _>( - cht::size(), - cht_number, - cht_range.map(|num| self.hash(num)), - ) - .map(Some) - } - - fn changes_trie_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result> { - let cht_number = match cht::block_to_cht_number(cht_size, block) { - Some(number) => number, - None => return Ok(None), - }; - - let cht_start: NumberFor = cht::start_number(cht::size(), cht_number); - - let mut current_num = cht_start; - let cht_range = ::std::iter::from_fn(|| { - let old_current_num = current_num; - current_num = current_num + One::one(); - Some(old_current_num) - }); - - cht::compute_root::, _>( - cht::size(), - cht_number, - cht_range.map(|num| self.changes_trie_root(BlockId::Number(num))), - ) - .map(Some) - } -} - /// Database transaction pub struct BlockImportOperation { old_state: SyncingCachingState, Block>, @@ -802,9 +711,6 @@ pub struct BlockImportOperation { storage_updates: StorageCollection, child_storage_updates: ChildStorageCollection, offchain_storage_updates: OffchainChangesCollection, - changes_trie_updates: MemoryDB>, - changes_trie_build_cache_update: Option>>, - changes_trie_config_update: Option>, pending_block: Option>, aux_ops: Vec<(Vec, Option>)>, finalized_blocks: Vec<(BlockId, Option)>, @@ -840,7 +746,11 @@ impl BlockImportOperation { } } - fn apply_new_state(&mut self, storage: Storage) -> ClientResult { + fn apply_new_state( + &mut self, + storage: Storage, + state_version: StateVersion, + ) -> ClientResult { if storage.top.keys().any(|k| well_known_keys::is_child_storage_key(&k)) { return Err(sp_blockchain::Error::InvalidState.into()) } @@ -852,25 +762,13 @@ impl BlockImportOperation { ) }); - let mut changes_trie_config = None; let (root, transaction) = self.old_state.full_storage_root( - storage.top.iter().map(|(k, v)| { - if &k[..] == well_known_keys::CHANGES_TRIE_CONFIG { - changes_trie_config = Some(Decode::decode(&mut &v[..])); - } - (&k[..], Some(&v[..])) - }), + storage.top.iter().map(|(k, v)| (&k[..], Some(&v[..]))), child_delta, + state_version, ); - let changes_trie_config = match changes_trie_config { - Some(Ok(c)) => Some(c), - Some(Err(_)) => return Err(sp_blockchain::Error::InvalidState.into()), - None => None, - }; - self.db_updates = transaction; - self.changes_trie_config_update = Some(changes_trie_config); Ok(root) } } @@ -893,11 +791,6 @@ impl sc_client_api::backend::BlockImportOperation leaf_state: NewBlockState, ) -> ClientResult<()> { assert!(self.pending_block.is_none(), "Only one block per operation is allowed"); - if let Some(changes_trie_config_update) = - changes_tries_storage::extract_new_configuration(&header) - { - self.changes_trie_config_update = Some(changes_trie_config_update.clone()); - } self.pending_block = Some(PendingBlock { header, body, indexed_body, justifications, leaf_state }); Ok(()) @@ -912,27 +805,27 @@ impl sc_client_api::backend::BlockImportOperation Ok(()) } - fn reset_storage(&mut self, storage: Storage) -> ClientResult { - let root = self.apply_new_state(storage)?; + fn reset_storage( + &mut self, + storage: Storage, + state_version: StateVersion, + ) -> ClientResult { + let root = self.apply_new_state(storage, state_version)?; self.commit_state = true; Ok(root) } - fn set_genesis_state(&mut self, storage: Storage, commit: bool) -> ClientResult { - let root = self.apply_new_state(storage)?; + fn set_genesis_state( + &mut self, + storage: Storage, + commit: bool, + state_version: StateVersion, + ) -> ClientResult { + let root = self.apply_new_state(storage, state_version)?; self.commit_state = commit; Ok(root) } - fn update_changes_trie( - &mut self, - update: ChangesTrieTransaction, NumberFor>, - ) -> ClientResult<()> { - self.changes_trie_updates = update.0; - self.changes_trie_build_cache_update = Some(update.1); - Ok(()) - } - fn insert_aux(&mut self, ops: I) -> ClientResult<()> where I: IntoIterator, Option>)>, @@ -1031,7 +924,8 @@ impl EmptyStorage { pub fn new() -> Self { let mut root = Block::Hash::default(); let mut mdb = MemoryDB::>::default(); - sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); + // both triedbmut are the same on empty storage. + sp_state_machine::TrieDBMutV1::>::new(&mut mdb, &mut root); EmptyStorage(root) } } @@ -1070,13 +964,14 @@ impl FrozenForDuration { F: FnOnce() -> T, { let mut lock = self.value.lock(); - if lock.at.elapsed() > self.duration || lock.value.is_none() { + let now = std::time::Instant::now(); + if now.saturating_duration_since(lock.at) > self.duration || lock.value.is_none() { let new_value = f(); - lock.at = std::time::Instant::now(); + lock.at = now; lock.value = Some(new_value.clone()); new_value } else { - lock.value.as_ref().expect("checked with lock above").clone() + lock.value.as_ref().expect("Checked with in branch above; qed").clone() } } } @@ -1088,14 +983,12 @@ impl FrozenForDuration { pub struct Backend { storage: Arc>, offchain_storage: offchain::LocalStorage, - changes_tries_storage: DbChangesTrieStorage, blockchain: BlockchainDb, canonicalization_delay: u64, shared_cache: SharedCache, import_lock: Arc>, is_archive: bool, keep_blocks: KeepBlocks, - transaction_storage: TransactionStorageMode, io_stats: FrozenForDuration<(kvdb::IoStats, StateUsageInfo)>, state_usage: Arc, genesis_state: RwLock>>>, @@ -1113,20 +1006,12 @@ impl Backend { /// Create new memory-backed client backend for tests. #[cfg(any(test, feature = "test-helpers"))] pub fn new_test(keep_blocks: u32, canonicalization_delay: u64) -> Self { - Self::new_test_with_tx_storage( - keep_blocks, - canonicalization_delay, - TransactionStorageMode::BlockBody, - ) + Self::new_test_with_tx_storage(keep_blocks, canonicalization_delay) } /// Create new memory-backed client backend for tests. #[cfg(any(test, feature = "test-helpers"))] - pub fn new_test_with_tx_storage( - keep_blocks: u32, - canonicalization_delay: u64, - transaction_storage: TransactionStorageMode, - ) -> Self { + pub fn new_test_with_tx_storage(keep_blocks: u32, canonicalization_delay: u64) -> Self { let db = kvdb_memorydb::create(crate::utils::NUM_COLUMNS); let db = sp_database::as_database(db); let db_setting = DatabaseSettings { @@ -1135,20 +1020,35 @@ impl Backend { state_pruning: PruningMode::keep_blocks(keep_blocks), source: DatabaseSource::Custom(db), keep_blocks: KeepBlocks::Some(keep_blocks), - transaction_storage, }; Self::new(db_setting, canonicalization_delay).expect("failed to create test-db") } + /// Expose the Database that is used by this backend. + /// The second argument is the Column that stores the State. + /// + /// Should only be needed for benchmarking. + #[cfg(any(feature = "runtime-benchmarks"))] + pub fn expose_db(&self) -> (Arc>, sp_database::ColumnId) { + (self.storage.db.clone(), columns::STATE) + } + + /// Expose the Storage that is used by this backend. + /// + /// Should only be needed for benchmarking. + #[cfg(any(feature = "runtime-benchmarks"))] + pub fn expose_storage(&self) -> Arc>> { + self.storage.clone() + } + fn from_database( db: Arc>, canonicalization_delay: u64, config: &DatabaseSettings, ) -> ClientResult { let is_archive_pruning = config.state_pruning.is_archive(); - let blockchain = BlockchainDb::new(db.clone(), config.transaction_storage.clone())?; - let meta = blockchain.meta.clone(); + let blockchain = BlockchainDb::new(db.clone())?; let map_e = |e: sc_state_db::Error| sp_blockchain::Error::from_state_db(e); let state_db: StateDb<_, _> = StateDb::new( config.state_pruning.clone(), @@ -1159,22 +1059,10 @@ impl Backend { let storage_db = StorageDb { db: db.clone(), state_db, prefix_keys: !db.supports_ref_counting() }; let offchain_storage = offchain::LocalStorage::new(db.clone()); - let changes_tries_storage = DbChangesTrieStorage::new( - db, - blockchain.header_metadata_cache.clone(), - columns::META, - columns::CHANGES_TRIE, - columns::KEY_LOOKUP, - columns::HEADER, - columns::CACHE, - meta, - if is_archive_pruning { None } else { Some(MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR) }, - )?; let backend = Backend { storage: Arc::new(storage_db), offchain_storage, - changes_tries_storage, blockchain, canonicalization_delay, shared_cache: new_shared_cache( @@ -1186,7 +1074,6 @@ impl Backend { io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1)), state_usage: Arc::new(StateUsageStats::new()), keep_blocks: config.keep_blocks.clone(), - transaction_storage: config.transaction_storage.clone(), genesis_state: RwLock::new(None), }; @@ -1311,7 +1198,6 @@ impl Backend { header: &Block::Header, last_finalized: Option, justification: Option, - changes_trie_cache_ops: &mut Option>, finalization_displaced: &mut Option>>, ) -> ClientResult> { // TODO: ensure best chain contains this block. @@ -1319,15 +1205,7 @@ impl Backend { self.ensure_sequential_finalization(header, last_finalized)?; let with_state = sc_client_api::Backend::have_state_at(self, &hash, number); - self.note_finalized( - transaction, - false, - header, - *hash, - changes_trie_cache_ops, - finalization_displaced, - with_state, - )?; + self.note_finalized(transaction, header, *hash, finalization_displaced, with_state)?; if let Some(justification) = justification { transaction.set_from_vec( @@ -1388,11 +1266,11 @@ impl Backend { operation.apply_offchain(&mut transaction); let mut meta_updates = Vec::with_capacity(operation.finalized_blocks.len()); - let mut last_finalized_hash = self.blockchain.meta.read().finalized_hash; - let mut last_finalized_num = self.blockchain.meta.read().finalized_number; - let best_num = self.blockchain.meta.read().best_number; + let (best_num, mut last_finalized_hash, mut last_finalized_num, mut block_gap) = { + let meta = self.blockchain.meta.read(); + (meta.best_number, meta.finalized_hash, meta.finalized_number, meta.block_gap.clone()) + }; - let mut changes_trie_cache_ops = None; for (block, justification) in operation.finalized_blocks { let block_hash = self.blockchain.expect_block_hash_from_id(&block)?; let block_header = self.blockchain.expect_header(BlockId::Hash(block_hash))?; @@ -1402,7 +1280,6 @@ impl Backend { &block_header, Some(last_finalized_hash), justification, - &mut changes_trie_cache_ops, &mut finalization_displaced_leaves, )?); last_finalized_hash = block_hash; @@ -1430,26 +1307,18 @@ impl Backend { transaction.set_from_vec(columns::HEADER, &lookup_key, pending_block.header.encode()); if let Some(body) = pending_block.body { - match self.transaction_storage { - TransactionStorageMode::BlockBody => { - transaction.set_from_vec(columns::BODY, &lookup_key, body.encode()); - }, - TransactionStorageMode::StorageChain => { - let body = - apply_index_ops::(&mut transaction, body, operation.index_ops); - transaction.set_from_vec(columns::BODY, &lookup_key, body); - }, + // If we have any index operations we save block in the new format with indexed + // extrinsic headers Otherwise we save the body as a single blob. + if operation.index_ops.is_empty() { + transaction.set_from_vec(columns::BODY, &lookup_key, body.encode()); + } else { + let body = + apply_index_ops::(&mut transaction, body, operation.index_ops); + transaction.set_from_vec(columns::BODY_INDEX, &lookup_key, body); } } if let Some(body) = pending_block.indexed_body { - match self.transaction_storage { - TransactionStorageMode::BlockBody => { - debug!(target: "db", "Commit: ignored indexed block body"); - }, - TransactionStorageMode::StorageChain => { - apply_indexed_body::(&mut transaction, body); - }, - } + apply_indexed_body::(&mut transaction, body); } if let Some(justifications) = pending_block.justifications { transaction.set_from_vec( @@ -1460,18 +1329,8 @@ impl Backend { } if number.is_zero() { - transaction.set_from_vec( - columns::META, - meta_keys::FINALIZED_BLOCK, - lookup_key.clone(), - ); transaction.set(columns::META, meta_keys::GENESIS_HASH, hash.as_ref()); - // for tests, because config is set from within the reset_storage - if operation.changes_trie_config_update.is_none() { - operation.changes_trie_config_update = Some(None); - } - if operation.commit_state { transaction.set_from_vec(columns::META, meta_keys::FINALIZED_STATE, lookup_key); } else { @@ -1493,10 +1352,7 @@ impl Backend { let mut removal: u64 = 0; let mut bytes_removal: u64 = 0; for (mut key, (val, rc)) in operation.db_updates.drain() { - if !self.storage.prefix_keys { - // Strip prefix - key.drain(0..key.len() - DB_HASH_LEN); - }; + self.storage.db.sanitize_key(&mut key); if rc > 0 { ops += 1; bytes += key.len() as u64 + val.len() as u64; @@ -1565,15 +1421,15 @@ impl Backend { let finalized = number_u64 == 0 || pending_block.leaf_state.is_final(); finalized } else { - number.is_zero() || pending_block.leaf_state.is_final() + (number.is_zero() && last_finalized_num.is_zero()) || + pending_block.leaf_state.is_final() }; let header = &pending_block.header; let is_best = pending_block.leaf_state.is_best(); - let changes_trie_updates = operation.changes_trie_updates; debug!(target: "db", - "DB Commit {:?} ({}), best={}, state={}, existing={}", - hash, number, is_best, operation.commit_state, existing_header, + "DB Commit {:?} ({}), best={}, state={}, existing={}, finalized={}", + hash, number, is_best, operation.commit_state, existing_header, finalized, ); self.state_usage.merge_sm(operation.old_state.usage_info()); @@ -1585,10 +1441,8 @@ impl Backend { self.ensure_sequential_finalization(header, Some(last_finalized_hash))?; self.note_finalized( &mut transaction, - true, header, hash, - &mut changes_trie_cache_ops, &mut finalization_displaced_leaves, operation.commit_state, )?; @@ -1598,22 +1452,8 @@ impl Backend { } if !existing_header { - let changes_trie_config_update = operation.changes_trie_config_update; - changes_trie_cache_ops = Some(self.changes_tries_storage.commit( - &mut transaction, - changes_trie_updates, - cache::ComplexBlockId::new( - *header.parent_hash(), - if number.is_zero() { Zero::zero() } else { number - One::one() }, - ), - cache::ComplexBlockId::new(hash, number), - header, - finalized, - changes_trie_config_update, - changes_trie_cache_ops, - )?); - - { + // Add a new leaf if the block has the potential to be finalized. + if number > last_finalized_num || last_finalized_num.is_zero() { let mut leaves = self.blockchain.leaves.write(); leaves.import(hash, number, parent_hash); leaves.prepare_transaction( @@ -1639,6 +1479,41 @@ impl Backend { children, ); } + + if let Some((mut start, end)) = block_gap { + if number == start { + start += One::one(); + utils::insert_number_to_key_mapping( + &mut transaction, + columns::KEY_LOOKUP, + number, + hash, + )?; + } + if start > end { + transaction.remove(columns::META, meta_keys::BLOCK_GAP); + block_gap = None; + debug!(target: "db", "Removed block gap."); + } else { + block_gap = Some((start, end)); + debug!(target: "db", "Update block gap. {:?}", block_gap); + transaction.set( + columns::META, + meta_keys::BLOCK_GAP, + &(start, end).encode(), + ); + } + } else if number > best_num + One::one() && + number > One::one() && self + .blockchain + .header(BlockId::hash(parent_hash))? + .is_none() + { + let gap = (best_num + One::one(), number - One::one()); + transaction.set(columns::META, meta_keys::BLOCK_GAP, &gap.encode()); + block_gap = Some(gap); + debug!(target: "db", "Detected block gap {:?}", block_gap); + } } meta_updates.push(MetaUpdate { @@ -1704,11 +1579,6 @@ impl Backend { ); } - if let Some(changes_trie_build_cache_update) = operation.changes_trie_build_cache_update { - self.changes_tries_storage.commit_build_cache(changes_trie_build_cache_update); - } - self.changes_tries_storage.post_commit(changes_trie_cache_ops); - if let Some((enacted, retracted)) = cache_update { self.shared_cache.write().sync(&enacted, &retracted); } @@ -1716,6 +1586,7 @@ impl Backend { for m in meta_updates { self.blockchain.update_meta(m); } + self.blockchain.update_block_gap(block_gap); Ok(()) } @@ -1726,10 +1597,8 @@ impl Backend { fn note_finalized( &self, transaction: &mut Transaction, - is_inserted: bool, f_header: &Block::Header, f_hash: Block::Hash, - changes_trie_cache_ops: &mut Option>, displaced: &mut Option>>, with_state: bool, ) -> ClientResult<()> { @@ -1754,18 +1623,6 @@ impl Backend { apply_state_commit(transaction, commit); } - if !f_num.is_zero() { - let new_changes_trie_cache_ops = self.changes_tries_storage.finalize( - transaction, - *f_header.parent_hash(), - f_hash, - f_num, - if is_inserted { Some(&f_header) } else { None }, - changes_trie_cache_ops.take(), - )?; - *changes_trie_cache_ops = Some(new_changes_trie_cache_ops); - } - let new_displaced = self.blockchain.leaves.write().finalize_height(f_num); self.prune_blocks(transaction, f_num, &new_displaced)?; match displaced { @@ -1796,7 +1653,7 @@ impl Backend { let mut hash = h.clone(); // Follow displaced chains back until we reach a finalized block. // Since leaves are discarded due to finality, they can't have parents - // that are canonical, but not yet finalized. So we stop deletig as soon as + // that are canonical, but not yet finalized. So we stop deleting as soon as // we reach canonical chain. while self.blockchain.hash(number)? != Some(hash.clone()) { let id = BlockId::::hash(hash.clone()); @@ -1819,36 +1676,37 @@ impl Backend { transaction: &mut Transaction, id: BlockId, ) -> ClientResult<()> { - match read_db(&*self.storage.db, columns::KEY_LOOKUP, columns::BODY, id)? { - Some(body) => { - debug!(target: "db", "Removing block #{}", id); - utils::remove_from_db( - transaction, - &*self.storage.db, - columns::KEY_LOOKUP, - columns::BODY, - id, - )?; - match self.transaction_storage { - TransactionStorageMode::BlockBody => {}, - TransactionStorageMode::StorageChain => { - match Vec::::decode(&mut &body[..]) { - Ok(body) => - for ExtrinsicHeader { indexed_hash, .. } in body { - if indexed_hash != Default::default() { - transaction.release(columns::TRANSACTION, indexed_hash); - } - }, - Err(err) => - return Err(sp_blockchain::Error::Backend(format!( - "Error decoding body list: {}", - err - ))), + debug!(target: "db", "Removing block #{}", id); + utils::remove_from_db( + transaction, + &*self.storage.db, + columns::KEY_LOOKUP, + columns::BODY, + id, + )?; + if let Some(index) = + read_db(&*self.storage.db, columns::KEY_LOOKUP, columns::BODY_INDEX, id)? + { + utils::remove_from_db( + transaction, + &*self.storage.db, + columns::KEY_LOOKUP, + columns::BODY_INDEX, + id, + )?; + match Vec::>::decode(&mut &index[..]) { + Ok(index) => + for ex in index { + if let DbExtrinsic::Indexed { hash, .. } = ex { + transaction.release(columns::TRANSACTION, hash); } }, - } - }, - None => return Ok(()), + Err(err) => + return Err(sp_blockchain::Error::Backend(format!( + "Error decoding body list: {}", + err + ))), + } } Ok(()) } @@ -1890,7 +1748,7 @@ fn apply_index_ops( body: Vec, ops: Vec, ) -> Vec { - let mut extrinsic_headers: Vec = Vec::with_capacity(body.len()); + let mut extrinsic_index: Vec> = Vec::with_capacity(body.len()); let mut index_map = HashMap::new(); let mut renewed_map = HashMap::new(); for op in ops { @@ -1904,37 +1762,44 @@ fn apply_index_ops( } } for (index, extrinsic) in body.into_iter().enumerate() { - let extrinsic = extrinsic.encode(); - let extrinsic_header = if let Some(hash) = renewed_map.get(&(index as u32)) { + let db_extrinsic = if let Some(hash) = renewed_map.get(&(index as u32)) { // Bump ref counter + let extrinsic = extrinsic.encode(); transaction.reference(columns::TRANSACTION, DbHash::from_slice(hash.as_ref())); - ExtrinsicHeader { indexed_hash: hash.clone(), data: extrinsic } + DbExtrinsic::Indexed { hash: hash.clone(), header: extrinsic } } else { match index_map.get(&(index as u32)) { - Some((hash, size)) if *size as usize <= extrinsic.len() => { - let offset = extrinsic.len() - *size as usize; - transaction.store( - columns::TRANSACTION, - DbHash::from_slice(hash.as_ref()), - extrinsic[offset..].to_vec(), - ); - ExtrinsicHeader { - indexed_hash: DbHash::from_slice(hash.as_ref()), - data: extrinsic[..offset].to_vec(), + Some((hash, size)) => { + let encoded = extrinsic.encode(); + if *size as usize <= encoded.len() { + let offset = encoded.len() - *size as usize; + transaction.store( + columns::TRANSACTION, + DbHash::from_slice(hash.as_ref()), + encoded[offset..].to_vec(), + ); + DbExtrinsic::Indexed { + hash: DbHash::from_slice(hash.as_ref()), + header: encoded[..offset].to_vec(), + } + } else { + // Invalid indexed slice. Just store full data and don't index anything. + DbExtrinsic::Full(extrinsic) } }, - _ => ExtrinsicHeader { indexed_hash: Default::default(), data: extrinsic }, + _ => DbExtrinsic::Full(extrinsic), } }; - extrinsic_headers.push(extrinsic_header); + extrinsic_index.push(db_extrinsic); } debug!( target: "db", - "DB transaction index: {} inserted, {} renewed", + "DB transaction index: {} inserted, {} renewed, {} full", index_map.len(), - renewed_map.len() + renewed_map.len(), + extrinsic_index.len() - index_map.len() - renewed_map.len(), ); - extrinsic_headers.encode() + extrinsic_index.encode() } fn apply_indexed_body(transaction: &mut Transaction, body: Vec>) { @@ -1992,9 +1857,6 @@ impl sc_client_api::backend::Backend for Backend { storage_updates: Default::default(), child_storage_updates: Default::default(), offchain_storage_updates: Default::default(), - changes_trie_config_update: None, - changes_trie_updates: MemoryDB::default(), - changes_trie_build_cache_update: None, aux_ops: Vec::new(), finalized_blocks: Vec::new(), set_head: None, @@ -2045,19 +1907,16 @@ impl sc_client_api::backend::Backend for Backend { let header = self.blockchain.expect_header(block)?; let mut displaced = None; - let mut changes_trie_cache_ops = None; let m = self.finalize_block_with_transaction( &mut transaction, &hash, &header, None, justification, - &mut changes_trie_cache_ops, &mut displaced, )?; self.storage.db.commit(transaction)?; self.blockchain.update_meta(m); - self.changes_tries_storage.post_commit(changes_trie_cache_ops); Ok(()) } @@ -2104,10 +1963,6 @@ impl sc_client_api::backend::Backend for Backend { Ok(()) } - fn changes_trie_storage(&self) -> Option<&dyn PrunableStateChangesTrieStorage> { - Some(&self.changes_tries_storage) - } - fn offchain_storage(&self) -> Option { Some(self.offchain_storage.clone()) } @@ -2122,7 +1977,7 @@ impl sc_client_api::backend::Backend for Backend { }); let database_cache = MemorySize::from_bytes(0); let state_cache = - MemorySize::from_bytes((*&self.shared_cache).read().used_storage_cache_size()); + MemorySize::from_bytes(self.shared_cache.read().used_storage_cache_size()); let state_db = self.storage.state_db.memory_info(); Some(UsageInfo { @@ -2164,7 +2019,6 @@ impl sc_client_api::backend::Backend for Backend { return Ok(c.saturated_into::>()) } let mut transaction = Transaction::new(); - let removed_number = best_number; let removed = self.blockchain.header(BlockId::Number(best_number))?.ok_or_else(|| { sp_blockchain::Error::UnknownBlock(format!( @@ -2197,10 +2051,6 @@ impl sc_client_api::backend::Backend for Backend { let key = utils::number_and_hash_to_lookup_key(best_number.clone(), &best_hash)?; - let changes_trie_cache_ops = self.changes_tries_storage.revert( - &mut transaction, - &cache::ComplexBlockId::new(removed.hash(), removed_number), - )?; if update_finalized { transaction.set_from_vec( columns::META, @@ -2239,7 +2089,6 @@ impl sc_client_api::backend::Backend for Backend { best_hash, ); self.storage.db.commit(transaction)?; - self.changes_tries_storage.post_commit(Some(changes_trie_cache_ops)); self.blockchain.update_meta(MetaUpdate { hash: best_hash, number: best_number, @@ -2301,11 +2150,6 @@ impl sc_client_api::backend::Backend for Backend { apply_state_commit(&mut transaction, commit); } transaction.remove(columns::KEY_LOOKUP, hash.as_ref()); - let changes_trie_cache_ops = self - .changes_tries_storage - .revert(&mut transaction, &cache::ComplexBlockId::new(*hash, hdr.number))?; - - self.changes_tries_storage.post_commit(Some(changes_trie_cache_ops)); leaves.revert(hash.clone(), hdr.number); leaves.prepare_transaction(&mut transaction, columns::META, meta_keys::LEAF_PREFIX); self.storage.db.commit(transaction)?; @@ -2417,32 +2261,16 @@ pub(crate) mod tests { use sp_blockchain::{lowest_common_ancestor, tree_route}; use sp_core::H256; use sp_runtime::{ - generic::DigestItem, testing::{Block as RawBlock, ExtrinsicWrapper, Header}, traits::{BlakeTwo256, Hash}, - ConsensusEngineId, + ConsensusEngineId, StateVersion, }; - use sp_state_machine::{TrieDBMut, TrieMut}; const CONS0_ENGINE_ID: ConsensusEngineId = *b"CON0"; const CONS1_ENGINE_ID: ConsensusEngineId = *b"CON1"; pub(crate) type Block = RawBlock>; - pub fn prepare_changes(changes: Vec<(Vec, Vec)>) -> (H256, MemoryDB) { - let mut changes_root = H256::default(); - let mut changes_trie_update = MemoryDB::::default(); - { - let mut trie = - TrieDBMut::::new(&mut changes_trie_update, &mut changes_root); - for (key, value) in changes { - trie.insert(&key, &value).unwrap(); - } - } - - (changes_root, changes_trie_update) - } - pub fn insert_header( backend: &Backend, number: u64, @@ -2451,30 +2279,25 @@ pub(crate) mod tests { extrinsics_root: H256, ) -> H256 { insert_block(backend, number, parent_hash, changes, extrinsics_root, Vec::new(), None) + .unwrap() } pub fn insert_block( backend: &Backend, number: u64, parent_hash: H256, - changes: Option, Vec)>>, + _changes: Option, Vec)>>, extrinsics_root: H256, body: Vec>, transaction_index: Option>, - ) -> H256 { + ) -> Result { use sp_runtime::testing::Digest; - let mut digest = Digest::default(); - let mut changes_trie_update = Default::default(); - if let Some(changes) = changes { - let (root, update) = prepare_changes(changes); - digest.push(DigestItem::ChangesTrieRoot(root)); - changes_trie_update = update; - } + let digest = Digest::default(); let header = Header { number, parent_hash, - state_root: BlakeTwo256::trie_root(Vec::new()), + state_root: BlakeTwo256::trie_root(Vec::new(), StateVersion::V1), digest, extrinsics_root, }; @@ -2491,10 +2314,31 @@ pub(crate) mod tests { if let Some(index) = transaction_index { op.update_transaction_index(index).unwrap(); } - op.update_changes_trie((changes_trie_update, ChangesTrieCacheAction::Clear)) - .unwrap(); - backend.commit_operation(op).unwrap(); + backend.commit_operation(op)?; + + Ok(header_hash) + } + + pub fn insert_header_no_head( + backend: &Backend, + number: u64, + parent_hash: H256, + extrinsics_root: H256, + ) -> H256 { + use sp_runtime::testing::Digest; + let digest = Digest::default(); + let header = Header { + number, + parent_hash, + state_root: BlakeTwo256::trie_root(Vec::new(), StateVersion::V1), + digest, + extrinsics_root, + }; + let header_hash = header.hash(); + let mut op = backend.begin_operation().unwrap(); + op.set_block_data(header, None, None, None, NewBlockState::Normal).unwrap(); + backend.commit_operation(op).unwrap(); header_hash } @@ -2543,7 +2387,6 @@ pub(crate) mod tests { state_pruning: PruningMode::keep_blocks(1), source: DatabaseSource::Custom(backing), keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorageMode::BlockBody, }, 0, ) @@ -2556,6 +2399,10 @@ pub(crate) mod tests { #[test] fn set_state_data() { + set_state_data_inner(StateVersion::V0); + set_state_data_inner(StateVersion::V1); + } + fn set_state_data_inner(state_version: StateVersion) { let db = Backend::::new_test(2, 0); let hash = { let mut op = db.begin_operation().unwrap(); @@ -2571,15 +2418,18 @@ pub(crate) mod tests { header.state_root = op .old_state - .storage_root(storage.iter().map(|(x, y)| (&x[..], Some(&y[..])))) + .storage_root(storage.iter().map(|(x, y)| (&x[..], Some(&y[..]))), state_version) .0 .into(); let hash = header.hash(); - op.reset_storage(Storage { - top: storage.into_iter().collect(), - children_default: Default::default(), - }) + op.reset_storage( + Storage { + top: storage.into_iter().collect(), + children_default: Default::default(), + }, + state_version, + ) .unwrap(); op.set_block_data(header.clone(), Some(vec![]), None, None, NewBlockState::Best) .unwrap(); @@ -2608,9 +2458,10 @@ pub(crate) mod tests { let storage = vec![(vec![1, 3, 5], None), (vec![5, 5, 5], Some(vec![4, 5, 6]))]; - let (root, overlay) = op - .old_state - .storage_root(storage.iter().map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..])))); + let (root, overlay) = op.old_state.storage_root( + storage.iter().map(|(k, v)| (k.as_slice(), v.as_ref().map(|v| &v[..]))), + state_version, + ); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); @@ -2631,6 +2482,7 @@ pub(crate) mod tests { #[test] fn delete_only_when_negative_rc() { sp_tracing::try_init_simple(); + let state_version = StateVersion::default(); let key; let backend = Backend::::new_test(1, 0); @@ -2647,13 +2499,14 @@ pub(crate) mod tests { extrinsics_root: Default::default(), }; - header.state_root = op.old_state.storage_root(std::iter::empty()).0.into(); + header.state_root = + op.old_state.storage_root(std::iter::empty(), state_version).0.into(); let hash = header.hash(); - op.reset_storage(Storage { - top: Default::default(), - children_default: Default::default(), - }) + op.reset_storage( + Storage { top: Default::default(), children_default: Default::default() }, + state_version, + ) .unwrap(); key = op.db_updates.insert(EMPTY_PREFIX, b"hello"); @@ -2687,7 +2540,7 @@ pub(crate) mod tests { header.state_root = op .old_state - .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y)))) + .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y))), state_version) .0 .into(); let hash = header.hash(); @@ -2724,7 +2577,7 @@ pub(crate) mod tests { header.state_root = op .old_state - .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y)))) + .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y))), state_version) .0 .into(); let hash = header.hash(); @@ -2758,7 +2611,7 @@ pub(crate) mod tests { header.state_root = op .old_state - .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y)))) + .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y))), state_version) .0 .into(); @@ -3093,6 +2946,7 @@ pub(crate) mod tests { #[test] fn storage_hash_is_cached_correctly() { + let state_version = StateVersion::default(); let backend = Backend::::new_test(10, 10); let hash0 = { @@ -3112,15 +2966,18 @@ pub(crate) mod tests { header.state_root = op .old_state - .storage_root(storage.iter().map(|(x, y)| (&x[..], Some(&y[..])))) + .storage_root(storage.iter().map(|(x, y)| (&x[..], Some(&y[..]))), state_version) .0 .into(); let hash = header.hash(); - op.reset_storage(Storage { - top: storage.into_iter().collect(), - children_default: Default::default(), - }) + op.reset_storage( + Storage { + top: storage.into_iter().collect(), + children_default: Default::default(), + }, + state_version, + ) .unwrap(); op.set_block_data(header.clone(), Some(vec![]), None, None, NewBlockState::Best) .unwrap(); @@ -3149,9 +3006,10 @@ pub(crate) mod tests { let storage = vec![(b"test".to_vec(), Some(b"test2".to_vec()))]; - let (root, overlay) = op - .old_state - .storage_root(storage.iter().map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..])))); + let (root, overlay) = op.old_state.storage_root( + storage.iter().map(|(k, v)| (k.as_slice(), v.as_ref().map(|v| &v[..]))), + state_version, + ); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); let hash = header.hash(); @@ -3168,7 +3026,6 @@ pub(crate) mod tests { { let header = backend.blockchain().header(BlockId::Hash(hash1)).unwrap().unwrap(); let mut op = backend.begin_operation().unwrap(); - backend.begin_state_operation(&mut op, BlockId::Hash(hash0)).unwrap(); op.set_block_data(header, None, None, None, NewBlockState::Best).unwrap(); backend.commit_operation(op).unwrap(); } @@ -3197,79 +3054,45 @@ pub(crate) mod tests { } } - #[test] - fn header_cht_root_works() { - use sc_client_api::ProvideChtRoots; - - let backend = Backend::::new_test(10, 10); - - // insert 1 + SIZE + SIZE + 1 blocks so that CHT#0 is created - let mut prev_hash = - insert_header(&backend, 0, Default::default(), None, Default::default()); - let cht_size: u64 = cht::size(); - for i in 1..1 + cht_size + cht_size + 1 { - prev_hash = insert_header(&backend, i, prev_hash, None, Default::default()); - } - - let blockchain = backend.blockchain(); - - let cht_root_1 = blockchain - .header_cht_root(cht_size, cht::start_number(cht_size, 0)) - .unwrap() - .unwrap(); - let cht_root_2 = blockchain - .header_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2) - .unwrap() - .unwrap(); - let cht_root_3 = blockchain - .header_cht_root(cht_size, cht::end_number(cht_size, 0)) - .unwrap() - .unwrap(); - assert_eq!(cht_root_1, cht_root_2); - assert_eq!(cht_root_2, cht_root_3); - } - #[test] fn prune_blocks_on_finalize() { - for storage in &[TransactionStorageMode::BlockBody, TransactionStorageMode::StorageChain] { - let backend = Backend::::new_test_with_tx_storage(2, 0, *storage); - let mut blocks = Vec::new(); - let mut prev_hash = Default::default(); - for i in 0..5 { - let hash = insert_block( - &backend, - i, - prev_hash, - None, - Default::default(), - vec![i.into()], - None, - ); - blocks.push(hash); - prev_hash = hash; - } + let backend = Backend::::new_test_with_tx_storage(2, 0); + let mut blocks = Vec::new(); + let mut prev_hash = Default::default(); + for i in 0..5 { + let hash = insert_block( + &backend, + i, + prev_hash, + None, + Default::default(), + vec![i.into()], + None, + ) + .unwrap(); + blocks.push(hash); + prev_hash = hash; + } - { - let mut op = backend.begin_operation().unwrap(); - backend.begin_state_operation(&mut op, BlockId::Hash(blocks[4])).unwrap(); - for i in 1..5 { - op.mark_finalized(BlockId::Hash(blocks[i]), None).unwrap(); - } - backend.commit_operation(op).unwrap(); + { + let mut op = backend.begin_operation().unwrap(); + backend.begin_state_operation(&mut op, BlockId::Hash(blocks[4])).unwrap(); + for i in 1..5 { + op.mark_finalized(BlockId::Hash(blocks[i]), None).unwrap(); } - let bc = backend.blockchain(); - assert_eq!(None, bc.body(BlockId::hash(blocks[0])).unwrap()); - assert_eq!(None, bc.body(BlockId::hash(blocks[1])).unwrap()); - assert_eq!(None, bc.body(BlockId::hash(blocks[2])).unwrap()); - assert_eq!(Some(vec![3.into()]), bc.body(BlockId::hash(blocks[3])).unwrap()); - assert_eq!(Some(vec![4.into()]), bc.body(BlockId::hash(blocks[4])).unwrap()); + backend.commit_operation(op).unwrap(); } + let bc = backend.blockchain(); + assert_eq!(None, bc.body(BlockId::hash(blocks[0])).unwrap()); + assert_eq!(None, bc.body(BlockId::hash(blocks[1])).unwrap()); + assert_eq!(None, bc.body(BlockId::hash(blocks[2])).unwrap()); + assert_eq!(Some(vec![3.into()]), bc.body(BlockId::hash(blocks[3])).unwrap()); + assert_eq!(Some(vec![4.into()]), bc.body(BlockId::hash(blocks[4])).unwrap()); } #[test] fn prune_blocks_on_finalize_with_fork() { - let backend = - Backend::::new_test_with_tx_storage(2, 10, TransactionStorageMode::StorageChain); + let backend = Backend::::new_test_with_tx_storage(2, 10); let mut blocks = Vec::new(); let mut prev_hash = Default::default(); for i in 0..5 { @@ -3281,7 +3104,8 @@ pub(crate) mod tests { Default::default(), vec![i.into()], None, - ); + ) + .unwrap(); blocks.push(hash); prev_hash = hash; } @@ -3295,7 +3119,8 @@ pub(crate) mod tests { sp_core::H256::random(), vec![2.into()], None, - ); + ) + .unwrap(); insert_block( &backend, 3, @@ -3304,7 +3129,8 @@ pub(crate) mod tests { H256::random(), vec![3.into(), 11.into()], None, - ); + ) + .unwrap(); let mut op = backend.begin_operation().unwrap(); backend.begin_state_operation(&mut op, BlockId::Hash(blocks[4])).unwrap(); op.mark_head(BlockId::Hash(blocks[4])).unwrap(); @@ -3325,10 +3151,86 @@ pub(crate) mod tests { assert_eq!(Some(vec![4.into()]), bc.body(BlockId::hash(blocks[4])).unwrap()); } + #[test] + fn indexed_data_block_body() { + let backend = Backend::::new_test_with_tx_storage(1, 10); + + let x0 = ExtrinsicWrapper::from(0u64).encode(); + let x1 = ExtrinsicWrapper::from(1u64).encode(); + let x0_hash = as sp_core::Hasher>::hash(&x0[1..]); + let x1_hash = as sp_core::Hasher>::hash(&x1[1..]); + let index = vec![ + IndexOperation::Insert { + extrinsic: 0, + hash: x0_hash.as_ref().to_vec(), + size: (x0.len() - 1) as u32, + }, + IndexOperation::Insert { + extrinsic: 1, + hash: x1_hash.as_ref().to_vec(), + size: (x1.len() - 1) as u32, + }, + ]; + let hash = insert_block( + &backend, + 0, + Default::default(), + None, + Default::default(), + vec![0u64.into(), 1u64.into()], + Some(index), + ) + .unwrap(); + let bc = backend.blockchain(); + assert_eq!(bc.indexed_transaction(&x0_hash).unwrap().unwrap(), &x0[1..]); + assert_eq!(bc.indexed_transaction(&x1_hash).unwrap().unwrap(), &x1[1..]); + + // Push one more blocks and make sure block is pruned and transaction index is cleared. + insert_block(&backend, 1, hash, None, Default::default(), vec![], None).unwrap(); + backend.finalize_block(BlockId::Number(1), None).unwrap(); + assert_eq!(bc.body(BlockId::Number(0)).unwrap(), None); + assert_eq!(bc.indexed_transaction(&x0_hash).unwrap(), None); + assert_eq!(bc.indexed_transaction(&x1_hash).unwrap(), None); + } + + #[test] + fn index_invalid_size() { + let backend = Backend::::new_test_with_tx_storage(1, 10); + + let x0 = ExtrinsicWrapper::from(0u64).encode(); + let x1 = ExtrinsicWrapper::from(1u64).encode(); + let x0_hash = as sp_core::Hasher>::hash(&x0[..]); + let x1_hash = as sp_core::Hasher>::hash(&x1[..]); + let index = vec![ + IndexOperation::Insert { + extrinsic: 0, + hash: x0_hash.as_ref().to_vec(), + size: (x0.len()) as u32, + }, + IndexOperation::Insert { + extrinsic: 1, + hash: x1_hash.as_ref().to_vec(), + size: (x1.len() + 1) as u32, + }, + ]; + insert_block( + &backend, + 0, + Default::default(), + None, + Default::default(), + vec![0u64.into(), 1u64.into()], + Some(index), + ) + .unwrap(); + let bc = backend.blockchain(); + assert_eq!(bc.indexed_transaction(&x0_hash).unwrap().unwrap(), &x0[..]); + assert_eq!(bc.indexed_transaction(&x1_hash).unwrap(), None); + } + #[test] fn renew_transaction_storage() { - let backend = - Backend::::new_test_with_tx_storage(2, 10, TransactionStorageMode::StorageChain); + let backend = Backend::::new_test_with_tx_storage(2, 10); let mut blocks = Vec::new(); let mut prev_hash = Default::default(); let x1 = ExtrinsicWrapper::from(0u64).encode(); @@ -3353,7 +3255,8 @@ pub(crate) mod tests { Default::default(), vec![i.into()], Some(index), - ); + ) + .unwrap(); blocks.push(hash); prev_hash = hash; } @@ -3374,8 +3277,7 @@ pub(crate) mod tests { #[test] fn remove_leaf_block_works() { - let backend = - Backend::::new_test_with_tx_storage(2, 10, TransactionStorageMode::StorageChain); + let backend = Backend::::new_test_with_tx_storage(2, 10); let mut blocks = Vec::new(); let mut prev_hash = Default::default(); for i in 0..2 { @@ -3387,7 +3289,8 @@ pub(crate) mod tests { Default::default(), vec![i.into()], None, - ); + ) + .unwrap(); blocks.push(hash); prev_hash = hash; } @@ -3401,7 +3304,8 @@ pub(crate) mod tests { sp_core::H256::random(), vec![42.into()], None, - ); + ) + .unwrap(); assert!(backend.remove_leaf_block(&best_hash).is_err()); assert!(backend.have_state_at(&prev_hash, 1)); backend.remove_leaf_block(&prev_hash).unwrap(); @@ -3425,7 +3329,7 @@ pub(crate) mod tests { let header = Header { number: 1, parent_hash: block0, - state_root: BlakeTwo256::trie_root(Vec::new()), + state_root: BlakeTwo256::trie_root(Vec::new(), StateVersion::V1), digest: Default::default(), extrinsics_root: Default::default(), }; @@ -3437,7 +3341,7 @@ pub(crate) mod tests { let header = Header { number: 2, parent_hash: block1, - state_root: BlakeTwo256::trie_root(Vec::new()), + state_root: BlakeTwo256::trie_root(Vec::new(), StateVersion::V1), digest: Default::default(), extrinsics_root: Default::default(), }; @@ -3460,7 +3364,7 @@ pub(crate) mod tests { let header = Header { number: 1, parent_hash: block0, - state_root: BlakeTwo256::trie_root(Vec::new()), + state_root: BlakeTwo256::trie_root(Vec::new(), StateVersion::V1), digest: Default::default(), extrinsics_root: Default::default(), }; @@ -3471,4 +3375,36 @@ pub(crate) mod tests { assert_eq!(backend.blockchain().info().finalized_hash, block1); } + + #[test] + fn test_import_existing_state_fails() { + let backend: Backend = Backend::new_test(10, 10); + let genesis = + insert_block(&backend, 0, Default::default(), None, Default::default(), vec![], None) + .unwrap(); + + insert_block(&backend, 1, genesis, None, Default::default(), vec![], None).unwrap(); + let err = insert_block(&backend, 1, genesis, None, Default::default(), vec![], None) + .err() + .unwrap(); + match err { + sp_blockchain::Error::StateDatabase(m) if m == "Block already exists" => (), + e @ _ => panic!("Unexpected error {:?}", e), + } + } + + #[test] + fn test_leaves_not_created_for_ancient_blocks() { + let backend: Backend = Backend::new_test(10, 10); + let block0 = insert_header(&backend, 0, Default::default(), None, Default::default()); + + let block1_a = insert_header(&backend, 1, block0, None, Default::default()); + let block2_a = insert_header(&backend, 2, block1_a, None, Default::default()); + backend.finalize_block(BlockId::hash(block1_a), None).unwrap(); + assert_eq!(backend.blockchain().leaves().unwrap(), vec![block2_a]); + + // Insert a fork prior to finalization point. Leave should not be created. + insert_header_no_head(&backend, 1, block0, [1; 32].into()); + assert_eq!(backend.blockchain().leaves().unwrap(), vec![block2_a]); + } } diff --git a/client/db/src/light.rs b/client/db/src/light.rs deleted file mode 100644 index bf2da5c61d05..000000000000 --- a/client/db/src/light.rs +++ /dev/null @@ -1,1328 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! RocksDB-based light client blockchain storage. - -use parking_lot::RwLock; -use std::{collections::HashMap, convert::TryInto, sync::Arc}; - -use crate::{ - cache::{ComplexBlockId, DbCache, DbCacheSync, EntryType as CacheEntryType}, - utils::{self, block_id_to_lookup_key, meta_keys, read_db, read_meta, DatabaseType, Meta}, - DatabaseSettings, DbHash, FrozenForDuration, -}; -use codec::{Decode, Encode}; -use log::{debug, trace, warn}; -use sc_client_api::{ - backend::{AuxStore, NewBlockState, ProvideChtRoots}, - blockchain::{BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo}, - cht, Storage, UsageInfo, -}; -use sp_blockchain::{ - well_known_cache_keys, CachedHeaderMetadata, Error as ClientError, - HeaderBackend as BlockchainHeaderBackend, HeaderMetadata, HeaderMetadataCache, - Result as ClientResult, -}; -use sp_database::{Database, Transaction}; -use sp_runtime::{ - generic::{BlockId, DigestItem}, - traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor, One, Zero}, -}; - -pub(crate) mod columns { - pub const META: u32 = crate::utils::COLUMN_META; - pub const KEY_LOOKUP: u32 = 1; - pub const HEADER: u32 = 2; - pub const CACHE: u32 = 3; - pub const CHT: u32 = 4; - pub const AUX: u32 = 5; -} - -/// Prefix for headers CHT. -const HEADER_CHT_PREFIX: u8 = 0; -/// Prefix for changes tries roots CHT. -const CHANGES_TRIE_CHT_PREFIX: u8 = 1; - -/// Light blockchain storage. Stores most recent headers + CHTs for older headers. -/// Locks order: meta, cache. -pub struct LightStorage { - db: Arc>, - meta: RwLock, Block::Hash>>, - cache: Arc>, - header_metadata_cache: Arc>, - io_stats: FrozenForDuration, -} - -impl LightStorage { - /// Create new storage with given settings. - pub fn new(config: DatabaseSettings) -> ClientResult { - let db = crate::utils::open_database::(&config, DatabaseType::Light)?; - Self::from_kvdb(db as Arc<_>) - } - - /// Create new memory-backed `LightStorage` for tests. - #[cfg(any(test, feature = "test-helpers"))] - pub fn new_test() -> Self { - let db = Arc::new(sp_database::MemDb::default()); - Self::from_kvdb(db as Arc<_>).expect("failed to create test-db") - } - - fn from_kvdb(db: Arc>) -> ClientResult { - let meta = read_meta::(&*db, columns::HEADER)?; - let header_metadata_cache = Arc::new(HeaderMetadataCache::default()); - let cache = DbCache::new( - db.clone(), - header_metadata_cache.clone(), - columns::KEY_LOOKUP, - columns::HEADER, - columns::CACHE, - meta.genesis_hash, - ComplexBlockId::new(meta.finalized_hash, meta.finalized_number), - ); - - Ok(LightStorage { - db, - meta: RwLock::new(meta), - cache: Arc::new(DbCacheSync(RwLock::new(cache))), - header_metadata_cache, - io_stats: FrozenForDuration::new(std::time::Duration::from_secs(1)), - }) - } - - #[cfg(test)] - pub(crate) fn cache(&self) -> &DbCacheSync { - &self.cache - } - - fn update_meta( - &self, - hash: Block::Hash, - number: NumberFor, - is_best: bool, - is_finalized: bool, - ) { - let mut meta = self.meta.write(); - - if number.is_zero() { - meta.genesis_hash = hash; - meta.finalized_hash = hash; - } - - if is_best { - meta.best_number = number; - meta.best_hash = hash; - } - - if is_finalized { - meta.finalized_number = number; - meta.finalized_hash = hash; - } - } -} - -impl BlockchainHeaderBackend for LightStorage -where - Block: BlockT, -{ - fn header(&self, id: BlockId) -> ClientResult> { - utils::read_header(&*self.db, columns::KEY_LOOKUP, columns::HEADER, id) - } - - fn info(&self) -> BlockchainInfo { - let meta = self.meta.read(); - BlockchainInfo { - best_hash: meta.best_hash, - best_number: meta.best_number, - genesis_hash: meta.genesis_hash.clone(), - finalized_hash: meta.finalized_hash, - finalized_number: meta.finalized_number, - finalized_state: if meta.finalized_hash != Default::default() { - Some((meta.genesis_hash, Zero::zero())) - } else { - None - }, - number_leaves: 1, - } - } - - fn status(&self, id: BlockId) -> ClientResult { - let exists = match id { - BlockId::Hash(_) => - read_db(&*self.db, columns::KEY_LOOKUP, columns::HEADER, id)?.is_some(), - BlockId::Number(n) => n <= self.meta.read().best_number, - }; - match exists { - true => Ok(BlockStatus::InChain), - false => Ok(BlockStatus::Unknown), - } - } - - fn number(&self, hash: Block::Hash) -> ClientResult>> { - if let Some(lookup_key) = - block_id_to_lookup_key::(&*self.db, columns::KEY_LOOKUP, BlockId::Hash(hash))? - { - let number = utils::lookup_key_to_number(&lookup_key)?; - Ok(Some(number)) - } else { - Ok(None) - } - } - - fn hash(&self, number: NumberFor) -> ClientResult> { - Ok(self.header(BlockId::Number(number))?.map(|header| header.hash().clone())) - } -} - -impl HeaderMetadata for LightStorage { - type Error = ClientError; - - fn header_metadata( - &self, - hash: Block::Hash, - ) -> Result, Self::Error> { - self.header_metadata_cache.header_metadata(hash).map_or_else( - || { - self.header(BlockId::hash(hash))? - .map(|header| { - let header_metadata = CachedHeaderMetadata::from(&header); - self.header_metadata_cache - .insert_header_metadata(header_metadata.hash, header_metadata.clone()); - header_metadata - }) - .ok_or_else(|| { - ClientError::UnknownBlock(format!("header not found in db: {}", hash)) - }) - }, - Ok, - ) - } - - fn insert_header_metadata(&self, hash: Block::Hash, metadata: CachedHeaderMetadata) { - self.header_metadata_cache.insert_header_metadata(hash, metadata) - } - - fn remove_header_metadata(&self, hash: Block::Hash) { - self.header_metadata_cache.remove_header_metadata(hash); - } -} - -impl LightStorage { - // Get block changes trie root, if available. - fn changes_trie_root(&self, block: BlockId) -> ClientResult> { - self.header(block).map(|header| { - header.and_then(|header| header.digest().log(DigestItem::as_changes_trie_root).cloned()) - }) - } - - /// Handle setting head within a transaction. `route_to` should be the last - /// block that existed in the database. `best_to` should be the best block - /// to be set. - /// - /// In the case where the new best block is a block to be imported, `route_to` - /// should be the parent of `best_to`. In the case where we set an existing block - /// to be best, `route_to` should equal to `best_to`. - fn set_head_with_transaction( - &self, - transaction: &mut Transaction, - route_to: Block::Hash, - best_to: (NumberFor, Block::Hash), - ) -> ClientResult<()> { - let lookup_key = utils::number_and_hash_to_lookup_key(best_to.0, &best_to.1)?; - - // handle reorg. - let meta = self.meta.read(); - if meta.best_hash != Default::default() { - let tree_route = sp_blockchain::tree_route(self, meta.best_hash, route_to)?; - - // update block number to hash lookup entries. - for retracted in tree_route.retracted() { - if retracted.hash == meta.finalized_hash { - // TODO: can we recover here? - warn!( - "Safety failure: reverting finalized block {:?}", - (&retracted.number, &retracted.hash) - ); - } - - utils::remove_number_to_key_mapping( - transaction, - columns::KEY_LOOKUP, - retracted.number, - )?; - } - - for enacted in tree_route.enacted() { - utils::insert_number_to_key_mapping( - transaction, - columns::KEY_LOOKUP, - enacted.number, - enacted.hash, - )?; - } - } - - transaction.set_from_vec(columns::META, meta_keys::BEST_BLOCK, lookup_key); - utils::insert_number_to_key_mapping( - transaction, - columns::KEY_LOOKUP, - best_to.0, - best_to.1, - )?; - - Ok(()) - } - - // Note that a block is finalized. Only call with child of last finalized block. - fn note_finalized( - &self, - transaction: &mut Transaction, - header: &Block::Header, - hash: Block::Hash, - ) -> ClientResult<()> { - let meta = self.meta.read(); - if &meta.finalized_hash != header.parent_hash() { - return Err(::sp_blockchain::Error::NonSequentialFinalization(format!( - "Last finalized {:?} not parent of {:?}", - meta.finalized_hash, hash - )) - .into()) - } - - let lookup_key = utils::number_and_hash_to_lookup_key(header.number().clone(), hash)?; - transaction.set_from_vec(columns::META, meta_keys::FINALIZED_BLOCK, lookup_key); - - // build new CHT(s) if required - if let Some(new_cht_number) = cht::is_build_required(cht::size(), *header.number()) { - let new_cht_start: NumberFor = cht::start_number(cht::size(), new_cht_number); - - let mut current_num = new_cht_start; - let cht_range = ::std::iter::from_fn(|| { - let old_current_num = current_num; - current_num = current_num + One::one(); - Some(old_current_num) - }); - - let new_header_cht_root = cht::compute_root::, _>( - cht::size(), - new_cht_number, - cht_range.map(|num| self.hash(num)), - )?; - transaction.set( - columns::CHT, - &cht_key(HEADER_CHT_PREFIX, new_cht_start)?, - new_header_cht_root.as_ref(), - ); - - // if the header includes changes trie root, let's build a changes tries roots CHT - if header.digest().log(DigestItem::as_changes_trie_root).is_some() { - let mut current_num = new_cht_start; - let cht_range = std::iter::from_fn(|| { - let old_current_num = current_num; - current_num = current_num + One::one(); - Some(old_current_num) - }); - let new_changes_trie_cht_root = - cht::compute_root::, _>( - cht::size(), - new_cht_number, - cht_range.map(|num| self.changes_trie_root(BlockId::Number(num))), - )?; - transaction.set( - columns::CHT, - &cht_key(CHANGES_TRIE_CHT_PREFIX, new_cht_start)?, - new_changes_trie_cht_root.as_ref(), - ); - } - - // prune headers that are replaced with CHT - let mut prune_block = new_cht_start; - let new_cht_end = cht::end_number(cht::size(), new_cht_number); - trace!(target: "db", "Replacing blocks [{}..{}] with CHT#{}", - new_cht_start, new_cht_end, new_cht_number); - - while prune_block <= new_cht_end { - if let Some(hash) = self.hash(prune_block)? { - let lookup_key = block_id_to_lookup_key::(&*self.db, columns::KEY_LOOKUP, BlockId::Number(prune_block))? - .expect("retrieved hash for `prune_block` right above. therefore retrieving lookup key must succeed. q.e.d."); - utils::remove_key_mappings( - transaction, - columns::KEY_LOOKUP, - prune_block, - hash, - )?; - transaction.remove(columns::HEADER, &lookup_key); - } - prune_block += One::one(); - } - } - - Ok(()) - } - - /// Read CHT root of given type for the block. - fn read_cht_root( - &self, - cht_type: u8, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult> { - let no_cht_for_block = || ClientError::Backend(format!("Missing CHT for block {}", block)); - - let meta = self.meta.read(); - let max_cht_number = cht::max_cht_number(cht_size, meta.finalized_number); - let cht_number = cht::block_to_cht_number(cht_size, block).ok_or_else(no_cht_for_block)?; - match max_cht_number { - Some(max_cht_number) if cht_number <= max_cht_number => (), - _ => return Ok(None), - } - - let cht_start = cht::start_number(cht_size, cht_number); - self.db - .get(columns::CHT, &cht_key(cht_type, cht_start)?) - .ok_or_else(no_cht_for_block) - .and_then(|hash| Block::Hash::decode(&mut &*hash).map_err(|_| no_cht_for_block())) - .map(Some) - } -} - -impl AuxStore for LightStorage -where - Block: BlockT, -{ - fn insert_aux< - 'a, - 'b: 'a, - 'c: 'a, - I: IntoIterator, - D: IntoIterator, - >( - &self, - insert: I, - delete: D, - ) -> ClientResult<()> { - let mut transaction = Transaction::new(); - for (k, v) in insert { - transaction.set(columns::AUX, k, v); - } - for k in delete { - transaction.remove(columns::AUX, k); - } - self.db.commit(transaction)?; - - Ok(()) - } - - fn get_aux(&self, key: &[u8]) -> ClientResult>> { - Ok(self.db.get(columns::AUX, key)) - } -} - -impl Storage for LightStorage -where - Block: BlockT, -{ - fn import_header( - &self, - header: Block::Header, - mut cache_at: HashMap>, - leaf_state: NewBlockState, - aux_ops: Vec<(Vec, Option>)>, - ) -> ClientResult<()> { - let mut transaction = Transaction::new(); - - let hash = header.hash(); - let number = *header.number(); - let parent_hash = *header.parent_hash(); - - for (key, maybe_val) in aux_ops { - match maybe_val { - Some(val) => transaction.set_from_vec(columns::AUX, &key, val), - None => transaction.remove(columns::AUX, &key), - } - } - - // blocks are keyed by number + hash. - let lookup_key = utils::number_and_hash_to_lookup_key(number, &hash)?; - - if leaf_state.is_best() { - self.set_head_with_transaction(&mut transaction, parent_hash, (number, hash))?; - } - - utils::insert_hash_to_key_mapping(&mut transaction, columns::KEY_LOOKUP, number, hash)?; - transaction.set_from_vec(columns::HEADER, &lookup_key, header.encode()); - - let header_metadata = CachedHeaderMetadata::from(&header); - self.header_metadata_cache - .insert_header_metadata(header.hash().clone(), header_metadata); - - let is_genesis = number.is_zero(); - if is_genesis { - self.cache.0.write().set_genesis_hash(hash); - transaction.set(columns::META, meta_keys::GENESIS_HASH, hash.as_ref()); - } - - let finalized = match leaf_state { - _ if is_genesis => true, - NewBlockState::Final => true, - _ => false, - }; - - if finalized { - self.note_finalized(&mut transaction, &header, hash)?; - } - - // update changes trie configuration cache - if !cache_at.contains_key(&well_known_cache_keys::CHANGES_TRIE_CONFIG) { - if let Some(new_configuration) = - crate::changes_tries_storage::extract_new_configuration(&header) - { - cache_at - .insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_configuration.encode()); - } - } - - { - let mut cache = self.cache.0.write(); - let cache_ops = cache - .transaction(&mut transaction) - .on_block_insert( - ComplexBlockId::new( - *header.parent_hash(), - if number.is_zero() { Zero::zero() } else { number - One::one() }, - ), - ComplexBlockId::new(hash, number), - cache_at, - if finalized { CacheEntryType::Final } else { CacheEntryType::NonFinal }, - )? - .into_ops(); - - debug!("Light DB Commit {:?} ({})", hash, number); - - self.db.commit(transaction)?; - cache.commit(cache_ops).expect( - "only fails if cache with given name isn't loaded yet; cache is already loaded \ - because there are cache_ops; qed", - ); - } - - self.update_meta(hash, number, leaf_state.is_best(), finalized); - - Ok(()) - } - - fn set_head(&self, id: BlockId) -> ClientResult<()> { - if let Some(header) = self.header(id)? { - let hash = header.hash(); - let number = header.number(); - - let mut transaction = Transaction::new(); - self.set_head_with_transaction( - &mut transaction, - hash.clone(), - (number.clone(), hash.clone()), - )?; - self.db.commit(transaction)?; - self.update_meta(hash, header.number().clone(), true, false); - - Ok(()) - } else { - Err(ClientError::UnknownBlock(format!("Cannot set head {:?}", id))) - } - } - - fn finalize_header(&self, id: BlockId) -> ClientResult<()> { - if let Some(header) = self.header(id)? { - let mut transaction = Transaction::new(); - let hash = header.hash(); - let number = *header.number(); - self.note_finalized(&mut transaction, &header, hash.clone())?; - { - let mut cache = self.cache.0.write(); - let cache_ops = cache - .transaction(&mut transaction) - .on_block_finalize( - ComplexBlockId::new( - *header.parent_hash(), - if number.is_zero() { Zero::zero() } else { number - One::one() }, - ), - ComplexBlockId::new(hash, number), - )? - .into_ops(); - - self.db.commit(transaction)?; - cache.commit(cache_ops).expect( - "only fails if cache with given name isn't loaded yet; cache is already loaded \ - because there are cache_ops; qed", - ); - } - self.update_meta(hash, header.number().clone(), false, true); - - Ok(()) - } else { - Err(ClientError::UnknownBlock(format!("Cannot finalize block {:?}", id))) - } - } - - fn last_finalized(&self) -> ClientResult { - Ok(self.meta.read().finalized_hash.clone()) - } - - fn cache(&self) -> Option>> { - Some(self.cache.clone()) - } - - fn usage_info(&self) -> Option { - use sc_client_api::{IoInfo, MemoryInfo, MemorySize}; - - // TODO: reimplement IO stats - let database_cache = MemorySize::from_bytes(0); - let io_stats = self.io_stats.take_or_else(|| kvdb::IoStats::empty()); - - Some(UsageInfo { - memory: MemoryInfo { - database_cache, - state_cache: Default::default(), - state_db: Default::default(), - }, - io: IoInfo { - transactions: io_stats.transactions, - bytes_read: io_stats.bytes_read, - bytes_written: io_stats.bytes_written, - writes: io_stats.writes, - reads: io_stats.reads, - average_transaction_size: io_stats.avg_transaction_size() as u64, - // Light client does not track those - state_reads: 0, - state_writes: 0, - state_reads_cache: 0, - state_writes_cache: 0, - state_writes_nodes: 0, - }, - }) - } -} - -impl ProvideChtRoots for LightStorage -where - Block: BlockT, -{ - fn header_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult> { - self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block) - } - - fn changes_trie_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult> { - self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block) - } -} - -/// Build the key for inserting header-CHT at given block. -fn cht_key>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> { - let mut key = [cht_type; 5]; - key[1..].copy_from_slice(&utils::number_index_key(block)?); - Ok(key) -} - -#[cfg(test)] -pub(crate) mod tests { - use super::*; - use sc_client_api::cht; - use sp_blockchain::{lowest_common_ancestor, tree_route}; - use sp_core::ChangesTrieConfiguration; - use sp_runtime::{ - generic::{ChangesTrieSignal, DigestItem}, - testing::{Block as RawBlock, ExtrinsicWrapper, Header, H256 as Hash}, - }; - - type Block = RawBlock>; - type AuthorityId = sp_core::ed25519::Public; - - pub fn default_header(parent: &Hash, number: u64) -> Header { - Header { - number: number.into(), - parent_hash: *parent, - state_root: Hash::random(), - digest: Default::default(), - extrinsics_root: Default::default(), - } - } - - fn header_with_changes_trie(parent: &Hash, number: u64) -> Header { - let mut header = default_header(parent, number); - header - .digest - .logs - .push(DigestItem::ChangesTrieRoot([(number % 256) as u8; 32].into())); - header - } - - fn header_with_extrinsics_root(parent: &Hash, number: u64, extrinsics_root: Hash) -> Header { - let mut header = default_header(parent, number); - header.extrinsics_root = extrinsics_root; - header - } - - pub fn insert_block Header>( - db: &LightStorage, - cache: HashMap>, - mut header: F, - ) -> Hash { - let header = header(); - let hash = header.hash(); - db.import_header(header, cache, NewBlockState::Best, Vec::new()).unwrap(); - hash - } - - fn insert_final_block Header>( - db: &LightStorage, - cache: HashMap>, - header: F, - ) -> Hash { - let header = header(); - let hash = header.hash(); - db.import_header(header, cache, NewBlockState::Final, Vec::new()).unwrap(); - hash - } - - fn insert_non_best_block Header>( - db: &LightStorage, - cache: HashMap>, - header: F, - ) -> Hash { - let header = header(); - let hash = header.hash(); - db.import_header(header, cache, NewBlockState::Normal, Vec::new()).unwrap(); - hash - } - - #[test] - fn returns_known_header() { - let db = LightStorage::new_test(); - let known_hash = - insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0)); - let header_by_hash = db.header(BlockId::Hash(known_hash)).unwrap().unwrap(); - let header_by_number = db.header(BlockId::Number(0)).unwrap().unwrap(); - assert_eq!(header_by_hash, header_by_number); - } - - #[test] - fn does_not_return_unknown_header() { - let db = LightStorage::::new_test(); - assert!(db.header(BlockId::Hash(Hash::from_low_u64_be(1))).unwrap().is_none()); - assert!(db.header(BlockId::Number(0)).unwrap().is_none()); - } - - #[test] - fn returns_info() { - let db = LightStorage::new_test(); - let genesis_hash = - insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0)); - let info = db.info(); - assert_eq!(info.best_hash, genesis_hash); - assert_eq!(info.best_number, 0); - assert_eq!(info.genesis_hash, genesis_hash); - let best_hash = insert_block(&db, HashMap::new(), || default_header(&genesis_hash, 1)); - let info = db.info(); - assert_eq!(info.best_hash, best_hash); - assert_eq!(info.best_number, 1); - assert_eq!(info.genesis_hash, genesis_hash); - } - - #[test] - fn returns_block_status() { - let db = LightStorage::new_test(); - let genesis_hash = - insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0)); - assert_eq!(db.status(BlockId::Hash(genesis_hash)).unwrap(), BlockStatus::InChain); - assert_eq!(db.status(BlockId::Number(0)).unwrap(), BlockStatus::InChain); - assert_eq!( - db.status(BlockId::Hash(Hash::from_low_u64_be(1))).unwrap(), - BlockStatus::Unknown - ); - assert_eq!(db.status(BlockId::Number(1)).unwrap(), BlockStatus::Unknown); - } - - #[test] - fn returns_block_hash() { - let db = LightStorage::new_test(); - let genesis_hash = - insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0)); - assert_eq!(db.hash(0).unwrap(), Some(genesis_hash)); - assert_eq!(db.hash(1).unwrap(), None); - } - - #[test] - fn import_header_works() { - let raw_db = Arc::new(sp_database::MemDb::default()); - let db = LightStorage::from_kvdb(raw_db.clone()).unwrap(); - - let genesis_hash = - insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0)); - assert_eq!(raw_db.count(columns::HEADER), 1); - assert_eq!(raw_db.count(columns::KEY_LOOKUP), 2); - - let _ = insert_block(&db, HashMap::new(), || default_header(&genesis_hash, 1)); - assert_eq!(raw_db.count(columns::HEADER), 2); - assert_eq!(raw_db.count(columns::KEY_LOOKUP), 4); - } - - #[test] - fn finalized_ancient_headers_are_replaced_with_cht() { - fn insert_headers Header>( - header_producer: F, - ) -> (Arc, LightStorage) { - let raw_db = Arc::new(sp_database::MemDb::default()); - let db = LightStorage::from_kvdb(raw_db.clone()).unwrap(); - let cht_size: u64 = cht::size(); - let ucht_size: usize = cht_size as _; - - // insert genesis block header (never pruned) - let mut prev_hash = - insert_final_block(&db, HashMap::new(), || header_producer(&Default::default(), 0)); - - // insert SIZE blocks && ensure that nothing is pruned - - for number in 0..cht::size() { - prev_hash = - insert_block(&db, HashMap::new(), || header_producer(&prev_hash, 1 + number)); - } - assert_eq!(raw_db.count(columns::HEADER), 1 + ucht_size); - assert_eq!(raw_db.count(columns::CHT), 0); - - // insert next SIZE blocks && ensure that nothing is pruned - for number in 0..(cht_size as _) { - prev_hash = insert_block(&db, HashMap::new(), || { - header_producer(&prev_hash, 1 + cht_size + number) - }); - } - assert_eq!(raw_db.count(columns::HEADER), 1 + ucht_size + ucht_size); - assert_eq!(raw_db.count(columns::CHT), 0); - - // insert block #{2 * cht::size() + 1} && check that new CHT is created + headers of - // this CHT are pruned nothing is yet finalized, so nothing is pruned. - prev_hash = insert_block(&db, HashMap::new(), || { - header_producer(&prev_hash, 1 + cht_size + cht_size) - }); - assert_eq!(raw_db.count(columns::HEADER), 2 + ucht_size + ucht_size); - assert_eq!(raw_db.count(columns::CHT), 0); - - // now finalize the block. - for i in (0..(ucht_size + ucht_size)).map(|i| i + 1) { - db.finalize_header(BlockId::Number(i as _)).unwrap(); - } - db.finalize_header(BlockId::Hash(prev_hash)).unwrap(); - (raw_db, db) - } - - // when headers are created without changes tries roots - let (raw_db, db) = insert_headers(default_header); - let cht_size: u64 = cht::size(); - assert_eq!(raw_db.count(columns::HEADER), (1 + cht_size + 1) as usize); - assert_eq!(raw_db.count(columns::KEY_LOOKUP), (2 * (1 + cht_size + 1)) as usize); - assert_eq!(raw_db.count(columns::CHT), 1); - assert!((0..cht_size as _).all(|i| db.header(BlockId::Number(1 + i)).unwrap().is_none())); - assert!(db.header_cht_root(cht_size, cht_size / 2).unwrap().is_some()); - assert!(db.header_cht_root(cht_size, cht_size + cht_size / 2).unwrap().is_none()); - assert!(db.changes_trie_cht_root(cht_size, cht_size / 2).is_err()); - assert!(db.changes_trie_cht_root(cht_size, cht_size + cht_size / 2).unwrap().is_none()); - - // when headers are created with changes tries roots - let (raw_db, db) = insert_headers(header_with_changes_trie); - assert_eq!(raw_db.count(columns::HEADER), (1 + cht_size + 1) as usize); - assert_eq!(raw_db.count(columns::CHT), 2); - assert!((0..cht_size as _).all(|i| db.header(BlockId::Number(1 + i)).unwrap().is_none())); - assert!(db.header_cht_root(cht_size, cht_size / 2).unwrap().is_some()); - assert!(db.header_cht_root(cht_size, cht_size + cht_size / 2).unwrap().is_none()); - assert!(db.changes_trie_cht_root(cht_size, cht_size / 2).unwrap().is_some()); - assert!(db.changes_trie_cht_root(cht_size, cht_size + cht_size / 2).unwrap().is_none()); - } - - #[test] - fn get_cht_fails_for_genesis_block() { - assert!(LightStorage::::new_test().header_cht_root(cht::size(), 0).is_err()); - } - - #[test] - fn get_cht_fails_for_non_existent_cht() { - let cht_size: u64 = cht::size(); - assert!(LightStorage::::new_test() - .header_cht_root(cht_size, cht_size / 2) - .unwrap() - .is_none()); - } - - #[test] - fn get_cht_works() { - let db = LightStorage::new_test(); - - // insert 1 + SIZE + SIZE + 1 blocks so that CHT#0 is created - let mut prev_hash = insert_final_block(&db, HashMap::new(), || { - header_with_changes_trie(&Default::default(), 0) - }); - let cht_size: u64 = cht::size(); - let ucht_size: usize = cht_size as _; - for i in 1..1 + ucht_size + ucht_size + 1 { - prev_hash = insert_block(&db, HashMap::new(), || { - header_with_changes_trie(&prev_hash, i as u64) - }); - db.finalize_header(BlockId::Hash(prev_hash)).unwrap(); - } - - let cht_root_1 = - db.header_cht_root(cht_size, cht::start_number(cht_size, 0)).unwrap().unwrap(); - let cht_root_2 = db - .header_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2) - .unwrap() - .unwrap(); - let cht_root_3 = - db.header_cht_root(cht_size, cht::end_number(cht_size, 0)).unwrap().unwrap(); - assert_eq!(cht_root_1, cht_root_2); - assert_eq!(cht_root_2, cht_root_3); - - let cht_root_1 = db - .changes_trie_cht_root(cht_size, cht::start_number(cht_size, 0)) - .unwrap() - .unwrap(); - let cht_root_2 = db - .changes_trie_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2) - .unwrap() - .unwrap(); - let cht_root_3 = db - .changes_trie_cht_root(cht_size, cht::end_number(cht_size, 0)) - .unwrap() - .unwrap(); - assert_eq!(cht_root_1, cht_root_2); - assert_eq!(cht_root_2, cht_root_3); - } - - #[test] - fn tree_route_works() { - let db = LightStorage::new_test(); - let block0 = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0)); - - // fork from genesis: 3 prong. - let a1 = insert_block(&db, HashMap::new(), || default_header(&block0, 1)); - let a2 = insert_block(&db, HashMap::new(), || default_header(&a1, 2)); - let a3 = insert_block(&db, HashMap::new(), || default_header(&a2, 3)); - - // fork from genesis: 2 prong. - let b1 = insert_block(&db, HashMap::new(), || { - header_with_extrinsics_root(&block0, 1, Hash::from([1; 32])) - }); - let b2 = insert_block(&db, HashMap::new(), || default_header(&b1, 2)); - - { - let tree_route = tree_route(&db, a3, b2).unwrap(); - - assert_eq!(tree_route.common_block().hash, block0); - assert_eq!( - tree_route.retracted().iter().map(|r| r.hash).collect::>(), - vec![a3, a2, a1] - ); - assert_eq!( - tree_route.enacted().iter().map(|r| r.hash).collect::>(), - vec![b1, b2] - ); - } - - { - let tree_route = tree_route(&db, a1, a3).unwrap(); - - assert_eq!(tree_route.common_block().hash, a1); - assert!(tree_route.retracted().is_empty()); - assert_eq!( - tree_route.enacted().iter().map(|r| r.hash).collect::>(), - vec![a2, a3] - ); - } - - { - let tree_route = tree_route(&db, a3, a1).unwrap(); - - assert_eq!(tree_route.common_block().hash, a1); - assert_eq!( - tree_route.retracted().iter().map(|r| r.hash).collect::>(), - vec![a3, a2] - ); - assert!(tree_route.enacted().is_empty()); - } - - { - let tree_route = tree_route(&db, a2, a2).unwrap(); - - assert_eq!(tree_route.common_block().hash, a2); - assert!(tree_route.retracted().is_empty()); - assert!(tree_route.enacted().is_empty()); - } - } - - #[test] - fn lowest_common_ancestor_works() { - let db = LightStorage::new_test(); - let block0 = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0)); - - // fork from genesis: 3 prong. - let a1 = insert_block(&db, HashMap::new(), || default_header(&block0, 1)); - let a2 = insert_block(&db, HashMap::new(), || default_header(&a1, 2)); - let a3 = insert_block(&db, HashMap::new(), || default_header(&a2, 3)); - - // fork from genesis: 2 prong. - let b1 = insert_block(&db, HashMap::new(), || { - header_with_extrinsics_root(&block0, 1, Hash::from([1; 32])) - }); - let b2 = insert_block(&db, HashMap::new(), || default_header(&b1, 2)); - - { - let lca = lowest_common_ancestor(&db, a3, b2).unwrap(); - - assert_eq!(lca.hash, block0); - assert_eq!(lca.number, 0); - } - - { - let lca = lowest_common_ancestor(&db, a1, a3).unwrap(); - - assert_eq!(lca.hash, a1); - assert_eq!(lca.number, 1); - } - - { - let lca = lowest_common_ancestor(&db, a3, a1).unwrap(); - - assert_eq!(lca.hash, a1); - assert_eq!(lca.number, 1); - } - - { - let lca = lowest_common_ancestor(&db, a2, a3).unwrap(); - - assert_eq!(lca.hash, a2); - assert_eq!(lca.number, 2); - } - - { - let lca = lowest_common_ancestor(&db, a2, a1).unwrap(); - - assert_eq!(lca.hash, a1); - assert_eq!(lca.number, 1); - } - - { - let lca = lowest_common_ancestor(&db, a2, a2).unwrap(); - - assert_eq!(lca.hash, a2); - assert_eq!(lca.number, 2); - } - } - - #[test] - fn authorities_are_cached() { - let db = LightStorage::new_test(); - - fn run_checks( - db: &LightStorage, - max: u64, - checks: &[(u64, Option>)], - ) { - for (at, expected) in checks.iter().take_while(|(at, _)| *at <= max) { - let actual = authorities(db.cache(), BlockId::Number(*at)); - assert_eq!(*expected, actual); - } - } - - fn same_authorities() -> HashMap> { - HashMap::new() - } - - fn make_authorities( - authorities: Vec, - ) -> HashMap> { - let mut map = HashMap::new(); - map.insert(well_known_cache_keys::AUTHORITIES, authorities.encode()); - map - } - - fn authorities( - cache: &dyn BlockchainCache, - at: BlockId, - ) -> Option> { - cache - .get_at(&well_known_cache_keys::AUTHORITIES, &at) - .unwrap_or(None) - .and_then(|(_, _, val)| Decode::decode(&mut &val[..]).ok()) - } - - let auth1 = || AuthorityId::from_raw([1u8; 32]); - let auth2 = || AuthorityId::from_raw([2u8; 32]); - let auth3 = || AuthorityId::from_raw([3u8; 32]); - let auth4 = || AuthorityId::from_raw([4u8; 32]); - let auth5 = || AuthorityId::from_raw([5u8; 32]); - let auth6 = || AuthorityId::from_raw([6u8; 32]); - - let (hash2, hash6) = { - // first few blocks are instantly finalized - // B0(None) -> B1(None) -> B2(1) -> B3(1) -> B4(1, 2) -> B5(1, 2) -> B6(1, 2) - let checks = vec![ - (0, None), - (1, None), - (2, Some(vec![auth1()])), - (3, Some(vec![auth1()])), - (4, Some(vec![auth1(), auth2()])), - (5, Some(vec![auth1(), auth2()])), - (6, Some(vec![auth1(), auth2()])), - ]; - - let hash0 = insert_final_block(&db, same_authorities(), || { - default_header(&Default::default(), 0) - }); - run_checks(&db, 0, &checks); - let hash1 = insert_final_block(&db, same_authorities(), || default_header(&hash0, 1)); - run_checks(&db, 1, &checks); - let hash2 = insert_final_block(&db, make_authorities(vec![auth1()]), || { - default_header(&hash1, 2) - }); - run_checks(&db, 2, &checks); - let hash3 = insert_final_block(&db, make_authorities(vec![auth1()]), || { - default_header(&hash2, 3) - }); - run_checks(&db, 3, &checks); - let hash4 = insert_final_block(&db, make_authorities(vec![auth1(), auth2()]), || { - default_header(&hash3, 4) - }); - run_checks(&db, 4, &checks); - let hash5 = insert_final_block(&db, make_authorities(vec![auth1(), auth2()]), || { - default_header(&hash4, 5) - }); - run_checks(&db, 5, &checks); - let hash6 = insert_final_block(&db, same_authorities(), || default_header(&hash5, 6)); - run_checks(&db, 6, &checks); - - (hash2, hash6) - }; - - { - // some older non-best blocks are inserted - // ... -> B2(1) -> B2_1(1) -> B2_2(2) - // => the cache ignores all writes before best finalized block - let hash2_1 = insert_non_best_block(&db, make_authorities(vec![auth1()]), || { - default_header(&hash2, 3) - }); - assert_eq!(None, authorities(db.cache(), BlockId::Hash(hash2_1))); - let hash2_2 = - insert_non_best_block(&db, make_authorities(vec![auth1(), auth2()]), || { - default_header(&hash2_1, 4) - }); - assert_eq!(None, authorities(db.cache(), BlockId::Hash(hash2_2))); - } - - let (hash7, hash8, hash6_1, hash6_2, hash6_1_1, hash6_1_2) = { - // inserting non-finalized blocks - // B6(None) -> B7(3) -> B8(3) - // \> B6_1(4) -> B6_2(4) - // \> B6_1_1(5) - // \> B6_1_2(6) -> B6_1_3(7) - - let hash7 = - insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash6, 7)); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - let hash8 = - insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash7, 8)); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - let hash6_1 = - insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6, 7)); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - let hash6_1_1 = insert_non_best_block(&db, make_authorities(vec![auth5()]), || { - default_header(&hash6_1, 8) - }); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); - let hash6_1_2 = insert_non_best_block(&db, make_authorities(vec![auth6()]), || { - default_header(&hash6_1, 8) - }); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); - let hash6_2 = - insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6_1, 8)); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); - - (hash7, hash8, hash6_1, hash6_2, hash6_1_1, hash6_1_2) - }; - - { - // finalize block hash6_1 - db.finalize_header(BlockId::Hash(hash6_1)).unwrap(); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), None); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), None); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); - // finalize block hash6_2 - db.finalize_header(BlockId::Hash(hash6_2)).unwrap(); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), None); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), None); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()])); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), None); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), None); - assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()])); - } - } - - #[test] - fn database_is_reopened() { - let db = LightStorage::new_test(); - let hash0 = - insert_final_block(&db, HashMap::new(), || default_header(&Default::default(), 0)); - assert_eq!(db.info().best_hash, hash0); - assert_eq!(db.header(BlockId::Hash(hash0)).unwrap().unwrap().hash(), hash0); - - let db = db.db; - let db = LightStorage::from_kvdb(db).unwrap(); - assert_eq!(db.info().best_hash, hash0); - assert_eq!(db.header(BlockId::Hash::(hash0)).unwrap().unwrap().hash(), hash0); - } - - #[test] - fn aux_store_works() { - let db = LightStorage::::new_test(); - - // insert aux1 + aux2 using direct store access - db.insert_aux(&[(&[1][..], &[101][..]), (&[2][..], &[102][..])], ::std::iter::empty()) - .unwrap(); - - // check aux values - assert_eq!(db.get_aux(&[1]).unwrap(), Some(vec![101])); - assert_eq!(db.get_aux(&[2]).unwrap(), Some(vec![102])); - assert_eq!(db.get_aux(&[3]).unwrap(), None); - - // delete aux1 + insert aux3 using import operation - db.import_header( - default_header(&Default::default(), 0), - HashMap::new(), - NewBlockState::Best, - vec![(vec![3], Some(vec![103])), (vec![1], None)], - ) - .unwrap(); - - // check aux values - assert_eq!(db.get_aux(&[1]).unwrap(), None); - assert_eq!(db.get_aux(&[2]).unwrap(), Some(vec![102])); - assert_eq!(db.get_aux(&[3]).unwrap(), Some(vec![103])); - } - - #[test] - fn cache_can_be_initialized_after_genesis_inserted() { - let (genesis_hash, storage) = { - let db = LightStorage::::new_test(); - - // before cache is initialized => Err - assert!(db.cache().get_at(b"test", &BlockId::Number(0)).is_err()); - - // insert genesis block (no value for cache is provided) - let mut genesis_hash = None; - insert_block(&db, HashMap::new(), || { - let header = default_header(&Default::default(), 0); - genesis_hash = Some(header.hash()); - header - }); - - // after genesis is inserted => None - assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(), None); - - // initialize cache - db.cache().initialize(b"test", vec![42]).unwrap(); - - // after genesis is inserted + cache is initialized => Some - assert_eq!( - db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(), - Some(((0, genesis_hash.unwrap()), None, vec![42])), - ); - - (genesis_hash, db.db) - }; - - // restart && check that after restart value is read from the cache - let db = - LightStorage::::from_kvdb(storage as Arc<_>).expect("failed to create test-db"); - assert_eq!( - db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(), - Some(((0, genesis_hash.unwrap()), None, vec![42])), - ); - } - - #[test] - fn changes_trie_configuration_is_tracked_on_light_client() { - let db = LightStorage::::new_test(); - - let new_config = Some(ChangesTrieConfiguration::new(2, 2)); - - // insert block#0 && block#1 (no value for cache is provided) - let hash0 = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0)); - assert_eq!( - db.cache() - .get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(0)) - .unwrap() - .map(|(_, _, v)| ChangesTrieConfiguration::decode(&mut &v[..]).unwrap()), - None, - ); - - // insert configuration at block#1 (starts from block#2) - insert_block(&db, HashMap::new(), || { - let mut header = default_header(&hash0, 1); - header.digest_mut().push(DigestItem::ChangesTrieSignal( - ChangesTrieSignal::NewConfiguration(new_config.clone()), - )); - header - }); - assert_eq!( - db.cache() - .get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(1)) - .unwrap() - .map(|(_, _, v)| Option::::decode(&mut &v[..]).unwrap()), - Some(new_config), - ); - } -} diff --git a/client/db/src/offchain.rs b/client/db/src/offchain.rs index c31273ff07c6..4f0a77ce5756 100644 --- a/client/db/src/offchain.rs +++ b/client/db/src/offchain.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -92,7 +92,7 @@ impl sp_core::offchain::OffchainStorage for LocalStorage { { let _key_guard = key_lock.lock(); let val = self.db.get(columns::OFFCHAIN, &key); - is_set = val.as_ref().map(|x| &**x) == old_value; + is_set = val.as_deref() == old_value; if is_set { self.set(prefix, item_key, new_value) diff --git a/client/db/src/parity_db.rs b/client/db/src/parity_db.rs index 1b645ca9fb2b..f88e6f2e9167 100644 --- a/client/db/src/parity_db.rs +++ b/client/db/src/parity_db.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,7 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . use crate::{ - columns, light, + columns, utils::{DatabaseType, NUM_COLUMNS}, }; /// A `Database` adapter for parity-db. @@ -38,20 +38,22 @@ pub fn open>( path: &std::path::Path, db_type: DatabaseType, create: bool, + upgrade: bool, ) -> parity_db::Result>> { let mut config = parity_db::Options::with_columns(path, NUM_COLUMNS as u8); match db_type { DatabaseType::Full => { - let indexes = [ + let compressed = [ columns::STATE, columns::HEADER, columns::BODY, + columns::BODY_INDEX, columns::TRANSACTION, columns::JUSTIFICATIONS, ]; - for i in indexes { + for i in compressed { let mut column = &mut config.columns[i as usize]; column.compression = parity_db::CompressionType::Lz4; } @@ -60,11 +62,19 @@ pub fn open>( state_col.ref_counted = true; state_col.preimage = true; state_col.uniform = true; + + let mut tx_col = &mut config.columns[columns::TRANSACTION as usize]; + tx_col.ref_counted = true; + tx_col.preimage = true; + tx_col.uniform = true; }, - DatabaseType::Light => { - config.columns[light::columns::HEADER as usize].compression = - parity_db::CompressionType::Lz4; - }, + } + + if upgrade { + log::info!("Upgrading database metadata."); + if let Some(meta) = parity_db::Options::load_metadata(path)? { + config.write_metadata_with_version(path, &meta.salt, Some(meta.version))?; + } } let db = if create { @@ -102,4 +112,8 @@ impl> Database for DbAdapter { fn supports_ref_counting(&self) -> bool { true } + + fn sanitize_key(&self, key: &mut Vec) { + let _prefix = key.drain(0..key.len() - crate::DB_HASH_LEN); + } } diff --git a/client/db/src/stats.rs b/client/db/src/stats.rs index 9223142ef5ab..f6c14568236e 100644 --- a/client/db/src/stats.rs +++ b/client/db/src/stats.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index 5fef0e5b12d0..504708737637 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -26,7 +26,10 @@ use linked_hash_map::{Entry, LinkedHashMap}; use log::trace; use parking_lot::{RwLock, RwLockUpgradableReadGuard}; use sp_core::{hexdisplay::HexDisplay, storage::ChildInfo}; -use sp_runtime::traits::{Block as BlockT, HashFor, Header, NumberFor}; +use sp_runtime::{ + traits::{Block as BlockT, HashFor, Header, NumberFor}, + StateVersion, +}; use sp_state_machine::{ backend::Backend as StateBackend, ChildStorageCollection, StorageCollection, StorageKey, StorageValue, TrieBackend, @@ -673,22 +676,24 @@ impl>, B: BlockT> StateBackend> for Cachin fn storage_root<'a>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord, { - self.state.storage_root(delta) + self.state.storage_root(delta, state_version) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord, { - self.state.child_storage_root(child_info, delta) + self.state.child_storage_root(child_info, delta, state_version) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -871,22 +876,24 @@ impl>, B: BlockT> StateBackend> fn storage_root<'a>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord, { - self.caching_state().storage_root(delta) + self.caching_state().storage_root(delta, state_version) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord, { - self.caching_state().child_storage_root(child_info, delta) + self.caching_state().child_storage_root(child_info, delta, state_version) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -1182,7 +1189,10 @@ mod tests { let shared = new_shared_cache::(256 * 1024, (0, 1)); let mut backend = InMemoryBackend::::default(); - backend.insert(std::iter::once((None, vec![(key.clone(), Some(vec![1]))]))); + backend.insert( + std::iter::once((None, vec![(key.clone(), Some(vec![1]))])), + Default::default(), + ); let mut s = CachingState::new(backend.clone(), shared.clone(), Some(root_parent)); s.cache.sync_cache( diff --git a/client/db/src/upgrade.rs b/client/db/src/upgrade.rs index 0f3578ad99a3..ec91a753ed87 100644 --- a/client/db/src/upgrade.rs +++ b/client/db/src/upgrade.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -33,11 +33,12 @@ use sp_runtime::traits::Block as BlockT; const VERSION_FILE_NAME: &'static str = "db_version"; /// Current db version. -const CURRENT_VERSION: u32 = 3; +const CURRENT_VERSION: u32 = 4; /// Number of columns in v1. const V1_NUM_COLUMNS: u32 = 11; const V2_NUM_COLUMNS: u32 = 12; +const V3_NUM_COLUMNS: u32 = 12; /// Database upgrade errors. #[derive(Debug)] @@ -68,7 +69,7 @@ impl fmt::Display for UpgradeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { UpgradeError::UnknownDatabaseVersion => { - write!(f, "Database version cannot be read from exisiting db_version file") + write!(f, "Database version cannot be read from existing db_version file") }, UpgradeError::MissingDatabaseVersionFile => write!(f, "Missing database version file"), UpgradeError::UnsupportedVersion(version) => { @@ -92,9 +93,16 @@ pub fn upgrade_db(db_path: &Path, db_type: DatabaseType) -> Upgra 0 => return Err(UpgradeError::UnsupportedVersion(db_version)), 1 => { migrate_1_to_2::(db_path, db_type)?; - migrate_2_to_3::(db_path, db_type)? + migrate_2_to_3::(db_path, db_type)?; + migrate_3_to_4::(db_path, db_type)?; + }, + 2 => { + migrate_2_to_3::(db_path, db_type)?; + migrate_3_to_4::(db_path, db_type)?; + }, + 3 => { + migrate_3_to_4::(db_path, db_type)?; }, - 2 => migrate_2_to_3::(db_path, db_type)?, CURRENT_VERSION => (), _ => return Err(UpgradeError::FutureDatabaseVersion(db_version)), } @@ -139,6 +147,15 @@ fn migrate_2_to_3(db_path: &Path, _db_type: DatabaseType) -> Upgr Ok(()) } +/// Migration from version3 to version4: +/// 1) the number of columns has changed from 12 to 13; +/// 2) BODY_INDEX column is added; +fn migrate_3_to_4(db_path: &Path, _db_type: DatabaseType) -> UpgradeResult<()> { + let db_cfg = DatabaseConfig::with_columns(V3_NUM_COLUMNS); + let db = Database::open(&db_cfg, db_path)?; + db.add_column().map_err(Into::into) +} + /// Reads current database version from the file at given path. /// If the file does not exist returns 0. fn current_version(path: &Path) -> UpgradeResult { @@ -173,9 +190,7 @@ fn version_file_path(path: &Path) -> PathBuf { #[cfg(test)] mod tests { use super::*; - use crate::{ - tests::Block, DatabaseSettings, DatabaseSource, KeepBlocks, TransactionStorageMode, - }; + use crate::{tests::Block, DatabaseSettings, DatabaseSource, KeepBlocks}; use sc_state_db::PruningMode; fn create_db(db_path: &Path, version: Option) { @@ -194,7 +209,6 @@ mod tests { state_pruning: PruningMode::ArchiveAll, source: DatabaseSource::RocksDb { path: db_path.to_owned(), cache_size: 128 }, keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorageMode::BlockBody, }, db_type, ) @@ -229,4 +243,16 @@ mod tests { assert_eq!(current_version(&db_path).unwrap(), CURRENT_VERSION); } } + + #[test] + fn upgrade_to_4_works() { + let db_type = DatabaseType::Full; + for version_from_file in &[None, Some(1), Some(2), Some(3)] { + let db_dir = tempfile::TempDir::new().unwrap(); + let db_path = db_dir.path().join(db_type.as_str()); + create_db(&db_path, *version_from_file); + open_database(&db_path, db_type).unwrap(); + assert_eq!(current_version(&db_path).unwrap(), CURRENT_VERSION); + } + } } diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs index ea22c774f463..1798838ecc15 100644 --- a/client/db/src/utils.rs +++ b/client/db/src/utils.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,7 +19,7 @@ //! Db-based backend utility structures and functions, used by both //! full and light storages. -use std::{convert::TryInto, fmt, fs, io, path::Path, sync::Arc}; +use std::{fmt, fs, io, path::Path, sync::Arc}; use log::{debug, info}; @@ -40,7 +40,7 @@ use sp_trie::DBValue; feature = "test-helpers", test ))] -pub const NUM_COLUMNS: u32 = 12; +pub const NUM_COLUMNS: u32 = 13; /// Meta column. The set of keys in the column is shared by full && light storages. pub const COLUMN_META: u32 = 0; @@ -54,10 +54,8 @@ pub mod meta_keys { pub const FINALIZED_BLOCK: &[u8; 5] = b"final"; /// Last finalized state key. pub const FINALIZED_STATE: &[u8; 6] = b"fstate"; - /// Meta information prefix for list-based caches. - pub const CACHE_META_PREFIX: &[u8; 5] = b"cache"; - /// Meta information for changes tries key. - pub const CHANGES_TRIES_META: &[u8; 5] = b"ctrie"; + /// Block gap. + pub const BLOCK_GAP: &[u8; 3] = b"gap"; /// Genesis block hash. pub const GENESIS_HASH: &[u8; 3] = b"gen"; /// Leaves prefix list key. @@ -81,6 +79,8 @@ pub struct Meta { pub genesis_hash: H, /// Finalized state, if any pub finalized_state: Option<(H, N)>, + /// Block gap, start and end inclusive, if any. + pub block_gap: Option<(N, N)>, } /// A block lookup key: used for canonical lookup from block number to hash @@ -91,8 +91,6 @@ pub type NumberIndexKey = [u8; 4]; pub enum DatabaseType { /// Full node database. Full, - /// Light node database. - Light, } /// Convert block number into short lookup key (LE representation) for @@ -120,19 +118,6 @@ where Ok(lookup_key) } -/// Convert block lookup key into block number. -/// all block lookup keys start with the block number. -pub fn lookup_key_to_number(key: &[u8]) -> sp_blockchain::Result -where - N: From, -{ - if key.len() < 4 { - return Err(sp_blockchain::Error::Backend("Invalid block key".into())) - } - Ok((key[0] as u32) << 24 | (key[1] as u32) << 16 | (key[2] as u32) << 8 | (key[3] as u32)) - .map(Into::into) -} - /// Delete number to hash mapping in DB transaction. pub fn remove_number_to_key_mapping>( transaction: &mut Transaction, @@ -143,18 +128,6 @@ pub fn remove_number_to_key_mapping>( Ok(()) } -/// Remove key mappings. -pub fn remove_key_mappings, H: AsRef<[u8]>>( - transaction: &mut Transaction, - key_lookup_col: u32, - number: N, - hash: H, -) -> sp_blockchain::Result<()> { - remove_number_to_key_mapping(transaction, key_lookup_col, number)?; - transaction.remove(key_lookup_col, hash.as_ref()); - Ok(()) -} - /// Place a number mapping into the database. This maps number to current perceived /// block hash at that position. pub fn insert_number_to_key_mapping + Clone, H: AsRef<[u8]>>( @@ -279,7 +252,7 @@ impl From for sp_blockchain::Error { #[cfg(feature = "with-parity-db")] impl From for OpenDbError { fn from(err: parity_db::Error) -> Self { - if err.to_string().contains("use open_or_create") { + if matches!(err, parity_db::Error::DatabaseNotFound) { OpenDbError::DoesNotExist } else { OpenDbError::Internal(err.to_string()) @@ -299,8 +272,15 @@ impl From for OpenDbError { #[cfg(feature = "with-parity-db")] fn open_parity_db(path: &Path, db_type: DatabaseType, create: bool) -> OpenDbResult { - let db = crate::parity_db::open(path, db_type, create)?; - Ok(db) + match crate::parity_db::open(path, db_type, create, false) { + Ok(db) => Ok(db), + Err(parity_db::Error::InvalidConfiguration(_)) => { + log::warn!("Invalid parity db configuration, attempting database metadata update."); + // Try to update the database with the new config + Ok(crate::parity_db::open(path, db_type, create, true)?) + }, + Err(e) => Err(e.into()), + } } #[cfg(not(feature = "with-parity-db"))] @@ -353,18 +333,6 @@ fn open_kvdb_rocksdb( other_col_budget, ); }, - DatabaseType::Light => { - let col_budget = cache_size / (NUM_COLUMNS as usize); - for i in 0..NUM_COLUMNS { - memory_budget.insert(i, col_budget); - } - log::trace!( - target: "db", - "Open RocksDB light database at {:?}, column cache: {} MiB", - path, - col_budget, - ); - }, } db_config.memory_budget = memory_budget; @@ -420,8 +388,7 @@ fn maybe_migrate_to_type_subdir( // See if there's a file identifying a rocksdb or paritydb folder in the parent dir and // the target path ends in a role specific directory if (basedir.join("db_version").exists() || basedir.join("metadata").exists()) && - (p.ends_with(DatabaseType::Full.as_str()) || - p.ends_with(DatabaseType::Light.as_str())) + (p.ends_with(DatabaseType::Full.as_str())) { // Try to open the database to check if the current `DatabaseType` matches the type of // database stored in the target directory and close the database on success. @@ -497,18 +464,6 @@ pub fn read_header( } } -/// Required header from the database. -pub fn require_header( - db: &dyn Database, - col_index: u32, - col: u32, - id: BlockId, -) -> sp_blockchain::Result { - read_header(db, col_index, col, id).and_then(|header| { - header.ok_or_else(|| sp_blockchain::Error::UnknownBlock(format!("Require header: {}", id))) - }) -} - /// Read meta from the database. pub fn read_meta( db: &dyn Database, @@ -527,6 +482,7 @@ where finalized_number: Zero::zero(), genesis_hash: Default::default(), finalized_state: None, + block_gap: None, }), }; @@ -541,7 +497,7 @@ where "Opened blockchain db, fetched {} = {:?} ({})", desc, hash, - header.number() + header.number(), ); Ok((hash, *header.number())) } else { @@ -558,6 +514,10 @@ where } else { None }; + let block_gap = db + .get(COLUMN_META, meta_keys::BLOCK_GAP) + .and_then(|d| Decode::decode(&mut d.as_slice()).ok()); + debug!(target: "db", "block_gap={:?}", block_gap); Ok(Meta { best_hash, @@ -566,6 +526,7 @@ where finalized_number, genesis_hash, finalized_state, + block_gap, }) } @@ -588,7 +549,6 @@ impl DatabaseType { pub fn as_str(&self) -> &'static str { match *self { DatabaseType::Full => "full", - DatabaseType::Light => "light", } } } @@ -620,7 +580,7 @@ impl<'a, 'b> codec::Input for JoinInput<'a, 'b> { #[cfg(test)] mod tests { use super::*; - use crate::{KeepBlocks, TransactionStorageMode}; + use crate::KeepBlocks; use codec::Input; use sc_state_db::PruningMode; use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper}; @@ -659,23 +619,12 @@ mod tests { assert!(old_db_path.join(db_type.as_str()).join(db_check_file).exists()); } - check_dir_for_db_type( - DatabaseType::Light, - DatabaseSource::RocksDb { path: PathBuf::new(), cache_size: 128 }, - "db_version", - ); check_dir_for_db_type( DatabaseType::Full, DatabaseSource::RocksDb { path: PathBuf::new(), cache_size: 128 }, "db_version", ); - #[cfg(feature = "with-parity-db")] - check_dir_for_db_type( - DatabaseType::Light, - DatabaseSource::ParityDb { path: PathBuf::new() }, - "metadata", - ); #[cfg(feature = "with-parity-db")] check_dir_for_db_type( DatabaseType::Full, @@ -699,16 +648,8 @@ mod tests { assert!(!old_db_path.join("light/db_version").exists()); assert!(!old_db_path.join("full/db_version").exists()); } - let source = DatabaseSource::RocksDb { - path: old_db_path.join(DatabaseType::Light.as_str()), - cache_size: 128, - }; - let settings = db_settings(source); - let db_res = open_database::(&settings, DatabaseType::Light); - assert!(db_res.is_err(), "Opening a light database in full role should fail"); // assert nothing was changed assert!(old_db_path.join("db_version").exists()); - assert!(!old_db_path.join("light/db_version").exists()); assert!(!old_db_path.join("full/db_version").exists()); } } @@ -725,7 +666,6 @@ mod tests { #[test] fn database_type_as_str_works() { assert_eq!(DatabaseType::Full.as_str(), "full"); - assert_eq!(DatabaseType::Light.as_str(), "light"); } #[test] @@ -756,7 +696,6 @@ mod tests { state_pruning: PruningMode::ArchiveAll, source, keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorageMode::BlockBody, } } diff --git a/client/executor/Cargo.toml b/client/executor/Cargo.toml index b7e2595b8e16..8e10de1b04a9 100644 --- a/client/executor/Cargo.toml +++ b/client/executor/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-executor" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "A crate that provides means of executing/dispatching calls into the runtime." documentation = "https://docs.rs/sc-executor" @@ -14,39 +14,45 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +codec = { package = "parity-scale-codec", version = "3.0.0" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } sp-tasks = { version = "4.0.0-dev", path = "../../primitives/tasks" } -sp-trie = { version = "4.0.0-dev", path = "../../primitives/trie" } -sp-version = { version = "4.0.0-dev", path = "../../primitives/version" } -sp-panic-handler = { version = "3.0.0", path = "../../primitives/panic-handler" } -wasmi = "0.9.0" +sp-trie = { version = "6.0.0", path = "../../primitives/trie" } +sp-version = { version = "5.0.0", path = "../../primitives/version" } +sp-panic-handler = { version = "4.0.0", path = "../../primitives/panic-handler" } +wasmi = "0.9.1" lazy_static = "1.4.0" sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } -sp-wasm-interface = { version = "4.0.0-dev", path = "../../primitives/wasm-interface" } -sp-runtime-interface = { version = "4.0.0-dev", path = "../../primitives/runtime-interface" } -sp-externalities = { version = "0.10.0-dev", path = "../../primitives/externalities" } +sp-wasm-interface = { version = "6.0.0", path = "../../primitives/wasm-interface" } +sp-runtime-interface = { version = "6.0.0", path = "../../primitives/runtime-interface" } +sp-externalities = { version = "0.12.0", path = "../../primitives/externalities" } sc-executor-common = { version = "0.10.0-dev", path = "common" } sc-executor-wasmi = { version = "0.10.0-dev", path = "wasmi" } sc-executor-wasmtime = { version = "0.10.0-dev", path = "wasmtime", optional = true } -parking_lot = "0.11.1" -log = "0.4.8" -libsecp256k1 = "0.6" +parking_lot = "0.12.0" +sp-core-hashing-proc-macro = { version = "5.0.0", path = "../../primitives/core/hashing/proc-macro" } +lru = "0.7.5" +tracing = "0.1.29" [dev-dependencies] wat = "1.0" -hex-literal = "0.3.1" +hex-literal = "0.3.4" sc-runtime-test = { version = "2.0.0", path = "runtime-test" } substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" } -sp-state-machine = { version = "0.10.0-dev", path = "../../primitives/state-machine" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sp-maybe-compressed-blob = { version = "4.0.0-dev", path = "../../primitives/maybe-compressed-blob" } +sp-state-machine = { version = "0.12.0", path = "../../primitives/state-machine" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } +sp-maybe-compressed-blob = { version = "4.1.0-dev", path = "../../primitives/maybe-compressed-blob" } sc-tracing = { version = "4.0.0-dev", path = "../tracing" } -tracing = "0.1.25" tracing-subscriber = "0.2.19" paste = "1.0" -regex = "1" +regex = "1.5.5" +criterion = "0.3" +env_logger = "0.9" + +[[bench]] +name = "bench" +harness = false [features] default = ["std"] diff --git a/client/executor/benches/bench.rs b/client/executor/benches/bench.rs new file mode 100644 index 000000000000..49ea8be50624 --- /dev/null +++ b/client/executor/benches/bench.rs @@ -0,0 +1,136 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use criterion::{criterion_group, criterion_main, Criterion}; + +use sc_executor_common::{runtime_blob::RuntimeBlob, wasm_runtime::WasmModule}; +use sc_runtime_test::wasm_binary_unwrap as test_runtime; +use sp_wasm_interface::HostFunctions as _; +use std::sync::Arc; + +enum Method { + Interpreted, + #[cfg(feature = "wasmtime")] + Compiled { + fast_instance_reuse: bool, + }, +} + +// This is just a bog-standard Kusama runtime with the extra `test_empty_return` +// function copy-pasted from the test runtime. +fn kusama_runtime() -> &'static [u8] { + include_bytes!("kusama_runtime.wasm") +} + +fn initialize(runtime: &[u8], method: Method) -> Arc { + let blob = RuntimeBlob::uncompress_if_needed(runtime).unwrap(); + let host_functions = sp_io::SubstrateHostFunctions::host_functions(); + let heap_pages = 2048; + let allow_missing_func_imports = true; + + match method { + Method::Interpreted => sc_executor_wasmi::create_runtime( + blob, + heap_pages, + host_functions, + allow_missing_func_imports, + ) + .map(|runtime| -> Arc { Arc::new(runtime) }), + #[cfg(feature = "wasmtime")] + Method::Compiled { fast_instance_reuse } => + sc_executor_wasmtime::create_runtime::( + blob, + sc_executor_wasmtime::Config { + max_memory_size: None, + allow_missing_func_imports, + cache_path: None, + semantics: sc_executor_wasmtime::Semantics { + extra_heap_pages: heap_pages, + fast_instance_reuse, + deterministic_stack_limit: None, + canonicalize_nans: false, + parallel_compilation: true, + }, + }, + ) + .map(|runtime| -> Arc { Arc::new(runtime) }), + } + .unwrap() +} + +fn bench_call_instance(c: &mut Criterion) { + let _ = env_logger::try_init(); + + #[cfg(feature = "wasmtime")] + { + let runtime = initialize(test_runtime(), Method::Compiled { fast_instance_reuse: true }); + c.bench_function("call_instance_test_runtime_with_fast_instance_reuse", |b| { + let mut instance = runtime.new_instance().unwrap(); + b.iter(|| instance.call_export("test_empty_return", &[0]).unwrap()) + }); + } + + #[cfg(feature = "wasmtime")] + { + let runtime = initialize(test_runtime(), Method::Compiled { fast_instance_reuse: false }); + c.bench_function("call_instance_test_runtime_without_fast_instance_reuse", |b| { + let mut instance = runtime.new_instance().unwrap(); + b.iter(|| instance.call_export("test_empty_return", &[0]).unwrap()); + }); + } + + #[cfg(feature = "wasmtime")] + { + let runtime = initialize(kusama_runtime(), Method::Compiled { fast_instance_reuse: true }); + c.bench_function("call_instance_kusama_runtime_with_fast_instance_reuse", |b| { + let mut instance = runtime.new_instance().unwrap(); + b.iter(|| instance.call_export("test_empty_return", &[0]).unwrap()) + }); + } + + #[cfg(feature = "wasmtime")] + { + let runtime = initialize(kusama_runtime(), Method::Compiled { fast_instance_reuse: false }); + c.bench_function("call_instance_kusama_runtime_without_fast_instance_reuse", |b| { + let mut instance = runtime.new_instance().unwrap(); + b.iter(|| instance.call_export("test_empty_return", &[0]).unwrap()); + }); + } + + { + let runtime = initialize(test_runtime(), Method::Interpreted); + c.bench_function("call_instance_test_runtime_interpreted", |b| { + let mut instance = runtime.new_instance().unwrap(); + b.iter(|| instance.call_export("test_empty_return", &[0]).unwrap()) + }); + } + + { + let runtime = initialize(kusama_runtime(), Method::Interpreted); + c.bench_function("call_instance_kusama_runtime_interpreted", |b| { + let mut instance = runtime.new_instance().unwrap(); + b.iter(|| instance.call_export("test_empty_return", &[0]).unwrap()) + }); + } +} + +criterion_group! { + name = benches; + config = Criterion::default(); + targets = bench_call_instance +} +criterion_main!(benches); diff --git a/client/executor/benches/kusama_runtime.wasm b/client/executor/benches/kusama_runtime.wasm new file mode 100755 index 000000000000..3470237fb5ae Binary files /dev/null and b/client/executor/benches/kusama_runtime.wasm differ diff --git a/client/executor/common/Cargo.toml b/client/executor/common/Cargo.toml index c4fc8c27f754..149d9fdc236c 100644 --- a/client/executor/common/Cargo.toml +++ b/client/executor/common/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-executor-common" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "A set of common definitions that are needed for defining execution engines." documentation = "https://docs.rs/sc-executor-common/" @@ -14,24 +14,20 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -derive_more = "0.99.2" -pwasm-utils = "0.18.0" -codec = { package = "parity-scale-codec", version = "2.0.0" } -wasmi = "0.9.0" -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sc-allocator = { version = "4.0.0-dev", path = "../../allocator" } -sp-wasm-interface = { version = "4.0.0-dev", path = "../../../primitives/wasm-interface" } -sp-maybe-compressed-blob = { version = "4.0.0-dev", path = "../../../primitives/maybe-compressed-blob" } -sp-serializer = { version = "3.0.0", path = "../../../primitives/serializer" } -thiserror = "1.0.21" +wasm-instrument = "0.1" +codec = { package = "parity-scale-codec", version = "3.0.0" } +wasmi = "0.9.1" +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sc-allocator = { version = "4.1.0-dev", path = "../../allocator" } +sp-wasm-interface = { version = "6.0.0", path = "../../../primitives/wasm-interface" } +sp-maybe-compressed-blob = { version = "4.1.0-dev", path = "../../../primitives/maybe-compressed-blob" } +sp-serializer = { version = "4.0.0-dev", path = "../../../primitives/serializer" } +thiserror = "1.0.30" environmental = "1.1.3" - -wasmer = { version = "1.0", optional = true } -wasmer-compiler-singlepass = { version = "1.0", optional = true } +wasmer = { version = "2.2", optional = true, features = ["singlepass"] } [features] default = [] wasmer-sandbox = [ "wasmer", - "wasmer-compiler-singlepass", ] diff --git a/client/executor/common/src/error.rs b/client/executor/common/src/error.rs index 6ad4802e57a8..5ffcafd7e92c 100644 --- a/client/executor/common/src/error.rs +++ b/client/executor/common/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -31,12 +31,12 @@ pub enum Error { #[error("Unserializable data encountered")] InvalidData(#[from] sp_serializer::Error), - #[error(transparent)] - Trap(#[from] wasmi::Trap), - #[error(transparent)] Wasmi(#[from] wasmi::Error), + #[error("Sandbox error: {0}")] + Sandbox(String), + #[error("Error calling api function: {0}")] ApiError(Box), @@ -108,6 +108,12 @@ pub enum Error { #[error("Invalid initializer expression provided {0}")] InvalidInitializerExpression(String), + + #[error("Execution aborted due to panic: {0}")] + AbortedDueToPanic(MessageWithBacktrace), + + #[error("Execution aborted due to trap: {0}")] + AbortedDueToTrap(MessageWithBacktrace), } impl wasmi::HostError for Error {} @@ -125,28 +131,73 @@ impl From for Error { } /// Type for errors occurring during Wasm runtime construction. -#[derive(Debug, derive_more::Display)] +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] pub enum WasmError { - /// Code could not be read from the state. + #[error("Code could not be read from the state.")] CodeNotFound, - /// Failure to reinitialize runtime instance from snapshot. + + #[error("Failure to reinitialize runtime instance from snapshot.")] ApplySnapshotFailed, + /// Failure to erase the wasm memory. /// /// Depending on the implementation might mean failure of allocating memory. + #[error("Failure to erase the wasm memory: {0}")] ErasingFailed(String), - /// Wasm code failed validation. + + #[error("Wasm code failed validation.")] InvalidModule, - /// Wasm code could not be deserialized. + + #[error("Wasm code could not be deserialized.")] CantDeserializeWasm, - /// The module does not export a linear memory named `memory`. + + #[error("The module does not export a linear memory named `memory`.")] InvalidMemory, - /// The number of heap pages requested is disallowed by the module. + + #[error("The number of heap pages requested is disallowed by the module.")] InvalidHeapPages, + /// Instantiation error. + #[error("{0}")] Instantiation(String), + /// Other error happenend. + #[error("{0}")] Other(String), } -impl std::error::Error for WasmError {} +/// An error message with an attached backtrace. +#[derive(Debug)] +pub struct MessageWithBacktrace { + /// The error message. + pub message: String, + + /// The backtrace associated with the error message. + pub backtrace: Option, +} + +impl std::fmt::Display for MessageWithBacktrace { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + fmt.write_str(&self.message)?; + if let Some(ref backtrace) = self.backtrace { + fmt.write_str("\nWASM backtrace:\n")?; + backtrace.backtrace_string.fmt(fmt)?; + } + + Ok(()) + } +} + +/// A WASM backtrace. +#[derive(Debug)] +pub struct Backtrace { + /// The string containing the backtrace. + pub backtrace_string: String, +} + +impl std::fmt::Display for Backtrace { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + fmt.write_str(&self.backtrace_string) + } +} diff --git a/client/executor/common/src/lib.rs b/client/executor/common/src/lib.rs index 99b927e06203..b69883afbaac 100644 --- a/client/executor/common/src/lib.rs +++ b/client/executor/common/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/executor/common/src/runtime_blob/data_segments_snapshot.rs b/client/executor/common/src/runtime_blob/data_segments_snapshot.rs index 5c3fedbdc963..b44370e681b1 100644 --- a/client/executor/common/src/runtime_blob/data_segments_snapshot.rs +++ b/client/executor/common/src/runtime_blob/data_segments_snapshot.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,8 +18,8 @@ use super::RuntimeBlob; use crate::error::{self, Error}; -use pwasm_utils::parity_wasm::elements::Instruction; use std::mem; +use wasm_instrument::parity_wasm::elements::Instruction; /// This is a snapshot of data segments specialzied for a particular instantiation. /// diff --git a/client/executor/common/src/runtime_blob/globals_snapshot.rs b/client/executor/common/src/runtime_blob/globals_snapshot.rs index a25fa6f9fd63..207fa751bc40 100644 --- a/client/executor/common/src/runtime_blob/globals_snapshot.rs +++ b/client/executor/common/src/runtime_blob/globals_snapshot.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/executor/common/src/runtime_blob/mod.rs b/client/executor/common/src/runtime_blob/mod.rs index 1af2708d3eb4..4b163bbaaf32 100644 --- a/client/executor/common/src/runtime_blob/mod.rs +++ b/client/executor/common/src/runtime_blob/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/executor/common/src/runtime_blob/runtime_blob.rs b/client/executor/common/src/runtime_blob/runtime_blob.rs index 6fb9303e0775..649ff51f287e 100644 --- a/client/executor/common/src/runtime_blob/runtime_blob.rs +++ b/client/executor/common/src/runtime_blob/runtime_blob.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -17,9 +17,12 @@ // along with this program. If not, see . use crate::error::WasmError; -use pwasm_utils::{ +use wasm_instrument::{ export_mutable_globals, - parity_wasm::elements::{deserialize_buffer, serialize, DataSegment, Internal, Module}, + parity_wasm::elements::{ + deserialize_buffer, serialize, DataSegment, ExportEntry, External, Internal, MemorySection, + MemoryType, Module, Section, + }, }; /// A bunch of information collected from a WebAssembly module. @@ -84,7 +87,7 @@ impl RuntimeBlob { /// depth of the wasm operand stack. pub fn inject_stack_depth_metering(self, stack_depth_limit: u32) -> Result { let injected_module = - pwasm_utils::stack_height::inject_limiter(self.raw_module, stack_depth_limit).map_err( + wasm_instrument::inject_stack_limiter(self.raw_module, stack_depth_limit).map_err( |e| WasmError::Other(format!("cannot inject the stack limiter: {:?}", e)), )?; @@ -104,6 +107,85 @@ impl RuntimeBlob { .unwrap_or_default() } + /// Converts a WASM memory import into a memory section and exports it. + /// + /// Does nothing if there's no memory import. + /// + /// May return an error in case the WASM module is invalid. + pub fn convert_memory_import_into_export(&mut self) -> Result<(), WasmError> { + let import_section = match self.raw_module.import_section_mut() { + Some(import_section) => import_section, + None => return Ok(()), + }; + + let import_entries = import_section.entries_mut(); + for index in 0..import_entries.len() { + let entry = &import_entries[index]; + let memory_ty = match entry.external() { + External::Memory(memory_ty) => *memory_ty, + _ => continue, + }; + + let memory_name = entry.field().to_owned(); + import_entries.remove(index); + + self.raw_module + .insert_section(Section::Memory(MemorySection::with_entries(vec![memory_ty]))) + .map_err(|error| { + WasmError::Other(format!( + "can't convert a memory import into an export: failed to insert a new memory section: {}", + error + )) + })?; + + if self.raw_module.export_section_mut().is_none() { + // A module without an export section is somewhat unrealistic, but let's do this + // just in case to cover all of our bases. + self.raw_module + .insert_section(Section::Export(Default::default())) + .expect("an export section can be always inserted if it doesn't exist; qed"); + } + self.raw_module + .export_section_mut() + .expect("export section already existed or we just added it above, so it always exists; qed") + .entries_mut() + .push(ExportEntry::new(memory_name, Internal::Memory(0))); + + break + } + + Ok(()) + } + + /// Increases the number of memory pages requested by the WASM blob by + /// the given amount of `extra_heap_pages`. + /// + /// Will return an error in case there is no memory section present, + /// or if the memory section is empty. + /// + /// Only modifies the initial size of the memory; the maximum is unmodified + /// unless it's smaller than the initial size, in which case it will be increased + /// so that it's at least as big as the initial size. + pub fn add_extra_heap_pages_to_memory_section( + &mut self, + extra_heap_pages: u32, + ) -> Result<(), WasmError> { + let memory_section = self + .raw_module + .memory_section_mut() + .ok_or_else(|| WasmError::Other("no memory section found".into()))?; + + if memory_section.entries().is_empty() { + return Err(WasmError::Other("memory section is empty".into())) + } + for memory_ty in memory_section.entries_mut() { + let min = memory_ty.limits().initial().saturating_add(extra_heap_pages); + let max = memory_ty.limits().maximum().map(|max| std::cmp::max(min, max)); + *memory_ty = MemoryType::new(min, max); + } + Ok(()) + } + /// Returns an iterator of all globals which were exported by [`expose_mutable_globals`]. pub(super) fn exported_internal_global_names<'module>( &'module self, diff --git a/client/executor/common/src/sandbox.rs b/client/executor/common/src/sandbox.rs index b62729424125..a2c1f602b1c9 100644 --- a/client/executor/common/src/sandbox.rs +++ b/client/executor/common/src/sandbox.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,24 +20,30 @@ //! //! Sandboxing is backed by wasmi and wasmer, depending on the configuration. +#[cfg(feature = "wasmer-sandbox")] +mod wasmer_backend; + +mod wasmi_backend; + use crate::{ - error::{Error, Result}, + error::{self, Result}, util, }; -use codec::{Decode, Encode}; +use codec::Decode; use sp_core::sandbox as sandbox_primitives; use sp_wasm_interface::{FunctionContext, Pointer, WordSize}; use std::{collections::HashMap, rc::Rc}; -use wasmi::{ - memory_units::Pages, Externals, ImportResolver, MemoryInstance, Module, ModuleInstance, - RuntimeArgs, RuntimeValue, Trap, TrapKind, -}; #[cfg(feature = "wasmer-sandbox")] -use crate::util::wasmer::MemoryWrapper as WasmerMemoryWrapper; -use crate::util::wasmi::MemoryWrapper as WasmiMemoryWrapper; +use wasmer_backend::{ + instantiate as wasmer_instantiate, invoke as wasmer_invoke, new_memory as wasmer_new_memory, + Backend as WasmerBackend, MemoryWrapper as WasmerMemoryWrapper, +}; -environmental::environmental!(SandboxContextStore: trait SandboxContext); +use wasmi_backend::{ + instantiate as wasmi_instantiate, invoke as wasmi_invoke, new_memory as wasmi_new_memory, + MemoryWrapper as WasmiMemoryWrapper, +}; /// Index of a function inside the supervisor. /// @@ -109,63 +115,6 @@ impl Imports { } } -impl ImportResolver for Imports { - fn resolve_func( - &self, - module_name: &str, - field_name: &str, - signature: &::wasmi::Signature, - ) -> std::result::Result { - let idx = self.func_by_name(module_name, field_name).ok_or_else(|| { - wasmi::Error::Instantiation(format!("Export {}:{} not found", module_name, field_name)) - })?; - - Ok(wasmi::FuncInstance::alloc_host(signature.clone(), idx.0)) - } - - fn resolve_memory( - &self, - module_name: &str, - field_name: &str, - _memory_type: &::wasmi::MemoryDescriptor, - ) -> std::result::Result { - let mem = self.memory_by_name(module_name, field_name).ok_or_else(|| { - wasmi::Error::Instantiation(format!("Export {}:{} not found", module_name, field_name)) - })?; - - let wrapper = mem.as_wasmi().ok_or_else(|| { - wasmi::Error::Instantiation(format!( - "Unsupported non-wasmi export {}:{}", - module_name, field_name - )) - })?; - - // Here we use inner memory reference only to resolve - // the imports without accessing the memory contents. - let mem = unsafe { wrapper.clone_inner() }; - - Ok(mem) - } - - fn resolve_global( - &self, - module_name: &str, - field_name: &str, - _global_type: &::wasmi::GlobalDescriptor, - ) -> std::result::Result { - Err(wasmi::Error::Instantiation(format!("Export {}:{} not found", module_name, field_name))) - } - - fn resolve_table( - &self, - module_name: &str, - field_name: &str, - _table_type: &::wasmi::TableDescriptor, - ) -> std::result::Result { - Err(wasmi::Error::Instantiation(format!("Export {}:{} not found", module_name, field_name))) - } -} - /// The sandbox context used to execute sandboxed functions. pub trait SandboxContext { /// Invoke a function in the supervisor environment. @@ -205,132 +154,6 @@ pub struct GuestExternals<'a> { state: u32, } -/// Construct trap error from specified message -fn trap(msg: &'static str) -> Trap { - TrapKind::Host(Box::new(Error::Other(msg.into()))).into() -} - -fn deserialize_result( - mut serialized_result: &[u8], -) -> std::result::Result, Trap> { - use self::sandbox_primitives::HostError; - use sp_wasm_interface::ReturnValue; - let result_val = std::result::Result::::decode(&mut serialized_result) - .map_err(|_| trap("Decoding Result failed!"))?; - - match result_val { - Ok(return_value) => Ok(match return_value { - ReturnValue::Unit => None, - ReturnValue::Value(typed_value) => Some(RuntimeValue::from(typed_value)), - }), - Err(HostError) => Err(trap("Supervisor function returned sandbox::HostError")), - } -} - -impl<'a> Externals for GuestExternals<'a> { - fn invoke_index( - &mut self, - index: usize, - args: RuntimeArgs, - ) -> std::result::Result, Trap> { - SandboxContextStore::with(|sandbox_context| { - // Make `index` typesafe again. - let index = GuestFuncIndex(index); - - // Convert function index from guest to supervisor space - let func_idx = self.sandbox_instance - .guest_to_supervisor_mapping - .func_by_guest_index(index) - .expect( - "`invoke_index` is called with indexes registered via `FuncInstance::alloc_host`; - `FuncInstance::alloc_host` is called with indexes that were obtained from `guest_to_supervisor_mapping`; - `func_by_guest_index` called with `index` can't return `None`; - qed" - ); - - // Serialize arguments into a byte vector. - let invoke_args_data: Vec = args - .as_ref() - .iter() - .cloned() - .map(sp_wasm_interface::Value::from) - .collect::>() - .encode(); - - let state = self.state; - - // Move serialized arguments inside the memory, invoke dispatch thunk and - // then free allocated memory. - let invoke_args_len = invoke_args_data.len() as WordSize; - let invoke_args_ptr = sandbox_context - .supervisor_context() - .allocate_memory(invoke_args_len) - .map_err(|_| trap("Can't allocate memory in supervisor for the arguments"))?; - - let deallocate = |supervisor_context: &mut dyn FunctionContext, ptr, fail_msg| { - supervisor_context.deallocate_memory(ptr).map_err(|_| trap(fail_msg)) - }; - - if sandbox_context - .supervisor_context() - .write_memory(invoke_args_ptr, &invoke_args_data) - .is_err() - { - deallocate( - sandbox_context.supervisor_context(), - invoke_args_ptr, - "Failed dealloction after failed write of invoke arguments", - )?; - return Err(trap("Can't write invoke args into memory")) - } - - let result = sandbox_context.invoke( - invoke_args_ptr, - invoke_args_len, - state, - func_idx, - ); - - deallocate( - sandbox_context.supervisor_context(), - invoke_args_ptr, - "Can't deallocate memory for dispatch thunk's invoke arguments", - )?; - let result = result?; - - // dispatch_thunk returns pointer to serialized arguments. - // Unpack pointer and len of the serialized result data. - let (serialized_result_val_ptr, serialized_result_val_len) = { - // Cast to u64 to use zero-extension. - let v = result as u64; - let ptr = (v as u64 >> 32) as u32; - let len = (v & 0xFFFFFFFF) as u32; - (Pointer::new(ptr), len) - }; - - let serialized_result_val = sandbox_context - .supervisor_context() - .read_memory(serialized_result_val_ptr, serialized_result_val_len) - .map_err(|_| trap("Can't read the serialized result from dispatch thunk")); - - deallocate( - sandbox_context.supervisor_context(), - serialized_result_val_ptr, - "Can't deallocate memory for dispatch thunk's result", - ) - .and_then(|_| serialized_result_val) - .and_then(|serialized_result_val| deserialize_result(&serialized_result_val)) - }).expect("SandboxContextStore is set when invoking sandboxed functions; qed") - } -} - -fn with_guest_externals(sandbox_instance: &SandboxInstance, state: u32, f: F) -> R -where - F: FnOnce(&mut GuestExternals) -> R, -{ - f(&mut GuestExternals { sandbox_instance, state }) -} - /// Module instance in terms of selected backend enum BackendInstance { /// Wasmi module instance @@ -370,74 +193,18 @@ impl SandboxInstance { /// these syscall implementations. pub fn invoke( &self, - - // function to call that is exported from the module export_name: &str, - - // arguments passed to the function - args: &[RuntimeValue], - - // arbitraty context data of the call + args: &[sp_wasm_interface::Value], state: u32, - sandbox_context: &mut dyn SandboxContext, - ) -> std::result::Result, wasmi::Error> { + ) -> std::result::Result, error::Error> { match &self.backend_instance { BackendInstance::Wasmi(wasmi_instance) => - with_guest_externals(self, state, |guest_externals| { - let wasmi_result = SandboxContextStore::using(sandbox_context, || { - wasmi_instance.invoke_export(export_name, args, guest_externals) - })?; - - Ok(wasmi_result) - }), + wasmi_invoke(self, wasmi_instance, export_name, args, state, sandbox_context), #[cfg(feature = "wasmer-sandbox")] - BackendInstance::Wasmer(wasmer_instance) => { - let function = wasmer_instance - .exports - .get_function(export_name) - .map_err(|error| wasmi::Error::Function(error.to_string()))?; - - let args: Vec = args - .iter() - .map(|v| match *v { - RuntimeValue::I32(val) => wasmer::Val::I32(val), - RuntimeValue::I64(val) => wasmer::Val::I64(val), - RuntimeValue::F32(val) => wasmer::Val::F32(val.into()), - RuntimeValue::F64(val) => wasmer::Val::F64(val.into()), - }) - .collect(); - - let wasmer_result = SandboxContextStore::using(sandbox_context, || { - function.call(&args).map_err(|error| wasmi::Error::Function(error.to_string())) - })?; - - if wasmer_result.len() > 1 { - return Err(wasmi::Error::Function( - "multiple return types are not supported yet".into(), - )) - } - - wasmer_result - .first() - .map(|wasm_value| { - let wasmer_value = match *wasm_value { - wasmer::Val::I32(val) => RuntimeValue::I32(val), - wasmer::Val::I64(val) => RuntimeValue::I64(val), - wasmer::Val::F32(val) => RuntimeValue::F32(val.into()), - wasmer::Val::F64(val) => RuntimeValue::F64(val.into()), - _ => - return Err(wasmi::Error::Function(format!( - "Unsupported return value: {:?}", - wasm_value, - ))), - }; - - Ok(wasmer_value) - }) - .transpose() - }, + BackendInstance::Wasmer(wasmer_instance) => + wasmer_invoke(wasmer_instance, export_name, args, state, sandbox_context), } } @@ -484,6 +251,8 @@ pub enum InstantiationError { /// Module is well-formed, instantiated and linked, but while executing the start function /// a trap was generated. StartTrapped, + /// The code was compiled with a CPU feature not available on the host. + CpuFeature, } fn decode_environment_definition( @@ -634,12 +403,6 @@ impl util::MemoryTransfer for Memory { } } -/// Wasmer specific context -#[cfg(feature = "wasmer-sandbox")] -struct WasmerBackend { - store: wasmer::Store, -} - /// Information specific to a particular execution backend enum BackendContext { /// Wasmi specific context @@ -659,13 +422,8 @@ impl BackendContext { SandboxBackend::TryWasmer => BackendContext::Wasmi, #[cfg(feature = "wasmer-sandbox")] - SandboxBackend::Wasmer | SandboxBackend::TryWasmer => { - let compiler = wasmer_compiler_singlepass::Singlepass::default(); - - BackendContext::Wasmer(WasmerBackend { - store: wasmer::Store::new(&wasmer::JIT::new(compiler).engine()), - }) - }, + SandboxBackend::Wasmer | SandboxBackend::TryWasmer => + BackendContext::Wasmer(WasmerBackend::new()), } } } @@ -709,19 +467,10 @@ impl Store

{ }; let memory = match &backend_context { - BackendContext::Wasmi => Memory::Wasmi(WasmiMemoryWrapper::new(MemoryInstance::alloc( - Pages(initial as usize), - maximum.map(|m| Pages(m as usize)), - )?)), + BackendContext::Wasmi => wasmi_new_memory(initial, maximum)?, #[cfg(feature = "wasmer-sandbox")] - BackendContext::Wasmer(context) => { - let ty = wasmer::MemoryType::new(initial, maximum, false); - Memory::Wasmer(WasmerMemoryWrapper::new( - wasmer::Memory::new(&context.store, ty) - .map_err(|_| Error::InvalidMemoryReference)?, - )) - }, + BackendContext::Wasmer(context) => wasmer_new_memory(context, initial, maximum)?, }; let mem_idx = memories.len(); @@ -827,12 +576,11 @@ impl Store
{ sandbox_context: &mut dyn SandboxContext, ) -> std::result::Result { let sandbox_instance = match self.backend_context { - BackendContext::Wasmi => - Self::instantiate_wasmi(wasm, guest_env, state, sandbox_context)?, + BackendContext::Wasmi => wasmi_instantiate(wasm, guest_env, state, sandbox_context)?, #[cfg(feature = "wasmer-sandbox")] BackendContext::Wasmer(ref context) => - Self::instantiate_wasmer(&context, wasm, guest_env, state, sandbox_context)?, + wasmer_instantiate(&context, wasm, guest_env, state, sandbox_context)?, }; Ok(UnregisteredInstance { sandbox_instance }) @@ -850,241 +598,4 @@ impl
Store
{ self.instances.push(Some((sandbox_instance, dispatch_thunk))); instance_idx as u32 } - - fn instantiate_wasmi( - wasm: &[u8], - guest_env: GuestEnvironment, - state: u32, - sandbox_context: &mut dyn SandboxContext, - ) -> std::result::Result, InstantiationError> { - let wasmi_module = - Module::from_buffer(wasm).map_err(|_| InstantiationError::ModuleDecoding)?; - let wasmi_instance = ModuleInstance::new(&wasmi_module, &guest_env.imports) - .map_err(|_| InstantiationError::Instantiation)?; - - let sandbox_instance = Rc::new(SandboxInstance { - // In general, it's not a very good idea to use `.not_started_instance()` for - // anything but for extracting memory and tables. But in this particular case, we - // are extracting for the purpose of running `start` function which should be ok. - backend_instance: BackendInstance::Wasmi(wasmi_instance.not_started_instance().clone()), - guest_to_supervisor_mapping: guest_env.guest_to_supervisor_mapping, - }); - - with_guest_externals(&sandbox_instance, state, |guest_externals| { - SandboxContextStore::using(sandbox_context, || { - wasmi_instance - .run_start(guest_externals) - .map_err(|_| InstantiationError::StartTrapped) - }) - - // Note: no need to run start on wasmtime instance, since it's done - // automatically - })?; - - Ok(sandbox_instance) - } - - #[cfg(feature = "wasmer-sandbox")] - fn instantiate_wasmer( - context: &WasmerBackend, - wasm: &[u8], - guest_env: GuestEnvironment, - state: u32, - sandbox_context: &mut dyn SandboxContext, - ) -> std::result::Result, InstantiationError> { - let module = wasmer::Module::new(&context.store, wasm) - .map_err(|_| InstantiationError::ModuleDecoding)?; - - type Exports = HashMap; - let mut exports_map = Exports::new(); - - for import in module.imports().into_iter() { - match import.ty() { - // Nothing to do here - wasmer::ExternType::Global(_) | wasmer::ExternType::Table(_) => (), - - wasmer::ExternType::Memory(_) => { - let exports = exports_map - .entry(import.module().to_string()) - .or_insert(wasmer::Exports::new()); - - let memory = guest_env - .imports - .memory_by_name(import.module(), import.name()) - .ok_or(InstantiationError::ModuleDecoding)?; - - let mut wasmer_memory_ref = memory.as_wasmer().expect( - "memory is created by wasmer; \ - exported by the same module and backend; \ - thus the operation can't fail; \ - qed", - ); - - // This is safe since we're only instantiating the module and populating - // the export table, so no memory access can happen at this time. - // All subsequent memory accesses should happen through the wrapper, - // that enforces the memory access protocol. - let wasmer_memory = unsafe { wasmer_memory_ref.clone_inner() }; - - exports.insert(import.name(), wasmer::Extern::Memory(wasmer_memory)); - }, - - wasmer::ExternType::Function(func_ty) => { - let guest_func_index = - guest_env.imports.func_by_name(import.module(), import.name()); - - let guest_func_index = if let Some(index) = guest_func_index { - index - } else { - // Missing import (should we abort here?) - continue - }; - - let supervisor_func_index = guest_env - .guest_to_supervisor_mapping - .func_by_guest_index(guest_func_index) - .ok_or(InstantiationError::ModuleDecoding)?; - - let function = Self::wasmer_dispatch_function( - supervisor_func_index, - &context.store, - func_ty, - state, - ); - - let exports = exports_map - .entry(import.module().to_string()) - .or_insert(wasmer::Exports::new()); - - exports.insert(import.name(), wasmer::Extern::Function(function)); - }, - } - } - - let mut import_object = wasmer::ImportObject::new(); - for (module_name, exports) in exports_map.into_iter() { - import_object.register(module_name, exports); - } - - let instance = SandboxContextStore::using(sandbox_context, || { - wasmer::Instance::new(&module, &import_object).map_err(|error| match error { - wasmer::InstantiationError::Link(_) => InstantiationError::Instantiation, - wasmer::InstantiationError::Start(_) => InstantiationError::StartTrapped, - wasmer::InstantiationError::HostEnvInitialization(_) => - InstantiationError::EnvironmentDefinitionCorrupted, - }) - })?; - - Ok(Rc::new(SandboxInstance { - backend_instance: BackendInstance::Wasmer(instance), - guest_to_supervisor_mapping: guest_env.guest_to_supervisor_mapping, - })) - } - - #[cfg(feature = "wasmer-sandbox")] - fn wasmer_dispatch_function( - supervisor_func_index: SupervisorFuncIndex, - store: &wasmer::Store, - func_ty: &wasmer::FunctionType, - state: u32, - ) -> wasmer::Function { - wasmer::Function::new(store, func_ty, move |params| { - SandboxContextStore::with(|sandbox_context| { - use sp_wasm_interface::Value; - - // Serialize arguments into a byte vector. - let invoke_args_data = params - .iter() - .map(|val| match val { - wasmer::Val::I32(val) => Ok(Value::I32(*val)), - wasmer::Val::I64(val) => Ok(Value::I64(*val)), - wasmer::Val::F32(val) => Ok(Value::F32(f32::to_bits(*val))), - wasmer::Val::F64(val) => Ok(Value::F64(f64::to_bits(*val))), - _ => Err(wasmer::RuntimeError::new(format!( - "Unsupported function argument: {:?}", - val - ))), - }) - .collect::, _>>()? - .encode(); - - // Move serialized arguments inside the memory, invoke dispatch thunk and - // then free allocated memory. - let invoke_args_len = invoke_args_data.len() as WordSize; - let invoke_args_ptr = sandbox_context - .supervisor_context() - .allocate_memory(invoke_args_len) - .map_err(|_| { - wasmer::RuntimeError::new( - "Can't allocate memory in supervisor for the arguments", - ) - })?; - - let deallocate = |fe: &mut dyn FunctionContext, ptr, fail_msg| { - fe.deallocate_memory(ptr).map_err(|_| wasmer::RuntimeError::new(fail_msg)) - }; - - if sandbox_context - .supervisor_context() - .write_memory(invoke_args_ptr, &invoke_args_data) - .is_err() - { - deallocate( - sandbox_context.supervisor_context(), - invoke_args_ptr, - "Failed dealloction after failed write of invoke arguments", - )?; - - return Err(wasmer::RuntimeError::new("Can't write invoke args into memory")) - } - - // Perform the actuall call - let serialized_result = sandbox_context - .invoke(invoke_args_ptr, invoke_args_len, state, supervisor_func_index) - .map_err(|e| wasmer::RuntimeError::new(e.to_string()))?; - - // dispatch_thunk returns pointer to serialized arguments. - // Unpack pointer and len of the serialized result data. - let (serialized_result_val_ptr, serialized_result_val_len) = { - // Cast to u64 to use zero-extension. - let v = serialized_result as u64; - let ptr = (v as u64 >> 32) as u32; - let len = (v & 0xFFFFFFFF) as u32; - (Pointer::new(ptr), len) - }; - - let serialized_result_val = sandbox_context - .supervisor_context() - .read_memory(serialized_result_val_ptr, serialized_result_val_len) - .map_err(|_| { - wasmer::RuntimeError::new( - "Can't read the serialized result from dispatch thunk", - ) - }); - - let deserialized_result = deallocate( - sandbox_context.supervisor_context(), - serialized_result_val_ptr, - "Can't deallocate memory for dispatch thunk's result", - ) - .and_then(|_| serialized_result_val) - .and_then(|serialized_result_val| { - deserialize_result(&serialized_result_val) - .map_err(|e| wasmer::RuntimeError::new(e.to_string())) - })?; - - if let Some(value) = deserialized_result { - Ok(vec![match value { - RuntimeValue::I32(val) => wasmer::Val::I32(val), - RuntimeValue::I64(val) => wasmer::Val::I64(val), - RuntimeValue::F32(val) => wasmer::Val::F32(val.into()), - RuntimeValue::F64(val) => wasmer::Val::F64(val.into()), - }]) - } else { - Ok(vec![]) - } - }) - .expect("SandboxContextStore is set when invoking sandboxed functions; qed") - }) - } } diff --git a/client/executor/common/src/sandbox/wasmer_backend.rs b/client/executor/common/src/sandbox/wasmer_backend.rs new file mode 100644 index 000000000000..44b43757148d --- /dev/null +++ b/client/executor/common/src/sandbox/wasmer_backend.rs @@ -0,0 +1,434 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Wasmer specific impls for sandbox + +use crate::{ + error::{Error, Result}, + sandbox::Memory, + util::{checked_range, MemoryTransfer}, +}; +use codec::{Decode, Encode}; +use sp_core::sandbox::HostError; +use sp_wasm_interface::{FunctionContext, Pointer, ReturnValue, Value, WordSize}; +use std::{cell::RefCell, collections::HashMap, rc::Rc}; +use wasmer::RuntimeError; + +use crate::sandbox::{ + BackendInstance, GuestEnvironment, InstantiationError, SandboxContext, SandboxInstance, + SupervisorFuncIndex, +}; + +environmental::environmental!(SandboxContextStore: trait SandboxContext); + +/// Wasmer specific context +pub struct Backend { + store: wasmer::Store, +} + +impl Backend { + pub fn new() -> Self { + let compiler = wasmer::Singlepass::default(); + Backend { store: wasmer::Store::new(&wasmer::Universal::new(compiler).engine()) } + } +} + +/// Invoke a function within a sandboxed module +pub fn invoke( + instance: &wasmer::Instance, + export_name: &str, + args: &[Value], + _state: u32, + sandbox_context: &mut dyn SandboxContext, +) -> std::result::Result, Error> { + let function = instance + .exports + .get_function(export_name) + .map_err(|error| Error::Sandbox(error.to_string()))?; + + let args: Vec = args + .iter() + .map(|v| match *v { + Value::I32(val) => wasmer::Val::I32(val), + Value::I64(val) => wasmer::Val::I64(val), + Value::F32(val) => wasmer::Val::F32(f32::from_bits(val)), + Value::F64(val) => wasmer::Val::F64(f64::from_bits(val)), + }) + .collect(); + + let wasmer_result = SandboxContextStore::using(sandbox_context, || { + function.call(&args).map_err(|error| Error::Sandbox(error.to_string())) + })?; + + match wasmer_result.as_ref() { + [] => Ok(None), + + [wasm_value] => { + let wasmer_value = match *wasm_value { + wasmer::Val::I32(val) => Value::I32(val), + wasmer::Val::I64(val) => Value::I64(val), + wasmer::Val::F32(val) => Value::F32(f32::to_bits(val)), + wasmer::Val::F64(val) => Value::F64(f64::to_bits(val)), + _ => + return Err(Error::Sandbox(format!( + "Unsupported return value: {:?}", + wasm_value, + ))), + }; + + Ok(Some(wasmer_value)) + }, + + _ => Err(Error::Sandbox("multiple return types are not supported yet".into())), + } +} + +/// Instantiate a module within a sandbox context +pub fn instantiate( + context: &Backend, + wasm: &[u8], + guest_env: GuestEnvironment, + state: u32, + sandbox_context: &mut dyn SandboxContext, +) -> std::result::Result, InstantiationError> { + let module = wasmer::Module::new(&context.store, wasm) + .map_err(|_| InstantiationError::ModuleDecoding)?; + + type Exports = HashMap; + let mut exports_map = Exports::new(); + + for import in module.imports().into_iter() { + match import.ty() { + // Nothing to do here + wasmer::ExternType::Global(_) | wasmer::ExternType::Table(_) => (), + + wasmer::ExternType::Memory(_) => { + let exports = exports_map + .entry(import.module().to_string()) + .or_insert(wasmer::Exports::new()); + + let memory = guest_env + .imports + .memory_by_name(import.module(), import.name()) + .ok_or(InstantiationError::ModuleDecoding)?; + + let wasmer_memory_ref = memory.as_wasmer().expect( + "memory is created by wasmer; \ + exported by the same module and backend; \ + thus the operation can't fail; \ + qed", + ); + + // This is safe since we're only instantiating the module and populating + // the export table, so no memory access can happen at this time. + // All subsequent memory accesses should happen through the wrapper, + // that enforces the memory access protocol. + // + // We take exclusive lock to ensure that we're the only one here, + // since during instantiation phase the memory should only be created + // and not yet accessed. + let wasmer_memory = wasmer_memory_ref + .buffer + .try_borrow_mut() + .map_err(|_| InstantiationError::EnvironmentDefinitionCorrupted)? + .clone(); + + exports.insert(import.name(), wasmer::Extern::Memory(wasmer_memory)); + }, + + wasmer::ExternType::Function(func_ty) => { + let guest_func_index = + guest_env.imports.func_by_name(import.module(), import.name()); + + let guest_func_index = if let Some(index) = guest_func_index { + index + } else { + // Missing import (should we abort here?) + continue + }; + + let supervisor_func_index = guest_env + .guest_to_supervisor_mapping + .func_by_guest_index(guest_func_index) + .ok_or(InstantiationError::ModuleDecoding)?; + + let function = + dispatch_function(supervisor_func_index, &context.store, func_ty, state); + + let exports = exports_map + .entry(import.module().to_string()) + .or_insert(wasmer::Exports::new()); + + exports.insert(import.name(), wasmer::Extern::Function(function)); + }, + } + } + + let mut import_object = wasmer::ImportObject::new(); + for (module_name, exports) in exports_map.into_iter() { + import_object.register(module_name, exports); + } + + let instance = SandboxContextStore::using(sandbox_context, || { + wasmer::Instance::new(&module, &import_object).map_err(|error| match error { + wasmer::InstantiationError::Link(_) => InstantiationError::Instantiation, + wasmer::InstantiationError::Start(_) => InstantiationError::StartTrapped, + wasmer::InstantiationError::HostEnvInitialization(_) => + InstantiationError::EnvironmentDefinitionCorrupted, + wasmer::InstantiationError::CpuFeature(_) => InstantiationError::CpuFeature, + }) + })?; + + Ok(Rc::new(SandboxInstance { + backend_instance: BackendInstance::Wasmer(instance), + guest_to_supervisor_mapping: guest_env.guest_to_supervisor_mapping, + })) +} + +fn dispatch_function( + supervisor_func_index: SupervisorFuncIndex, + store: &wasmer::Store, + func_ty: &wasmer::FunctionType, + state: u32, +) -> wasmer::Function { + wasmer::Function::new(store, func_ty, move |params| { + SandboxContextStore::with(|sandbox_context| { + // Serialize arguments into a byte vector. + let invoke_args_data = params + .iter() + .map(|val| match val { + wasmer::Val::I32(val) => Ok(Value::I32(*val)), + wasmer::Val::I64(val) => Ok(Value::I64(*val)), + wasmer::Val::F32(val) => Ok(Value::F32(f32::to_bits(*val))), + wasmer::Val::F64(val) => Ok(Value::F64(f64::to_bits(*val))), + _ => + Err(RuntimeError::new(format!("Unsupported function argument: {:?}", val))), + }) + .collect::, _>>()? + .encode(); + + // Move serialized arguments inside the memory, invoke dispatch thunk and + // then free allocated memory. + let invoke_args_len = invoke_args_data.len() as WordSize; + let invoke_args_ptr = + sandbox_context.supervisor_context().allocate_memory(invoke_args_len).map_err( + |_| RuntimeError::new("Can't allocate memory in supervisor for the arguments"), + )?; + + let deallocate = |fe: &mut dyn FunctionContext, ptr, fail_msg| { + fe.deallocate_memory(ptr).map_err(|_| RuntimeError::new(fail_msg)) + }; + + if sandbox_context + .supervisor_context() + .write_memory(invoke_args_ptr, &invoke_args_data) + .is_err() + { + deallocate( + sandbox_context.supervisor_context(), + invoke_args_ptr, + "Failed dealloction after failed write of invoke arguments", + )?; + + return Err(RuntimeError::new("Can't write invoke args into memory")) + } + + // Perform the actuall call + let serialized_result = sandbox_context + .invoke(invoke_args_ptr, invoke_args_len, state, supervisor_func_index) + .map_err(|e| RuntimeError::new(e.to_string())); + + deallocate( + sandbox_context.supervisor_context(), + invoke_args_ptr, + "Failed dealloction after invoke", + )?; + + let serialized_result = serialized_result?; + + // dispatch_thunk returns pointer to serialized arguments. + // Unpack pointer and len of the serialized result data. + let (serialized_result_val_ptr, serialized_result_val_len) = { + // Cast to u64 to use zero-extension. + let v = serialized_result as u64; + let ptr = (v as u64 >> 32) as u32; + let len = (v & 0xFFFFFFFF) as u32; + (Pointer::new(ptr), len) + }; + + let serialized_result_val = sandbox_context + .supervisor_context() + .read_memory(serialized_result_val_ptr, serialized_result_val_len) + .map_err(|_| { + RuntimeError::new("Can't read the serialized result from dispatch thunk") + }); + + deallocate( + sandbox_context.supervisor_context(), + serialized_result_val_ptr, + "Can't deallocate memory for dispatch thunk's result", + )?; + + let serialized_result_val = serialized_result_val?; + + let deserialized_result = std::result::Result::::decode( + &mut serialized_result_val.as_slice(), + ) + .map_err(|_| RuntimeError::new("Decoding Result failed!"))? + .map_err(|_| RuntimeError::new("Supervisor function returned sandbox::HostError"))?; + + let result = match deserialized_result { + ReturnValue::Value(Value::I32(val)) => vec![wasmer::Val::I32(val)], + ReturnValue::Value(Value::I64(val)) => vec![wasmer::Val::I64(val)], + ReturnValue::Value(Value::F32(val)) => vec![wasmer::Val::F32(f32::from_bits(val))], + ReturnValue::Value(Value::F64(val)) => vec![wasmer::Val::F64(f64::from_bits(val))], + + ReturnValue::Unit => vec![], + }; + + Ok(result) + }) + .expect("SandboxContextStore is set when invoking sandboxed functions; qed") + }) +} + +/// Allocate new memory region +pub fn new_memory( + context: &Backend, + initial: u32, + maximum: Option, +) -> crate::error::Result { + let ty = wasmer::MemoryType::new(initial, maximum, false); + let memory = Memory::Wasmer(MemoryWrapper::new( + wasmer::Memory::new(&context.store, ty).map_err(|_| Error::InvalidMemoryReference)?, + )); + + Ok(memory) +} + +/// In order to enforce memory access protocol to the backend memory +/// we wrap it with `RefCell` and encapsulate all memory operations. +#[derive(Debug, Clone)] +pub struct MemoryWrapper { + buffer: Rc>, +} + +impl MemoryWrapper { + /// Take ownership of the memory region and return a wrapper object + pub fn new(memory: wasmer::Memory) -> Self { + Self { buffer: Rc::new(RefCell::new(memory)) } + } + + /// Returns linear memory of the wasm instance as a slice. + /// + /// # Safety + /// + /// Wasmer doesn't provide comprehensive documentation about the exact behavior of the data + /// pointer. If a dynamic style heap is used the base pointer of the heap can change. Since + /// growing, we cannot guarantee the lifetime of the returned slice reference. + unsafe fn memory_as_slice(memory: &wasmer::Memory) -> &[u8] { + let ptr = memory.data_ptr() as *const _; + + let len: usize = memory.data_size().try_into().expect( + "maximum memory object size never exceeds pointer size on any architecture; \ + usize by design and definition is enough to store any memory object size \ + possible on current achitecture; thus the conversion can not fail; qed", + ); + + if len == 0 { + &[] + } else { + core::slice::from_raw_parts(ptr, len) + } + } + + /// Returns linear memory of the wasm instance as a slice. + /// + /// # Safety + /// + /// See `[memory_as_slice]`. In addition to those requirements, since a mutable reference is + /// returned it must be ensured that only one mutable and no shared references to memory + /// exists at the same time. + unsafe fn memory_as_slice_mut(memory: &mut wasmer::Memory) -> &mut [u8] { + let ptr = memory.data_ptr(); + + let len: usize = memory.data_size().try_into().expect( + "maximum memory object size never exceeds pointer size on any architecture; \ + usize by design and definition is enough to store any memory object size \ + possible on current achitecture; thus the conversion can not fail; qed", + ); + + if len == 0 { + &mut [] + } else { + core::slice::from_raw_parts_mut(ptr, len) + } + } +} + +impl MemoryTransfer for MemoryWrapper { + fn read(&self, source_addr: Pointer, size: usize) -> Result> { + let memory = self.buffer.borrow(); + + let data_size: usize = memory.data_size().try_into().expect( + "maximum memory object size never exceeds pointer size on any architecture; \ + usize by design and definition is enough to store any memory object size \ + possible on current achitecture; thus the conversion can not fail; qed", + ); + + let range = checked_range(source_addr.into(), size, data_size) + .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; + + let mut buffer = vec![0; range.len()]; + self.read_into(source_addr, &mut buffer)?; + + Ok(buffer) + } + + fn read_into(&self, source_addr: Pointer, destination: &mut [u8]) -> Result<()> { + unsafe { + let memory = self.buffer.borrow(); + + // This should be safe since we don't grow up memory while caching this reference + // and we give up the reference before returning from this function. + let source = Self::memory_as_slice(&memory); + + let range = checked_range(source_addr.into(), destination.len(), source.len()) + .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; + + destination.copy_from_slice(&source[range]); + Ok(()) + } + } + + fn write_from(&self, dest_addr: Pointer, source: &[u8]) -> Result<()> { + unsafe { + let memory = &mut self.buffer.borrow_mut(); + + // This should be safe since we don't grow up memory while caching this reference + // and we give up the reference before returning from this function. + let destination = Self::memory_as_slice_mut(memory); + + let range = checked_range(dest_addr.into(), source.len(), destination.len()) + .ok_or_else(|| Error::Other("memory write is out of bounds".into()))?; + + destination[range].copy_from_slice(source); + Ok(()) + } + } +} diff --git a/client/executor/common/src/sandbox/wasmi_backend.rs b/client/executor/common/src/sandbox/wasmi_backend.rs new file mode 100644 index 000000000000..92bb0e1e398e --- /dev/null +++ b/client/executor/common/src/sandbox/wasmi_backend.rs @@ -0,0 +1,323 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Wasmi specific impls for sandbox + +use codec::{Decode, Encode}; +use sp_core::sandbox::HostError; +use sp_wasm_interface::{FunctionContext, Pointer, ReturnValue, Value, WordSize}; +use std::rc::Rc; + +use wasmi::{ + memory_units::Pages, ImportResolver, MemoryInstance, Module, ModuleInstance, RuntimeArgs, + RuntimeValue, Trap, TrapKind, +}; + +use crate::{ + error::{self, Error}, + sandbox::{ + BackendInstance, GuestEnvironment, GuestExternals, GuestFuncIndex, Imports, + InstantiationError, Memory, SandboxContext, SandboxInstance, + }, + util::{checked_range, MemoryTransfer}, +}; + +environmental::environmental!(SandboxContextStore: trait SandboxContext); + +/// Construct trap error from specified message +fn trap(msg: &'static str) -> Trap { + TrapKind::Host(Box::new(Error::Other(msg.into()))).into() +} + +impl ImportResolver for Imports { + fn resolve_func( + &self, + module_name: &str, + field_name: &str, + signature: &wasmi::Signature, + ) -> std::result::Result { + let idx = self.func_by_name(module_name, field_name).ok_or_else(|| { + wasmi::Error::Instantiation(format!("Export {}:{} not found", module_name, field_name)) + })?; + + Ok(wasmi::FuncInstance::alloc_host(signature.clone(), idx.0)) + } + + fn resolve_memory( + &self, + module_name: &str, + field_name: &str, + _memory_type: &wasmi::MemoryDescriptor, + ) -> std::result::Result { + let mem = self.memory_by_name(module_name, field_name).ok_or_else(|| { + wasmi::Error::Instantiation(format!("Export {}:{} not found", module_name, field_name)) + })?; + + let wrapper = mem.as_wasmi().ok_or_else(|| { + wasmi::Error::Instantiation(format!( + "Unsupported non-wasmi export {}:{}", + module_name, field_name + )) + })?; + + // Here we use inner memory reference only to resolve the imports + // without accessing the memory contents. All subsequent memory accesses + // should happen through the wrapper, that enforces the memory access protocol. + let mem = wrapper.0.clone(); + + Ok(mem) + } + + fn resolve_global( + &self, + module_name: &str, + field_name: &str, + _global_type: &wasmi::GlobalDescriptor, + ) -> std::result::Result { + Err(wasmi::Error::Instantiation(format!("Export {}:{} not found", module_name, field_name))) + } + + fn resolve_table( + &self, + module_name: &str, + field_name: &str, + _table_type: &wasmi::TableDescriptor, + ) -> std::result::Result { + Err(wasmi::Error::Instantiation(format!("Export {}:{} not found", module_name, field_name))) + } +} + +/// Allocate new memory region +pub fn new_memory(initial: u32, maximum: Option) -> crate::error::Result { + let memory = Memory::Wasmi(MemoryWrapper::new( + MemoryInstance::alloc(Pages(initial as usize), maximum.map(|m| Pages(m as usize))) + .map_err(|error| Error::Sandbox(error.to_string()))?, + )); + + Ok(memory) +} + +/// Wasmi provides direct access to its memory using slices. +/// +/// This wrapper limits the scope where the slice can be taken to +#[derive(Debug, Clone)] +pub struct MemoryWrapper(wasmi::MemoryRef); + +impl MemoryWrapper { + /// Take ownership of the memory region and return a wrapper object + fn new(memory: wasmi::MemoryRef) -> Self { + Self(memory) + } +} + +impl MemoryTransfer for MemoryWrapper { + fn read(&self, source_addr: Pointer, size: usize) -> error::Result> { + self.0.with_direct_access(|source| { + let range = checked_range(source_addr.into(), size, source.len()) + .ok_or_else(|| error::Error::Other("memory read is out of bounds".into()))?; + + Ok(Vec::from(&source[range])) + }) + } + + fn read_into(&self, source_addr: Pointer, destination: &mut [u8]) -> error::Result<()> { + self.0.with_direct_access(|source| { + let range = checked_range(source_addr.into(), destination.len(), source.len()) + .ok_or_else(|| error::Error::Other("memory read is out of bounds".into()))?; + + destination.copy_from_slice(&source[range]); + Ok(()) + }) + } + + fn write_from(&self, dest_addr: Pointer, source: &[u8]) -> error::Result<()> { + self.0.with_direct_access_mut(|destination| { + let range = checked_range(dest_addr.into(), source.len(), destination.len()) + .ok_or_else(|| error::Error::Other("memory write is out of bounds".into()))?; + + destination[range].copy_from_slice(source); + Ok(()) + }) + } +} + +impl<'a> wasmi::Externals for GuestExternals<'a> { + fn invoke_index( + &mut self, + index: usize, + args: RuntimeArgs, + ) -> std::result::Result, Trap> { + SandboxContextStore::with(|sandbox_context| { + // Make `index` typesafe again. + let index = GuestFuncIndex(index); + + // Convert function index from guest to supervisor space + let func_idx = self.sandbox_instance + .guest_to_supervisor_mapping + .func_by_guest_index(index) + .expect( + "`invoke_index` is called with indexes registered via `FuncInstance::alloc_host`; + `FuncInstance::alloc_host` is called with indexes that were obtained from `guest_to_supervisor_mapping`; + `func_by_guest_index` called with `index` can't return `None`; + qed" + ); + + // Serialize arguments into a byte vector. + let invoke_args_data: Vec = args + .as_ref() + .iter() + .cloned() + .map(sp_wasm_interface::Value::from) + .collect::>() + .encode(); + + let state = self.state; + + // Move serialized arguments inside the memory, invoke dispatch thunk and + // then free allocated memory. + let invoke_args_len = invoke_args_data.len() as WordSize; + let invoke_args_ptr = sandbox_context + .supervisor_context() + .allocate_memory(invoke_args_len) + .map_err(|_| trap("Can't allocate memory in supervisor for the arguments"))?; + + let deallocate = |supervisor_context: &mut dyn FunctionContext, ptr, fail_msg| { + supervisor_context.deallocate_memory(ptr).map_err(|_| trap(fail_msg)) + }; + + if sandbox_context + .supervisor_context() + .write_memory(invoke_args_ptr, &invoke_args_data) + .is_err() + { + deallocate( + sandbox_context.supervisor_context(), + invoke_args_ptr, + "Failed dealloction after failed write of invoke arguments", + )?; + return Err(trap("Can't write invoke args into memory")) + } + + let result = sandbox_context.invoke( + invoke_args_ptr, + invoke_args_len, + state, + func_idx, + ); + + deallocate( + sandbox_context.supervisor_context(), + invoke_args_ptr, + "Can't deallocate memory for dispatch thunk's invoke arguments", + )?; + let result = result?; + + // dispatch_thunk returns pointer to serialized arguments. + // Unpack pointer and len of the serialized result data. + let (serialized_result_val_ptr, serialized_result_val_len) = { + // Cast to u64 to use zero-extension. + let v = result as u64; + let ptr = (v as u64 >> 32) as u32; + let len = (v & 0xFFFFFFFF) as u32; + (Pointer::new(ptr), len) + }; + + let serialized_result_val = sandbox_context + .supervisor_context() + .read_memory(serialized_result_val_ptr, serialized_result_val_len) + .map_err(|_| trap("Can't read the serialized result from dispatch thunk")); + + deallocate( + sandbox_context.supervisor_context(), + serialized_result_val_ptr, + "Can't deallocate memory for dispatch thunk's result", + ) + .and_then(|_| serialized_result_val) + .and_then(|serialized_result_val| { + let result_val = std::result::Result::::decode(&mut serialized_result_val.as_slice()) + .map_err(|_| trap("Decoding Result failed!"))?; + + match result_val { + Ok(return_value) => Ok(match return_value { + ReturnValue::Unit => None, + ReturnValue::Value(typed_value) => Some(RuntimeValue::from(typed_value)), + }), + Err(HostError) => Err(trap("Supervisor function returned sandbox::HostError")), + } + }) + }).expect("SandboxContextStore is set when invoking sandboxed functions; qed") + } +} + +fn with_guest_externals(sandbox_instance: &SandboxInstance, state: u32, f: F) -> R +where + F: FnOnce(&mut GuestExternals) -> R, +{ + f(&mut GuestExternals { sandbox_instance, state }) +} + +/// Instantiate a module within a sandbox context +pub fn instantiate( + wasm: &[u8], + guest_env: GuestEnvironment, + state: u32, + sandbox_context: &mut dyn SandboxContext, +) -> std::result::Result, InstantiationError> { + let wasmi_module = Module::from_buffer(wasm).map_err(|_| InstantiationError::ModuleDecoding)?; + let wasmi_instance = ModuleInstance::new(&wasmi_module, &guest_env.imports) + .map_err(|_| InstantiationError::Instantiation)?; + + let sandbox_instance = Rc::new(SandboxInstance { + // In general, it's not a very good idea to use `.not_started_instance()` for + // anything but for extracting memory and tables. But in this particular case, we + // are extracting for the purpose of running `start` function which should be ok. + backend_instance: BackendInstance::Wasmi(wasmi_instance.not_started_instance().clone()), + guest_to_supervisor_mapping: guest_env.guest_to_supervisor_mapping, + }); + + with_guest_externals(&sandbox_instance, state, |guest_externals| { + SandboxContextStore::using(sandbox_context, || { + wasmi_instance + .run_start(guest_externals) + .map_err(|_| InstantiationError::StartTrapped) + }) + })?; + + Ok(sandbox_instance) +} + +/// Invoke a function within a sandboxed module +pub fn invoke( + instance: &SandboxInstance, + module: &wasmi::ModuleRef, + export_name: &str, + args: &[Value], + state: u32, + sandbox_context: &mut dyn SandboxContext, +) -> std::result::Result, error::Error> { + with_guest_externals(instance, state, |guest_externals| { + SandboxContextStore::using(sandbox_context, || { + let args = args.iter().cloned().map(Into::into).collect::>(); + + module + .invoke_export(export_name, &args, guest_externals) + .map(|result| result.map(Into::into)) + .map_err(|error| error::Error::Sandbox(error.to_string())) + }) + }) +} diff --git a/client/executor/common/src/util.rs b/client/executor/common/src/util.rs index ffbeb8c7ab53..fbae01b556fb 100644 --- a/client/executor/common/src/util.rs +++ b/client/executor/common/src/util.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,7 +18,7 @@ //! Utilities used by all backends -use crate::error::{Error, Result}; +use crate::error::Result; use sp_wasm_interface::Pointer; use std::ops::Range; @@ -50,192 +50,3 @@ pub trait MemoryTransfer { /// Returns an error if the write would go out of the memory bounds. fn write_from(&self, dest_addr: Pointer, source: &[u8]) -> Result<()>; } - -/// Safe wrapper over wasmi memory reference -pub mod wasmi { - use super::*; - - /// Wasmi provides direct access to its memory using slices. - /// - /// This wrapper limits the scope where the slice can be taken to - #[derive(Debug, Clone)] - pub struct MemoryWrapper(::wasmi::MemoryRef); - - impl MemoryWrapper { - /// Take ownership of the memory region and return a wrapper object - pub fn new(memory: ::wasmi::MemoryRef) -> Self { - Self(memory) - } - - /// Clone the underlying memory object - /// - /// # Safety - /// - /// The sole purpose of `MemoryRef` is to protect the memory from uncontrolled - /// access. By returning the memory object "as is" we bypass all of the checks. - /// - /// Intended to use only during module initialization. - pub unsafe fn clone_inner(&self) -> ::wasmi::MemoryRef { - self.0.clone() - } - } - - impl super::MemoryTransfer for MemoryWrapper { - fn read(&self, source_addr: Pointer, size: usize) -> Result> { - self.0.with_direct_access(|source| { - let range = checked_range(source_addr.into(), size, source.len()) - .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; - - Ok(Vec::from(&source[range])) - }) - } - - fn read_into(&self, source_addr: Pointer, destination: &mut [u8]) -> Result<()> { - self.0.with_direct_access(|source| { - let range = checked_range(source_addr.into(), destination.len(), source.len()) - .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; - - destination.copy_from_slice(&source[range]); - Ok(()) - }) - } - - fn write_from(&self, dest_addr: Pointer, source: &[u8]) -> Result<()> { - self.0.with_direct_access_mut(|destination| { - let range = checked_range(dest_addr.into(), source.len(), destination.len()) - .ok_or_else(|| Error::Other("memory write is out of bounds".into()))?; - - destination[range].copy_from_slice(source); - Ok(()) - }) - } - } -} - -// Routines specific to Wasmer runtime. Since sandbox can be invoked from both -/// wasmi and wasmtime runtime executors, we need to have a way to deal with sanbox -/// backends right from the start. -#[cfg(feature = "wasmer-sandbox")] -pub mod wasmer { - use super::checked_range; - use crate::error::{Error, Result}; - use sp_wasm_interface::Pointer; - use std::{cell::RefCell, convert::TryInto, rc::Rc}; - - /// In order to enforce memory access protocol to the backend memory - /// we wrap it with `RefCell` and encapsulate all memory operations. - #[derive(Debug, Clone)] - pub struct MemoryWrapper { - buffer: Rc>, - } - - impl MemoryWrapper { - /// Take ownership of the memory region and return a wrapper object - pub fn new(memory: wasmer::Memory) -> Self { - Self { buffer: Rc::new(RefCell::new(memory)) } - } - - /// Returns linear memory of the wasm instance as a slice. - /// - /// # Safety - /// - /// Wasmer doesn't provide comprehensive documentation about the exact behavior of the data - /// pointer. If a dynamic style heap is used the base pointer of the heap can change. Since - /// growing, we cannot guarantee the lifetime of the returned slice reference. - unsafe fn memory_as_slice(memory: &wasmer::Memory) -> &[u8] { - let ptr = memory.data_ptr() as *const _; - let len: usize = - memory.data_size().try_into().expect("data size should fit into usize"); - - if len == 0 { - &[] - } else { - core::slice::from_raw_parts(ptr, len) - } - } - - /// Returns linear memory of the wasm instance as a slice. - /// - /// # Safety - /// - /// See `[memory_as_slice]`. In addition to those requirements, since a mutable reference is - /// returned it must be ensured that only one mutable and no shared references to memory - /// exists at the same time. - unsafe fn memory_as_slice_mut(memory: &wasmer::Memory) -> &mut [u8] { - let ptr = memory.data_ptr(); - let len: usize = - memory.data_size().try_into().expect("data size should fit into usize"); - - if len == 0 { - &mut [] - } else { - core::slice::from_raw_parts_mut(ptr, len) - } - } - - /// Clone the underlying memory object - /// - /// # Safety - /// - /// The sole purpose of `MemoryRef` is to protect the memory from uncontrolled - /// access. By returning the memory object "as is" we bypass all of the checks. - /// - /// Intended to use only during module initialization. - /// - /// # Panics - /// - /// Will panic if `MemoryRef` is currently in use. - pub unsafe fn clone_inner(&mut self) -> wasmer::Memory { - // We take exclusive lock to ensure that we're the only one here - self.buffer.borrow_mut().clone() - } - } - - impl super::MemoryTransfer for MemoryWrapper { - fn read(&self, source_addr: Pointer, size: usize) -> Result> { - let memory = self.buffer.borrow(); - - let data_size = memory.data_size().try_into().expect("data size does not fit"); - - let range = checked_range(source_addr.into(), size, data_size) - .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; - - let mut buffer = vec![0; range.len()]; - self.read_into(source_addr, &mut buffer)?; - - Ok(buffer) - } - - fn read_into(&self, source_addr: Pointer, destination: &mut [u8]) -> Result<()> { - unsafe { - let memory = self.buffer.borrow(); - - // This should be safe since we don't grow up memory while caching this reference - // and we give up the reference before returning from this function. - let source = Self::memory_as_slice(&memory); - - let range = checked_range(source_addr.into(), destination.len(), source.len()) - .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; - - destination.copy_from_slice(&source[range]); - Ok(()) - } - } - - fn write_from(&self, dest_addr: Pointer, source: &[u8]) -> Result<()> { - unsafe { - let memory = self.buffer.borrow_mut(); - - // This should be safe since we don't grow up memory while caching this reference - // and we give up the reference before returning from this function. - let destination = Self::memory_as_slice_mut(&memory); - - let range = checked_range(dest_addr.into(), source.len(), destination.len()) - .ok_or_else(|| Error::Other("memory write is out of bounds".into()))?; - - destination[range].copy_from_slice(source); - Ok(()) - } - } - } -} diff --git a/client/executor/common/src/wasm_runtime.rs b/client/executor/common/src/wasm_runtime.rs index 1e9f1225518a..d43ad758b144 100644 --- a/client/executor/common/src/wasm_runtime.rs +++ b/client/executor/common/src/wasm_runtime.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/executor/runtime-test/Cargo.toml b/client/executor/runtime-test/Cargo.toml index a4fbc88cf566..352ffdf7a65c 100644 --- a/client/executor/runtime-test/Cargo.toml +++ b/client/executor/runtime-test/Cargo.toml @@ -2,23 +2,24 @@ name = "sc-runtime-test" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" build = "build.rs" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" publish = false -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/core" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +sp-core = { version = "6.0.0", default-features = false, path = "../../../primitives/core" } +sp-io = { version = "6.0.0", default-features = false, path = "../../../primitives/io", features = ["improved_panic_error_reporting"] } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } sp-sandbox = { version = "0.10.0-dev", default-features = false, path = "../../../primitives/sandbox" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/std" } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } sp-tasks = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/tasks" } +paste = "1.0.6" [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } diff --git a/client/executor/runtime-test/build.rs b/client/executor/runtime-test/build.rs index 9456d6bc90f4..27f931a542d4 100644 --- a/client/executor/runtime-test/build.rs +++ b/client/executor/runtime-test/build.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs index 2b5699fa3f77..0c61d6fcd38a 100644 --- a/client/executor/runtime-test/src/lib.rs +++ b/client/executor/runtime-test/src/lib.rs @@ -30,7 +30,7 @@ use sp_runtime::{ traits::{BlakeTwo256, Hash}, }; #[cfg(not(feature = "std"))] -use sp_sandbox::Value; +use sp_sandbox::{SandboxEnvironmentBuilder, SandboxInstance, SandboxMemory, Value}; extern "C" { #[allow(dead_code)] @@ -113,7 +113,9 @@ sp_core::wasm_export_functions! { } } - fn test_exhaust_heap() -> Vec { Vec::with_capacity(16777216) } + fn test_allocate_vec(size: u32) -> Vec { + Vec::with_capacity(size as usize) + } fn test_fp_f32add(a: [u8; 4], b: [u8; 4]) -> [u8; 4] { let a = f32::from_le_bytes(a); @@ -180,64 +182,10 @@ sp_core::wasm_export_functions! { b"one"[..].into(), b"two"[..].into(), ], + sp_core::storage::StateVersion::V1, ).as_ref().to_vec() } - fn test_sandbox(code: Vec) -> bool { - execute_sandboxed(&code, &[]).is_ok() - } - - fn test_sandbox_args(code: Vec) -> bool { - execute_sandboxed( - &code, - &[ - Value::I32(0x12345678), - Value::I64(0x1234567887654321), - ], - ).is_ok() - } - - fn test_sandbox_return_val(code: Vec) -> bool { - let ok = match execute_sandboxed( - &code, - &[ - Value::I32(0x1336), - ] - ) { - Ok(sp_sandbox::ReturnValue::Value(Value::I32(0x1337))) => true, - _ => false, - }; - - ok - } - - fn test_sandbox_instantiate(code: Vec) -> u8 { - let env_builder = sp_sandbox::EnvironmentDefinitionBuilder::new(); - let code = match sp_sandbox::Instance::new(&code, &env_builder, &mut ()) { - Ok(_) => 0, - Err(sp_sandbox::Error::Module) => 1, - Err(sp_sandbox::Error::Execution) => 2, - Err(sp_sandbox::Error::OutOfBounds) => 3, - }; - - code - } - - fn test_sandbox_get_global_val(code: Vec) -> i64 { - let env_builder = sp_sandbox::EnvironmentDefinitionBuilder::new(); - let instance = if let Ok(i) = sp_sandbox::Instance::new(&code, &env_builder, &mut ()) { - i - } else { - return 20; - }; - - match instance.get_global_val("test_global") { - Some(sp_sandbox::Value::I64(val)) => val, - None => 30, - _ => 40, - } - } - fn test_offchain_index_set() { sp_io::offchain_index::set(b"k", b"v"); } @@ -387,6 +335,22 @@ sp_core::wasm_export_functions! { fn test_panic_in_spawned() { sp_tasks::spawn(tasks::panicker, vec![]).join(); } + + fn test_return_i8() -> i8 { + -66 + } + + fn test_take_i8(value: i8) { + assert_eq!(value, -66); + } + + fn test_abort_on_panic() { + sp_io::panic_handler::abort_on_panic("test_abort_on_panic called"); + } + + fn test_unreachable_intrinsic() { + core::arch::wasm32::unreachable() + } } #[cfg(not(feature = "std"))] @@ -408,15 +372,112 @@ mod tasks { } } +/// A macro to define a test entrypoint for each available sandbox executor. +macro_rules! wasm_export_sandbox_test_functions { + ( + $( + fn $name:ident( + $( $arg_name:ident: $arg_ty:ty ),* $(,)? + ) $( -> $ret_ty:ty )? where T: SandboxInstance<$state:ty> $(,)? + { $( $fn_impl:tt )* } + )* + ) => { + $( + #[cfg(not(feature = "std"))] + fn $name( $($arg_name: $arg_ty),* ) $( -> $ret_ty )? where T: SandboxInstance<$state> { + $( $fn_impl )* + } + + paste::paste! { + sp_core::wasm_export_functions! { + fn [<$name _host>]( $($arg_name: $arg_ty),* ) $( -> $ret_ty )? { + $name::>( $( $arg_name ),* ) + } + + fn [<$name _embedded>]( $($arg_name: $arg_ty),* ) $( -> $ret_ty )? { + $name::>( $( $arg_name ),* ) + } + } + } + )* + }; +} + +wasm_export_sandbox_test_functions! { + fn test_sandbox(code: Vec) -> bool + where + T: SandboxInstance, + { + execute_sandboxed::(&code, &[]).is_ok() + } + + fn test_sandbox_args(code: Vec) -> bool + where + T: SandboxInstance, + { + execute_sandboxed::(&code, &[Value::I32(0x12345678), Value::I64(0x1234567887654321)]) + .is_ok() + } + + fn test_sandbox_return_val(code: Vec) -> bool + where + T: SandboxInstance, + { + let ok = match execute_sandboxed::(&code, &[Value::I32(0x1336)]) { + Ok(sp_sandbox::ReturnValue::Value(Value::I32(0x1337))) => true, + _ => false, + }; + + ok + } + + fn test_sandbox_instantiate(code: Vec) -> u8 + where + T: SandboxInstance<()>, + { + let env_builder = T::EnvironmentBuilder::new(); + let code = match T::new(&code, &env_builder, &mut ()) { + Ok(_) => 0, + Err(sp_sandbox::Error::Module) => 1, + Err(sp_sandbox::Error::Execution) => 2, + Err(sp_sandbox::Error::OutOfBounds) => 3, + }; + + code + } + + fn test_sandbox_get_global_val(code: Vec) -> i64 + where + T: SandboxInstance<()>, + { + let env_builder = T::EnvironmentBuilder::new(); + let instance = if let Ok(i) = T::new(&code, &env_builder, &mut ()) { + i + } else { + return 20 + }; + + match instance.get_global_val("test_global") { + Some(sp_sandbox::Value::I64(val)) => val, + None => 30, + _ => 40, + } + } +} + +#[cfg(not(feature = "std"))] +struct State { + counter: u32, +} + #[cfg(not(feature = "std"))] -fn execute_sandboxed( +fn execute_sandboxed( code: &[u8], args: &[Value], -) -> Result { - struct State { - counter: u32, - } - +) -> Result +where + T: sp_sandbox::SandboxInstance, +{ fn env_assert( _e: &mut State, args: &[Value], @@ -446,10 +507,10 @@ fn execute_sandboxed( let mut state = State { counter: 0 }; let env_builder = { - let mut env_builder = sp_sandbox::EnvironmentDefinitionBuilder::new(); + let mut env_builder = T::EnvironmentBuilder::new(); env_builder.add_host_func("env", "assert", env_assert); env_builder.add_host_func("env", "inc_counter", env_inc_counter); - let memory = match sp_sandbox::Memory::new(1, Some(16)) { + let memory = match T::Memory::new(1, Some(16)) { Ok(m) => m, Err(_) => unreachable!( " @@ -462,7 +523,7 @@ fn execute_sandboxed( env_builder }; - let mut instance = sp_sandbox::Instance::new(code, &env_builder, &mut state)?; + let mut instance = T::new(code, &env_builder, &mut state)?; let result = instance.invoke("call", args, &mut state); result.map_err(|_| sp_sandbox::HostError) diff --git a/client/executor/src/integration_tests/linux.rs b/client/executor/src/integration_tests/linux.rs index 38e57707e9e6..8775a35cb83c 100644 --- a/client/executor/src/integration_tests/linux.rs +++ b/client/executor/src/integration_tests/linux.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/executor/src/integration_tests/linux/smaps.rs b/client/executor/src/integration_tests/linux/smaps.rs index b23a188b93a2..665155ff81ea 100644 --- a/client/executor/src/integration_tests/linux/smaps.rs +++ b/client/executor/src/integration_tests/linux/smaps.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index fe964f47ba37..75b458a399e3 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -22,7 +22,7 @@ mod sandbox; use codec::{Decode, Encode}; use hex_literal::hex; -use sc_executor_common::{runtime_blob::RuntimeBlob, wasm_runtime::WasmModule}; +use sc_executor_common::{error::Error, runtime_blob::RuntimeBlob, wasm_runtime::WasmModule}; use sc_runtime_test::wasm_binary_unwrap; use sp_core::{ blake2_128, blake2_256, ed25519, map, @@ -33,14 +33,13 @@ use sp_core::{ }; use sp_runtime::traits::BlakeTwo256; use sp_state_machine::TestExternalities as CoreTestExternalities; -use sp_trie::{trie_types::Layout, TrieConfiguration}; -use sp_wasm_interface::HostFunctions as _; +use sp_trie::{LayoutV1 as Layout, TrieConfiguration}; use std::sync::Arc; use tracing_subscriber::layer::SubscriberExt; use crate::WasmExecutionMethod; -pub type TestExternalities = CoreTestExternalities; +pub type TestExternalities = CoreTestExternalities; type HostFunctions = sp_io::SubstrateHostFunctions; /// Simple macro that runs a given method as test with the available wasm execution methods. @@ -71,21 +70,63 @@ macro_rules! test_wasm_execution { }; } +/// A macro to run a given test for each available WASM execution method *and* for each +/// sandbox execution method. +#[macro_export] +macro_rules! test_wasm_execution_sandbox { + ($method_name:ident) => { + paste::item! { + #[test] + fn [<$method_name _interpreted_host_executor>]() { + $method_name(WasmExecutionMethod::Interpreted, "_host"); + } + + #[test] + fn [<$method_name _interpreted_embedded_executor>]() { + $method_name(WasmExecutionMethod::Interpreted, "_embedded"); + } + + #[test] + #[cfg(feature = "wasmtime")] + fn [<$method_name _compiled_host_executor>]() { + $method_name(WasmExecutionMethod::Compiled, "_host"); + } + + #[test] + #[cfg(feature = "wasmtime")] + fn [<$method_name _compiled_embedded_executor>]() { + $method_name(WasmExecutionMethod::Compiled, "_embedded"); + } + } + }; + + (interpreted_only $method_name:ident) => { + paste::item! { + #[test] + fn [<$method_name _interpreted_host_executor>]() { + $method_name(WasmExecutionMethod::Interpreted, "_host"); + } + } + + paste::item! { + #[test] + fn [<$method_name _interpreted_embedded_executor>]() { + $method_name(WasmExecutionMethod::Interpreted, "_embedded"); + } + } + }; +} + fn call_in_wasm( function: &str, call_data: &[u8], execution_method: WasmExecutionMethod, ext: &mut E, -) -> Result, String> { - let executor = crate::WasmExecutor::new( - execution_method, - Some(1024), - HostFunctions::host_functions(), - 8, - None, - ); +) -> Result, Error> { + let executor = + crate::WasmExecutor::::new(execution_method, Some(1024), 8, None, 2); executor.uncached_call( - RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(), + RuntimeBlob::uncompress_if_needed(wasm_binary_unwrap()).unwrap(), ext, true, function, @@ -107,25 +148,16 @@ fn call_not_existing_function(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - match call_in_wasm( - "test_calling_missing_external", - &[], - wasm_method, - &mut ext, - ) { - Ok(_) => panic!("was expected an `Err`"), - Err(e) => { - match wasm_method { - WasmExecutionMethod::Interpreted => assert_eq!( - &format!("{:?}", e), - "\"Trap: Trap { kind: Host(Other(\\\"Function `missing_external` is only a stub. Calling a stub is not allowed.\\\")) }\"" - ), + match call_in_wasm("test_calling_missing_external", &[], wasm_method, &mut ext).unwrap_err() { + Error::AbortedDueToTrap(error) => { + let expected = match wasm_method { + WasmExecutionMethod::Interpreted => "Trap: Host(Other(\"Function `missing_external` is only a stub. Calling a stub is not allowed.\"))", #[cfg(feature = "wasmtime")] - WasmExecutionMethod::Compiled => assert!( - format!("{:?}", e).contains("Wasm execution trapped: call to a missing function env:missing_external") - ), - } - } + WasmExecutionMethod::Compiled => "call to a missing function env:missing_external" + }; + assert_eq!(error.message, expected); + }, + error => panic!("unexpected error: {:?}", error), } } @@ -134,25 +166,18 @@ fn call_yet_another_not_existing_function(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - match call_in_wasm( - "test_calling_yet_another_missing_external", - &[], - wasm_method, - &mut ext, - ) { - Ok(_) => panic!("was expected an `Err`"), - Err(e) => { - match wasm_method { - WasmExecutionMethod::Interpreted => assert_eq!( - &format!("{:?}", e), - "\"Trap: Trap { kind: Host(Other(\\\"Function `yet_another_missing_external` is only a stub. Calling a stub is not allowed.\\\")) }\"" - ), + match call_in_wasm("test_calling_yet_another_missing_external", &[], wasm_method, &mut ext) + .unwrap_err() + { + Error::AbortedDueToTrap(error) => { + let expected = match wasm_method { + WasmExecutionMethod::Interpreted => "Trap: Host(Other(\"Function `yet_another_missing_external` is only a stub. Calling a stub is not allowed.\"))", #[cfg(feature = "wasmtime")] - WasmExecutionMethod::Compiled => assert!( - format!("{:?}", e).contains("Wasm execution trapped: call to a missing function env:yet_another_missing_external") - ), - } - } + WasmExecutionMethod::Compiled => "call to a missing function env:yet_another_missing_external" + }; + assert_eq!(error.message, expected); + }, + error => panic!("unexpected error: {:?}", error), } } @@ -174,21 +199,22 @@ fn panicking_should_work(wasm_method: WasmExecutionMethod) { test_wasm_execution!(storage_should_work); fn storage_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); + // Test value must be bigger than 32 bytes + // to test the trie versioning. + let value = vec![7u8; 60]; { let mut ext = ext.ext(); ext.set_storage(b"foo".to_vec(), b"bar".to_vec()); - let output = - call_in_wasm("test_data_in", &b"Hello world".to_vec().encode(), wasm_method, &mut ext) - .unwrap(); + let output = call_in_wasm("test_data_in", &value.encode(), wasm_method, &mut ext).unwrap(); assert_eq!(output, b"all ok!".to_vec().encode()); } let expected = TestExternalities::new(sp_core::storage::Storage { top: map![ - b"input".to_vec() => b"Hello world".to_vec(), + b"input".to_vec() => value, b"foo".to_vec() => b"bar".to_vec(), b"baz".to_vec() => b"bar".to_vec() ], @@ -427,36 +453,47 @@ test_wasm_execution!(should_trap_when_heap_exhausted); fn should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); - let executor = crate::WasmExecutor::new( + let executor = crate::WasmExecutor::::new( wasm_method, Some(17), // `17` is the initial number of pages compiled into the binary. - HostFunctions::host_functions(), 8, None, + 2, ); let err = executor .uncached_call( - RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(), + RuntimeBlob::uncompress_if_needed(wasm_binary_unwrap()).unwrap(), &mut ext.ext(), true, - "test_exhaust_heap", - &[0], + "test_allocate_vec", + &16777216_u32.encode(), ) .unwrap_err(); - assert!(err.contains("Allocator ran out of space")); + match err { + #[cfg(feature = "wasmtime")] + Error::AbortedDueToTrap(error) if wasm_method == WasmExecutionMethod::Compiled => { + assert_eq!( + error.message, + r#"host code panicked while being called by the runtime: Failed to allocate memory: "Allocator ran out of space""# + ); + }, + Error::RuntimePanicked(error) if wasm_method == WasmExecutionMethod::Interpreted => { + assert_eq!(error, r#"Failed to allocate memory: "Allocator ran out of space""#); + }, + error => panic!("unexpected error: {:?}", error), + } } fn mk_test_runtime(wasm_method: WasmExecutionMethod, pages: u64) -> Arc { - let blob = RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]) + let blob = RuntimeBlob::uncompress_if_needed(wasm_binary_unwrap()) .expect("failed to create a runtime blob out of test runtime"); - crate::wasm_runtime::create_wasm_runtime_with_code( + crate::wasm_runtime::create_wasm_runtime_with_code::( wasm_method, pages, blob, - HostFunctions::host_functions(), true, None, ) @@ -540,12 +577,12 @@ fn heap_is_reset_between_calls(wasm_method: WasmExecutionMethod) { test_wasm_execution!(parallel_execution); fn parallel_execution(wasm_method: WasmExecutionMethod) { - let executor = std::sync::Arc::new(crate::WasmExecutor::new( + let executor = std::sync::Arc::new(crate::WasmExecutor::::new( wasm_method, Some(1024), - HostFunctions::host_functions(), 8, None, + 2, )); let threads: Vec<_> = (0..8) .map(|_| { @@ -556,7 +593,7 @@ fn parallel_execution(wasm_method: WasmExecutionMethod) { assert_eq!( executor .uncached_call( - RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(), + RuntimeBlob::uncompress_if_needed(wasm_binary_unwrap()).unwrap(), &mut ext, true, "test_twox_128", @@ -582,11 +619,11 @@ fn wasm_tracing_should_work(wasm_method: WasmExecutionMethod) { struct TestTraceHandler(Arc>>); impl sc_tracing::TraceHandler for TestTraceHandler { - fn handle_span(&self, sd: SpanDatum) { - self.0.lock().unwrap().push(sd); + fn handle_span(&self, sd: &SpanDatum) { + self.0.lock().unwrap().push(sd.clone()); } - fn handle_event(&self, _event: TraceEvent) {} + fn handle_event(&self, _event: &TraceEvent) {} } let traces = Arc::new(Mutex::new(Vec::new())); @@ -650,5 +687,130 @@ fn panic_in_spawned_instance_panics_on_joining_its_result(wasm_method: WasmExecu let error_result = call_in_wasm("test_panic_in_spawned", &[], wasm_method, &mut ext).unwrap_err(); - assert!(format!("{}", error_result).contains("Spawned task")); + assert!(error_result.to_string().contains("Spawned task")); +} + +test_wasm_execution!(memory_is_cleared_between_invocations); +fn memory_is_cleared_between_invocations(wasm_method: WasmExecutionMethod) { + // This is based on the code generated by compiling a runtime *without* + // the `-C link-arg=--import-memory` using the following code and then + // disassembling the resulting blob with `wasm-dis`: + // + // ``` + // #[no_mangle] + // #[cfg(not(feature = "std"))] + // pub fn returns_no_bss_mutable_static(_: *mut u8, _: usize) -> u64 { + // static mut COUNTER: usize = 0; + // let output = unsafe { + // COUNTER += 1; + // COUNTER as u64 + // }; + // sp_core::to_substrate_wasm_fn_return_value(&output) + // } + // ``` + // + // This results in the BSS section to *not* be emitted, hence the executor has no way + // of knowing about the `static` variable's existence, so this test will fail if the linear + // memory is not properly cleared between invocations. + let binary = wat::parse_str(r#" + (module + (type $i32_=>_i32 (func (param i32) (result i32))) + (type $i32_i32_=>_i64 (func (param i32 i32) (result i64))) + (import "env" "ext_allocator_malloc_version_1" (func $ext_allocator_malloc_version_1 (param i32) (result i32))) + (global $__stack_pointer (mut i32) (i32.const 1048576)) + (global $global$1 i32 (i32.const 1048580)) + (global $global$2 i32 (i32.const 1048592)) + (memory $0 17) + (export "memory" (memory $0)) + (export "returns_no_bss_mutable_static" (func $returns_no_bss_mutable_static)) + (export "__data_end" (global $global$1)) + (export "__heap_base" (global $global$2)) + (func $returns_no_bss_mutable_static (param $0 i32) (param $1 i32) (result i64) + (local $2 i32) + (local $3 i32) + (i32.store offset=1048576 + (i32.const 0) + (local.tee $2 + (i32.add + (i32.load offset=1048576 (i32.const 0)) + (i32.const 1) + ) + ) + ) + (i64.store + (local.tee $3 + (call $ext_allocator_malloc_version_1 (i32.const 8)) + ) + (i64.extend_i32_u (local.get $2)) + ) + (i64.or + (i64.extend_i32_u (local.get $3)) + (i64.const 34359738368) + ) + ) + )"#).unwrap(); + + let runtime = crate::wasm_runtime::create_wasm_runtime_with_code::( + wasm_method, + 1024, + RuntimeBlob::uncompress_if_needed(&binary[..]).unwrap(), + true, + None, + ) + .unwrap(); + + let mut instance = runtime.new_instance().unwrap(); + let res = instance.call_export("returns_no_bss_mutable_static", &[0]).unwrap(); + assert_eq!(1, u64::decode(&mut &res[..]).unwrap()); + + let res = instance.call_export("returns_no_bss_mutable_static", &[0]).unwrap(); + assert_eq!(1, u64::decode(&mut &res[..]).unwrap()); +} + +test_wasm_execution!(return_i8); +fn return_i8(wasm_method: WasmExecutionMethod) { + let mut ext = TestExternalities::default(); + let mut ext = ext.ext(); + + assert_eq!( + call_in_wasm("test_return_i8", &[], wasm_method, &mut ext).unwrap(), + (-66_i8).encode() + ); +} + +test_wasm_execution!(take_i8); +fn take_i8(wasm_method: WasmExecutionMethod) { + let mut ext = TestExternalities::default(); + let mut ext = ext.ext(); + + call_in_wasm("test_take_i8", &(-66_i8).encode(), wasm_method, &mut ext).unwrap(); +} + +test_wasm_execution!(abort_on_panic); +fn abort_on_panic(wasm_method: WasmExecutionMethod) { + let mut ext = TestExternalities::default(); + let mut ext = ext.ext(); + + match call_in_wasm("test_abort_on_panic", &[], wasm_method, &mut ext).unwrap_err() { + Error::AbortedDueToPanic(error) => assert_eq!(error.message, "test_abort_on_panic called"), + error => panic!("unexpected error: {:?}", error), + } +} + +test_wasm_execution!(unreachable_intrinsic); +fn unreachable_intrinsic(wasm_method: WasmExecutionMethod) { + let mut ext = TestExternalities::default(); + let mut ext = ext.ext(); + + match call_in_wasm("test_unreachable_intrinsic", &[], wasm_method, &mut ext).unwrap_err() { + Error::AbortedDueToTrap(error) => { + let expected = match wasm_method { + WasmExecutionMethod::Interpreted => "Trap: Unreachable", + #[cfg(feature = "wasmtime")] + WasmExecutionMethod::Compiled => "wasm trap: wasm `unreachable` instruction executed", + }; + assert_eq!(error.message, expected); + }, + error => panic!("unexpected error: {:?}", error), + } } diff --git a/client/executor/src/integration_tests/sandbox.rs b/client/executor/src/integration_tests/sandbox.rs index aacd493297cc..643db5097c6a 100644 --- a/client/executor/src/integration_tests/sandbox.rs +++ b/client/executor/src/integration_tests/sandbox.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -17,12 +17,12 @@ // along with this program. If not, see . use super::{call_in_wasm, TestExternalities}; -use crate::{test_wasm_execution, WasmExecutionMethod}; +use crate::{test_wasm_execution_sandbox, WasmExecutionMethod}; use codec::Encode; -test_wasm_execution!(sandbox_should_work); -fn sandbox_should_work(wasm_method: WasmExecutionMethod) { +test_wasm_execution_sandbox!(sandbox_should_work); +fn sandbox_should_work(wasm_method: WasmExecutionMethod, fn_suffix: &str) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); @@ -51,11 +51,14 @@ fn sandbox_should_work(wasm_method: WasmExecutionMethod) { .unwrap() .encode(); - assert_eq!(call_in_wasm("test_sandbox", &code, wasm_method, &mut ext).unwrap(), true.encode()); + assert_eq!( + call_in_wasm(&format!("test_sandbox{}", fn_suffix), &code, wasm_method, &mut ext).unwrap(), + true.encode() + ); } -test_wasm_execution!(sandbox_trap); -fn sandbox_trap(wasm_method: WasmExecutionMethod) { +test_wasm_execution_sandbox!(sandbox_trap); +fn sandbox_trap(wasm_method: WasmExecutionMethod, fn_suffix: &str) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); @@ -72,11 +75,14 @@ fn sandbox_trap(wasm_method: WasmExecutionMethod) { ) .unwrap(); - assert_eq!(call_in_wasm("test_sandbox", &code, wasm_method, &mut ext).unwrap(), vec![0]); + assert_eq!( + call_in_wasm(&format!("test_sandbox{}", fn_suffix), &code, wasm_method, &mut ext).unwrap(), + vec![0] + ); } -test_wasm_execution!(start_called); -fn start_called(wasm_method: WasmExecutionMethod) { +test_wasm_execution_sandbox!(start_called); +fn start_called(wasm_method: WasmExecutionMethod, fn_suffix: &str) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); @@ -111,11 +117,14 @@ fn start_called(wasm_method: WasmExecutionMethod) { .unwrap() .encode(); - assert_eq!(call_in_wasm("test_sandbox", &code, wasm_method, &mut ext).unwrap(), true.encode()); + assert_eq!( + call_in_wasm(&format!("test_sandbox{}", fn_suffix), &code, wasm_method, &mut ext).unwrap(), + true.encode() + ); } -test_wasm_execution!(invoke_args); -fn invoke_args(wasm_method: WasmExecutionMethod) { +test_wasm_execution_sandbox!(invoke_args); +fn invoke_args(wasm_method: WasmExecutionMethod, fn_suffix: &str) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); @@ -147,13 +156,14 @@ fn invoke_args(wasm_method: WasmExecutionMethod) { .encode(); assert_eq!( - call_in_wasm("test_sandbox_args", &code, wasm_method, &mut ext,).unwrap(), + call_in_wasm(&format!("test_sandbox_args{}", fn_suffix), &code, wasm_method, &mut ext,) + .unwrap(), true.encode(), ); } -test_wasm_execution!(return_val); -fn return_val(wasm_method: WasmExecutionMethod) { +test_wasm_execution_sandbox!(return_val); +fn return_val(wasm_method: WasmExecutionMethod, fn_suffix: &str) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); @@ -173,13 +183,19 @@ fn return_val(wasm_method: WasmExecutionMethod) { .encode(); assert_eq!( - call_in_wasm("test_sandbox_return_val", &code, wasm_method, &mut ext,).unwrap(), + call_in_wasm( + &format!("test_sandbox_return_val{}", fn_suffix), + &code, + wasm_method, + &mut ext, + ) + .unwrap(), true.encode(), ); } -test_wasm_execution!(unlinkable_module); -fn unlinkable_module(wasm_method: WasmExecutionMethod) { +test_wasm_execution_sandbox!(unlinkable_module); +fn unlinkable_module(wasm_method: WasmExecutionMethod, fn_suffix: &str) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); @@ -197,13 +213,19 @@ fn unlinkable_module(wasm_method: WasmExecutionMethod) { .encode(); assert_eq!( - call_in_wasm("test_sandbox_instantiate", &code, wasm_method, &mut ext,).unwrap(), + call_in_wasm( + &format!("test_sandbox_instantiate{}", fn_suffix), + &code, + wasm_method, + &mut ext, + ) + .unwrap(), 1u8.encode(), ); } -test_wasm_execution!(corrupted_module); -fn corrupted_module(wasm_method: WasmExecutionMethod) { +test_wasm_execution_sandbox!(corrupted_module); +fn corrupted_module(wasm_method: WasmExecutionMethod, fn_suffix: &str) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); @@ -211,13 +233,19 @@ fn corrupted_module(wasm_method: WasmExecutionMethod) { let code = vec![0u8, 0, 0, 0, 1, 0, 0, 0].encode(); assert_eq!( - call_in_wasm("test_sandbox_instantiate", &code, wasm_method, &mut ext,).unwrap(), + call_in_wasm( + &format!("test_sandbox_instantiate{}", fn_suffix), + &code, + wasm_method, + &mut ext, + ) + .unwrap(), 1u8.encode(), ); } -test_wasm_execution!(start_fn_ok); -fn start_fn_ok(wasm_method: WasmExecutionMethod) { +test_wasm_execution_sandbox!(start_fn_ok); +fn start_fn_ok(wasm_method: WasmExecutionMethod, fn_suffix: &str) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); @@ -238,13 +266,19 @@ fn start_fn_ok(wasm_method: WasmExecutionMethod) { .encode(); assert_eq!( - call_in_wasm("test_sandbox_instantiate", &code, wasm_method, &mut ext,).unwrap(), + call_in_wasm( + &format!("test_sandbox_instantiate{}", fn_suffix), + &code, + wasm_method, + &mut ext, + ) + .unwrap(), 0u8.encode(), ); } -test_wasm_execution!(start_fn_traps); -fn start_fn_traps(wasm_method: WasmExecutionMethod) { +test_wasm_execution_sandbox!(start_fn_traps); +fn start_fn_traps(wasm_method: WasmExecutionMethod, fn_suffix: &str) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); @@ -266,13 +300,19 @@ fn start_fn_traps(wasm_method: WasmExecutionMethod) { .encode(); assert_eq!( - call_in_wasm("test_sandbox_instantiate", &code, wasm_method, &mut ext,).unwrap(), + call_in_wasm( + &format!("test_sandbox_instantiate{}", fn_suffix), + &code, + wasm_method, + &mut ext, + ) + .unwrap(), 2u8.encode(), ); } -test_wasm_execution!(get_global_val_works); -fn get_global_val_works(wasm_method: WasmExecutionMethod) { +test_wasm_execution_sandbox!(get_global_val_works); +fn get_global_val_works(wasm_method: WasmExecutionMethod, fn_suffix: &str) { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); @@ -287,7 +327,13 @@ fn get_global_val_works(wasm_method: WasmExecutionMethod) { .encode(); assert_eq!( - call_in_wasm("test_sandbox_get_global_val", &code, wasm_method, &mut ext,).unwrap(), + call_in_wasm( + &format!("test_sandbox_get_global_val{}", fn_suffix), + &code, + wasm_method, + &mut ext, + ) + .unwrap(), 500i64.encode(), ); } diff --git a/client/executor/src/lib.rs b/client/executor/src/lib.rs index 041db87bc82a..5cd04b9e4ee6 100644 --- a/client/executor/src/lib.rs +++ b/client/executor/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -67,23 +67,22 @@ mod tests { use sc_executor_common::runtime_blob::RuntimeBlob; use sc_runtime_test::wasm_binary_unwrap; use sp_io::TestExternalities; - use sp_wasm_interface::HostFunctions; #[test] fn call_in_interpreted_wasm_works() { let mut ext = TestExternalities::default(); let mut ext = ext.ext(); - let executor = WasmExecutor::new( + let executor = WasmExecutor::::new( WasmExecutionMethod::Interpreted, Some(8), - sp_io::SubstrateHostFunctions::host_functions(), 8, None, + 2, ); let res = executor .uncached_call( - RuntimeBlob::uncompress_if_needed(&wasm_binary_unwrap()[..]).unwrap(), + RuntimeBlob::uncompress_if_needed(wasm_binary_unwrap()).unwrap(), &mut ext, true, "test_empty_return", diff --git a/client/executor/src/native_executor.rs b/client/executor/src/native_executor.rs index d912fc0fd13c..669780f2a4b6 100644 --- a/client/executor/src/native_executor.rs +++ b/client/executor/src/native_executor.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -24,6 +24,7 @@ use crate::{ use std::{ collections::HashMap, + marker::PhantomData, panic::{AssertUnwindSafe, UnwindSafe}, path::PathBuf, result, @@ -34,7 +35,6 @@ use std::{ }; use codec::{Decode, Encode}; -use log::trace; use sc_executor_common::{ runtime_blob::RuntimeBlob, wasm_runtime::{InvokeMethod, WasmInstance, WasmModule}, @@ -46,7 +46,7 @@ use sp_core::{ use sp_externalities::ExternalitiesExt as _; use sp_tasks::new_async_externalities; use sp_version::{GetNativeVersion, NativeVersion, RuntimeVersion}; -use sp_wasm_interface::{Function, HostFunctions}; +use sp_wasm_interface::{ExtendedHostFunctions, HostFunctions}; /// Default num of pages for the heap const DEFAULT_HEAP_PAGES: u64 = 2048; @@ -91,22 +91,36 @@ pub trait NativeExecutionDispatch: Send + Sync { /// An abstraction over Wasm code executor. Supports selecting execution backend and /// manages runtime cache. -#[derive(Clone)] -pub struct WasmExecutor { +pub struct WasmExecutor { /// Method used to execute fallback Wasm code. method: WasmExecutionMethod, /// The number of 64KB pages to allocate for Wasm execution. default_heap_pages: u64, - /// The host functions registered with this instance. - host_functions: Arc>, /// WASM runtime cache. cache: Arc, /// The path to a directory which the executor can leverage for a file cache, e.g. put there /// compiled artifacts. cache_path: Option, + + phantom: PhantomData, +} + +impl Clone for WasmExecutor { + fn clone(&self) -> Self { + Self { + method: self.method, + default_heap_pages: self.default_heap_pages, + cache: self.cache.clone(), + cache_path: self.cache_path.clone(), + phantom: self.phantom, + } + } } -impl WasmExecutor { +impl WasmExecutor +where + H: HostFunctions, +{ /// Create new instance. /// /// # Parameters @@ -127,16 +141,20 @@ impl WasmExecutor { pub fn new( method: WasmExecutionMethod, default_heap_pages: Option, - host_functions: Vec<&'static dyn Function>, max_runtime_instances: usize, cache_path: Option, + runtime_cache_size: u8, ) -> Self { WasmExecutor { method, default_heap_pages: default_heap_pages.unwrap_or(DEFAULT_HEAP_PAGES), - host_functions: Arc::new(host_functions), - cache: Arc::new(RuntimeCache::new(max_runtime_instances, cache_path.clone())), + cache: Arc::new(RuntimeCache::new( + max_runtime_instances, + cache_path.clone(), + runtime_cache_size, + )), cache_path, + phantom: PhantomData, } } @@ -168,12 +186,11 @@ impl WasmExecutor { AssertUnwindSafe<&mut dyn Externalities>, ) -> Result>, { - match self.cache.with_instance( + match self.cache.with_instance::( runtime_code, ext, self.method, self.default_heap_pages, - &*self.host_functions, allow_missing_host_functions, |module, instance, version, ext| { let module = AssertUnwindSafe(module); @@ -202,20 +219,18 @@ impl WasmExecutor { allow_missing_host_functions: bool, export_name: &str, call_data: &[u8], - ) -> std::result::Result, String> { - let module = crate::wasm_runtime::create_wasm_runtime_with_code( + ) -> std::result::Result, Error> { + let module = crate::wasm_runtime::create_wasm_runtime_with_code::( self.method, self.default_heap_pages, runtime_blob, - self.host_functions.to_vec(), allow_missing_host_functions, self.cache_path.as_deref(), ) - .map_err(|e| format!("Failed to create module: {:?}", e))?; + .map_err(|e| format!("Failed to create module: {}", e))?; - let instance = module - .new_instance() - .map_err(|e| format!("Failed to create instance: {:?}", e))?; + let instance = + module.new_instance().map_err(|e| format!("Failed to create instance: {}", e))?; let mut instance = AssertUnwindSafe(instance); let mut ext = AssertUnwindSafe(ext); @@ -226,11 +241,13 @@ impl WasmExecutor { instance.call_export(export_name, call_data) }) .and_then(|r| r) - .map_err(|e| e.to_string()) } } -impl sp_core::traits::ReadRuntimeVersion for WasmExecutor { +impl sp_core::traits::ReadRuntimeVersion for WasmExecutor +where + H: HostFunctions, +{ fn read_runtime_version( &self, wasm_code: &[u8], @@ -261,10 +278,14 @@ impl sp_core::traits::ReadRuntimeVersion for WasmExecutor { "Core_version", &[], ) + .map_err(|e| e.to_string()) } } -impl CodeExecutor for WasmExecutor { +impl CodeExecutor for WasmExecutor +where + H: HostFunctions, +{ type Error = Error; fn call< @@ -279,6 +300,12 @@ impl CodeExecutor for WasmExecutor { _use_native: bool, _native_call: Option, ) -> (Result>, bool) { + tracing::trace!( + target: "executor", + %method, + "Executing function", + ); + let result = self.with_instance( runtime_code, ext, @@ -294,7 +321,10 @@ impl CodeExecutor for WasmExecutor { } } -impl RuntimeVersionOf for WasmExecutor { +impl RuntimeVersionOf for WasmExecutor +where + H: HostFunctions, +{ fn runtime_version( &self, ext: &mut dyn Externalities, @@ -308,13 +338,17 @@ impl RuntimeVersionOf for WasmExecutor { /// A generic `CodeExecutor` implementation that uses a delegate to determine wasm code equivalence /// and dispatch to native code when possible, falling back on `WasmExecutor` when not. -pub struct NativeElseWasmExecutor { +pub struct NativeElseWasmExecutor +where + D: NativeExecutionDispatch, +{ /// Dummy field to avoid the compiler complaining about us not using `D`. _dummy: std::marker::PhantomData, /// Native runtime version info. native_version: NativeVersion, /// Fallback wasm executor. - wasm: WasmExecutor, + wasm: + WasmExecutor>, } impl NativeElseWasmExecutor { @@ -330,27 +364,14 @@ impl NativeElseWasmExecutor { fallback_method: WasmExecutionMethod, default_heap_pages: Option, max_runtime_instances: usize, + runtime_cache_size: u8, ) -> Self { - let extended = D::ExtendHostFunctions::host_functions(); - let mut host_functions = sp_io::SubstrateHostFunctions::host_functions() - .into_iter() - // filter out any host function overrides provided. - .filter(|host_fn| { - extended - .iter() - .find(|ext_host_fn| host_fn.name() == ext_host_fn.name()) - .is_none() - }) - .collect::>(); - - // Add the custom host functions provided by the user. - host_functions.extend(extended); let wasm_executor = WasmExecutor::new( fallback_method, default_heap_pages, - host_functions, max_runtime_instances, None, + runtime_cache_size, ); NativeElseWasmExecutor { @@ -399,16 +420,17 @@ impl RuntimeSpawn for RuntimeInstanceSpawn { let scheduler = self.scheduler.clone(); self.scheduler.spawn( "executor-extra-runtime-instance", + None, Box::pin(async move { let module = AssertUnwindSafe(module); let async_ext = match new_async_externalities(scheduler.clone()) { Ok(val) => val, Err(e) => { - log::error!( + tracing::error!( target: "executor", - "Failed to setup externalities for async context: {}", - e, + error = %e, + "Failed to setup externalities for async context.", ); // This will drop sender and receiver end will panic @@ -421,10 +443,10 @@ impl RuntimeSpawn for RuntimeInstanceSpawn { )) { Ok(val) => val, Err(e) => { - log::error!( + tracing::error!( target: "executor", - "Failed to setup runtime extension for async externalities: {}", - e, + error = %e, + "Failed to setup runtime extension for async externalities", ); // This will drop sender and receiver end will panic @@ -438,12 +460,19 @@ impl RuntimeSpawn for RuntimeInstanceSpawn { // pool of instances should be used. // // https://github.com/paritytech/substrate/issues/7354 - let mut instance = - module.new_instance().expect("Failed to create new instance from module"); - - instance + let mut instance = match module.new_instance() { + Ok(instance) => instance, + Err(error) => { + panic!("failed to create new instance from module: {}", error) + }, + }; + + match instance .call(InvokeMethod::TableWithWrapper { dispatcher_ref, func }, &data[..]) - .expect("Failed to invoke instance.") + { + Ok(result) => result, + Err(error) => panic!("failed to invoke instance: {}", error), + } }); match result { @@ -453,7 +482,7 @@ impl RuntimeSpawn for RuntimeInstanceSpawn { Err(error) => { // If execution is panicked, the `join` in the original runtime code will // panic as well, since the sender is dropped without sending anything. - log::error!("Call error in spawned task: {:?}", error); + tracing::error!(error = %error, "Call error in spawned task"); }, } }), @@ -495,10 +524,10 @@ fn preregister_builtin_ext(module: Arc) { RuntimeInstanceSpawn::with_externalities_and_module(module, ext) { if let Err(e) = ext.register_extension(RuntimeSpawnExt(Box::new(runtime_spawn))) { - trace!( + tracing::trace!( target: "executor", - "Failed to register `RuntimeSpawnExt` instance on externalities: {:?}", - e, + error = ?e, + "Failed to register `RuntimeSpawnExt` instance on externalities", ) } } @@ -520,6 +549,12 @@ impl CodeExecutor for NativeElseWasmExecut use_native: bool, native_call: Option, ) -> (Result>, bool) { + tracing::trace!( + target: "executor", + function = %method, + "Executing function", + ); + let mut used_native = false; let result = self.wasm.with_instance( runtime_code, @@ -535,11 +570,11 @@ impl CodeExecutor for NativeElseWasmExecut match (use_native, can_call_with, native_call) { (_, false, _) | (false, _, _) => { if !can_call_with { - trace!( + tracing::trace!( target: "executor", - "Request for native execution failed (native: {}, chain: {})", - self.native_version.runtime_version, - onchain_version, + native = %self.native_version.runtime_version, + chain = %onchain_version, + "Request for native execution failed", ); } @@ -549,12 +584,11 @@ impl CodeExecutor for NativeElseWasmExecut }) }, (true, true, Some(call)) => { - trace!( + tracing::trace!( target: "executor", - "Request for native execution with native call succeeded \ - (native: {}, chain: {}).", - self.native_version.runtime_version, - onchain_version, + native = %self.native_version.runtime_version, + chain = %onchain_version, + "Request for native execution with native call succeeded" ); used_native = true; @@ -564,11 +598,11 @@ impl CodeExecutor for NativeElseWasmExecut Ok(res) }, _ => { - trace!( + tracing::trace!( target: "executor", - "Request for native execution succeeded (native: {}, chain: {})", - self.native_version.runtime_version, - onchain_version + native = %self.native_version.runtime_version, + chain = %onchain_version, + "Request for native execution succeeded", ); used_native = true; @@ -635,9 +669,23 @@ mod tests { WasmExecutionMethod::Interpreted, None, 8, + 2, ); + + fn extract_host_functions( + _: &WasmExecutor, + ) -> Vec<&'static dyn sp_wasm_interface::Function> + where + H: HostFunctions, + { + H::host_functions() + } + my_interface::HostFunctions::host_functions().iter().for_each(|function| { - assert_eq!(executor.wasm.host_functions.iter().filter(|f| f == &function).count(), 2); + assert_eq!( + extract_host_functions(&executor.wasm).iter().filter(|f| f == &function).count(), + 2 + ); }); my_interface::say_hello_world("hey"); diff --git a/client/executor/src/wasm_runtime.rs b/client/executor/src/wasm_runtime.rs index b3a981d9e082..952130e98087 100644 --- a/client/executor/src/wasm_runtime.rs +++ b/client/executor/src/wasm_runtime.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,6 +23,7 @@ use crate::error::{Error, WasmError}; use codec::Decode; +use lru::LruCache; use parking_lot::Mutex; use sc_executor_common::{ runtime_blob::RuntimeBlob, @@ -36,7 +37,7 @@ use std::{ sync::Arc, }; -use sp_wasm_interface::Function; +use sp_wasm_interface::HostFunctions; /// Specification of different methods of executing the runtime Wasm code. #[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)] @@ -54,25 +55,29 @@ impl Default for WasmExecutionMethod { } } -/// A Wasm runtime object along with its cached runtime version. -struct VersionedRuntime { +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +struct VersionedRuntimeId { /// Runtime code hash. code_hash: Vec, /// Wasm runtime type. wasm_method: WasmExecutionMethod, - /// Shared runtime that can spawn instances. - module: Arc, /// The number of WebAssembly heap pages this instance was created with. heap_pages: u64, +} + +/// A Wasm runtime object along with its cached runtime version. +struct VersionedRuntime { + /// Shared runtime that can spawn instances. + module: Arc, /// Runtime version according to `Core_version` if any. version: Option, /// Cached instance pool. - instances: Vec>>>, + instances: Arc>>>>, } impl VersionedRuntime { /// Run the given closure `f` with an instance of this runtime. - fn with_instance<'c, R, F>(&self, ext: &mut dyn Externalities, f: F) -> Result + fn with_instance(&self, ext: &mut dyn Externalities, f: F) -> Result where F: FnOnce( &Arc, @@ -98,23 +103,23 @@ impl VersionedRuntime { let result = f(&self.module, &mut *instance, self.version.as_ref(), ext); if let Err(e) = &result { if new_inst { - log::warn!( + tracing::warn!( target: "wasm-runtime", - "Fresh runtime instance failed with {:?}", - e, + error = %e, + "Fresh runtime instance failed", ) } else { - log::warn!( + tracing::warn!( target: "wasm-runtime", - "Evicting failed runtime instance: {:?}", - e, + error = %e, + "Evicting failed runtime instance", ); } } else { *locked = Some(instance); if new_inst { - log::debug!( + tracing::debug!( target: "wasm-runtime", "Allocated WASM instance {}/{}", index + 1, @@ -126,7 +131,7 @@ impl VersionedRuntime { result }, None => { - log::warn!(target: "wasm-runtime", "Ran out of free WASM instances"); + tracing::warn!(target: "wasm-runtime", "Ran out of free WASM instances"); // Allocate a new instance let mut instance = self.module.new_instance()?; @@ -137,8 +142,6 @@ impl VersionedRuntime { } } -const MAX_RUNTIMES: usize = 2; - /// Cache for the runtimes. /// /// When an instance is requested for the first time it is added to this cache. Metadata is kept @@ -149,12 +152,12 @@ const MAX_RUNTIMES: usize = 2; /// the memory reset to the initial memory. So, one runtime instance is reused for every fetch /// request. /// -/// The size of cache is equal to `MAX_RUNTIMES`. +/// The size of cache is configurable via the cli option `--runtime-cache-size`. pub struct RuntimeCache { /// A cache of runtimes along with metadata. /// /// Runtimes sorted by recent usage. The most recently used is at the front. - runtimes: Mutex<[Option>; MAX_RUNTIMES]>, + runtimes: Mutex>>, /// The size of the instances cache for each runtime. max_runtime_instances: usize, cache_path: Option, @@ -163,13 +166,24 @@ pub struct RuntimeCache { impl RuntimeCache { /// Creates a new instance of a runtimes cache. /// - /// `max_runtime_instances` specifies the number of runtime instances preserved in an in-memory - /// cache. + /// `max_runtime_instances` specifies the number of instances per runtime preserved in an + /// in-memory cache. /// /// `cache_path` allows to specify an optional directory where the executor can store files /// for caching. - pub fn new(max_runtime_instances: usize, cache_path: Option) -> RuntimeCache { - RuntimeCache { runtimes: Default::default(), max_runtime_instances, cache_path } + /// + /// `runtime_cache_size` specifies the number of different runtimes versions preserved in an + /// in-memory cache. + pub fn new( + max_runtime_instances: usize, + cache_path: Option, + runtime_cache_size: u8, + ) -> RuntimeCache { + RuntimeCache { + runtimes: Mutex::new(LruCache::new(runtime_cache_size.into())), + max_runtime_instances, + cache_path, + } } /// Prepares a WASM module instance and executes given function for it. @@ -185,14 +199,14 @@ impl RuntimeCache { /// /// `wasm_method` - Type of WASM backend to use. /// - /// `host_functions` - The host functions that should be registered for the Wasm runtime. - /// /// `allow_missing_func_imports` - Ignore missing function imports. /// /// `max_runtime_instances` - The size of the instances cache. /// /// `f` - Function to execute. /// + /// `H` - A compile-time list of host functions to expose to the runtime. + /// /// # Returns result of `f` wrapped in an additional result. /// In case of failure one of two errors can be returned: /// @@ -200,17 +214,17 @@ impl RuntimeCache { /// /// `Error::InvalidMemoryReference` is returned if no memory export with the /// identifier `memory` can be found in the runtime. - pub fn with_instance<'c, R, F>( + pub fn with_instance<'c, H, R, F>( &self, runtime_code: &'c RuntimeCode<'c>, ext: &mut dyn Externalities, wasm_method: WasmExecutionMethod, default_heap_pages: u64, - host_functions: &[&'static dyn Function], allow_missing_func_imports: bool, f: F, ) -> Result, Error> where + H: HostFunctions, F: FnOnce( &Arc, &mut dyn WasmInstance, @@ -221,141 +235,114 @@ impl RuntimeCache { let code_hash = &runtime_code.hash; let heap_pages = runtime_code.heap_pages.unwrap_or(default_heap_pages); + let versioned_runtime_id = + VersionedRuntimeId { code_hash: code_hash.clone(), heap_pages, wasm_method }; + let mut runtimes = self.runtimes.lock(); // this must be released prior to calling f - let pos = runtimes.iter().position(|r| { - r.as_ref().map_or(false, |r| { - r.wasm_method == wasm_method && - r.code_hash == *code_hash && - r.heap_pages == heap_pages - }) - }); + let versioned_runtime = if let Some(versioned_runtime) = runtimes.get(&versioned_runtime_id) + { + versioned_runtime.clone() + } else { + let code = runtime_code.fetch_runtime_code().ok_or(WasmError::CodeNotFound)?; + + let time = std::time::Instant::now(); + + let result = create_versioned_wasm_runtime::( + &code, + ext, + wasm_method, + heap_pages, + allow_missing_func_imports, + self.max_runtime_instances, + self.cache_path.as_deref(), + ); + + match result { + Ok(ref result) => { + tracing::debug!( + target: "wasm-runtime", + "Prepared new runtime version {:?} in {} ms.", + result.version, + time.elapsed().as_millis(), + ); + }, + Err(ref err) => { + tracing::warn!(target: "wasm-runtime", error = ?err, "Cannot create a runtime"); + }, + } - let runtime = match pos { - Some(n) => runtimes[n] - .clone() - .expect("`position` only returns `Some` for entries that are `Some`"), - None => { - let code = runtime_code.fetch_runtime_code().ok_or(WasmError::CodeNotFound)?; - - let time = std::time::Instant::now(); - - let result = create_versioned_wasm_runtime( - &code, - code_hash.clone(), - ext, - wasm_method, - heap_pages, - host_functions.into(), - allow_missing_func_imports, - self.max_runtime_instances, - self.cache_path.as_deref(), - ); - - match result { - Ok(ref result) => { - log::debug!( - target: "wasm-runtime", - "Prepared new runtime version {:?} in {} ms.", - result.version, - time.elapsed().as_millis(), - ); - }, - Err(ref err) => { - log::warn!(target: "wasm-runtime", "Cannot create a runtime: {:?}", err); - }, - } + let versioned_runtime = Arc::new(result?); - Arc::new(result?) - }, + // Save new versioned wasm runtime in cache + runtimes.put(versioned_runtime_id, versioned_runtime.clone()); + + versioned_runtime }; - // Rearrange runtimes by last recently used. - match pos { - Some(0) => {}, - Some(n) => - for i in (1..n + 1).rev() { - runtimes.swap(i, i - 1); - }, - None => { - runtimes[MAX_RUNTIMES - 1] = Some(runtime.clone()); - for i in (1..MAX_RUNTIMES).rev() { - runtimes.swap(i, i - 1); - } - }, - } + // Lock must be released prior to calling f drop(runtimes); - Ok(runtime.with_instance(ext, f)) + Ok(versioned_runtime.with_instance(ext, f)) } } /// Create a wasm runtime with the given `code`. -pub fn create_wasm_runtime_with_code( +pub fn create_wasm_runtime_with_code( wasm_method: WasmExecutionMethod, heap_pages: u64, blob: RuntimeBlob, - host_functions: Vec<&'static dyn Function>, allow_missing_func_imports: bool, cache_path: Option<&Path>, -) -> Result, WasmError> { +) -> Result, WasmError> +where + H: HostFunctions, +{ match wasm_method { WasmExecutionMethod::Interpreted => { // Wasmi doesn't have any need in a cache directory. // // We drop the cache_path here to silence warnings that cache_path is not used if // compiling without the `wasmtime` flag. - drop(cache_path); + let _ = cache_path; sc_executor_wasmi::create_runtime( blob, heap_pages, - host_functions, + H::host_functions(), allow_missing_func_imports, ) .map(|runtime| -> Arc { Arc::new(runtime) }) }, #[cfg(feature = "wasmtime")] - WasmExecutionMethod::Compiled => sc_executor_wasmtime::create_runtime( + WasmExecutionMethod::Compiled => sc_executor_wasmtime::create_runtime::( blob, sc_executor_wasmtime::Config { - heap_pages: heap_pages as u32, - max_memory_pages: None, + max_memory_size: None, allow_missing_func_imports, cache_path: cache_path.map(ToOwned::to_owned), semantics: sc_executor_wasmtime::Semantics { + extra_heap_pages: heap_pages, fast_instance_reuse: true, deterministic_stack_limit: None, canonicalize_nans: false, + parallel_compilation: true, }, }, - host_functions, ) .map(|runtime| -> Arc { Arc::new(runtime) }), } } fn decode_version(mut version: &[u8]) -> Result { - let v: RuntimeVersion = sp_api::OldRuntimeVersion::decode(&mut &version[..]) - .map_err(|_| { - WasmError::Instantiation( - "failed to decode \"Core_version\" result using old runtime version".into(), - ) - })? - .into(); - - let core_api_id = sp_core::hashing::blake2_64(b"Core"); - if v.has_api_with(&core_api_id, |v| v >= 3) { - sp_api::RuntimeVersion::decode(&mut version).map_err(|_| { - WasmError::Instantiation("failed to decode \"Core_version\" result".into()) - }) - } else { - Ok(v) - } + Decode::decode(&mut version).map_err(|_| { + WasmError::Instantiation( + "failed to decode \"Core_version\" result using old runtime version".into(), + ) + }) } fn decode_runtime_apis(apis: &[u8]) -> Result, WasmError> { use sp_api::RUNTIME_API_INFO_SIZE; - use std::convert::TryFrom; apis.chunks(RUNTIME_API_INFO_SIZE) .map(|chunk| { @@ -375,17 +362,25 @@ fn decode_runtime_apis(apis: &[u8]) -> Result, WasmError> { /// sections, `Err` will be returned. pub fn read_embedded_version(blob: &RuntimeBlob) -> Result, WasmError> { if let Some(mut version_section) = blob.custom_section_contents("runtime_version") { - // We do not use `decode_version` here because the runtime_version section is not supposed - // to ever contain a legacy version. Apart from that `decode_version` relies on presence - // of a special API in the `apis` field to treat the input as a non-legacy version. However - // the structure found in the `runtime_version` always contain an empty `apis` field. - // Therefore the version read will be mistakenly treated as an legacy one. - let mut decoded_version = sp_api::RuntimeVersion::decode(&mut version_section) - .map_err(|_| WasmError::Instantiation("failed to decode version section".into()))?; - - // Don't stop on this and check if there is a special section that encodes all runtime APIs. - if let Some(apis_section) = blob.custom_section_contents("runtime_apis") { - decoded_version.apis = decode_runtime_apis(apis_section)?.into(); + let apis = blob + .custom_section_contents("runtime_apis") + .map(decode_runtime_apis) + .transpose()? + .map(Into::into); + + let core_version = apis.as_ref().and_then(|apis| sp_version::core_version_from_apis(apis)); + // We do not use `RuntimeVersion::decode` here because that `decode_version` relies on + // presence of a special API in the `apis` field to treat the input as a non-legacy version. + // However the structure found in the `runtime_version` always contain an empty `apis` + // field. Therefore the version read will be mistakenly treated as an legacy one. + let mut decoded_version = sp_version::RuntimeVersion::decode_with_version_hint( + &mut version_section, + core_version, + ) + .map_err(|_| WasmError::Instantiation("failed to decode version section".into()))?; + + if let Some(apis) = apis { + decoded_version.apis = apis; } Ok(Some(decoded_version)) @@ -394,17 +389,18 @@ pub fn read_embedded_version(blob: &RuntimeBlob) -> Result( code: &[u8], - code_hash: Vec, ext: &mut dyn Externalities, wasm_method: WasmExecutionMethod, heap_pages: u64, - host_functions: Vec<&'static dyn Function>, allow_missing_func_imports: bool, max_instances: usize, cache_path: Option<&Path>, -) -> Result { +) -> Result +where + H: HostFunctions, +{ // The incoming code may be actually compressed. We decompress it here and then work with // the uncompressed code from now on. let blob = sc_executor_common::runtime_blob::RuntimeBlob::uncompress_if_needed(&code)?; @@ -414,11 +410,10 @@ fn create_versioned_wasm_runtime( // runtime. let mut version: Option<_> = read_embedded_version(&blob)?; - let runtime = create_wasm_runtime_with_code( + let runtime = create_wasm_runtime_with_code::( wasm_method, heap_pages, blob, - host_functions, allow_missing_func_imports, cache_path, )?; @@ -449,7 +444,7 @@ fn create_versioned_wasm_runtime( let mut instances = Vec::with_capacity(max_instances); instances.resize_with(max_instances, || Mutex::new(None)); - Ok(VersionedRuntime { code_hash, module: runtime, version, heap_pages, wasm_method, instances }) + Ok(VersionedRuntime { module: runtime, version, instances: Arc::new(instances) }) } #[cfg(test)] @@ -457,9 +452,20 @@ mod tests { use super::*; use codec::Encode; use sp_api::{Core, RuntimeApiInfo}; + use sp_runtime::RuntimeString; use sp_wasm_interface::HostFunctions; use substrate_test_runtime::Block; + #[derive(Encode)] + pub struct OldRuntimeVersion { + pub spec_name: RuntimeString, + pub impl_name: RuntimeString, + pub authoring_version: u32, + pub spec_version: u32, + pub impl_version: u32, + pub apis: sp_version::ApisVec, + } + #[test] fn host_functions_are_equal() { let host_functions = sp_io::SubstrateHostFunctions::host_functions(); @@ -470,7 +476,7 @@ mod tests { #[test] fn old_runtime_version_decodes() { - let old_runtime_version = sp_api::OldRuntimeVersion { + let old_runtime_version = OldRuntimeVersion { spec_name: "test".into(), impl_name: "test".into(), authoring_version: 1, @@ -481,11 +487,12 @@ mod tests { let version = decode_version(&old_runtime_version.encode()).unwrap(); assert_eq!(1, version.transaction_version); + assert_eq!(0, version.state_version); } #[test] fn old_runtime_version_decodes_fails_with_version_3() { - let old_runtime_version = sp_api::OldRuntimeVersion { + let old_runtime_version = OldRuntimeVersion { spec_name: "test".into(), impl_name: "test".into(), authoring_version: 1, @@ -507,10 +514,27 @@ mod tests { impl_version: 1, apis: sp_api::create_apis_vec!([(>::ID, 3)]), transaction_version: 3, + state_version: 4, }; let version = decode_version(&old_runtime_version.encode()).unwrap(); assert_eq!(3, version.transaction_version); + assert_eq!(0, version.state_version); + + let old_runtime_version = sp_api::RuntimeVersion { + spec_name: "test".into(), + impl_name: "test".into(), + authoring_version: 1, + spec_version: 1, + impl_version: 1, + apis: sp_api::create_apis_vec!([(>::ID, 4)]), + transaction_version: 3, + state_version: 4, + }; + + let version = decode_version(&old_runtime_version.encode()).unwrap(); + assert_eq!(3, version.transaction_version); + assert_eq!(4, version.state_version); } #[test] @@ -520,15 +544,15 @@ mod tests { sp_maybe_compressed_blob::CODE_BLOB_BOMB_LIMIT, ) .expect("Decompressing works"); - let runtime_version = RuntimeVersion { spec_name: "test_replace".into(), impl_name: "test_replace".into(), authoring_version: 100, spec_version: 100, impl_version: 100, - apis: sp_api::create_apis_vec!([(>::ID, 3)]), + apis: sp_api::create_apis_vec!([(>::ID, 4)]), transaction_version: 100, + state_version: 1, }; let embedded = sp_version::embed::embed_runtime_version(&wasm, runtime_version.clone()) diff --git a/client/executor/wasmi/Cargo.toml b/client/executor/wasmi/Cargo.toml index 324b2bdd0bae..cab254f1c71f 100644 --- a/client/executor/wasmi/Cargo.toml +++ b/client/executor/wasmi/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-executor-wasmi" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "This crate provides an implementation of `WasmRuntime` that is baked by wasmi." documentation = "https://docs.rs/sc-executor-wasmi" @@ -15,11 +15,11 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] log = "0.4.8" -wasmi = "0.9.0" -codec = { package = "parity-scale-codec", version = "2.0.0" } +wasmi = "0.9.1" +codec = { package = "parity-scale-codec", version = "3.0.0" } sc-executor-common = { version = "0.10.0-dev", path = "../common" } -sc-allocator = { version = "4.0.0-dev", path = "../../allocator" } -sp-wasm-interface = { version = "4.0.0-dev", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "4.0.0-dev", path = "../../../primitives/runtime-interface" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } +sc-allocator = { version = "4.1.0-dev", path = "../../allocator" } +sp-wasm-interface = { version = "6.0.0", path = "../../../primitives/wasm-interface" } +sp-runtime-interface = { version = "6.0.0", path = "../../../primitives/runtime-interface" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } scoped-tls = "1.0" diff --git a/client/executor/wasmi/src/lib.rs b/client/executor/wasmi/src/lib.rs index ceab07c2f71c..97c73c3454a4 100644 --- a/client/executor/wasmi/src/lib.rs +++ b/client/executor/wasmi/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,7 +21,7 @@ use codec::{Decode, Encode}; use log::{debug, error, trace}; use sc_executor_common::{ - error::{Error, WasmError}, + error::{Error, MessageWithBacktrace, WasmError}, runtime_blob::{DataSegmentsSnapshot, RuntimeBlob}, sandbox, util::MemoryTransfer, @@ -48,6 +48,7 @@ struct FunctionExecutor { host_functions: Arc>, allow_missing_func_imports: bool, missing_functions: Arc>, + panic_message: Option, } impl FunctionExecutor { @@ -69,6 +70,7 @@ impl FunctionExecutor { host_functions, allow_missing_func_imports, missing_functions, + panic_message: None, }) } } @@ -100,7 +102,7 @@ impl<'a> sandbox::SandboxContext for SandboxContext<'a> { match result { Ok(Some(RuntimeValue::I64(val))) => Ok(val), Ok(_) => return Err("Supervisor function returned unexpected result!".into()), - Err(err) => Err(Error::Trap(err)), + Err(err) => Err(Error::Sandbox(err.to_string())), } } @@ -133,6 +135,10 @@ impl FunctionContext for FunctionExecutor { fn sandbox(&mut self) -> &mut dyn Sandbox { self } + + fn register_panic_error_message(&mut self, message: &str) { + self.panic_message = Some(message.to_owned()); + } } impl Sandbox for FunctionExecutor { @@ -213,7 +219,6 @@ impl Sandbox for FunctionExecutor { let args = Vec::::decode(&mut args) .map_err(|_| "Can't decode serialized arguments for the invocation")? .into_iter() - .map(Into::into) .collect::>(); let instance = @@ -502,12 +507,31 @@ fn call_in_wasm_module( let offset = function_executor.allocate_memory(data.len() as u32)?; function_executor.write_memory(offset, data)?; + fn convert_trap(executor: &mut FunctionExecutor, trap: wasmi::Trap) -> Error { + if let Some(message) = executor.panic_message.take() { + Error::AbortedDueToPanic(MessageWithBacktrace { message, backtrace: None }) + } else { + Error::AbortedDueToTrap(MessageWithBacktrace { + message: trap.to_string(), + backtrace: None, + }) + } + } + let result = match method { - InvokeMethod::Export(method) => module_instance.invoke_export( - method, - &[I32(u32::from(offset) as i32), I32(data.len() as i32)], - &mut function_executor, - ), + InvokeMethod::Export(method) => module_instance + .invoke_export( + method, + &[I32(u32::from(offset) as i32), I32(data.len() as i32)], + &mut function_executor, + ) + .map_err(|error| { + if let wasmi::Error::Trap(trap) = error { + convert_trap(&mut function_executor, trap) + } else { + error.into() + } + }), InvokeMethod::Table(func_ref) => { let func = table .ok_or(Error::NoTable)? @@ -518,7 +542,7 @@ fn call_in_wasm_module( &[I32(u32::from(offset) as i32), I32(data.len() as i32)], &mut function_executor, ) - .map_err(Into::into) + .map_err(|trap| convert_trap(&mut function_executor, trap)) }, InvokeMethod::TableWithWrapper { dispatcher_ref, func } => { let dispatcher = table @@ -531,7 +555,7 @@ fn call_in_wasm_module( &[I32(func as _), I32(u32::from(offset) as i32), I32(data.len() as i32)], &mut function_executor, ) - .map_err(Into::into) + .map_err(|trap| convert_trap(&mut function_executor, trap)) }, }; diff --git a/client/executor/wasmtime/Cargo.toml b/client/executor/wasmtime/Cargo.toml index edf174752426..f0155204f544 100644 --- a/client/executor/wasmtime/Cargo.toml +++ b/client/executor/wasmtime/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-executor-wasmtime" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Defines a `WasmRuntime` that uses the Wasmtime JIT to execute." readme = "README.md" @@ -13,19 +13,24 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -libc = "0.2.90" +libc = "0.2.121" cfg-if = "1.0" log = "0.4.8" parity-wasm = "0.42.0" -codec = { package = "parity-scale-codec", version = "2.0.0" } +codec = { package = "parity-scale-codec", version = "3.0.0" } sc-executor-common = { version = "0.10.0-dev", path = "../common" } -sp-wasm-interface = { version = "4.0.0-dev", path = "../../../primitives/wasm-interface" } -sp-runtime-interface = { version = "4.0.0-dev", path = "../../../primitives/runtime-interface" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sc-allocator = { version = "4.0.0-dev", path = "../../allocator" } -wasmtime = { version = "0.29.0", default-features = false, features = ["cache", "parallel-compilation"] } +sp-wasm-interface = { version = "6.0.0", path = "../../../primitives/wasm-interface", features = ["wasmtime"] } +sp-runtime-interface = { version = "6.0.0", path = "../../../primitives/runtime-interface" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sc-allocator = { version = "4.1.0-dev", path = "../../allocator" } +wasmtime = { version = "0.33.0", default-features = false, features = [ + "cache", + "cranelift", + "jitdump", + "parallel-compilation", +] } [dev-dependencies] sc-runtime-test = { version = "2.0.0", path = "../runtime-test" } -sp-io = { version = "4.0.0-dev", path = "../../../primitives/io" } +sp-io = { version = "6.0.0", path = "../../../primitives/io" } wat = "1.0" diff --git a/client/executor/wasmtime/build.rs b/client/executor/wasmtime/build.rs new file mode 100644 index 000000000000..c514b0041bc3 --- /dev/null +++ b/client/executor/wasmtime/build.rs @@ -0,0 +1,25 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use std::env; + +fn main() { + if let Ok(profile) = env::var("PROFILE") { + println!("cargo:rustc-cfg=build_type=\"{}\"", profile); + } +} diff --git a/client/executor/wasmtime/src/host.rs b/client/executor/wasmtime/src/host.rs index 4edb9f9c423f..23deacbf9362 100644 --- a/client/executor/wasmtime/src/host.rs +++ b/client/executor/wasmtime/src/host.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,7 +19,7 @@ //! This module defines `HostState` and `HostContext` structs which provide logic and state //! required for execution of host. -use crate::{instance_wrapper::InstanceWrapper, runtime::StoreData}; +use crate::{runtime::StoreData, util}; use codec::{Decode, Encode}; use log::trace; use sc_allocator::FreeingBumpHeapAllocator; @@ -30,110 +30,128 @@ use sc_executor_common::{ }; use sp_core::sandbox as sandbox_primitives; use sp_wasm_interface::{FunctionContext, MemoryId, Pointer, Sandbox, WordSize}; -use std::{cell::RefCell, rc::Rc}; use wasmtime::{Caller, Func, Val}; +// The sandbox store is inside of a Option>> so that we can temporarily borrow it. +struct SandboxStore(Option>>); + +// There are a bunch of `Rc`s within the sandbox store, however we only manipulate +// those within one thread so this should be safe. +unsafe impl Send for SandboxStore {} + /// The state required to construct a HostContext context. The context only lasts for one host /// call, whereas the state is maintained for the duration of a Wasm runtime call, which may make /// many different host calls that must share state. pub struct HostState { - /// We need some interior mutability here since the host state is shared between all host - /// function handlers and the wasmtime backend's `impl WasmRuntime`. - /// - /// Furthermore, because of recursive calls (e.g. runtime can create and call an sandboxed - /// instance which in turn can call the runtime back) we have to be very careful with borrowing - /// those. - /// - /// Basically, most of the interactions should do temporary borrow immediately releasing the - /// borrow after performing necessary queries/changes. - sandbox_store: Rc>>, - allocator: RefCell, - instance: Rc, + sandbox_store: SandboxStore, + allocator: FreeingBumpHeapAllocator, + panic_message: Option, } impl HostState { /// Constructs a new `HostState`. - pub fn new(allocator: FreeingBumpHeapAllocator, instance: Rc) -> Self { + pub fn new(allocator: FreeingBumpHeapAllocator) -> Self { HostState { - sandbox_store: Rc::new(RefCell::new(sandbox::Store::new( + sandbox_store: SandboxStore(Some(Box::new(sandbox::Store::new( sandbox::SandboxBackend::TryWasmer, - ))), - allocator: RefCell::new(allocator), - instance, + )))), + allocator, + panic_message: None, } } - /// Materialize `HostContext` that can be used to invoke a substrate host `dyn Function`. - pub(crate) fn materialize<'a, 'b, 'c>( - &'a self, - caller: &'b mut Caller<'c, StoreData>, - ) -> HostContext<'a, 'b, 'c> { - HostContext { host_state: self, caller } + /// Takes the error message out of the host state, leaving a `None` in its place. + pub fn take_panic_message(&mut self) -> Option { + self.panic_message.take() } } /// A `HostContext` implements `FunctionContext` for making host calls from a Wasmtime /// runtime. The `HostContext` exists only for the lifetime of the call and borrows state from /// a longer-living `HostState`. -pub(crate) struct HostContext<'a, 'b, 'c> { - host_state: &'a HostState, - caller: &'b mut Caller<'c, StoreData>, +pub(crate) struct HostContext<'a> { + pub(crate) caller: Caller<'a, StoreData>, } -impl<'a, 'b, 'c> std::ops::Deref for HostContext<'a, 'b, 'c> { - type Target = HostState; - fn deref(&self) -> &HostState { - self.host_state +impl<'a> HostContext<'a> { + fn host_state(&self) -> &HostState { + self.caller + .data() + .host_state() + .expect("host state is not empty when calling a function in wasm; qed") + } + + fn host_state_mut(&mut self) -> &mut HostState { + self.caller + .data_mut() + .host_state_mut() + .expect("host state is not empty when calling a function in wasm; qed") + } + + fn sandbox_store(&self) -> &sandbox::Store { + self.host_state() + .sandbox_store + .0 + .as_ref() + .expect("sandbox store is only empty when temporarily borrowed") + } + + fn sandbox_store_mut(&mut self) -> &mut sandbox::Store { + self.host_state_mut() + .sandbox_store + .0 + .as_mut() + .expect("sandbox store is only empty when temporarily borrowed") } } -impl<'a, 'b, 'c> sp_wasm_interface::FunctionContext for HostContext<'a, 'b, 'c> { +impl<'a> sp_wasm_interface::FunctionContext for HostContext<'a> { fn read_memory_into( &self, address: Pointer, dest: &mut [u8], ) -> sp_wasm_interface::Result<()> { - let ctx = &self.caller; - self.host_state - .instance - .read_memory_into(ctx, address, dest) - .map_err(|e| e.to_string()) + util::read_memory_into(&self.caller, address, dest).map_err(|e| e.to_string()) } fn write_memory(&mut self, address: Pointer, data: &[u8]) -> sp_wasm_interface::Result<()> { - let ctx = &mut self.caller; - self.host_state - .instance - .write_memory_from(ctx, address, data) - .map_err(|e| e.to_string()) + util::write_memory_from(&mut self.caller, address, data).map_err(|e| e.to_string()) } fn allocate_memory(&mut self, size: WordSize) -> sp_wasm_interface::Result> { - let ctx = &mut self.caller; - let allocator = &self.host_state.allocator; - - self.host_state - .instance - .allocate(ctx, &mut *allocator.borrow_mut(), size) + let memory = self.caller.data().memory(); + let (memory, data) = memory.data_and_store_mut(&mut self.caller); + data.host_state_mut() + .expect("host state is not empty when calling a function in wasm; qed") + .allocator + .allocate(memory, size) .map_err(|e| e.to_string()) } fn deallocate_memory(&mut self, ptr: Pointer) -> sp_wasm_interface::Result<()> { - let ctx = &mut self.caller; - let allocator = &self.host_state.allocator; - - self.host_state - .instance - .deallocate(ctx, &mut *allocator.borrow_mut(), ptr) + let memory = self.caller.data().memory(); + let (memory, data) = memory.data_and_store_mut(&mut self.caller); + data.host_state_mut() + .expect("host state is not empty when calling a function in wasm; qed") + .allocator + .deallocate(memory, ptr) .map_err(|e| e.to_string()) } fn sandbox(&mut self) -> &mut dyn Sandbox { self } + + fn register_panic_error_message(&mut self, message: &str) { + self.caller + .data_mut() + .host_state_mut() + .expect("host state is not empty when calling a function in wasm; qed") + .panic_message = Some(message.to_owned()); + } } -impl<'a, 'b, 'c> Sandbox for HostContext<'a, 'b, 'c> { +impl<'a> Sandbox for HostContext<'a> { fn memory_get( &mut self, memory_id: MemoryId, @@ -141,8 +159,7 @@ impl<'a, 'b, 'c> Sandbox for HostContext<'a, 'b, 'c> { buf_ptr: Pointer, buf_len: WordSize, ) -> sp_wasm_interface::Result { - let sandboxed_memory = - self.sandbox_store.borrow().memory(memory_id).map_err(|e| e.to_string())?; + let sandboxed_memory = self.sandbox_store().memory(memory_id).map_err(|e| e.to_string())?; let len = buf_len as usize; @@ -151,8 +168,7 @@ impl<'a, 'b, 'c> Sandbox for HostContext<'a, 'b, 'c> { Ok(buffer) => buffer, }; - let instance = self.instance.clone(); - if let Err(_) = instance.write_memory_from(&mut self.caller, buf_ptr, &buffer) { + if util::write_memory_from(&mut self.caller, buf_ptr, &buffer).is_err() { return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS) } @@ -166,17 +182,16 @@ impl<'a, 'b, 'c> Sandbox for HostContext<'a, 'b, 'c> { val_ptr: Pointer, val_len: WordSize, ) -> sp_wasm_interface::Result { - let sandboxed_memory = - self.sandbox_store.borrow().memory(memory_id).map_err(|e| e.to_string())?; + let sandboxed_memory = self.sandbox_store().memory(memory_id).map_err(|e| e.to_string())?; let len = val_len as usize; - let buffer = match self.instance.read_memory(&self.caller, val_ptr, len) { + let buffer = match util::read_memory(&self.caller, val_ptr, len) { Err(_) => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), Ok(buffer) => buffer, }; - if let Err(_) = sandboxed_memory.write_from(Pointer::new(offset as u32), &buffer) { + if sandboxed_memory.write_from(Pointer::new(offset as u32), &buffer).is_err() { return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS) } @@ -184,24 +199,18 @@ impl<'a, 'b, 'c> Sandbox for HostContext<'a, 'b, 'c> { } fn memory_teardown(&mut self, memory_id: MemoryId) -> sp_wasm_interface::Result<()> { - self.sandbox_store - .borrow_mut() - .memory_teardown(memory_id) - .map_err(|e| e.to_string()) + self.sandbox_store_mut().memory_teardown(memory_id).map_err(|e| e.to_string()) } fn memory_new(&mut self, initial: u32, maximum: u32) -> sp_wasm_interface::Result { - self.sandbox_store - .borrow_mut() - .new_memory(initial, maximum) - .map_err(|e| e.to_string()) + self.sandbox_store_mut().new_memory(initial, maximum).map_err(|e| e.to_string()) } fn invoke( &mut self, instance_id: u32, export_name: &str, - args: &[u8], + mut args: &[u8], return_val: Pointer, return_val_len: u32, state: u32, @@ -209,20 +218,15 @@ impl<'a, 'b, 'c> Sandbox for HostContext<'a, 'b, 'c> { trace!(target: "sp-sandbox", "invoke, instance_idx={}", instance_id); // Deserialize arguments and convert them into wasmi types. - let args = Vec::::decode(&mut &args[..]) + let args = Vec::::decode(&mut args) .map_err(|_| "Can't decode serialized arguments for the invocation")? .into_iter() - .map(Into::into) .collect::>(); - let instance = - self.sandbox_store.borrow().instance(instance_id).map_err(|e| e.to_string())?; + let instance = self.sandbox_store().instance(instance_id).map_err(|e| e.to_string())?; - let dispatch_thunk = self - .sandbox_store - .borrow() - .dispatch_thunk(instance_id) - .map_err(|e| e.to_string())?; + let dispatch_thunk = + self.sandbox_store().dispatch_thunk(instance_id).map_err(|e| e.to_string())?; let result = instance.invoke( export_name, @@ -249,8 +253,7 @@ impl<'a, 'b, 'c> Sandbox for HostContext<'a, 'b, 'c> { } fn instance_teardown(&mut self, instance_id: u32) -> sp_wasm_interface::Result<()> { - self.sandbox_store - .borrow_mut() + self.sandbox_store_mut() .instance_teardown(instance_id) .map_err(|e| e.to_string()) } @@ -264,14 +267,12 @@ impl<'a, 'b, 'c> Sandbox for HostContext<'a, 'b, 'c> { ) -> sp_wasm_interface::Result { // Extract a dispatch thunk from the instance's table by the specified index. let dispatch_thunk = { - let ctx = &mut self.caller; - let table_item = self - .host_state - .instance + let table = self + .caller + .data() .table() - .as_ref() - .ok_or_else(|| "Runtime doesn't have a table; sandbox is unavailable")? - .get(ctx, dispatch_thunk_id); + .ok_or_else(|| "Runtime doesn't have a table; sandbox is unavailable")?; + let table_item = table.get(&mut self.caller, dispatch_thunk_id); table_item .ok_or_else(|| "dispatch_thunk_id is out of bounds")? @@ -281,25 +282,39 @@ impl<'a, 'b, 'c> Sandbox for HostContext<'a, 'b, 'c> { .clone() }; - let guest_env = - match sandbox::GuestEnvironment::decode(&*self.sandbox_store.borrow(), raw_env_def) { - Ok(guest_env) => guest_env, - Err(_) => return Ok(sandbox_primitives::ERR_MODULE as u32), - }; + let guest_env = match sandbox::GuestEnvironment::decode(&self.sandbox_store(), raw_env_def) + { + Ok(guest_env) => guest_env, + Err(_) => return Ok(sandbox_primitives::ERR_MODULE as u32), + }; - let store = self.sandbox_store.clone(); - let store = &mut store.borrow_mut(); - let result = store - .instantiate( + let mut store = self + .host_state_mut() + .sandbox_store + .0 + .take() + .expect("sandbox store is only empty when borrowed"); + + // Catch any potential panics so that we can properly restore the sandbox store + // which we've destructively borrowed. + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + store.instantiate( wasm, guest_env, state, &mut SandboxContext { host_context: self, dispatch_thunk: dispatch_thunk.clone() }, ) - .map(|i| i.register(store, dispatch_thunk)); + })); + + self.host_state_mut().sandbox_store.0 = Some(store); + + let result = match result { + Ok(result) => result, + Err(error) => std::panic::resume_unwind(error), + }; let instance_idx_or_err_code = match result { - Ok(instance_idx) => instance_idx, + Ok(instance) => instance.register(&mut self.sandbox_store_mut(), dispatch_thunk), Err(sandbox::InstantiationError::StartTrapped) => sandbox_primitives::ERR_EXECUTION, Err(_) => sandbox_primitives::ERR_MODULE, }; @@ -312,20 +327,19 @@ impl<'a, 'b, 'c> Sandbox for HostContext<'a, 'b, 'c> { instance_idx: u32, name: &str, ) -> sp_wasm_interface::Result> { - self.sandbox_store - .borrow() + self.sandbox_store() .instance(instance_idx) .map(|i| i.get_global_val(name)) .map_err(|e| e.to_string()) } } -struct SandboxContext<'a, 'b, 'c, 'd> { - host_context: &'a mut HostContext<'b, 'c, 'd>, +struct SandboxContext<'a, 'b> { + host_context: &'a mut HostContext<'b>, dispatch_thunk: Func, } -impl<'a, 'b, 'c, 'd> sandbox::SandboxContext for SandboxContext<'a, 'b, 'c, 'd> { +impl<'a, 'b> sandbox::SandboxContext for SandboxContext<'a, 'b> { fn invoke( &mut self, invoke_args_ptr: Pointer, @@ -333,6 +347,7 @@ impl<'a, 'b, 'c, 'd> sandbox::SandboxContext for SandboxContext<'a, 'b, 'c, 'd> state: u32, func_idx: SupervisorFuncIndex, ) -> Result { + let mut ret_vals = [Val::null()]; let result = self.dispatch_thunk.call( &mut self.host_context.caller, &[ @@ -341,26 +356,16 @@ impl<'a, 'b, 'c, 'd> sandbox::SandboxContext for SandboxContext<'a, 'b, 'c, 'd> Val::I32(state as i32), Val::I32(usize::from(func_idx) as i32), ], + &mut ret_vals, ); match result { - Ok(ret_vals) => { - let ret_val = if ret_vals.len() != 1 { - return Err(format!( - "Supervisor function returned {} results, expected 1", - ret_vals.len() - ) - .into()) - } else { - &ret_vals[0] - }; - - if let Some(ret_val) = ret_val.i64() { + Ok(()) => + if let Some(ret_val) = ret_vals[0].i64() { Ok(ret_val) } else { return Err("Supervisor function returned unexpected result!".into()) - } - }, + }, Err(err) => Err(err.to_string().into()), } } diff --git a/client/executor/wasmtime/src/imports.rs b/client/executor/wasmtime/src/imports.rs index a9ef6e1f58a7..4aad57102931 100644 --- a/client/executor/wasmtime/src/imports.rs +++ b/client/executor/wasmtime/src/imports.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,36 +16,23 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::{ - runtime::{Store, StoreData}, - util, -}; +use crate::{host::HostContext, runtime::StoreData}; use sc_executor_common::error::WasmError; -use sp_wasm_interface::{Function, ValueType}; -use std::any::Any; -use wasmtime::{ - Caller, Extern, ExternType, Func, FuncType, ImportType, Limits, Memory, MemoryType, Module, - Trap, Val, -}; - -pub struct Imports { - /// Contains the index into `externs` where the memory import is stored if any. `None` if there - /// is none. - pub memory_import_index: Option, - pub externs: Vec, -} - -/// Goes over all imports of a module and prepares a vector of `Extern`s that can be used for -/// instantiation of the module. Returns an error if there are imports that cannot be satisfied. -pub(crate) fn resolve_imports( - store: &mut Store, +use sp_wasm_interface::{FunctionContext, HostFunctions}; +use std::collections::HashMap; +use wasmtime::{ExternType, FuncType, ImportType, Linker, Module, Trap}; + +/// Goes over all imports of a module and prepares the given linker for instantiation of the module. +/// Returns an error if there are imports that cannot be satisfied. +pub(crate) fn prepare_imports( + linker: &mut Linker, module: &Module, - host_functions: &[&'static dyn Function], - heap_pages: u32, allow_missing_func_imports: bool, -) -> Result { - let mut externs = vec![]; - let mut memory_import_index = None; +) -> Result<(), WasmError> +where + H: HostFunctions, +{ + let mut pending_func_imports = HashMap::new(); for import_ty in module.imports() { let name = import_name(&import_ty)?; @@ -57,238 +44,90 @@ pub(crate) fn resolve_imports( ))) } - let resolved = match name { - "memory" => { - memory_import_index = Some(externs.len()); - resolve_memory_import(store, &import_ty, heap_pages)? + match import_ty.ty() { + ExternType::Func(func_ty) => { + pending_func_imports.insert(name.to_owned(), (import_ty, func_ty)); }, _ => - resolve_func_import(store, &import_ty, host_functions, allow_missing_func_imports)?, + return Err(WasmError::Other(format!( + "host doesn't provide any non function imports: {}:{}", + import_ty.module(), + name, + ))), }; - externs.push(resolved); } - Ok(Imports { memory_import_index, externs }) -} - -/// When the module linking proposal is supported the import's name can be `None`. -/// Because we are not using this proposal we could safely unwrap the name. -/// However, we opt for an error in order to avoid panics at all costs. -fn import_name<'a, 'b: 'a>(import: &'a ImportType<'b>) -> Result<&'a str, WasmError> { - let name = import.name().ok_or_else(|| { - WasmError::Other("The module linking proposal is not supported.".to_owned()) - })?; - Ok(name) -} -fn resolve_memory_import( - store: &mut Store, - import_ty: &ImportType, - heap_pages: u32, -) -> Result { - let requested_memory_ty = match import_ty.ty() { - ExternType::Memory(memory_ty) => memory_ty, - _ => + let mut registry = Registry { linker, pending_func_imports }; + H::register_static(&mut registry)?; + + if !registry.pending_func_imports.is_empty() { + if allow_missing_func_imports { + for (name, (import_ty, func_ty)) in registry.pending_func_imports { + let error = format!("call to a missing function {}:{}", import_ty.module(), name); + log::debug!("Missing import: '{}' {:?}", name, func_ty); + linker + .func_new("env", &name, func_ty.clone(), move |_, _, _| { + Err(Trap::new(error.clone())) + }) + .expect("adding a missing import stub can only fail when the item already exists, and it is missing here; qed"); + } + } else { + let mut names = Vec::new(); + for (name, (import_ty, _)) in registry.pending_func_imports { + names.push(format!("'{}:{}'", import_ty.module(), name)); + } + let names = names.join(", "); return Err(WasmError::Other(format!( - "this import must be of memory type: {}:{}", - import_ty.module(), - import_name(&import_ty)?, - ))), - }; - - // Increment the min (a.k.a initial) number of pages by `heap_pages` and check if it exceeds the - // maximum specified by the import. - let initial = requested_memory_ty.limits().min().saturating_add(heap_pages); - if let Some(max) = requested_memory_ty.limits().max() { - if initial > max { - return Err(WasmError::Other(format!( - "incremented number of pages by heap_pages (total={}) is more than maximum requested\ - by the runtime wasm module {}", - initial, - max, + "runtime requires function imports which are not present on the host: {}", + names ))) } } - let memory_ty = MemoryType::new(Limits::new(initial, requested_memory_ty.limits().max())); - let memory = Memory::new(store, memory_ty).map_err(|e| { - WasmError::Other(format!( - "failed to create a memory during resolving of memory import: {}", - e, - )) - })?; - Ok(Extern::Memory(memory)) -} - -fn resolve_func_import( - store: &mut Store, - import_ty: &ImportType, - host_functions: &[&'static dyn Function], - allow_missing_func_imports: bool, -) -> Result { - let name = import_name(&import_ty)?; - - let func_ty = match import_ty.ty() { - ExternType::Func(func_ty) => func_ty, - _ => - return Err(WasmError::Other(format!( - "host doesn't provide any non function imports besides 'memory': {}:{}", - import_ty.module(), - name, - ))), - }; - - let host_func = match host_functions.iter().find(|host_func| host_func.name() == name) { - Some(host_func) => host_func, - None if allow_missing_func_imports => - return Ok(MissingHostFuncHandler::new(import_ty)?.into_extern(store, &func_ty)), - None => - return Err(WasmError::Other(format!( - "host doesn't provide such function: {}:{}", - import_ty.module(), - name, - ))), - }; - if &func_ty != &wasmtime_func_sig(*host_func) { - return Err(WasmError::Other(format!( - "signature mismatch for: {}:{}", - import_ty.module(), - name, - ))) - } - - Ok(HostFuncHandler::new(*host_func).into_extern(store)) + Ok(()) } -/// This structure implements `Callable` and acts as a bridge between wasmtime and -/// substrate host functions. -struct HostFuncHandler { - host_func: &'static dyn Function, +struct Registry<'a, 'b> { + linker: &'a mut Linker, + pending_func_imports: HashMap, FuncType)>, } -fn call_static<'a>( - static_func: &'static dyn Function, - wasmtime_params: &[Val], - wasmtime_results: &mut [Val], - mut caller: Caller<'a, StoreData>, -) -> Result<(), wasmtime::Trap> { - let unwind_result = { - let host_state = caller - .data() - .host_state() - .expect( - "host functions can be called only from wasm instance; - wasm instance is always called initializing context; - therefore host_ctx cannot be None; - qed - ", - ) - .clone(); +impl<'a, 'b> sp_wasm_interface::HostFunctionRegistry for Registry<'a, 'b> { + type State = StoreData; + type Error = WasmError; + type FunctionContext = HostContext<'a>; - let mut host_ctx = host_state.materialize(&mut caller); - - // `from_wasmtime_val` panics if it encounters a value that doesn't fit into the values - // available in substrate. - // - // This, however, cannot happen since the signature of this function is created from - // a `dyn Function` signature of which cannot have a non substrate value by definition. - let mut params = wasmtime_params.iter().cloned().map(util::from_wasmtime_val); - - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - static_func.execute(&mut host_ctx, &mut params) - })) - }; - - let execution_result = match unwind_result { - Ok(execution_result) => execution_result, - Err(err) => return Err(Trap::new(stringify_panic_payload(err))), - }; - - match execution_result { - Ok(Some(ret_val)) => { - debug_assert!( - wasmtime_results.len() == 1, - "wasmtime function signature, therefore the number of results, should always \ - correspond to the number of results returned by the host function", - ); - wasmtime_results[0] = util::into_wasmtime_val(ret_val); - Ok(()) - }, - Ok(None) => { - debug_assert!( - wasmtime_results.len() == 0, - "wasmtime function signature, therefore the number of results, should always \ - correspond to the number of results returned by the host function", - ); - Ok(()) - }, - Err(msg) => Err(Trap::new(msg)), - } -} - -impl HostFuncHandler { - fn new(host_func: &'static dyn Function) -> Self { - Self { host_func } - } - - fn into_extern(self, store: &mut Store) -> Extern { - let host_func = self.host_func; - let func_ty = wasmtime_func_sig(self.host_func); - let func = Func::new(store, func_ty, move |caller, params, result| { - call_static(host_func, params, result, caller) - }); - Extern::Func(func) - } -} - -/// A `Callable` handler for missing functions. -struct MissingHostFuncHandler { - module: String, - name: String, -} - -impl MissingHostFuncHandler { - fn new(import_ty: &ImportType) -> Result { - Ok(Self { - module: import_ty.module().to_string(), - name: import_name(import_ty)?.to_string(), - }) - } - - fn into_extern(self, store: &mut Store, func_ty: &FuncType) -> Extern { - let Self { module, name } = self; - let func = Func::new(store, func_ty.clone(), move |_, _, _| { - Err(Trap::new(format!("call to a missing function {}:{}", module, name))) - }); - Extern::Func(func) + fn with_function_context( + caller: wasmtime::Caller, + callback: impl FnOnce(&mut dyn FunctionContext) -> R, + ) -> R { + callback(&mut HostContext { caller }) } -} - -fn wasmtime_func_sig(func: &dyn Function) -> wasmtime::FuncType { - let signature = func.signature(); - let params = signature.args.iter().cloned().map(into_wasmtime_val_type); - - let results = signature.return_value.iter().cloned().map(into_wasmtime_val_type); - wasmtime::FuncType::new(params, results) -} + fn register_static( + &mut self, + fn_name: &str, + func: impl wasmtime::IntoFunc, + ) -> Result<(), Self::Error> { + if self.pending_func_imports.remove(fn_name).is_some() { + self.linker.func_wrap("env", fn_name, func).map_err(|error| { + WasmError::Other(format!( + "failed to register host function '{}' with the WASM linker: {}", + fn_name, error + )) + })?; + } -fn into_wasmtime_val_type(val_ty: ValueType) -> wasmtime::ValType { - match val_ty { - ValueType::I32 => wasmtime::ValType::I32, - ValueType::I64 => wasmtime::ValType::I64, - ValueType::F32 => wasmtime::ValType::F32, - ValueType::F64 => wasmtime::ValType::F64, + Ok(()) } } -/// Attempt to convert a opaque panic payload to a string. -fn stringify_panic_payload(payload: Box) -> String { - match payload.downcast::<&'static str>() { - Ok(msg) => msg.to_string(), - Err(payload) => match payload.downcast::() { - Ok(msg) => *msg, - // At least we tried... - Err(_) => "Box".to_string(), - }, - } +/// When the module linking proposal is supported the import's name can be `None`. +/// Because we are not using this proposal we could safely unwrap the name. +/// However, we opt for an error in order to avoid panics at all costs. +fn import_name<'a, 'b: 'a>(import: &'a ImportType<'b>) -> Result<&'a str, WasmError> { + let name = import.name().ok_or_else(|| { + WasmError::Other("The module linking proposal is not supported.".to_owned()) + })?; + Ok(name) } diff --git a/client/executor/wasmtime/src/instance_wrapper.rs b/client/executor/wasmtime/src/instance_wrapper.rs index ccfbb912b9a6..6abcbca1bba6 100644 --- a/client/executor/wasmtime/src/instance_wrapper.rs +++ b/client/executor/wasmtime/src/instance_wrapper.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,17 +19,15 @@ //! Defines data and logic needed for interaction with an WebAssembly instance of a substrate //! runtime module. -use crate::imports::Imports; - +use crate::runtime::{Store, StoreData}; use sc_executor_common::{ - error::{Error, Result}, - util::checked_range, + error::{Backtrace, Error, MessageWithBacktrace, Result, WasmError}, wasm_runtime::InvokeMethod, }; use sp_wasm_interface::{Pointer, Value, WordSize}; -use std::marker; use wasmtime::{ - AsContext, AsContextMut, Extern, Func, Global, Instance, Memory, Module, Table, Val, + AsContext, AsContextMut, Engine, Extern, Func, Global, Instance, InstancePre, Memory, Table, + Val, }; /// Invoked entrypoint format. @@ -56,25 +54,51 @@ pub struct EntryPoint { impl EntryPoint { /// Call this entry point. - pub fn call( + pub(crate) fn call( &self, - ctx: impl AsContextMut, + store: &mut Store, data_ptr: Pointer, data_len: WordSize, ) -> Result { let data_ptr = u32::from(data_ptr); let data_len = u32::from(data_len); - fn handle_trap(err: wasmtime::Trap) -> Error { - Error::from(format!("Wasm execution trapped: {}", err)) - } - match self.call_type { EntryPointType::Direct { ref entrypoint } => - entrypoint.call(ctx, (data_ptr, data_len)).map_err(handle_trap), + entrypoint.call(&mut *store, (data_ptr, data_len)), EntryPointType::Wrapped { func, ref dispatcher } => - dispatcher.call(ctx, (func, data_ptr, data_len)).map_err(handle_trap), + dispatcher.call(&mut *store, (func, data_ptr, data_len)), } + .map_err(|trap| { + let host_state = store + .data_mut() + .host_state + .as_mut() + .expect("host state cannot be empty while a function is being called; qed"); + + // The logic to print out a backtrace is somewhat complicated, + // so let's get wasmtime to print it out for us. + let mut backtrace_string = trap.to_string(); + let suffix = "\nwasm backtrace:"; + if let Some(index) = backtrace_string.find(suffix) { + // Get rid of the error message and just grab the backtrace, + // since we're storing the error message ourselves separately. + backtrace_string.replace_range(0..index + suffix.len(), ""); + } + + let backtrace = Backtrace { backtrace_string }; + if let Some(error) = host_state.take_panic_message() { + Error::AbortedDueToPanic(MessageWithBacktrace { + message: error, + backtrace: Some(backtrace), + }) + } else { + Error::AbortedDueToTrap(MessageWithBacktrace { + message: trap.display_reason().to_string(), + backtrace: Some(backtrace), + }) + } + }) } pub fn direct( @@ -107,18 +131,8 @@ impl EntryPoint { /// routines. pub struct InstanceWrapper { instance: Instance, - - // The memory instance of the `instance`. - // - // It is important to make sure that we don't make any copies of this to make it easier to - // proof See `memory_as_slice` and `memory_as_slice_mut`. memory: Memory, - - /// Indirect functions table of the module - table: Option, - - // Make this struct explicitly !Send & !Sync. - _not_send_nor_sync: marker::PhantomData<*const ()>, + store: Store, } fn extern_memory(extern_: &Extern) -> Option<&Memory> { @@ -149,71 +163,77 @@ fn extern_func(extern_: &Extern) -> Option<&Func> { } } +pub(crate) fn create_store(engine: &wasmtime::Engine, max_memory_size: Option) -> Store { + let limits = if let Some(max_memory_size) = max_memory_size { + wasmtime::StoreLimitsBuilder::new().memory_size(max_memory_size).build() + } else { + Default::default() + }; + + let mut store = + Store::new(engine, StoreData { limits, host_state: None, memory: None, table: None }); + if max_memory_size.is_some() { + store.limiter(|s| &mut s.limits); + } + store +} + impl InstanceWrapper { - /// Create a new instance wrapper from the given wasm module. - pub fn new( - module: &Module, - imports: &Imports, - heap_pages: u32, - mut ctx: impl AsContextMut, + pub(crate) fn new( + engine: &Engine, + instance_pre: &InstancePre, + max_memory_size: Option, ) -> Result { - let instance = Instance::new(&mut ctx, module, &imports.externs) - .map_err(|e| Error::from(format!("cannot instantiate: {}", e)))?; - - let memory = match imports.memory_import_index { - Some(memory_idx) => extern_memory(&imports.externs[memory_idx]) - .expect("only memory can be at the `memory_idx`; qed") - .clone(), - None => { - let memory = get_linear_memory(&instance, &mut ctx)?; - if !memory.grow(&mut ctx, heap_pages).is_ok() { - return Err("failed top increase the linear memory size".into()) - } - memory - }, - }; + let mut store = create_store(engine, max_memory_size); + let instance = instance_pre.instantiate(&mut store).map_err(|error| { + WasmError::Other( + format!("failed to instantiate a new WASM module instance: {}", error,), + ) + })?; + + let memory = get_linear_memory(&instance, &mut store)?; + let table = get_table(&instance, &mut store); - let table = get_table(&instance, ctx); + store.data_mut().memory = Some(memory); + store.data_mut().table = table; - Ok(Self { table, instance, memory, _not_send_nor_sync: marker::PhantomData }) + Ok(InstanceWrapper { instance, memory, store }) } /// Resolves a substrate entrypoint by the given name. /// /// An entrypoint must have a signature `(i32, i32) -> i64`, otherwise this function will return /// an error. - pub fn resolve_entrypoint( - &self, - method: InvokeMethod, - mut ctx: impl AsContextMut, - ) -> Result { + pub fn resolve_entrypoint(&mut self, method: InvokeMethod) -> Result { Ok(match method { InvokeMethod::Export(method) => { // Resolve the requested method and verify that it has a proper signature. - let export = self.instance.get_export(&mut ctx, method).ok_or_else(|| { - Error::from(format!("Exported method {} is not found", method)) - })?; + let export = + self.instance.get_export(&mut self.store, method).ok_or_else(|| { + Error::from(format!("Exported method {} is not found", method)) + })?; let func = extern_func(&export) .ok_or_else(|| Error::from(format!("Export {} is not a function", method)))? .clone(); - EntryPoint::direct(func, ctx).map_err(|_| { + EntryPoint::direct(func, &self.store).map_err(|_| { Error::from(format!("Exported function '{}' has invalid signature.", method)) })? }, InvokeMethod::Table(func_ref) => { let table = self .instance - .get_table(&mut ctx, "__indirect_function_table") + .get_table(&mut self.store, "__indirect_function_table") .ok_or(Error::NoTable)?; - let val = - table.get(&mut ctx, func_ref).ok_or(Error::NoTableEntryWithIndex(func_ref))?; + let val = table + .get(&mut self.store, func_ref) + .ok_or(Error::NoTableEntryWithIndex(func_ref))?; let func = val .funcref() .ok_or(Error::TableElementIsNotAFunction(func_ref))? .ok_or(Error::FunctionRefIsNull(func_ref))? .clone(); - EntryPoint::direct(func, ctx).map_err(|_| { + EntryPoint::direct(func, &self.store).map_err(|_| { Error::from(format!( "Function @{} in exported table has invalid signature for direct call.", func_ref, @@ -223,10 +243,10 @@ impl InstanceWrapper { InvokeMethod::TableWithWrapper { dispatcher_ref, func } => { let table = self .instance - .get_table(&mut ctx, "__indirect_function_table") + .get_table(&mut self.store, "__indirect_function_table") .ok_or(Error::NoTable)?; let val = table - .get(&mut ctx, dispatcher_ref) + .get(&mut self.store, dispatcher_ref) .ok_or(Error::NoTableEntryWithIndex(dispatcher_ref))?; let dispatcher = val .funcref() @@ -234,7 +254,7 @@ impl InstanceWrapper { .ok_or(Error::FunctionRefIsNull(dispatcher_ref))? .clone(); - EntryPoint::wrapped(dispatcher, func, ctx).map_err(|_| { + EntryPoint::wrapped(dispatcher, func, &self.store).map_err(|_| { Error::from(format!( "Function @{} in exported table has invalid signature for wrapped call.", dispatcher_ref, @@ -244,25 +264,20 @@ impl InstanceWrapper { }) } - /// Returns an indirect function table of this instance. - pub fn table(&self) -> Option<&Table> { - self.table.as_ref() - } - /// Reads `__heap_base: i32` global variable and returns it. /// /// If it doesn't exist, not a global or of not i32 type returns an error. - pub fn extract_heap_base(&self, mut ctx: impl AsContextMut) -> Result { + pub fn extract_heap_base(&mut self) -> Result { let heap_base_export = self .instance - .get_export(&mut ctx, "__heap_base") + .get_export(&mut self.store, "__heap_base") .ok_or_else(|| Error::from("__heap_base is not found"))?; let heap_base_global = extern_global(&heap_base_export) .ok_or_else(|| Error::from("__heap_base is not a global"))?; let heap_base = heap_base_global - .get(&mut ctx) + .get(&mut self.store) .i32() .ok_or_else(|| Error::from("__heap_base is not a i32"))?; @@ -270,15 +285,15 @@ impl InstanceWrapper { } /// Get the value from a global with the given `name`. - pub fn get_global_val(&self, mut ctx: impl AsContextMut, name: &str) -> Result> { - let global = match self.instance.get_export(&mut ctx, name) { + pub fn get_global_val(&mut self, name: &str) -> Result> { + let global = match self.instance.get_export(&mut self.store, name) { Some(global) => global, None => return Ok(None), }; let global = extern_global(&global).ok_or_else(|| format!("`{}` is not a global", name))?; - match global.get(ctx) { + match global.get(&mut self.store) { Val::I32(val) => Ok(Some(Value::I32(val))), Val::I64(val) => Ok(Some(Value::I64(val))), Val::F32(val) => Ok(Some(Value::F32(val))), @@ -288,8 +303,8 @@ impl InstanceWrapper { } /// Get a global with the given `name`. - pub fn get_global(&self, ctx: impl AsContextMut, name: &str) -> Option { - self.instance.get_global(ctx, name) + pub fn get_global(&mut self, name: &str) -> Option { + self.instance.get_global(&mut self.store, name) } } @@ -307,7 +322,7 @@ fn get_linear_memory(instance: &Instance, ctx: impl AsContextMut) -> Result Option
{ +fn get_table(instance: &Instance, ctx: &mut Store) -> Option
{ instance .get_export(ctx, "__indirect_function_table") .as_ref() @@ -317,97 +332,16 @@ fn get_table(instance: &Instance, ctx: impl AsContextMut) -> Option
{ /// Functions related to memory. impl InstanceWrapper { - /// Read data from a slice of memory into a newly allocated buffer. - /// - /// Returns an error if the read would go out of the memory bounds. - pub fn read_memory( - &self, - ctx: impl AsContext, - source_addr: Pointer, - size: usize, - ) -> Result> { - let range = checked_range(source_addr.into(), size, self.memory.data_size(&ctx)) - .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; - - let mut buffer = vec![0; range.len()]; - self.read_memory_into(ctx, source_addr, &mut buffer)?; - - Ok(buffer) - } - - /// Read data from the instance memory into a slice. - /// - /// Returns an error if the read would go out of the memory bounds. - pub fn read_memory_into( - &self, - ctx: impl AsContext, - address: Pointer, - dest: &mut [u8], - ) -> Result<()> { - let memory = self.memory.data(ctx.as_context()); - - let range = checked_range(address.into(), dest.len(), memory.len()) - .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; - dest.copy_from_slice(&memory[range]); - Ok(()) - } - - /// Write data to the instance memory from a slice. - /// - /// Returns an error if the write would go out of the memory bounds. - pub fn write_memory_from( - &self, - mut ctx: impl AsContextMut, - address: Pointer, - data: &[u8], - ) -> Result<()> { - let memory = self.memory.data_mut(ctx.as_context_mut()); - - let range = checked_range(address.into(), data.len(), memory.len()) - .ok_or_else(|| Error::Other("memory write is out of bounds".into()))?; - memory[range].copy_from_slice(data); - Ok(()) - } - - /// Allocate some memory of the given size. Returns pointer to the allocated memory region. - /// - /// Returns `Err` in case memory cannot be allocated. Refer to the allocator documentation - /// to get more details. - pub fn allocate( - &self, - mut ctx: impl AsContextMut, - allocator: &mut sc_allocator::FreeingBumpHeapAllocator, - size: WordSize, - ) -> Result> { - let memory = self.memory.data_mut(ctx.as_context_mut()); - - allocator.allocate(memory, size).map_err(Into::into) - } - - /// Deallocate the memory pointed by the given pointer. - /// - /// Returns `Err` in case the given memory region cannot be deallocated. - pub fn deallocate( - &self, - mut ctx: impl AsContextMut, - allocator: &mut sc_allocator::FreeingBumpHeapAllocator, - ptr: Pointer, - ) -> Result<()> { - let memory = self.memory.data_mut(ctx.as_context_mut()); - - allocator.deallocate(memory, ptr).map_err(Into::into) - } - /// Returns the pointer to the first byte of the linear memory for this instance. - pub fn base_ptr(&self, ctx: impl AsContext) -> *const u8 { - self.memory.data_ptr(ctx) + pub fn base_ptr(&self) -> *const u8 { + self.memory.data_ptr(&self.store) } - /// Removes physical backing from the allocated linear memory. This leads to returning the - /// memory back to the system. While the memory is zeroed this is considered as a side-effect - /// and is not relied upon. Thus this function acts as a hint. - pub fn decommit(&self, ctx: impl AsContext) { - if self.memory.data_size(&ctx) == 0 { + /// If possible removes physical backing from the allocated linear memory which + /// leads to returning the memory back to the system; this also zeroes the memory + /// as a side-effect. + pub fn decommit(&mut self) { + if self.memory.data_size(&self.store) == 0 { return } @@ -416,8 +350,8 @@ impl InstanceWrapper { use std::sync::Once; unsafe { - let ptr = self.memory.data_ptr(&ctx); - let len = self.memory.data_size(ctx); + let ptr = self.memory.data_ptr(&self.store); + let len = self.memory.data_size(&self.store); // Linux handles MADV_DONTNEED reliably. The result is that the given area // is unmapped and will be zeroed on the next pagefault. @@ -429,9 +363,65 @@ impl InstanceWrapper { std::io::Error::last_os_error(), ); }); + } else { + return; + } + } + } else if #[cfg(target_os = "macos")] { + use std::sync::Once; + + unsafe { + let ptr = self.memory.data_ptr(&self.store); + let len = self.memory.data_size(&self.store); + + // On MacOS we can simply overwrite memory mapping. + if libc::mmap( + ptr as _, + len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_FIXED | libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ) == libc::MAP_FAILED { + static LOGGED: Once = Once::new(); + LOGGED.call_once(|| { + log::warn!( + "Failed to decommit WASM instance memory through mmap: {}", + std::io::Error::last_os_error(), + ); + }); + } else { + return; } } } } + + // If we're on an unsupported OS or the memory couldn't have been + // decommited for some reason then just manually zero it out. + self.memory.data_mut(self.store.as_context_mut()).fill(0); + } + + pub(crate) fn store(&self) -> &Store { + &self.store } + + pub(crate) fn store_mut(&mut self) -> &mut Store { + &mut self.store + } +} + +#[test] +fn decommit_works() { + let engine = wasmtime::Engine::default(); + let code = wat::parse_str("(module (memory (export \"memory\") 1 4))").unwrap(); + let module = wasmtime::Module::new(&engine, code).unwrap(); + let linker = wasmtime::Linker::new(&engine); + let mut store = create_store(&engine, None); + let instance_pre = linker.instantiate_pre(&mut store, &module).unwrap(); + let mut wrapper = InstanceWrapper::new(&engine, &instance_pre, None).unwrap(); + unsafe { *wrapper.memory.data_ptr(&wrapper.store) = 42 }; + assert_eq!(unsafe { *wrapper.memory.data_ptr(&wrapper.store) }, 42); + wrapper.decommit(); + assert_eq!(unsafe { *wrapper.memory.data_ptr(&wrapper.store) }, 0); } diff --git a/client/executor/wasmtime/src/lib.rs b/client/executor/wasmtime/src/lib.rs index 8d7f93fecb30..c54c8305f3e4 100644 --- a/client/executor/wasmtime/src/lib.rs +++ b/client/executor/wasmtime/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,7 +16,17 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -/// ! Defines a `WasmRuntime` that uses the Wasmtime JIT to execute. +//! Defines a `WasmRuntime` that uses the Wasmtime JIT to execute. +//! +//! You can choose a profiling strategy at runtime with +//! environment variable `WASMTIME_PROFILING_STRATEGY`: +//! +//! | `WASMTIME_PROFILING_STRATEGY` | Effect | +//! |-------------|-------------------------| +//! | undefined | No profiling | +//! | `"jitdump"` | jitdump profiling | +//! | other value | No profiling (warning) | + mod host; mod imports; mod instance_wrapper; diff --git a/client/executor/wasmtime/src/runtime.rs b/client/executor/wasmtime/src/runtime.rs index 006f102926ec..fa3b567cc0ab 100644 --- a/client/executor/wasmtime/src/runtime.rs +++ b/client/executor/wasmtime/src/runtime.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,7 +20,6 @@ use crate::{ host::HostState, - imports::{resolve_imports, Imports}, instance_wrapper::{EntryPoint, InstanceWrapper}, util, }; @@ -34,75 +33,93 @@ use sc_executor_common::{ wasm_runtime::{InvokeMethod, WasmInstance, WasmModule}, }; use sp_runtime_interface::unpack_ptr_and_len; -use sp_wasm_interface::{Function, Pointer, Value, WordSize}; +use sp_wasm_interface::{HostFunctions, Pointer, Value, WordSize}; use std::{ path::{Path, PathBuf}, - rc::Rc, - sync::Arc, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, }; -use wasmtime::{AsContext, AsContextMut, Engine, StoreLimits}; +use wasmtime::{Engine, Memory, StoreLimits, Table}; pub(crate) struct StoreData { - /// The limits we aply to the store. We need to store it here to return a reference to this + /// The limits we apply to the store. We need to store it here to return a reference to this /// object when we have the limits enabled. - limits: StoreLimits, + pub(crate) limits: StoreLimits, /// This will only be set when we call into the runtime. - host_state: Option>, + pub(crate) host_state: Option, + /// This will be always set once the store is initialized. + pub(crate) memory: Option, + /// This will be set only if the runtime actually contains a table. + pub(crate) table: Option
, } impl StoreData { /// Returns a reference to the host state. - pub fn host_state(&self) -> Option<&Rc> { + pub fn host_state(&self) -> Option<&HostState> { self.host_state.as_ref() } + + /// Returns a mutable reference to the host state. + pub fn host_state_mut(&mut self) -> Option<&mut HostState> { + self.host_state.as_mut() + } + + /// Returns the host memory. + pub fn memory(&self) -> Memory { + self.memory.expect("memory is always set; qed") + } + + /// Returns the host table. + pub fn table(&self) -> Option
{ + self.table + } } pub(crate) type Store = wasmtime::Store; enum Strategy { FastInstanceReuse { - instance_wrapper: Rc, + instance_wrapper: InstanceWrapper, globals_snapshot: GlobalsSnapshot, data_segments_snapshot: Arc, heap_base: u32, - store: Store, }, RecreateInstance(InstanceCreator), } struct InstanceCreator { - store: Store, - module: Arc, - imports: Arc, - heap_pages: u32, + engine: wasmtime::Engine, + instance_pre: Arc>, + max_memory_size: Option, } impl InstanceCreator { fn instantiate(&mut self) -> Result { - InstanceWrapper::new(&*self.module, &*self.imports, self.heap_pages, &mut self.store) + InstanceWrapper::new(&self.engine, &self.instance_pre, self.max_memory_size) } } -struct InstanceGlobals<'a, C> { - ctx: &'a mut C, - instance: &'a InstanceWrapper, +struct InstanceGlobals<'a> { + instance: &'a mut InstanceWrapper, } -impl<'a, C: AsContextMut> runtime_blob::InstanceGlobals for InstanceGlobals<'a, C> { +impl<'a> runtime_blob::InstanceGlobals for InstanceGlobals<'a> { type Global = wasmtime::Global; fn get_global(&mut self, export_name: &str) -> Self::Global { self.instance - .get_global(&mut self.ctx, export_name) + .get_global(export_name) .expect("get_global is guaranteed to be called with an export name of a global; qed") } fn get_global_value(&mut self, global: &Self::Global) -> Value { - util::from_wasmtime_val(global.get(&mut self.ctx)) + util::from_wasmtime_val(global.get(&mut self.instance.store_mut())) } fn set_global_value(&mut self, global: &Self::Global, value: Value) { - global.set(&mut self.ctx, util::into_wasmtime_val(value)).expect( + global.set(&mut self.instance.store_mut(), util::into_wasmtime_val(value)).expect( "the value is guaranteed to be of the same value; the global is guaranteed to be mutable; qed", ); } @@ -117,54 +134,21 @@ struct InstanceSnapshotData { /// A `WasmModule` implementation using wasmtime to compile the runtime module to machine code /// and execute the compiled code. pub struct WasmtimeRuntime { - module: Arc, + engine: wasmtime::Engine, + instance_pre: Arc>, snapshot_data: Option, config: Config, - host_functions: Vec<&'static dyn Function>, - engine: Engine, -} - -impl WasmtimeRuntime { - /// Creates the store respecting the set limits. - fn new_store(&self) -> Store { - let limits = if let Some(max_memory_pages) = self.config.max_memory_pages { - wasmtime::StoreLimitsBuilder::new().memory_pages(max_memory_pages).build() - } else { - Default::default() - }; - - let mut store = Store::new(&self.engine, StoreData { limits, host_state: None }); - - if self.config.max_memory_pages.is_some() { - store.limiter(|s| &mut s.limits); - } - - store - } } impl WasmModule for WasmtimeRuntime { fn new_instance(&self) -> Result> { - let mut store = self.new_store(); - - // Scan all imports, find the matching host functions, and create stubs that adapt arguments - // and results. - // - // NOTE: Attentive reader may notice that this could've been moved in `WasmModule` creation. - // However, I am not sure if that's a good idea since it would be pushing our luck - // further by assuming that `Store` not only `Send` but also `Sync`. - let imports = resolve_imports( - &mut store, - &self.module, - &self.host_functions, - self.config.heap_pages, - self.config.allow_missing_func_imports, - )?; - let strategy = if let Some(ref snapshot_data) = self.snapshot_data { - let instance_wrapper = - InstanceWrapper::new(&self.module, &imports, self.config.heap_pages, &mut store)?; - let heap_base = instance_wrapper.extract_heap_base(&mut store)?; + let mut instance_wrapper = InstanceWrapper::new( + &self.engine, + &self.instance_pre, + self.config.max_memory_size, + )?; + let heap_base = instance_wrapper.extract_heap_base()?; // This function panics if the instance was created from a runtime blob different from // which the mutable globals were collected. Here, it is easy to see that there is only @@ -172,22 +156,20 @@ impl WasmModule for WasmtimeRuntime { // instance and collecting the mutable globals. let globals_snapshot = GlobalsSnapshot::take( &snapshot_data.mutable_globals, - &mut InstanceGlobals { ctx: &mut store, instance: &instance_wrapper }, + &mut InstanceGlobals { instance: &mut instance_wrapper }, ); Strategy::FastInstanceReuse { - instance_wrapper: Rc::new(instance_wrapper), + instance_wrapper, globals_snapshot, data_segments_snapshot: snapshot_data.data_segments_snapshot.clone(), heap_base, - store, } } else { Strategy::RecreateInstance(InstanceCreator { - imports: Arc::new(imports), - module: self.module.clone(), - store, - heap_pages: self.config.heap_pages, + engine: self.engine.clone(), + instance_pre: self.instance_pre.clone(), + max_memory_size: self.config.max_memory_size, }) }; @@ -201,68 +183,52 @@ pub struct WasmtimeInstance { strategy: Strategy, } -// This is safe because `WasmtimeInstance` does not leak reference to `self.imports` -// and all imports don't reference anything, other than host functions and memory -unsafe impl Send for WasmtimeInstance {} - impl WasmInstance for WasmtimeInstance { fn call(&mut self, method: InvokeMethod, data: &[u8]) -> Result> { match &mut self.strategy { Strategy::FastInstanceReuse { - instance_wrapper, + ref mut instance_wrapper, globals_snapshot, data_segments_snapshot, heap_base, - ref mut store, } => { - let entrypoint = instance_wrapper.resolve_entrypoint(method, &mut *store)?; + let entrypoint = instance_wrapper.resolve_entrypoint(method)?; data_segments_snapshot.apply(|offset, contents| { - instance_wrapper.write_memory_from(&mut *store, Pointer::new(offset), contents) + util::write_memory_from( + instance_wrapper.store_mut(), + Pointer::new(offset), + contents, + ) })?; - globals_snapshot - .apply(&mut InstanceGlobals { ctx: &mut *store, instance: &*instance_wrapper }); + globals_snapshot.apply(&mut InstanceGlobals { instance: instance_wrapper }); let allocator = FreeingBumpHeapAllocator::new(*heap_base); - let result = perform_call( - &mut *store, - data, - instance_wrapper.clone(), - entrypoint, - allocator, - ); + let result = perform_call(data, instance_wrapper, entrypoint, allocator); // Signal to the OS that we are done with the linear memory and that it can be // reclaimed. - instance_wrapper.decommit(&store); + instance_wrapper.decommit(); result }, Strategy::RecreateInstance(ref mut instance_creator) => { - let instance_wrapper = instance_creator.instantiate()?; - let heap_base = instance_wrapper.extract_heap_base(&mut instance_creator.store)?; - let entrypoint = - instance_wrapper.resolve_entrypoint(method, &mut instance_creator.store)?; + let mut instance_wrapper = instance_creator.instantiate()?; + let heap_base = instance_wrapper.extract_heap_base()?; + let entrypoint = instance_wrapper.resolve_entrypoint(method)?; let allocator = FreeingBumpHeapAllocator::new(heap_base); - perform_call( - &mut instance_creator.store, - data, - Rc::new(instance_wrapper), - entrypoint, - allocator, - ) + perform_call(data, &mut instance_wrapper, entrypoint, allocator) }, } } fn get_global_const(&mut self, name: &str) -> Result> { match &mut self.strategy { - Strategy::FastInstanceReuse { instance_wrapper, ref mut store, .. } => - instance_wrapper.get_global_val(&mut *store, name), - Strategy::RecreateInstance(ref mut instance_creator) => instance_creator - .instantiate()? - .get_global_val(&mut instance_creator.store, name), + Strategy::FastInstanceReuse { instance_wrapper, .. } => + instance_wrapper.get_global_val(name), + Strategy::RecreateInstance(ref mut instance_creator) => + instance_creator.instantiate()?.get_global_val(name), } } @@ -273,8 +239,8 @@ impl WasmInstance for WasmtimeInstance { // associated with it. None }, - Strategy::FastInstanceReuse { instance_wrapper, store, .. } => - Some(instance_wrapper.base_ptr(&store)), + Strategy::FastInstanceReuse { instance_wrapper, .. } => + Some(instance_wrapper.base_ptr()), } } } @@ -322,6 +288,23 @@ fn common_config(semantics: &Semantics) -> std::result::Result wasmtime::ProfilingStrategy::JitDump, + None => wasmtime::ProfilingStrategy::None, + Some(_) => { + // Remember if we have already logged a warning due to an unknown profiling strategy. + static UNKNOWN_PROFILING_STRATEGY: AtomicBool = AtomicBool::new(false); + // Make sure that the warning will not be relogged regularly. + if !UNKNOWN_PROFILING_STRATEGY.swap(true, Ordering::Relaxed) { + log::warn!("WASMTIME_PROFILING_STRATEGY is set to unknown value, ignored."); + } + wasmtime::ProfilingStrategy::None + }, + }; + config + .profiler(profiler) + .map_err(|e| WasmError::Instantiation(format!("fail to set profiler: {}", e)))?; + if let Some(DeterministicStackLimit { native_stack_max, .. }) = semantics.deterministic_stack_limit { @@ -330,6 +313,8 @@ fn common_config(semantics: &Semantics) -> std::result::Result std::result::Result std::result::Result, + pub max_memory_size: Option, /// The WebAssembly standard requires all imports of an instantiated module to be resolved, - /// othewise, the instantiation fails. If this option is set to `true`, then this behavior is + /// otherwise, the instantiation fails. If this option is set to `true`, then this behavior is /// overriden and imports that are requested by the module and not provided by the host /// functions will be resolved using stubs. These stubs will trap upon a call. pub allow_missing_func_imports: bool, @@ -486,13 +474,18 @@ enum CodeSupplyMode<'a> { /// Create a new `WasmtimeRuntime` given the code. This function performs translation from Wasm to /// machine code, which can be computationally heavy. -pub fn create_runtime( +/// +/// The `H` generic parameter is used to statically pass a set of host functions which are exposed +/// to the runtime. +pub fn create_runtime( blob: RuntimeBlob, config: Config, - host_functions: Vec<&'static dyn Function>, -) -> std::result::Result { +) -> std::result::Result +where + H: HostFunctions, +{ // SAFETY: this is safe because it doesn't use `CodeSupplyMode::Artifact`. - unsafe { do_create_runtime(CodeSupplyMode::Verbatim { blob }, config, host_functions) } + unsafe { do_create_runtime::(CodeSupplyMode::Verbatim { blob }, config) } } /// The same as [`create_runtime`] but takes a precompiled artifact, which makes this function @@ -506,23 +499,27 @@ pub fn create_runtime( /// /// It is ok though if the `compiled_artifact` was created by code of another version or with /// different configuration flags. In such case the caller will receive an `Err` deterministically. -pub unsafe fn create_runtime_from_artifact( +pub unsafe fn create_runtime_from_artifact( compiled_artifact: &[u8], config: Config, - host_functions: Vec<&'static dyn Function>, -) -> std::result::Result { - do_create_runtime(CodeSupplyMode::Artifact { compiled_artifact }, config, host_functions) +) -> std::result::Result +where + H: HostFunctions, +{ + do_create_runtime::(CodeSupplyMode::Artifact { compiled_artifact }, config) } /// # Safety /// /// This is only unsafe if called with [`CodeSupplyMode::Artifact`]. See /// [`create_runtime_from_artifact`] to get more details. -unsafe fn do_create_runtime( +unsafe fn do_create_runtime( code_supply_mode: CodeSupplyMode<'_>, config: Config, - host_functions: Vec<&'static dyn Function>, -) -> std::result::Result { +) -> std::result::Result +where + H: HostFunctions, +{ // Create the engine, store and finally the module from the given code. let mut wasmtime_config = common_config(&config.semantics)?; if let Some(ref cache_path) = config.cache_path { @@ -535,27 +532,25 @@ unsafe fn do_create_runtime( } let engine = Engine::new(&wasmtime_config) - .map_err(|e| WasmError::Other(format!("cannot create the engine for runtime: {}", e)))?; + .map_err(|e| WasmError::Other(format!("cannot create the wasmtime engine: {}", e)))?; let (module, snapshot_data) = match code_supply_mode { CodeSupplyMode::Verbatim { blob } => { - let blob = instrument(blob, &config.semantics)?; + let blob = prepare_blob_for_compilation(blob, &config.semantics)?; + let serialized_blob = blob.clone().serialize(); + + let module = wasmtime::Module::new(&engine, &serialized_blob) + .map_err(|e| WasmError::Other(format!("cannot create module: {}", e)))?; if config.semantics.fast_instance_reuse { let data_segments_snapshot = DataSegmentsSnapshot::take(&blob).map_err(|e| { WasmError::Other(format!("cannot take data segments snapshot: {}", e)) })?; let data_segments_snapshot = Arc::new(data_segments_snapshot); - let mutable_globals = ExposedMutableGlobalsSet::collect(&blob); - let module = wasmtime::Module::new(&engine, &blob.serialize()) - .map_err(|e| WasmError::Other(format!("cannot create module: {}", e)))?; - (module, Some(InstanceSnapshotData { data_segments_snapshot, mutable_globals })) } else { - let module = wasmtime::Module::new(&engine, &blob.serialize()) - .map_err(|e| WasmError::Other(format!("cannot create module: {}", e)))?; (module, None) } }, @@ -569,10 +564,18 @@ unsafe fn do_create_runtime( }, }; - Ok(WasmtimeRuntime { module: Arc::new(module), snapshot_data, config, host_functions, engine }) + let mut linker = wasmtime::Linker::new(&engine); + crate::imports::prepare_imports::(&mut linker, &module, config.allow_missing_func_imports)?; + + let mut store = crate::instance_wrapper::create_store(module.engine(), config.max_memory_size); + let instance_pre = linker + .instantiate_pre(&mut store, &module) + .map_err(|e| WasmError::Other(format!("cannot preinstantiate module: {}", e)))?; + + Ok(WasmtimeRuntime { engine, instance_pre: Arc::new(instance_pre), snapshot_data, config }) } -fn instrument( +fn prepare_blob_for_compilation( mut blob: RuntimeBlob, semantics: &Semantics, ) -> std::result::Result { @@ -585,6 +588,19 @@ fn instrument( blob.expose_mutable_globals(); } + // We don't actually need the memory to be imported so we can just convert any memory + // import into an export with impunity. This simplifies our code since `wasmtime` will + // now automatically take care of creating the memory for us, and it also allows us + // to potentially enable `wasmtime`'s instance pooling at a later date. (Imported + // memories are ineligible for pooling.) + blob.convert_memory_import_into_export()?; + blob.add_extra_heap_pages_to_memory_section( + semantics + .extra_heap_pages + .try_into() + .map_err(|e| WasmError::Other(format!("invalid `extra_heap_pages`: {}", e)))?, + )?; + Ok(blob) } @@ -594,7 +610,7 @@ pub fn prepare_runtime_artifact( blob: RuntimeBlob, semantics: &Semantics, ) -> std::result::Result, WasmError> { - let blob = instrument(blob, semantics)?; + let blob = prepare_blob_for_compilation(blob, semantics)?; let engine = Engine::new(&common_config(semantics)?) .map_err(|e| WasmError::Other(format!("cannot create the engine: {}", e)))?; @@ -605,50 +621,51 @@ pub fn prepare_runtime_artifact( } fn perform_call( - mut ctx: impl AsContextMut, data: &[u8], - instance_wrapper: Rc, + instance_wrapper: &mut InstanceWrapper, entrypoint: EntryPoint, mut allocator: FreeingBumpHeapAllocator, ) -> Result> { - let (data_ptr, data_len) = - inject_input_data(&mut ctx, &instance_wrapper, &mut allocator, data)?; + let (data_ptr, data_len) = inject_input_data(instance_wrapper, &mut allocator, data)?; - let host_state = HostState::new(allocator, instance_wrapper.clone()); + let host_state = HostState::new(allocator); // Set the host state before calling into wasm. - ctx.as_context_mut().data_mut().host_state = Some(Rc::new(host_state)); + instance_wrapper.store_mut().data_mut().host_state = Some(host_state); - let ret = entrypoint.call(&mut ctx, data_ptr, data_len).map(unpack_ptr_and_len); + let ret = entrypoint + .call(instance_wrapper.store_mut(), data_ptr, data_len) + .map(unpack_ptr_and_len); // Reset the host state - ctx.as_context_mut().data_mut().host_state = None; + instance_wrapper.store_mut().data_mut().host_state = None; let (output_ptr, output_len) = ret?; - let output = extract_output_data(ctx, &instance_wrapper, output_ptr, output_len)?; + let output = extract_output_data(instance_wrapper, output_ptr, output_len)?; Ok(output) } fn inject_input_data( - mut ctx: impl AsContextMut, - instance: &InstanceWrapper, + instance: &mut InstanceWrapper, allocator: &mut FreeingBumpHeapAllocator, data: &[u8], ) -> Result<(Pointer, WordSize)> { + let mut ctx = instance.store_mut(); + let memory = ctx.data().memory(); + let memory = memory.data_mut(&mut ctx); let data_len = data.len() as WordSize; - let data_ptr = instance.allocate(&mut ctx, allocator, data_len)?; - instance.write_memory_from(ctx, data_ptr, data)?; + let data_ptr = allocator.allocate(memory, data_len)?; + util::write_memory_from(instance.store_mut(), data_ptr, data)?; Ok((data_ptr, data_len)) } fn extract_output_data( - ctx: impl AsContext, instance: &InstanceWrapper, output_ptr: u32, output_len: u32, ) -> Result> { let mut output = vec![0; output_len as usize]; - instance.read_memory_into(ctx, Pointer::new(output_ptr), &mut output)?; + util::read_memory_into(instance.store(), Pointer::new(output_ptr), &mut output)?; Ok(output) } diff --git a/client/executor/wasmtime/src/tests.rs b/client/executor/wasmtime/src/tests.rs index 2a8bcc0b01b0..d5b92f2f24a7 100644 --- a/client/executor/wasmtime/src/tests.rs +++ b/client/executor/wasmtime/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -17,19 +17,20 @@ // along with this program. If not, see . use codec::{Decode as _, Encode as _}; -use sc_executor_common::{runtime_blob::RuntimeBlob, wasm_runtime::WasmModule}; +use sc_executor_common::{error::Error, runtime_blob::RuntimeBlob, wasm_runtime::WasmModule}; use sc_runtime_test::wasm_binary_unwrap; use std::sync::Arc; type HostFunctions = sp_io::SubstrateHostFunctions; struct RuntimeBuilder { - code: Option<&'static str>, + code: Option, fast_instance_reuse: bool, canonicalize_nans: bool, deterministic_stack: bool, - heap_pages: u32, - max_memory_pages: Option, + extra_heap_pages: u64, + max_memory_size: Option, + precompile_runtime: bool, } impl RuntimeBuilder { @@ -41,34 +42,44 @@ impl RuntimeBuilder { fast_instance_reuse: false, canonicalize_nans: false, deterministic_stack: false, - heap_pages: 1024, - max_memory_pages: None, + extra_heap_pages: 1024, + max_memory_size: None, + precompile_runtime: false, } } - fn use_wat(&mut self, code: &'static str) { + fn use_wat(&mut self, code: String) -> &mut Self { self.code = Some(code); + self } - fn canonicalize_nans(&mut self, canonicalize_nans: bool) { + fn canonicalize_nans(&mut self, canonicalize_nans: bool) -> &mut Self { self.canonicalize_nans = canonicalize_nans; + self } - fn deterministic_stack(&mut self, deterministic_stack: bool) { + fn deterministic_stack(&mut self, deterministic_stack: bool) -> &mut Self { self.deterministic_stack = deterministic_stack; + self } - fn max_memory_pages(&mut self, max_memory_pages: Option) { - self.max_memory_pages = max_memory_pages; + fn precompile_runtime(&mut self, precompile_runtime: bool) -> &mut Self { + self.precompile_runtime = precompile_runtime; + self } - fn build(self) -> Arc { + fn max_memory_size(&mut self, max_memory_size: Option) -> &mut Self { + self.max_memory_size = max_memory_size; + self + } + + fn build(&mut self) -> Arc { let blob = { let wasm: Vec; let wasm = match self.code { None => wasm_binary_unwrap(), - Some(wat) => { + Some(ref wat) => { wasm = wat::parse_str(wat).expect("wat parsing failed"); &wasm }, @@ -78,30 +89,31 @@ impl RuntimeBuilder { .expect("failed to create a runtime blob out of test runtime") }; - let rt = crate::create_runtime( - blob, - crate::Config { - heap_pages: self.heap_pages, - max_memory_pages: self.max_memory_pages, - allow_missing_func_imports: true, - cache_path: None, - semantics: crate::Semantics { - fast_instance_reuse: self.fast_instance_reuse, - deterministic_stack_limit: match self.deterministic_stack { - true => Some(crate::DeterministicStackLimit { - logical_max: 65536, - native_stack_max: 256 * 1024 * 1024, - }), - false => None, - }, - canonicalize_nans: self.canonicalize_nans, + let config = crate::Config { + max_memory_size: self.max_memory_size, + allow_missing_func_imports: true, + cache_path: None, + semantics: crate::Semantics { + fast_instance_reuse: self.fast_instance_reuse, + deterministic_stack_limit: match self.deterministic_stack { + true => Some(crate::DeterministicStackLimit { + logical_max: 65536, + native_stack_max: 256 * 1024 * 1024, + }), + false => None, }, + canonicalize_nans: self.canonicalize_nans, + parallel_compilation: true, + extra_heap_pages: self.extra_heap_pages, }, - { - use sp_wasm_interface::HostFunctions as _; - HostFunctions::host_functions() - }, - ) + }; + + let rt = if self.precompile_runtime { + let artifact = crate::prepare_runtime_artifact(blob, &config.semantics).unwrap(); + unsafe { crate::create_runtime_from_artifact::(&artifact, config) } + } else { + crate::create_runtime::(blob, config) + } .expect("cannot create runtime"); Arc::new(rt) as Arc @@ -110,11 +122,7 @@ impl RuntimeBuilder { #[test] fn test_nan_canonicalization() { - let runtime = { - let mut builder = RuntimeBuilder::new_on_demand(); - builder.canonicalize_nans(true); - builder.build() - }; + let runtime = RuntimeBuilder::new_on_demand().canonicalize_nans(true).build(); let mut instance = runtime.new_instance().expect("failed to instantiate a runtime"); @@ -153,157 +161,243 @@ fn test_nan_canonicalization() { fn test_stack_depth_reaching() { const TEST_GUARD_PAGE_SKIP: &str = include_str!("test-guard-page-skip.wat"); - let runtime = { - let mut builder = RuntimeBuilder::new_on_demand(); - builder.use_wat(TEST_GUARD_PAGE_SKIP); - builder.deterministic_stack(true); - builder.build() - }; + let runtime = RuntimeBuilder::new_on_demand() + .use_wat(TEST_GUARD_PAGE_SKIP.to_string()) + .deterministic_stack(true) + .build(); let mut instance = runtime.new_instance().expect("failed to instantiate a runtime"); - let err = instance.call_export("test-many-locals", &[]).unwrap_err(); + match instance.call_export("test-many-locals", &[]).unwrap_err() { + Error::AbortedDueToTrap(error) => { + let expected = "wasm trap: wasm `unreachable` instruction executed"; + assert_eq!(error.message, expected); + }, + error => panic!("unexpected error: {:?}", error), + } +} - assert!( - format!("{:?}", err).starts_with("Other(\"Wasm execution trapped: wasm trap: unreachable") - ); +#[test] +fn test_max_memory_pages_imported_memory_without_precompilation() { + test_max_memory_pages(true, false); } #[test] -fn test_max_memory_pages() { +fn test_max_memory_pages_exported_memory_without_precompilation() { + test_max_memory_pages(false, false); +} + +#[test] +fn test_max_memory_pages_imported_memory_with_precompilation() { + test_max_memory_pages(true, true); +} + +#[test] +fn test_max_memory_pages_exported_memory_with_precompilation() { + test_max_memory_pages(false, true); +} + +fn test_max_memory_pages(import_memory: bool, precompile_runtime: bool) { fn try_instantiate( - max_memory_pages: Option, - wat: &'static str, + max_memory_size: Option, + wat: String, + precompile_runtime: bool, ) -> Result<(), Box> { - let runtime = { - let mut builder = RuntimeBuilder::new_on_demand(); - builder.use_wat(wat); - builder.max_memory_pages(max_memory_pages); - builder.build() - }; + let runtime = RuntimeBuilder::new_on_demand() + .use_wat(wat) + .max_memory_size(max_memory_size) + .precompile_runtime(precompile_runtime) + .build(); let mut instance = runtime.new_instance()?; let _ = instance.call_export("main", &[])?; Ok(()) } + fn memory(initial: u32, maximum: Option, import: bool) -> String { + let memory = if let Some(maximum) = maximum { + format!("(memory $0 {} {})", initial, maximum) + } else { + format!("(memory $0 {})", initial) + }; + + if import { + format!("(import \"env\" \"memory\" {})", memory) + } else { + format!("{}\n(export \"memory\" (memory $0))", memory) + } + } + + const WASM_PAGE_SIZE: usize = 65536; + // check the old behavior if preserved. That is, if no limit is set we allow 4 GiB of memory. try_instantiate( None, - r#" - (module - ;; we want to allocate the maximum number of pages supported in wasm for this test. - ;; - ;; However, due to a bug in wasmtime (I think wasmi is also affected) it is only possible - ;; to allocate 65536 - 1 pages. - ;; - ;; Then, during creation of the Substrate Runtime instance, 1024 (heap_pages) pages are - ;; mounted. - ;; - ;; Thus 65535 = 64511 + 1024 - (import "env" "memory" (memory 64511)) - - (global (export "__heap_base") i32 (i32.const 0)) - (func (export "main") - (param i32 i32) (result i64) - (i64.const 0) + format!( + r#" + (module + {} + (global (export "__heap_base") i32 (i32.const 0)) + (func (export "main") + (param i32 i32) (result i64) + (i64.const 0) + ) ) - ) - "#, + "#, + /* + We want to allocate the maximum number of pages supported in wasm for this test. + However, due to a bug in wasmtime (I think wasmi is also affected) it is only possible + to allocate 65536 - 1 pages. + + Then, during creation of the Substrate Runtime instance, 1024 (heap_pages) pages are + mounted. + + Thus 65535 = 64511 + 1024 + */ + memory(64511, None, import_memory) + ), + precompile_runtime, ) .unwrap(); // max is not specified, therefore it's implied to be 65536 pages (4 GiB). // - // max_memory_pages = 1 (initial) + 1024 (heap_pages) + // max_memory_size = (1 (initial) + 1024 (heap_pages)) * WASM_PAGE_SIZE try_instantiate( - Some(1 + 1024), - r#" - (module - - (import "env" "memory" (memory 1)) ;; <- 1 initial, max is not specified - - (global (export "__heap_base") i32 (i32.const 0)) - (func (export "main") - (param i32 i32) (result i64) - (i64.const 0) + Some((1 + 1024) * WASM_PAGE_SIZE), + format!( + r#" + (module + {} + (global (export "__heap_base") i32 (i32.const 0)) + (func (export "main") + (param i32 i32) (result i64) + (i64.const 0) + ) ) - ) - "#, + "#, + // 1 initial, max is not specified. + memory(1, None, import_memory) + ), + precompile_runtime, ) .unwrap(); // max is specified explicitly to 2048 pages. try_instantiate( - Some(1 + 1024), - r#" - (module - - (import "env" "memory" (memory 1 2048)) ;; <- max is 2048 - - (global (export "__heap_base") i32 (i32.const 0)) - (func (export "main") - (param i32 i32) (result i64) - (i64.const 0) + Some((1 + 1024) * WASM_PAGE_SIZE), + format!( + r#" + (module + {} + (global (export "__heap_base") i32 (i32.const 0)) + (func (export "main") + (param i32 i32) (result i64) + (i64.const 0) + ) ) - ) - "#, + "#, + // Max is 2048. + memory(1, Some(2048), import_memory) + ), + precompile_runtime, ) .unwrap(); // memory grow should work as long as it doesn't exceed 1025 pages in total. try_instantiate( - Some(0 + 1024 + 25), - r#" - (module - (import "env" "memory" (memory 0)) ;; <- zero starting pages. - - (global (export "__heap_base") i32 (i32.const 0)) - (func (export "main") - (param i32 i32) (result i64) - - ;; assert(memory.grow returns != -1) - (if - (i32.eq - (memory.grow - (i32.const 25) + Some((0 + 1024 + 25) * WASM_PAGE_SIZE), + format!( + r#" + (module + {} + (global (export "__heap_base") i32 (i32.const 0)) + (func (export "main") + (param i32 i32) (result i64) + + ;; assert(memory.grow returns != -1) + (if + (i32.eq + (memory.grow + (i32.const 25) + ) + (i32.const -1) ) - (i32.const -1) + (unreachable) ) - (unreachable) - ) - (i64.const 0) + (i64.const 0) + ) ) - ) - "#, + "#, + // Zero starting pages. + memory(0, None, import_memory) + ), + precompile_runtime, ) .unwrap(); // We start with 1025 pages and try to grow at least one. try_instantiate( - Some(1 + 1024), - r#" - (module - (import "env" "memory" (memory 1)) ;; <- initial=1, meaning after heap pages mount the - ;; total will be already 1025 - (global (export "__heap_base") i32 (i32.const 0)) - (func (export "main") - (param i32 i32) (result i64) - - ;; assert(memory.grow returns == -1) - (if - (i32.ne - (memory.grow - (i32.const 1) + Some((1 + 1024) * WASM_PAGE_SIZE), + format!( + r#" + (module + {} + (global (export "__heap_base") i32 (i32.const 0)) + (func (export "main") + (param i32 i32) (result i64) + + ;; assert(memory.grow returns == -1) + (if + (i32.ne + (memory.grow + (i32.const 1) + ) + (i32.const -1) ) - (i32.const -1) + (unreachable) ) - (unreachable) - ) - (i64.const 0) + (i64.const 0) + ) ) - ) - "#, + "#, + // Initial=1, meaning after heap pages mount the total will be already 1025. + memory(1, None, import_memory) + ), + precompile_runtime, ) .unwrap(); } + +// This test takes quite a while to execute in a debug build (over 6 minutes on a TR 3970x) +// so it's ignored by default unless it was compiled with `--release`. +#[cfg_attr(build_type = "debug", ignore)] +#[test] +fn test_instances_without_reuse_are_not_leaked() { + let runtime = crate::create_runtime::( + RuntimeBlob::uncompress_if_needed(wasm_binary_unwrap()).unwrap(), + crate::Config { + max_memory_size: None, + allow_missing_func_imports: true, + cache_path: None, + semantics: crate::Semantics { + fast_instance_reuse: false, + deterministic_stack_limit: None, + canonicalize_nans: false, + parallel_compilation: true, + extra_heap_pages: 2048, + }, + }, + ) + .unwrap(); + + // As long as the `wasmtime`'s `Store` lives the instances spawned through it + // will live indefinitely. Currently it has a maximum limit of 10k instances, + // so let's spawn 10k + 1 of them to make sure our code doesn't keep the `Store` + // alive longer than it is necessary. (And since we disabled instance reuse + // a new instance will be spawned on each call.) + let mut instance = runtime.new_instance().unwrap(); + for _ in 0..10001 { + instance.call_export("test_empty_return", &[0]).unwrap(); + } +} diff --git a/client/executor/wasmtime/src/util.rs b/client/executor/wasmtime/src/util.rs index 2c135fe7a343..60ca598aee18 100644 --- a/client/executor/wasmtime/src/util.rs +++ b/client/executor/wasmtime/src/util.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,7 +16,13 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use sp_wasm_interface::Value; +use crate::runtime::StoreData; +use sc_executor_common::{ + error::{Error, Result}, + util::checked_range, +}; +use sp_wasm_interface::{Pointer, Value}; +use wasmtime::{AsContext, AsContextMut}; /// Converts a [`wasmtime::Val`] into a substrate runtime interface [`Value`]. /// @@ -41,3 +47,54 @@ pub fn into_wasmtime_val(value: Value) -> wasmtime::Val { Value::F64(f_bits) => wasmtime::Val::F64(f_bits), } } + +/// Read data from a slice of memory into a newly allocated buffer. +/// +/// Returns an error if the read would go out of the memory bounds. +pub(crate) fn read_memory( + ctx: impl AsContext, + source_addr: Pointer, + size: usize, +) -> Result> { + let range = + checked_range(source_addr.into(), size, ctx.as_context().data().memory().data_size(&ctx)) + .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; + + let mut buffer = vec![0; range.len()]; + read_memory_into(ctx, source_addr, &mut buffer)?; + + Ok(buffer) +} + +/// Read data from the instance memory into a slice. +/// +/// Returns an error if the read would go out of the memory bounds. +pub(crate) fn read_memory_into( + ctx: impl AsContext, + address: Pointer, + dest: &mut [u8], +) -> Result<()> { + let memory = ctx.as_context().data().memory().data(&ctx); + + let range = checked_range(address.into(), dest.len(), memory.len()) + .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; + dest.copy_from_slice(&memory[range]); + Ok(()) +} + +/// Write data to the instance memory from a slice. +/// +/// Returns an error if the write would go out of the memory bounds. +pub(crate) fn write_memory_from( + mut ctx: impl AsContextMut, + address: Pointer, + data: &[u8], +) -> Result<()> { + let memory = ctx.as_context().data().memory(); + let memory = memory.data_mut(&mut ctx); + + let range = checked_range(address.into(), data.len(), memory.len()) + .ok_or_else(|| Error::Other("memory write is out of bounds".into()))?; + memory[range].copy_from_slice(data); + Ok(()) +} diff --git a/client/finality-grandpa-warp-sync/Cargo.toml b/client/finality-grandpa-warp-sync/Cargo.toml deleted file mode 100644 index a444125fdfa1..000000000000 --- a/client/finality-grandpa-warp-sync/Cargo.toml +++ /dev/null @@ -1,35 +0,0 @@ -[package] -description = "A request-response protocol for handling grandpa warp sync requests" -name = "sc-finality-grandpa-warp-sync" -version = "0.10.0-dev" -license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -authors = ["Parity Technologies "] -edition = "2018" -homepage = "https://substrate.dev" -repository = "https://github.com/paritytech/substrate/" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0" } -derive_more = "0.99.11" -futures = "0.3.8" -log = "0.4.11" -prost = "0.8" -sc-client-api = { version = "4.0.0-dev", path = "../api" } -sc-finality-grandpa = { version = "0.10.0-dev", path = "../finality-grandpa" } -sc-network = { version = "0.10.0-dev", path = "../network" } -sc-service = { version = "0.10.0-dev", path = "../service" } -sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } -sp-finality-grandpa = { version = "4.0.0-dev", path = "../../primitives/finality-grandpa" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } - -[dev-dependencies] -finality-grandpa = { version = "0.14.4" } -rand = "0.8" -sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" } -sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } -sc-consensus = { version = "0.10.0-dev", path = "../../client/consensus/common" } -sp-keyring = { version = "4.0.0-dev", path = "../../primitives/keyring" } -substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml index 7fdd91e557ab..58dc22192c0c 100644 --- a/client/finality-grandpa/Cargo.toml +++ b/client/finality-grandpa/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-finality-grandpa" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Integration of the GRANDPA finality gadget into substrate." documentation = "https://docs.rs/sc-finality-grandpa" @@ -13,49 +13,53 @@ readme = "README.md" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] - [dependencies] -derive_more = "0.99.2" +thiserror = "1.0" dyn-clone = "1.0" fork-tree = { version = "3.0.0", path = "../../utils/fork-tree" } -futures = "0.3.9" +futures = "0.3.21" futures-timer = "3.0.1" +hex = "0.4.2" log = "0.4.8" -parking_lot = "0.11.1" +parking_lot = "0.12.0" rand = "0.8.4" -parity-scale-codec = { version = "2.0.0", features = ["derive"] } -sp-application-crypto = { version = "4.0.0-dev", path = "../../primitives/application-crypto" } -sp-arithmetic = { version = "4.0.0-dev", path = "../../primitives/arithmetic" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } +ahash = "0.7.6" +parity-scale-codec = { version = "3.0.0", features = ["derive"] } +sp-application-crypto = { version = "6.0.0", path = "../../primitives/application-crypto" } +sp-arithmetic = { version = "5.0.0", path = "../../primitives/arithmetic" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } +sc-chain-spec = { version = "4.0.0-dev", path = "../../client/chain-spec" } sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-keystore = { version = "0.12.0", path = "../../primitives/keystore" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" } sc-keystore = { version = "4.0.0-dev", path = "../keystore" } -serde_json = "1.0.68" +serde_json = "1.0.79" sc-client-api = { version = "4.0.0-dev", path = "../api" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sc-network = { version = "0.10.0-dev", path = "../network" } sc-network-gossip = { version = "0.10.0-dev", path = "../network-gossip" } sp-finality-grandpa = { version = "4.0.0-dev", path = "../../primitives/finality-grandpa" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.9.0" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.10.0-dev" } sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" } -finality-grandpa = { version = "0.14.4", features = ["derive-codec"] } +finality-grandpa = { version = "0.15.0", features = ["derive-codec"] } async-trait = "0.1.50" [dev-dependencies] assert_matches = "1.3.0" -finality-grandpa = { version = "0.14.1", features = [ +finality-grandpa = { version = "0.15.0", features = [ "derive-codec", "test-helpers", ] } sc-network = { version = "0.10.0-dev", path = "../network" } sc-network-test = { version = "0.8.0", path = "../network/test" } -sp-keyring = { version = "4.0.0-dev", path = "../../primitives/keyring" } +sp-keyring = { version = "6.0.0", path = "../../primitives/keyring" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } -tokio = "1.10" +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } + +serde = "1.0.136" +tokio = "1.17.0" tempfile = "3.1.0" diff --git a/client/finality-grandpa/rpc/Cargo.toml b/client/finality-grandpa/rpc/Cargo.toml index d2976ee71275..5e173e1a15fe 100644 --- a/client/finality-grandpa/rpc/Cargo.toml +++ b/client/finality-grandpa/rpc/Cargo.toml @@ -4,7 +4,7 @@ version = "0.10.0-dev" authors = ["Parity Technologies "] description = "RPC extensions for the GRANDPA finality gadget" repository = "https://github.com/paritytech/substrate/" -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" readme = "README.md" @@ -12,9 +12,9 @@ readme = "README.md" sc-finality-grandpa = { version = "0.10.0-dev", path = "../" } sc-rpc = { version = "4.0.0-dev", path = "../../rpc" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -finality-grandpa = { version = "0.14.4", features = ["derive-codec"] } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } +finality-grandpa = { version = "0.15.0", features = ["derive-codec"] } jsonrpc-core = "18.0.0" jsonrpc-core-client = "18.0.0" jsonrpc-derive = "18.0.0" @@ -23,8 +23,8 @@ futures = "0.3.16" serde = { version = "1.0.105", features = ["derive"] } serde_json = "1.0.50" log = "0.4.8" -derive_more = "0.99.2" -parity-scale-codec = { version = "2.0.0", features = ["derive"] } +thiserror = "1.0" +parity-scale-codec = { version = "3.0.0", features = ["derive"] } sc-client-api = { version = "4.0.0-dev", path = "../../api" } [dev-dependencies] @@ -32,7 +32,7 @@ sc-block-builder = { version = "0.10.0-dev", path = "../../block-builder" } sc-rpc = { version = "4.0.0-dev", path = "../../rpc", features = [ "test-helpers", ] } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } sp-finality-grandpa = { version = "4.0.0-dev", path = "../../../primitives/finality-grandpa" } -sp-keyring = { version = "4.0.0-dev", path = "../../../primitives/keyring" } +sp-keyring = { version = "6.0.0", path = "../../../primitives/keyring" } substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } diff --git a/client/finality-grandpa/rpc/src/error.rs b/client/finality-grandpa/rpc/src/error.rs index c812b78f3fd8..845b4d99dcc1 100644 --- a/client/finality-grandpa/rpc/src/error.rs +++ b/client/finality-grandpa/rpc/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,21 +16,21 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -#[derive(derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] /// Top-level error type for the RPC handler pub enum Error { /// The GRANDPA RPC endpoint is not ready. - #[display(fmt = "GRANDPA RPC endpoint not ready")] + #[error("GRANDPA RPC endpoint not ready")] EndpointNotReady, /// GRANDPA reports the authority set id to be larger than 32-bits. - #[display(fmt = "GRANDPA reports authority set id unreasonably large")] + #[error("GRANDPA reports authority set id unreasonably large")] AuthoritySetIdReportedAsUnreasonablyLarge, /// GRANDPA reports voter state with round id or weights larger than 32-bits. - #[display(fmt = "GRANDPA reports voter state as unreasonably large")] + #[error("GRANDPA reports voter state as unreasonably large")] VoterStateReportsUnreasonablyLargeNumbers, /// GRANDPA prove finality failed. - #[display(fmt = "GRANDPA prove finality rpc failed: {}", _0)] - ProveFinalityFailed(sc_finality_grandpa::FinalityProofError), + #[error("GRANDPA prove finality rpc failed: {0}")] + ProveFinalityFailed(#[from] sc_finality_grandpa::FinalityProofError), } /// The error codes returned by jsonrpc. diff --git a/client/finality-grandpa/rpc/src/finality.rs b/client/finality-grandpa/rpc/src/finality.rs index 62e3502fc718..f2be6d674b2f 100644 --- a/client/finality-grandpa/rpc/src/finality.rs +++ b/client/finality-grandpa/rpc/src/finality.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/finality-grandpa/rpc/src/lib.rs b/client/finality-grandpa/rpc/src/lib.rs index b8b8b2d95646..9c51bc3d226a 100644 --- a/client/finality-grandpa/rpc/src/lib.rs +++ b/client/finality-grandpa/rpc/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -171,7 +171,7 @@ where mod tests { use super::*; use jsonrpc_core::{types::Params, Notification, Output}; - use std::{collections::HashSet, convert::TryInto, sync::Arc}; + use std::{collections::HashSet, sync::Arc}; use parity_scale_codec::{Decode, Encode}; use sc_block_builder::{BlockBuilder, RecordProof}; @@ -179,7 +179,7 @@ mod tests { report, AuthorityId, FinalityProof, GrandpaJustification, GrandpaJustificationSender, }; use sp_blockchain::HeaderBackend; - use sp_core::crypto::Public; + use sp_core::crypto::ByteArray; use sp_keyring::Ed25519Keyring; use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use substrate_test_runtime_client::{ @@ -196,8 +196,8 @@ mod tests { } fn voters() -> HashSet { - let voter_id_1 = AuthorityId::from_slice(&[1; 32]); - let voter_id_2 = AuthorityId::from_slice(&[2; 32]); + let voter_id_1 = AuthorityId::from_slice(&[1; 32]).unwrap(); + let voter_id_2 = AuthorityId::from_slice(&[2; 32]).unwrap(); vec![voter_id_1, voter_id_2].into_iter().collect() } @@ -245,7 +245,7 @@ mod tests { impl ReportVoterState for TestVoterState { fn get(&self) -> Option> { - let voter_id_1 = AuthorityId::from_slice(&[1; 32]); + let voter_id_1 = AuthorityId::from_slice(&[1; 32]).unwrap(); let voters_best: HashSet<_> = vec![voter_id_1].into_iter().collect(); let best_round_state = sc_finality_grandpa::report::RoundState { @@ -469,7 +469,7 @@ mod tests { // Notify with a header and justification let justification = create_justification(); - justification_sender.notify(|| Ok(justification.clone())).unwrap(); + justification_sender.notify(|| Ok::<_, ()>(justification.clone())).unwrap(); // Inspect what we received let recv = futures::executor::block_on(receiver.take(1).collect::>()); diff --git a/client/finality-grandpa/rpc/src/notification.rs b/client/finality-grandpa/rpc/src/notification.rs index 68944e903e0f..6fb138357935 100644 --- a/client/finality-grandpa/rpc/src/notification.rs +++ b/client/finality-grandpa/rpc/src/notification.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/finality-grandpa/rpc/src/report.rs b/client/finality-grandpa/rpc/src/report.rs index fef8f2265995..24d0b5ab0d1d 100644 --- a/client/finality-grandpa/rpc/src/report.rs +++ b/client/finality-grandpa/rpc/src/report.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -87,8 +87,6 @@ impl RoundState { round_state: &report::RoundState, voters: &HashSet, ) -> Result { - use std::convert::TryInto; - let prevotes = &round_state.prevote_ids; let missing_prevotes = voters.difference(&prevotes).cloned().collect(); @@ -130,8 +128,6 @@ impl ReportedRoundStates { AuthoritySet: ReportAuthoritySet, VoterState: ReportVoterState, { - use std::convert::TryFrom; - let voter_state = voter_state.get().ok_or(Error::EndpointNotReady)?; let (set_id, current_voters) = authority_set.get(); diff --git a/client/finality-grandpa/src/authorities.rs b/client/finality-grandpa/src/authorities.rs index 6e5dfdd05e62..668fe5f26905 100644 --- a/client/finality-grandpa/src/authorities.rs +++ b/client/finality-grandpa/src/authorities.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,7 +21,7 @@ use std::{cmp::Ord, fmt::Debug, ops::Add}; use finality_grandpa::voter_set::VoterSet; -use fork_tree::ForkTree; +use fork_tree::{FilterAction, ForkTree}; use log::debug; use parity_scale_codec::{Decode, Encode}; use parking_lot::MappedMutexGuard; @@ -32,23 +32,22 @@ use sp_finality_grandpa::{AuthorityId, AuthorityList}; use crate::SetId; /// Error type returned on operations on the `AuthoritySet`. -#[derive(Debug, derive_more::Display)] +#[derive(Debug, thiserror::Error)] pub enum Error { - #[display(fmt = "Invalid authority set, either empty or with an authority weight set to 0.")] + #[error("Invalid authority set, either empty or with an authority weight set to 0.")] InvalidAuthoritySet, - #[display(fmt = "Client error during ancestry lookup: {}", _0)] + #[error("Client error during ancestry lookup: {0}")] Client(E), - #[display(fmt = "Duplicate authority set change.")] + #[error("Duplicate authority set change.")] DuplicateAuthoritySetChange, - #[display(fmt = "Multiple pending forced authority set changes are not allowed.")] + #[error("Multiple pending forced authority set changes are not allowed.")] MultiplePendingForcedAuthoritySetChanges, - #[display( - fmt = "A pending forced authority set change could not be applied since it must be applied \ - after the pending standard change at #{}", - _0 + #[error( + "A pending forced authority set change could not be applied since it must be applied \ + after the pending standard change at #{0}" )] ForcedAuthoritySetChangeDependencyUnsatisfied(N), - #[display(fmt = "Invalid operation in the pending changes tree: {}", _0)] + #[error("Invalid operation in the pending changes tree: {0}")] ForkTree(fork_tree::Error), } @@ -168,7 +167,7 @@ pub struct AuthoritySet { /// Track at which blocks the set id changed. This is useful when we need to prove finality for /// a given block since we can figure out what set the block belongs to and when the set /// started/ended. - authority_set_changes: AuthoritySetChanges, + pub(crate) authority_set_changes: AuthoritySetChanges, } impl AuthoritySet @@ -221,6 +220,37 @@ where pub(crate) fn current(&self) -> (u64, &[(AuthorityId, u64)]) { (self.set_id, &self.current_authorities[..]) } + + /// Revert to a specified block given its `hash` and `number`. + /// This removes all the authority set changes that were announced after + /// the revert point. + /// Revert point is identified by `number` and `hash`. + pub(crate) fn revert(&mut self, hash: H, number: N, is_descendent_of: &F) + where + F: Fn(&H, &H) -> Result, + { + let mut filter = |node_hash: &H, node_num: &N, _: &PendingChange| { + if number >= *node_num && + (is_descendent_of(node_hash, &hash).unwrap_or_default() || *node_hash == hash) + { + // Continue the search in this subtree. + FilterAction::KeepNode + } else if number < *node_num && is_descendent_of(&hash, node_hash).unwrap_or_default() { + // Found a node to be removed. + FilterAction::Remove + } else { + // Not a parent or child of the one we're looking for, stop processing this branch. + FilterAction::KeepTree + } + }; + + // Remove standard changes. + let _ = self.pending_standard_changes.drain_filter(&mut filter); + + // Remove forced changes. + self.pending_forced_changes + .retain(|change| !is_descendent_of(&hash, &change.canon_hash).unwrap_or_default()); + } } impl AuthoritySet @@ -714,6 +744,17 @@ impl AuthoritySetChanges { } } + pub(crate) fn insert(&mut self, block_number: N) { + let idx = self + .0 + .binary_search_by_key(&block_number, |(_, n)| n.clone()) + .unwrap_or_else(|b| b); + + let set_id = if idx == 0 { 0 } else { self.0[idx - 1].0 + 1 }; + assert!(idx == self.0.len() || self.0[idx].0 != set_id); + self.0.insert(idx, (set_id, block_number)); + } + /// Returns an iterator over all historical authority set changes starting at the given block /// number (excluded). The iterator yields a tuple representing the set id and the block number /// of the last block in that set. @@ -742,7 +783,7 @@ impl AuthoritySetChanges { #[cfg(test)] mod tests { use super::*; - use sp_core::crypto::Public; + use sp_core::crypto::{ByteArray, UncheckedFrom}; fn static_is_descendent_of(value: bool) -> impl Fn(&A, &A) -> Result { move |_, _| Ok(value) @@ -757,7 +798,7 @@ mod tests { #[test] fn current_limit_filters_min() { - let current_authorities = vec![(AuthorityId::from_slice(&[1; 32]), 1)]; + let current_authorities = vec![(AuthorityId::from_slice(&[1; 32]).unwrap(), 1)]; let mut authorities = AuthoritySet { current_authorities: current_authorities.clone(), @@ -791,7 +832,7 @@ mod tests { #[test] fn changes_iterated_in_pre_order() { - let current_authorities = vec![(AuthorityId::from_slice(&[1; 32]), 1)]; + let current_authorities = vec![(AuthorityId::from_slice(&[1; 32]).unwrap(), 1)]; let mut authorities = AuthoritySet { current_authorities: current_authorities.clone(), @@ -883,8 +924,8 @@ mod tests { authority_set_changes: AuthoritySetChanges::empty(), }; - let set_a = vec![(AuthorityId::from_slice(&[1; 32]), 5)]; - let set_b = vec![(AuthorityId::from_slice(&[2; 32]), 5)]; + let set_a = vec![(AuthorityId::from_slice(&[1; 32]).unwrap(), 5)]; + let set_b = vec![(AuthorityId::from_slice(&[2; 32]).unwrap(), 5)]; // two competing changes at the same height on different forks let change_a = PendingChange { @@ -966,8 +1007,8 @@ mod tests { authority_set_changes: AuthoritySetChanges::empty(), }; - let set_a = vec![(AuthorityId::from_slice(&[1; 32]), 5)]; - let set_c = vec![(AuthorityId::from_slice(&[2; 32]), 5)]; + let set_a = vec![(AuthorityId::from_slice(&[1; 32]).unwrap(), 5)]; + let set_c = vec![(AuthorityId::from_slice(&[2; 32]).unwrap(), 5)]; // two competing changes at the same height on different forks let change_a = PendingChange { @@ -1046,7 +1087,7 @@ mod tests { authority_set_changes: AuthoritySetChanges::empty(), }; - let set_a = vec![(AuthorityId::from_slice(&[1; 32]), 5)]; + let set_a = vec![(AuthorityId::from_slice(&[1; 32]).unwrap(), 5)]; let change_a = PendingChange { next_authorities: set_a.clone(), @@ -1117,8 +1158,8 @@ mod tests { authority_set_changes: AuthoritySetChanges::empty(), }; - let set_a = vec![(AuthorityId::from_slice(&[1; 32]), 5)]; - let set_b = vec![(AuthorityId::from_slice(&[2; 32]), 5)]; + let set_a = vec![(AuthorityId::from_slice(&[1; 32]).unwrap(), 5)]; + let set_b = vec![(AuthorityId::from_slice(&[2; 32]).unwrap(), 5)]; let change_a = PendingChange { next_authorities: set_a.clone(), @@ -1217,7 +1258,7 @@ mod tests { authority_set_changes: AuthoritySetChanges::empty(), }; - let set_a = vec![(AuthorityId::from_slice(&[1; 32]), 5)]; + let set_a = vec![(AuthorityId::from_slice(&[1; 32]).unwrap(), 5)]; // we create a forced change with no delay let change_a = PendingChange { @@ -1242,7 +1283,7 @@ mod tests { #[test] fn forced_changes_blocked_by_standard_changes() { - let set_a = vec![(AuthorityId::from_slice(&[1; 32]), 1)]; + let set_a = vec![(AuthorityId::from_slice(&[1; 32]).unwrap(), 1)]; let mut authorities = AuthoritySet { current_authorities: set_a.clone(), @@ -1367,7 +1408,7 @@ mod tests { #[test] fn next_change_works() { - let current_authorities = vec![(AuthorityId::from_slice(&[1; 32]), 1)]; + let current_authorities = vec![(AuthorityId::from_slice(&[1; 32]).unwrap(), 1)]; let mut authorities = AuthoritySet { current_authorities: current_authorities.clone(), @@ -1482,8 +1523,10 @@ mod tests { None, ); - let invalid_authorities_weight = - vec![(AuthorityId::from_slice(&[1; 32]), 5), (AuthorityId::from_slice(&[2; 32]), 0)]; + let invalid_authorities_weight = vec![ + (AuthorityId::from_slice(&[1; 32]).unwrap(), 5), + (AuthorityId::from_slice(&[2; 32]).unwrap(), 0), + ]; // authority weight of zero is invalid assert_eq!(AuthoritySet::<(), ()>::genesis(invalid_authorities_weight.clone()), None); @@ -1499,7 +1542,8 @@ mod tests { ); let mut authority_set = - AuthoritySet::<(), u64>::genesis(vec![(AuthorityId::from_slice(&[1; 32]), 5)]).unwrap(); + AuthoritySet::<(), u64>::genesis(vec![(AuthorityId::unchecked_from([1; 32]), 5)]) + .unwrap(); let invalid_change_empty_authorities = PendingChange { next_authorities: vec![], @@ -1539,7 +1583,7 @@ mod tests { #[test] fn cleans_up_stale_forced_changes_when_applying_standard_change() { - let current_authorities = vec![(AuthorityId::from_slice(&[1; 32]), 1)]; + let current_authorities = vec![(AuthorityId::from_slice(&[1; 32]).unwrap(), 1)]; let mut authorities = AuthoritySet { current_authorities: current_authorities.clone(), @@ -1632,6 +1676,18 @@ mod tests { assert_eq!(authorities.pending_forced_changes.first().unwrap().canon_hash, "D"); } + #[test] + fn authority_set_changes_insert() { + let mut authority_set_changes = AuthoritySetChanges::empty(); + authority_set_changes.append(0, 41); + authority_set_changes.append(1, 81); + authority_set_changes.append(4, 121); + + authority_set_changes.insert(101); + assert_eq!(authority_set_changes.get_set_id(100), AuthoritySetChangeId::Set(2, 101)); + assert_eq!(authority_set_changes.get_set_id(101), AuthoritySetChangeId::Set(2, 101)); + } + #[test] fn authority_set_changes_for_complete_data() { let mut authority_set_changes = AuthoritySetChanges::empty(); diff --git a/client/finality-grandpa/src/aux_schema.rs b/client/finality-grandpa/src/aux_schema.rs index bad01e6dfc62..0ac9ba9e64bd 100644 --- a/client/finality-grandpa/src/aux_schema.rs +++ b/client/finality-grandpa/src/aux_schema.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -100,7 +100,7 @@ where // previously we only supported at most one pending change per fork &|_, _| Ok(false), ) { - warn!(target: "afg", "Error migrating pending authority set change: {:?}.", err); + warn!(target: "afg", "Error migrating pending authority set change: {}", err); warn!(target: "afg", "Node is in a potentially inconsistent state."); } } @@ -498,15 +498,19 @@ pub(crate) fn load_authorities( #[cfg(test)] mod test { use super::*; - use sp_core::H256; + use sp_core::{crypto::UncheckedFrom, H256}; use sp_finality_grandpa::AuthorityId; use substrate_test_runtime_client; + fn dummy_id() -> AuthorityId { + AuthorityId::unchecked_from([1; 32]) + } + #[test] fn load_decode_from_v0_migrates_data_format() { let client = substrate_test_runtime_client::new(); - let authorities = vec![(AuthorityId::default(), 100)]; + let authorities = vec![(dummy_id(), 100)]; let set_id = 3; let round_number: RoundNumber = 42; let round_state = RoundState:: { @@ -595,7 +599,7 @@ mod test { fn load_decode_from_v1_migrates_data_format() { let client = substrate_test_runtime_client::new(); - let authorities = vec![(AuthorityId::default(), 100)]; + let authorities = vec![(dummy_id(), 100)]; let set_id = 3; let round_number: RoundNumber = 42; let round_state = RoundState:: { @@ -688,7 +692,7 @@ mod test { fn load_decode_from_v2_migrates_data_format() { let client = substrate_test_runtime_client::new(); - let authorities = vec![(AuthorityId::default(), 100)]; + let authorities = vec![(dummy_id(), 100)]; let set_id = 3; { diff --git a/client/finality-grandpa/src/communication/gossip.rs b/client/finality-grandpa/src/communication/gossip.rs index 2e50a3bac01d..c39e2e82a621 100644 --- a/client/finality-grandpa/src/communication/gossip.rs +++ b/client/finality-grandpa/src/communication/gossip.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -84,23 +84,23 @@ //! //! We only send polite messages to peers, -use parity_scale_codec::{Decode, Encode}; -use sc_network::{ObservedRole, PeerId, ReputationChange}; -use sc_network_gossip::{MessageIntent, ValidatorContext}; -use sp_finality_grandpa::AuthorityId; -use sp_runtime::traits::{Block as BlockT, NumberFor, Zero}; - +use ahash::{AHashMap, AHashSet}; use log::{debug, trace}; +use parity_scale_codec::{Decode, Encode}; use prometheus_endpoint::{register, CounterVec, Opts, PrometheusError, Registry, U64}; use rand::seq::SliceRandom; +use sc_network::{ObservedRole, PeerId, ReputationChange}; +use sc_network_gossip::{MessageIntent, ValidatorContext}; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sp_finality_grandpa::AuthorityId; +use sp_runtime::traits::{Block as BlockT, NumberFor, Zero}; use super::{benefit, cost, Round, SetId}; use crate::{environment, CatchUp, CompactCommit, SignedMessage}; use std::{ - collections::{HashMap, HashSet, VecDeque}, + collections::{HashSet, VecDeque}, time::{Duration, Instant}, }; @@ -260,7 +260,7 @@ const KEEP_RECENT_ROUNDS: usize = 3; struct KeepTopics { current_set: SetId, rounds: VecDeque<(Round, SetId)>, - reverse_map: HashMap, SetId)>, + reverse_map: AHashMap, SetId)>, } impl KeepTopics { @@ -268,7 +268,7 @@ impl KeepTopics { KeepTopics { current_set: SetId(0), rounds: VecDeque::with_capacity(KEEP_RECENT_ROUNDS + 2), - reverse_map: HashMap::new(), + reverse_map: Default::default(), } } @@ -290,7 +290,7 @@ impl KeepTopics { let _ = self.rounds.pop_front(); } - let mut map = HashMap::with_capacity(KEEP_RECENT_ROUNDS + 3); + let mut map = AHashMap::with_capacity(KEEP_RECENT_ROUNDS + 3); map.insert(super::global_topic::(self.current_set.0), (None, self.current_set)); for &(round, set) in &self.rounds { @@ -477,10 +477,10 @@ impl PeerInfo { /// The peers we're connected to in gossip. struct Peers { - inner: HashMap>, + inner: AHashMap>, /// The randomly picked set of `LUCKY_PEERS` we'll gossip to in the first stage of round /// gossiping. - first_stage_peers: HashSet, + first_stage_peers: AHashSet, /// The randomly picked set of peers we'll gossip to in the second stage of gossiping if the /// first stage didn't allow us to spread the voting data enough to conclude the round. This /// set should have size `sqrt(connected_peers)`. @@ -492,10 +492,10 @@ struct Peers { impl Default for Peers { fn default() -> Self { Peers { - inner: HashMap::new(), - first_stage_peers: HashSet::new(), - second_stage_peers: HashSet::new(), - lucky_light_peers: HashSet::new(), + inner: Default::default(), + first_stage_peers: Default::default(), + second_stage_peers: Default::default(), + lucky_light_peers: Default::default(), } } } @@ -608,15 +608,14 @@ impl Peers { } }); - let mut first_stage_peers = HashSet::new(); + let mut first_stage_peers = AHashSet::new(); let mut second_stage_peers = HashSet::new(); // we start by allocating authorities to the first stage set and when the minimum of // `LUCKY_PEERS / 2` is filled we start allocating to the second stage set. let half_lucky = LUCKY_PEERS / 2; let one_and_a_half_lucky = LUCKY_PEERS + half_lucky; - let mut n_authorities_added = 0; - for peer_id in shuffled_authorities { + for (n_authorities_added, peer_id) in shuffled_authorities.enumerate() { if n_authorities_added < half_lucky { first_stage_peers.insert(*peer_id); } else if n_authorities_added < one_and_a_half_lucky { @@ -624,8 +623,6 @@ impl Peers { } else { break } - - n_authorities_added += 1; } // fill up first and second sets with remaining peers (either full or authorities) @@ -802,7 +799,7 @@ impl Inner { Some(ref mut v) => if v.set_id == set_id { let diff_authorities = self.authorities.iter().collect::>() != - authorities.iter().collect(); + authorities.iter().collect::>(); if diff_authorities { debug!(target: "afg", @@ -1299,7 +1296,7 @@ impl Metrics { messages_validated: register( CounterVec::new( Opts::new( - "finality_grandpa_communication_gossip_validator_messages", + "substrate_finality_grandpa_communication_gossip_validator_messages", "Number of messages validated by the finality grandpa gossip validator.", ), &["message", "action"], @@ -1667,10 +1664,11 @@ pub(super) struct PeerReport { #[cfg(test)] mod tests { use super::{environment::SharedVoterSetState, *}; + use crate::communication; use sc_network::config::Role; use sc_network_gossip::Validator as GossipValidatorT; use sc_network_test::Block; - use sp_core::{crypto::Public, H256}; + use sp_core::{crypto::UncheckedFrom, H256}; // some random config (not really needed) fn config() -> crate::Config { @@ -1682,6 +1680,7 @@ mod tests { local_role: Role::Authority, observer_enabled: true, telemetry: None, + protocol_name: communication::grandpa_protocol_name::NAME.into(), } } @@ -1691,7 +1690,7 @@ mod tests { let base = (H256::zero(), 0); - let voters = vec![(AuthorityId::from_slice(&[1; 32]), 1)]; + let voters = vec![(AuthorityId::unchecked_from([1; 32]), 1)]; let voters = AuthoritySet::genesis(voters).unwrap(); let set_state = VoterSetState::live(0, &voters, base); @@ -1843,13 +1842,13 @@ mod tests { // messages from old rounds are expired. for round_num in 1u64..last_kept_round { - let topic = crate::communication::round_topic::(round_num, 1); + let topic = communication::round_topic::(round_num, 1); assert!(is_expired(topic, &[1, 2, 3])); } // messages from not-too-old rounds are not expired. for round_num in last_kept_round..10 { - let topic = crate::communication::round_topic::(round_num, 1); + let topic = communication::round_topic::(round_num, 1); assert!(!is_expired(topic, &[1, 2, 3])); } } @@ -1861,7 +1860,7 @@ mod tests { let (val, _) = GossipValidator::::new(config(), voter_set_state(), None, None); let set_id = 1; - let auth = AuthorityId::from_slice(&[1u8; 32]); + let auth = AuthorityId::unchecked_from([1u8; 32]); let peer = PeerId::random(); val.note_set(SetId(set_id), vec![auth.clone()], |_, _| {}); @@ -1878,8 +1877,8 @@ mod tests { target_hash: Default::default(), target_number: 10, }), - signature: Default::default(), - id: AuthorityId::from_slice(&[2u8; 32]), + signature: UncheckedFrom::unchecked_from([1; 64]), + id: UncheckedFrom::unchecked_from([2u8; 32]), }, }, ); @@ -1894,7 +1893,7 @@ mod tests { target_hash: Default::default(), target_number: 10, }), - signature: Default::default(), + signature: UncheckedFrom::unchecked_from([1; 64]), id: auth.clone(), }, }, @@ -1909,7 +1908,7 @@ mod tests { let (val, _) = GossipValidator::::new(config(), voter_set_state(), None, None); let set_id = 1; - let auth = AuthorityId::from_slice(&[1u8; 32]); + let auth = AuthorityId::unchecked_from([1u8; 32]); let peer = PeerId::random(); val.note_set(SetId(set_id), vec![auth.clone()], |_, _| {}); @@ -1972,7 +1971,7 @@ mod tests { let (val, _) = GossipValidator::::new(config(), set_state.clone(), None, None); let set_id = 1; - let auth = AuthorityId::from_slice(&[1u8; 32]); + let auth = AuthorityId::unchecked_from([1u8; 32]); let peer = PeerId::random(); val.note_set(SetId(set_id), vec![auth.clone()], |_, _| {}); @@ -2265,7 +2264,7 @@ mod tests { // we accept messages from rounds 9, 10 and 11 // therefore neither of those should be considered expired for round in &[9, 10, 11] { - assert!(!is_expired(crate::communication::round_topic::(*round, 1), &[])) + assert!(!is_expired(communication::round_topic::(*round, 1), &[])) } } @@ -2313,7 +2312,7 @@ mod tests { if message_allowed( peer, MessageIntent::Broadcast, - &crate::communication::round_topic::(1, 0), + &communication::round_topic::(1, 0), &[], ) { allowed += 1; @@ -2377,7 +2376,7 @@ mod tests { assert!(!val.message_allowed()( &light_peer, MessageIntent::Broadcast, - &crate::communication::round_topic::(1, 0), + &communication::round_topic::(1, 0), &[], )); @@ -2391,7 +2390,7 @@ mod tests { assert!(!val.message_allowed()( &light_peer, MessageIntent::Broadcast, - &crate::communication::round_topic::(1, 0), + &communication::round_topic::(1, 0), &[], )); @@ -2415,8 +2414,8 @@ mod tests { auth_data: Vec::new(), }; - crate::communication::gossip::GossipMessage::::Commit( - crate::communication::gossip::FullCommitMessage { + communication::gossip::GossipMessage::::Commit( + communication::gossip::FullCommitMessage { round: Round(2), set_id: SetId(0), message: commit, @@ -2429,7 +2428,7 @@ mod tests { assert!(val.message_allowed()( &light_peer, MessageIntent::Broadcast, - &crate::communication::global_topic::(0), + &communication::global_topic::(0), &commit, )); } @@ -2469,8 +2468,8 @@ mod tests { auth_data: Vec::new(), }; - crate::communication::gossip::GossipMessage::::Commit( - crate::communication::gossip::FullCommitMessage { + communication::gossip::GossipMessage::::Commit( + communication::gossip::FullCommitMessage { round: Round(1), set_id: SetId(1), message: commit, @@ -2488,7 +2487,7 @@ mod tests { assert!(message_allowed( &peer1, MessageIntent::Broadcast, - &crate::communication::global_topic::(1), + &communication::global_topic::(1), &commit, )); @@ -2497,7 +2496,7 @@ mod tests { assert!(!message_allowed( &peer2, MessageIntent::Broadcast, - &crate::communication::global_topic::(1), + &communication::global_topic::(1), &commit, )); } @@ -2514,8 +2513,8 @@ mod tests { auth_data: Vec::new(), }; - crate::communication::gossip::GossipMessage::::Commit( - crate::communication::gossip::FullCommitMessage { + communication::gossip::GossipMessage::::Commit( + communication::gossip::FullCommitMessage { round: Round(round), set_id: SetId(set_id), message: commit, @@ -2535,27 +2534,26 @@ mod tests { // a commit message for round 1 that finalizes the same height as we // have observed previously should not be expired - assert!( - !message_expired(crate::communication::global_topic::(1), &commit(1, 1, 2),) - ); + assert!(!message_expired(communication::global_topic::(1), &commit(1, 1, 2),)); // it should be expired if it is for a lower block - assert!(message_expired(crate::communication::global_topic::(1), &commit(1, 1, 1))); + assert!(message_expired(communication::global_topic::(1), &commit(1, 1, 1))); // or the same block height but from the previous round - assert!(message_expired(crate::communication::global_topic::(1), &commit(0, 1, 2))); + assert!(message_expired(communication::global_topic::(1), &commit(0, 1, 2))); } #[test] fn allow_noting_different_authorities_for_same_set() { let (val, _) = GossipValidator::::new(config(), voter_set_state(), None, None); - let a1 = vec![AuthorityId::from_slice(&[0; 32])]; + let a1 = vec![UncheckedFrom::unchecked_from([0; 32])]; val.note_set(SetId(1), a1.clone(), |_, _| {}); assert_eq!(val.inner().read().authorities, a1); - let a2 = vec![AuthorityId::from_slice(&[1; 32]), AuthorityId::from_slice(&[2; 32])]; + let a2 = + vec![UncheckedFrom::unchecked_from([1; 32]), UncheckedFrom::unchecked_from([2; 32])]; val.note_set(SetId(1), a2.clone(), |_, _| {}); assert_eq!(val.inner().read().authorities, a2); diff --git a/client/finality-grandpa/src/communication/mod.rs b/client/finality-grandpa/src/communication/mod.rs index c370e1d642d7..9c05774fffdf 100644 --- a/client/finality-grandpa/src/communication/mod.rs +++ b/client/finality-grandpa/src/communication/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -67,9 +67,27 @@ mod periodic; #[cfg(test)] pub(crate) mod tests; -/// Name of the notifications protocol used by Grandpa. Must be registered towards the networking -/// in order for Grandpa to properly function. -pub const GRANDPA_PROTOCOL_NAME: &'static str = "/paritytech/grandpa/1"; +pub mod grandpa_protocol_name { + use sc_chain_spec::ChainSpec; + + pub(crate) const NAME: &'static str = "/grandpa/1"; + /// Old names for the notifications protocol, used for backward compatibility. + pub(crate) const LEGACY_NAMES: [&'static str; 1] = ["/paritytech/grandpa/1"]; + + /// Name of the notifications protocol used by GRANDPA. + /// + /// Must be registered towards the networking in order for GRANDPA to properly function. + pub fn standard_name>( + genesis_hash: &Hash, + chain_spec: &Box, + ) -> std::borrow::Cow<'static, str> { + let chain_prefix = match chain_spec.fork_id() { + Some(fork_id) => format!("/{}/{}", hex::encode(genesis_hash), fork_id), + None => format!("/{}", hex::encode(genesis_hash)), + }; + format!("{}{}", chain_prefix, NAME).into() + } +} // cost scalars for reporting peers. mod cost { @@ -220,13 +238,14 @@ impl> NetworkBridge { prometheus_registry: Option<&Registry>, telemetry: Option, ) -> Self { + let protocol = config.protocol_name.clone(); let (validator, report_stream) = GossipValidator::new(config, set_state.clone(), prometheus_registry, telemetry.clone()); let validator = Arc::new(validator); let gossip_engine = Arc::new(Mutex::new(GossipEngine::new( service.clone(), - GRANDPA_PROTOCOL_NAME, + protocol, validator.clone(), prometheus_registry, ))); diff --git a/client/finality-grandpa/src/communication/periodic.rs b/client/finality-grandpa/src/communication/periodic.rs index 77e55ad652f6..e6d63beafc36 100644 --- a/client/finality-grandpa/src/communication/periodic.rs +++ b/client/finality-grandpa/src/communication/periodic.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/finality-grandpa/src/communication/tests.rs b/client/finality-grandpa/src/communication/tests.rs index 1fac0230b2a8..e41d21fc0684 100644 --- a/client/finality-grandpa/src/communication/tests.rs +++ b/client/finality-grandpa/src/communication/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -22,7 +22,7 @@ use super::{ gossip::{self, GossipValidator}, Round, SetId, VoterSet, }; -use crate::{communication::GRANDPA_PROTOCOL_NAME, environment::SharedVoterSetState}; +use crate::{communication::grandpa_protocol_name, environment::SharedVoterSetState}; use futures::prelude::*; use parity_scale_codec::Encode; use sc_network::{config::Role, Event as NetworkEvent, ObservedRole, PeerId}; @@ -97,7 +97,7 @@ impl sc_network_gossip::ValidatorContext for TestNetwork { >::write_notification( self, who.clone(), - GRANDPA_PROTOCOL_NAME.into(), + grandpa_protocol_name::NAME.into(), data, ); } @@ -148,6 +148,7 @@ fn config() -> crate::Config { local_role: Role::Authority, observer_enabled: true, telemetry: None, + protocol_name: grandpa_protocol_name::NAME.into(), } } @@ -155,13 +156,13 @@ fn config() -> crate::Config { fn voter_set_state() -> SharedVoterSetState { use crate::{authorities::AuthoritySet, environment::VoterSetState}; use finality_grandpa::round::State as RoundState; - use sp_core::{crypto::Public, H256}; + use sp_core::{crypto::ByteArray, H256}; use sp_finality_grandpa::AuthorityId; let state = RoundState::genesis((H256::zero(), 0)); let base = state.prevote_ghost.unwrap(); - let voters = vec![(AuthorityId::from_slice(&[1; 32]), 1)]; + let voters = vec![(AuthorityId::from_slice(&[1; 32]).unwrap(), 1)]; let voters = AuthoritySet::genesis(voters).unwrap(); let set_state = VoterSetState::live(0, &voters, base); @@ -286,7 +287,7 @@ fn good_commit_leads_to_relay() { // Add the sending peer and send the commit let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { remote: sender_id.clone(), - protocol: GRANDPA_PROTOCOL_NAME.into(), + protocol: grandpa_protocol_name::NAME.into(), negotiated_fallback: None, role: ObservedRole::Full, }); @@ -294,7 +295,7 @@ fn good_commit_leads_to_relay() { let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived { remote: sender_id.clone(), messages: vec![( - GRANDPA_PROTOCOL_NAME.into(), + grandpa_protocol_name::NAME.into(), commit_to_send.clone().into(), )], }); @@ -303,7 +304,7 @@ fn good_commit_leads_to_relay() { let receiver_id = sc_network::PeerId::random(); let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { remote: receiver_id.clone(), - protocol: GRANDPA_PROTOCOL_NAME.into(), + protocol: grandpa_protocol_name::NAME.into(), negotiated_fallback: None, role: ObservedRole::Full, }); @@ -321,7 +322,10 @@ fn good_commit_leads_to_relay() { sender.unbounded_send(NetworkEvent::NotificationsReceived { remote: receiver_id, - messages: vec![(GRANDPA_PROTOCOL_NAME.into(), msg.encode().into())], + messages: vec![( + grandpa_protocol_name::NAME.into(), + msg.encode().into(), + )], }) }; @@ -433,14 +437,14 @@ fn bad_commit_leads_to_report() { Event::EventStream(sender) => { let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { remote: sender_id.clone(), - protocol: GRANDPA_PROTOCOL_NAME.into(), + protocol: grandpa_protocol_name::NAME.into(), negotiated_fallback: None, role: ObservedRole::Full, }); let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived { remote: sender_id.clone(), messages: vec![( - GRANDPA_PROTOCOL_NAME.into(), + grandpa_protocol_name::NAME.into(), commit_to_send.clone().into(), )], }); @@ -531,3 +535,46 @@ fn peer_with_higher_view_leads_to_catch_up_request() { futures::executor::block_on(test); } + +fn local_chain_spec() -> Box { + use sc_chain_spec::{ChainSpec, GenericChainSpec}; + use serde::{Deserialize, Serialize}; + use sp_runtime::{BuildStorage, Storage}; + + #[derive(Debug, Serialize, Deserialize)] + struct Genesis(std::collections::BTreeMap); + impl BuildStorage for Genesis { + fn assimilate_storage(&self, storage: &mut Storage) -> Result<(), String> { + storage.top.extend( + self.0.iter().map(|(a, b)| (a.clone().into_bytes(), b.clone().into_bytes())), + ); + Ok(()) + } + } + let chain_spec = GenericChainSpec::::from_json_file(std::path::PathBuf::from( + "../chain-spec/res/chain_spec.json", + )) + .unwrap(); + chain_spec.cloned_box() +} + +#[test] +fn grandpa_protocol_name() { + let chain_spec = local_chain_spec(); + + // Create protocol name using random genesis hash. + let genesis_hash = sp_core::H256::random(); + let expected = format!("/{}/grandpa/1", hex::encode(genesis_hash)); + let proto_name = grandpa_protocol_name::standard_name(&genesis_hash, &chain_spec); + assert_eq!(proto_name.to_string(), expected); + + // Create protocol name using hardcoded genesis hash. Verify exact representation. + let genesis_hash = [ + 53, 79, 112, 97, 119, 217, 39, 202, 147, 138, 225, 38, 88, 182, 215, 185, 110, 88, 8, 53, + 125, 210, 158, 151, 50, 113, 102, 59, 245, 199, 221, 240, + ]; + let expected = + "/354f706177d927ca938ae12658b6d7b96e5808357dd29e973271663bf5c7ddf0/grandpa/1".to_string(); + let proto_name = grandpa_protocol_name::standard_name(&genesis_hash, &chain_spec); + assert_eq!(proto_name.to_string(), expected); +} diff --git a/client/finality-grandpa/src/environment.rs b/client/finality-grandpa/src/environment.rs index c79698902e97..6ffcdc719a16 100644 --- a/client/finality-grandpa/src/environment.rs +++ b/client/finality-grandpa/src/environment.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -404,19 +404,19 @@ impl Metrics { ) -> Result { Ok(Self { finality_grandpa_round: register( - Gauge::new("finality_grandpa_round", "Highest completed GRANDPA round.")?, + Gauge::new("substrate_finality_grandpa_round", "Highest completed GRANDPA round.")?, registry, )?, finality_grandpa_prevotes: register( Counter::new( - "finality_grandpa_prevotes_total", + "substrate_finality_grandpa_prevotes_total", "Total number of GRANDPA prevotes cast locally.", )?, registry, )?, finality_grandpa_precommits: register( Counter::new( - "finality_grandpa_precommits_total", + "substrate_finality_grandpa_precommits_total", "Total number of GRANDPA precommits cast locally.", )?, registry, @@ -609,7 +609,7 @@ where let tree_route = match tree_route_res { Ok(tree_route) => tree_route, Err(e) => { - debug!(target: "afg", "Encountered error computing ancestry between block {:?} and base {:?}: {:?}", + debug!(target: "afg", "Encountered error computing ancestry between block {:?} and base {:?}: {}", block, base, e); return Err(GrandpaError::NotDescendent) @@ -1098,7 +1098,7 @@ where ) { warn!(target: "afg", "Detected prevote equivocation in the finality worker: {:?}", equivocation); if let Err(err) = self.report_equivocation(equivocation.into()) { - warn!(target: "afg", "Error reporting prevote equivocation: {:?}", err); + warn!(target: "afg", "Error reporting prevote equivocation: {}", err); } } @@ -1109,7 +1109,7 @@ where ) { warn!(target: "afg", "Detected precommit equivocation in the finality worker: {:?}", equivocation); if let Err(err) = self.report_equivocation(equivocation.into()) { - warn!(target: "afg", "Error reporting precommit equivocation: {:?}", err); + warn!(target: "afg", "Error reporting precommit equivocation: {}", err); } } } @@ -1224,7 +1224,7 @@ where .or_else(|| Some((target_header.hash(), *target_header.number()))) }, Err(e) => { - debug!(target: "afg", "Encountered error finding best chain containing {:?}: {:?}", block, e); + warn!(target: "afg", "Encountered error finding best chain containing {:?}: {}", block, e); None }, }; @@ -1293,7 +1293,7 @@ where ) { if let Some(sender) = justification_sender { if let Err(err) = sender.notify(justification) { - warn!(target: "afg", "Error creating justification for subscriber: {:?}", err); + warn!(target: "afg", "Error creating justification for subscriber: {}", err); } } } @@ -1344,7 +1344,7 @@ where client .apply_finality(import_op, BlockId::Hash(hash), persisted_justification, true) .map_err(|e| { - warn!(target: "afg", "Error applying finality to block {:?}: {:?}", (hash, number), e); + warn!(target: "afg", "Error applying finality to block {:?}: {}", (hash, number), e); e })?; diff --git a/client/finality-grandpa/src/finality_proof.rs b/client/finality-grandpa/src/finality_proof.rs index 1e20c2edc3a6..03a4f2ff450a 100644 --- a/client/finality-grandpa/src/finality_proof.rs +++ b/client/finality-grandpa/src/finality_proof.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -132,17 +132,18 @@ pub struct FinalityProof { } /// Errors occurring when trying to prove finality -#[derive(Debug, derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] pub enum FinalityProofError { /// The requested block has not yet been finalized. - #[display(fmt = "Block not yet finalized")] + #[error("Block not yet finalized")] BlockNotYetFinalized, /// The requested block is not covered by authority set changes. Likely this means the block is /// in the latest authority set, and the subscription API is more appropriate. - #[display(fmt = "Block not covered by authority set changes")] + #[error("Block not covered by authority set changes")] BlockNotInAuthoritySetChanges, /// Errors originating from the client. - Client(sp_blockchain::Error), + #[error(transparent)] + Client(#[from] sp_blockchain::Error), } fn prove_finality( @@ -243,8 +244,8 @@ pub(crate) mod tests { use sc_block_builder::BlockBuilderProvider; use sc_client_api::{apply_aux, LockImportRun}; use sp_consensus::BlockOrigin; - use sp_core::crypto::Public; - use sp_finality_grandpa::{AuthorityId, GRANDPA_ENGINE_ID as ID}; + use sp_core::crypto::UncheckedFrom; + use sp_finality_grandpa::GRANDPA_ENGINE_ID as ID; use sp_keyring::Ed25519Keyring; use substrate_test_runtime_client::{ runtime::{Block, Header, H256}, @@ -350,7 +351,7 @@ pub(crate) mod tests { // When we can't decode proof from Vec check_finality_proof::( 1, - vec![(AuthorityId::from_slice(&[3u8; 32]), 1u64)], + vec![(UncheckedFrom::unchecked_from([3u8; 32]), 1u64)], vec![42], ) .unwrap_err(); @@ -361,7 +362,7 @@ pub(crate) mod tests { // When decoded proof has zero length check_finality_proof::( 1, - vec![(AuthorityId::from_slice(&[3u8; 32]), 1u64)], + vec![(UncheckedFrom::unchecked_from([3u8; 32]), 1u64)], Vec::>::new().encode(), ) .unwrap_err(); @@ -387,7 +388,7 @@ pub(crate) mod tests { check_finality_proof::( 1, - vec![(AuthorityId::from_slice(&[3u8; 32]), 1u64)], + vec![(UncheckedFrom::unchecked_from([3u8; 32]), 1u64)], finality_proof.encode(), ) .unwrap_err(); diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index 1c4d1b4e97b8..ae5839d0c24e 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,7 +19,7 @@ use std::{collections::HashMap, marker::PhantomData, sync::Arc}; use log::debug; -use parity_scale_codec::{Decode, Encode}; +use parity_scale_codec::Decode; use sc_client_api::{backend::Backend, utils::is_descendent_of}; use sc_consensus::{ @@ -35,7 +35,7 @@ use sp_core::hashing::twox_128; use sp_finality_grandpa::{ConsensusLog, GrandpaApi, ScheduledChange, SetId, GRANDPA_ENGINE_ID}; use sp_runtime::{ generic::{BlockId, OpaqueDigestItemId}, - traits::{Block as BlockT, DigestFor, Header as HeaderT, NumberFor, Zero}, + traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}, Justification, }; @@ -89,7 +89,6 @@ impl JustificationImport for GrandpaBlockImport where NumberFor: finality_grandpa::BlockNumberOps, - DigestFor: Encode, BE: Backend, Client: ClientForGrandpa, SC: SelectChain, @@ -229,7 +228,6 @@ pub fn find_forced_change( impl GrandpaBlockImport where NumberFor: finality_grandpa::BlockNumberOps, - DigestFor: Encode, BE: Backend, Client: ClientForGrandpa, Client::Api: GrandpaApi, @@ -515,7 +513,6 @@ where impl BlockImport for GrandpaBlockImport where NumberFor: finality_grandpa::BlockNumberOps, - DigestFor: Encode, BE: Backend, Client: ClientForGrandpa, Client::Api: GrandpaApi, @@ -551,6 +548,32 @@ where return self.import_state(block, new_cache).await } + if number <= self.inner.info().finalized_number { + // Importing an old block. Just save justifications and authority set changes + if self.check_new_change(&block.header, hash).is_some() { + if block.justifications.is_none() { + return Err(ConsensusError::ClientImport( + "Justification required when importing \ + an old block with authority set change." + .into(), + )) + } + assert!(block.justifications.is_some()); + let mut authority_set = self.authority_set.inner_locked(); + authority_set.authority_set_changes.insert(number); + crate::aux_schema::update_authority_set::( + &authority_set, + None, + |insert| { + block + .auxiliary + .extend(insert.iter().map(|(k, v)| (k.to_vec(), Some(v.to_vec())))) + }, + ); + } + return (&*self.inner).import_block(block, new_cache).await + } + // on initial sync we will restrict logging under info to avoid spam. let initial_sync = block.origin == BlockOrigin::NetworkInitialSync; @@ -575,7 +598,7 @@ where Err(e) => { debug!( target: "afg", - "Restoring old authority set after block import error: {:?}", + "Restoring old authority set after block import error: {}", e, ); pending_changes.revert(); @@ -640,8 +663,12 @@ where import_res.unwrap_or_else(|err| { if needs_justification { - debug!(target: "afg", "Imported block #{} that enacts authority set change with \ - invalid justification: {:?}, requesting justification from peers.", number, err); + debug!( + target: "afg", + "Requesting justification from peers due to imported block #{} that enacts authority set change with invalid justification: {}", + number, + err + ); imported_aux.bad_justification = true; imported_aux.needs_justification = true; } diff --git a/client/finality-grandpa/src/justification.rs b/client/finality-grandpa/src/justification.rs index a852c74d9d1a..39f24cb8ea57 100644 --- a/client/finality-grandpa/src/justification.rs +++ b/client/finality-grandpa/src/justification.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -185,7 +185,7 @@ impl GrandpaJustification { } } - let ancestry_hashes = + let ancestry_hashes: HashSet<_> = self.votes_ancestries.iter().map(|h: &Block::Header| h.hash()).collect(); if visited_hashes != ancestry_hashes { diff --git a/client/finality-grandpa/src/lib.rs b/client/finality-grandpa/src/lib.rs index 452659ced6a7..34d5b6bb1f70 100644 --- a/client/finality-grandpa/src/lib.rs +++ b/client/finality-grandpa/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -58,11 +58,12 @@ use futures::{prelude::*, StreamExt}; use log::{debug, error, info}; -use parity_scale_codec::{Decode, Encode}; +use parity_scale_codec::Decode; use parking_lot::RwLock; use prometheus_endpoint::{PrometheusError, Registry}; use sc_client_api::{ backend::{AuxStore, Backend}, + utils::is_descendent_of, BlockchainEvents, CallExecutor, ExecutionStrategy, ExecutorProvider, Finalizer, LockImportRun, StorageProvider, TransactionFor, }; @@ -71,13 +72,13 @@ use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; use sp_api::ProvideRuntimeApi; use sp_application_crypto::AppKey; -use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata}; +use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata, Result as ClientResult}; use sp_consensus::SelectChain; -use sp_core::crypto::Public; +use sp_core::crypto::ByteArray; use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, DigestFor, NumberFor, Zero}, + traits::{Block as BlockT, NumberFor, Zero}, }; pub use finality_grandpa::BlockNumberOps; @@ -123,6 +124,7 @@ pub mod warp_proof; pub use authorities::{AuthoritySet, AuthoritySetChanges, SharedAuthoritySet}; pub use aux_schema::best_justification; +pub use communication::grandpa_protocol_name::standard_name as protocol_standard_name; pub use finality_grandpa::voter::report; pub use finality_proof::{FinalityProof, FinalityProofError, FinalityProofProvider}; pub use import::{find_forced_change, find_scheduled_change, GrandpaBlockImport}; @@ -263,47 +265,52 @@ pub struct Config { pub keystore: Option, /// TelemetryHandle instance. pub telemetry: Option, + /// Chain specific GRANDPA protocol name. See [`crate::protocol_standard_name`]. + pub protocol_name: std::borrow::Cow<'static, str>, } impl Config { fn name(&self) -> &str { - self.name.as_ref().map(|s| s.as_str()).unwrap_or("") + self.name.as_deref().unwrap_or("") } } /// Errors that can occur while voting in GRANDPA. -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum Error { /// An error within grandpa. - Grandpa(GrandpaError), + #[error("grandpa error: {0}")] + Grandpa(#[from] GrandpaError), + /// A network error. + #[error("network error: {0}")] Network(String), + /// A blockchain error. + #[error("blockchain error: {0}")] Blockchain(String), + /// Could not complete a round on disk. - Client(ClientError), + #[error("could not complete a round on disk: {0}")] + Client(#[from] ClientError), + /// Could not sign outgoing message + #[error("could not sign outgoing message: {0}")] Signing(String), + /// An invariant has been violated (e.g. not finalizing pending change blocks in-order) + #[error("safety invariant has been violated: {0}")] Safety(String), + /// A timer failed to fire. + #[error("a timer failed to fire: {0}")] Timer(io::Error), + /// A runtime api request failed. + #[error("runtime API request failed: {0}")] RuntimeApi(sp_api::ApiError), } -impl From for Error { - fn from(e: GrandpaError) -> Self { - Error::Grandpa(e) - } -} - -impl From for Error { - fn from(e: ClientError) -> Self { - Error::Client(e) - } -} - /// Something which can determine if a block is known. pub(crate) trait BlockStatus { /// Return `Ok(Some(number))` or `Ok(None)` depending on whether the block @@ -319,7 +326,7 @@ where { fn block_number(&self, hash: Block::Hash) -> Result>, Error> { self.block_number_from_id(&BlockId::Hash(hash)) - .map_err(|e| Error::Blockchain(format!("{:?}", e))) + .map_err(|e| Error::Blockchain(e.to_string())) } } @@ -456,7 +463,7 @@ impl ::std::error::Error for CommandOrError impl fmt::Display for CommandOrError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { - CommandOrError::Error(ref e) => write!(f, "{:?}", e), + CommandOrError::Error(ref e) => write!(f, "{}", e), CommandOrError::VoterCommand(ref cmd) => write!(f, "{}", cmd), } } @@ -541,6 +548,24 @@ where ) } +/// A descriptor for an authority set hard fork. These are authority set changes +/// that are not signalled by the runtime and instead are defined off-chain +/// (hence the hard fork). +pub struct AuthoritySetHardFork { + /// The new authority set id. + pub set_id: SetId, + /// The block hash and number at which the hard fork should be applied. + pub block: (Block::Hash, NumberFor), + /// The authorities in the new set. + pub authorities: AuthorityList, + /// The latest block number that was finalized before this authority set + /// hard fork. When defined, the authority set change will be forced, i.e. + /// the node won't wait for the block above to be finalized before enacting + /// the change, and the given finalized number will be used as a base for + /// voting. + pub last_finalized: Option>, +} + /// Make block importer and link half necessary to tie the background voter to /// it. A vector of authority set hard forks can be passed, any authority set /// change signaled at the given block (either already signalled or in a further @@ -550,7 +575,7 @@ pub fn block_import_with_authority_set_hard_forks client: Arc, genesis_authorities_provider: &dyn GenesisAuthoritySetProvider, select_chain: SC, - authority_set_hard_forks: Vec<(SetId, (Block::Hash, NumberFor), AuthorityList)>, + authority_set_hard_forks: Vec>, telemetry: Option, ) -> Result<(GrandpaBlockImport, LinkHalf), ClientError> where @@ -580,19 +605,24 @@ where let (justification_sender, justification_stream) = GrandpaJustificationStream::channel(); - // create pending change objects with 0 delay and enacted on finality - // (i.e. standard changes) for each authority set hard fork. + // create pending change objects with 0 delay for each authority set hard fork. let authority_set_hard_forks = authority_set_hard_forks .into_iter() - .map(|(set_id, (hash, number), authorities)| { + .map(|fork| { + let delay_kind = if let Some(last_finalized) = fork.last_finalized { + authorities::DelayKind::Best { median_last_finalized: last_finalized } + } else { + authorities::DelayKind::Finalized + }; + ( - set_id, + fork.set_id, authorities::PendingChange { - next_authorities: authorities, + next_authorities: fork.authorities, delay: Zero::zero(), - canon_hash: hash, - canon_height: number, - delay_kind: authorities::DelayKind::Finalized, + canon_hash: fork.block.0, + canon_height: fork.block.1, + delay_kind, }, ) }) @@ -691,10 +721,14 @@ pub struct GrandpaParams { /// Returns the configuration value to put in /// [`sc_network::config::NetworkConfiguration::extra_sets`]. -pub fn grandpa_peers_set_config() -> sc_network::config::NonDefaultSetConfig { +/// For standard protocol name see [`crate::protocol_standard_name`]. +pub fn grandpa_peers_set_config( + protocol_name: std::borrow::Cow<'static, str>, +) -> sc_network::config::NonDefaultSetConfig { + use communication::grandpa_protocol_name; sc_network::config::NonDefaultSetConfig { - notifications_protocol: communication::GRANDPA_PROTOCOL_NAME.into(), - fallback_names: Vec::new(), + notifications_protocol: protocol_name, + fallback_names: grandpa_protocol_name::LEGACY_NAMES.iter().map(|&n| n.into()).collect(), // Notifications reach ~256kiB in size at the time of writing on Kusama and Polkadot. max_notification_size: 1024 * 1024, set_config: sc_network::config::SetConfig { @@ -718,7 +752,6 @@ where SC: SelectChain + 'static, VR: VotingRule + Clone + 'static, NumberFor: BlockNumberOps, - DigestFor: Encode, C: ClientForGrandpa + 'static, C::Api: GrandpaApi, { @@ -764,8 +797,8 @@ where let events = telemetry_on_connect.for_each(move |_| { let current_authorities = authorities.current_authorities(); let set_id = authorities.set_id(); - let authority_id = local_authority_id(¤t_authorities, conf.keystore.as_ref()) - .unwrap_or_default(); + let maybe_authority_id = + local_authority_id(¤t_authorities, conf.keystore.as_ref()); let authorities = current_authorities.iter().map(|(id, _)| id.to_string()).collect::>(); @@ -779,7 +812,7 @@ where telemetry; CONSENSUS_INFO; "afg.authority_set"; - "authority_id" => authority_id.to_string(), + "authority_id" => maybe_authority_id.map_or("".into(), |s| s.to_string()), "authority_set_id" => ?set_id, "authorities" => authorities, ); @@ -809,7 +842,7 @@ where Ok(()) => error!(target: "afg", "GRANDPA voter future has concluded naturally, this should be unreachable." ), - Err(e) => error!(target: "afg", "GRANDPA voter error: {:?}", e), + Err(e) => error!(target: "afg", "GRANDPA voter error: {}", e), }); // Make sure that `telemetry_task` doesn't accidentally finish and kill grandpa. @@ -918,8 +951,9 @@ where fn rebuild_voter(&mut self) { debug!(target: "afg", "{}: Starting new voter with set ID {}", self.env.config.name(), self.env.set_id); - let authority_id = local_authority_id(&self.env.voters, self.env.config.keystore.as_ref()) - .unwrap_or_default(); + let maybe_authority_id = + local_authority_id(&self.env.voters, self.env.config.keystore.as_ref()); + let authority_id = maybe_authority_id.map_or("".into(), |s| s.to_string()); telemetry!( self.telemetry; @@ -927,7 +961,7 @@ where "afg.starting_new_voter"; "name" => ?self.env.config.name(), "set_id" => ?self.env.set_id, - "authority_id" => authority_id.to_string(), + "authority_id" => authority_id, ); let chain_info = self.env.client.info(); @@ -944,7 +978,7 @@ where "afg.authority_set"; "number" => ?chain_info.finalized_number, "hash" => ?chain_info.finalized_hash, - "authority_id" => authority_id.to_string(), + "authority_id" => authority_id, "authority_set_id" => ?self.env.set_id, "authorities" => authorities, ); @@ -1129,3 +1163,49 @@ fn local_authority_id( .map(|(p, _)| p.clone()) }) } + +/// Reverts protocol aux data to at most the last finalized block. +/// In particular, standard and forced authority set changes announced after the +/// revert point are removed. +pub fn revert(client: Arc, blocks: NumberFor) -> ClientResult<()> +where + Block: BlockT, + Client: AuxStore + + HeaderMetadata + + HeaderBackend + + ProvideRuntimeApi, +{ + let best_number = client.info().best_number; + let finalized = client.info().finalized_number; + let revertible = blocks.min(best_number - finalized); + + let number = best_number - revertible; + let hash = client + .block_hash_from_id(&BlockId::Number(number))? + .ok_or(ClientError::Backend(format!( + "Unexpected hash lookup failure for block number: {}", + number + )))?; + + let info = client.info(); + let persistent_data: PersistentData = + aux_schema::load_persistent(&*client, info.genesis_hash, Zero::zero(), || unreachable!())?; + + let shared_authority_set = persistent_data.authority_set; + let mut authority_set = shared_authority_set.inner(); + + let is_descendent_of = is_descendent_of(&*client, None); + authority_set.revert(hash, number, &is_descendent_of); + + // The following has the side effect to properly reset the current voter state. + let (set_id, set_ref) = authority_set.current(); + let new_set = Some(NewAuthoritySet { + canon_hash: info.finalized_hash, + canon_number: info.finalized_number, + set_id, + authorities: set_ref.to_vec(), + }); + aux_schema::update_authority_set::(&authority_set, new_set.as_ref(), |values| { + client.insert_aux(values, None) + }) +} diff --git a/client/finality-grandpa/src/notification.rs b/client/finality-grandpa/src/notification.rs index 85d581bd5065..1d6e25e55dc6 100644 --- a/client/finality-grandpa/src/notification.rs +++ b/client/finality-grandpa/src/notification.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,61 +16,15 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use parking_lot::Mutex; -use std::sync::Arc; +use sc_utils::notification::{NotificationSender, NotificationStream, TracingKeyStr}; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; -use sp_runtime::traits::Block as BlockT; - -use crate::{justification::GrandpaJustification, Error}; - -// Stream of justifications returned when subscribing. -type JustificationStream = TracingUnboundedReceiver>; - -// Sending endpoint for notifying about justifications. -type JustificationSender = TracingUnboundedSender>; - -// Collection of channel sending endpoints shared with the receiver side so they can register -// themselves. -type SharedJustificationSenders = Arc>>>; +use crate::justification::GrandpaJustification; /// The sending half of the Grandpa justification channel(s). /// /// Used to send notifications about justifications generated /// at the end of a Grandpa round. -#[derive(Clone)] -pub struct GrandpaJustificationSender { - subscribers: SharedJustificationSenders, -} - -impl GrandpaJustificationSender { - /// The `subscribers` should be shared with a corresponding - /// `GrandpaJustificationStream`. - fn new(subscribers: SharedJustificationSenders) -> Self { - Self { subscribers } - } - - /// Send out a notification to all subscribers that a new justification - /// is available for a block. - pub fn notify( - &self, - justification: impl FnOnce() -> Result, Error>, - ) -> Result<(), Error> { - let mut subscribers = self.subscribers.lock(); - - // do an initial prune on closed subscriptions - subscribers.retain(|n| !n.is_closed()); - - // if there's no subscribers we avoid creating - // the justification which is a costly operation - if !subscribers.is_empty() { - let justification = justification()?; - subscribers.retain(|n| n.unbounded_send(justification.clone()).is_ok()); - } - - Ok(()) - } -} +pub type GrandpaJustificationSender = NotificationSender>; /// The receiving half of the Grandpa justification channel. /// @@ -78,33 +32,12 @@ impl GrandpaJustificationSender { /// at the end of a Grandpa round. /// The `GrandpaJustificationStream` entity stores the `SharedJustificationSenders` /// so it can be used to add more subscriptions. -#[derive(Clone)] -pub struct GrandpaJustificationStream { - subscribers: SharedJustificationSenders, -} - -impl GrandpaJustificationStream { - /// Creates a new pair of receiver and sender of justification notifications. - pub fn channel() -> (GrandpaJustificationSender, Self) { - let subscribers = Arc::new(Mutex::new(vec![])); - let receiver = GrandpaJustificationStream::new(subscribers.clone()); - let sender = GrandpaJustificationSender::new(subscribers.clone()); - (sender, receiver) - } +pub type GrandpaJustificationStream = + NotificationStream, GrandpaJustificationsTracingKey>; - /// Create a new receiver of justification notifications. - /// - /// The `subscribers` should be shared with a corresponding - /// `GrandpaJustificationSender`. - fn new(subscribers: SharedJustificationSenders) -> Self { - Self { subscribers } - } - - /// Subscribe to a channel through which justifications are sent - /// at the end of each Grandpa voting round. - pub fn subscribe(&self) -> JustificationStream { - let (sender, receiver) = tracing_unbounded("mpsc_justification_notification_stream"); - self.subscribers.lock().push(sender); - receiver - } +/// Provides tracing key for GRANDPA justifications stream. +#[derive(Clone)] +pub struct GrandpaJustificationsTracingKey; +impl TracingKeyStr for GrandpaJustificationsTracingKey { + const TRACING_KEY: &'static str = "mpsc_grandpa_justification_notification_stream"; } diff --git a/client/finality-grandpa/src/observer.rs b/client/finality-grandpa/src/observer.rs index 70a94cd50472..a7c951cc33db 100644 --- a/client/finality-grandpa/src/observer.rs +++ b/client/finality-grandpa/src/observer.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -203,7 +203,7 @@ where ); let observer_work = observer_work.map_ok(|_| ()).map_err(|e| { - warn!("GRANDPA Observer failed: {:?}", e); + warn!("GRANDPA Observer failed: {}", e); }); Ok(observer_work.map(drop)) diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index 1aef7cd1b017..5083cbfc21d1 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -43,6 +43,7 @@ use sp_finality_grandpa::{ use sp_keyring::Ed25519Keyring; use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; use sp_runtime::{ + codec::Encode, generic::{BlockId, DigestItem}, traits::{Block as BlockT, Header as HeaderT}, Justifications, @@ -55,7 +56,8 @@ use substrate_test_runtime_client::runtime::BlockNumber; use tokio::runtime::{Handle, Runtime}; use authorities::AuthoritySet; -use sc_block_builder::BlockBuilderProvider; +use communication::grandpa_protocol_name; +use sc_block_builder::{BlockBuilder, BlockBuilderProvider}; use sc_consensus::LongestChain; use sc_keystore::LocalKeystore; use sp_application_crypto::key_types::GRANDPA; @@ -96,7 +98,7 @@ impl GrandpaTestNet { impl GrandpaTestNet { fn add_authority_peer(&mut self) { self.add_full_peer_with_config(FullPeerConfig { - notifications_protocols: vec![communication::GRANDPA_PROTOCOL_NAME.into()], + notifications_protocols: vec![grandpa_protocol_name::NAME.into()], is_authority: true, ..Default::default() }) @@ -120,7 +122,7 @@ impl TestNetFactory for GrandpaTestNet { fn add_full_peer(&mut self) { self.add_full_peer_with_config(FullPeerConfig { - notifications_protocols: vec![communication::GRANDPA_PROTOCOL_NAME.into()], + notifications_protocols: vec![grandpa_protocol_name::NAME.into()], is_authority: false, ..Default::default() }) @@ -139,26 +141,16 @@ impl TestNetFactory for GrandpaTestNet { &self, client: PeersClient, ) -> (BlockImportAdapter, Option>, PeerData) { - match client { - PeersClient::Full(ref client, ref backend) => { - let (import, link) = block_import( - client.clone(), - &self.test_config, - LongestChain::new(backend.clone()), - None, - ) - .expect("Could not create block import for fresh peer."); - let justification_import = Box::new(import.clone()); - ( - BlockImportAdapter::new(import), - Some(justification_import), - Mutex::new(Some(link)), - ) - }, - PeersClient::Light(..) => { - panic!("Light client is not used in tests."); - }, - } + let (client, backend) = (client.as_client(), client.as_backend()); + let (import, link) = block_import( + client.clone(), + &self.test_config, + LongestChain::new(backend.clone()), + None, + ) + .expect("Could not create block import for fresh peer."); + let justification_import = Box::new(import.clone()); + (BlockImportAdapter::new(import), Some(justification_import), Mutex::new(Some(link))) } fn peer(&mut self, i: usize) -> &mut GrandpaPeer { @@ -283,6 +275,7 @@ fn initialize_grandpa( local_role: Role::Authority, observer_enabled: true, telemetry: None, + protocol_name: grandpa_protocol_name::NAME.into(), }, link, network: net_service, @@ -432,6 +425,7 @@ fn finalize_3_voters_1_full_observer() { local_role: Role::Authority, observer_enabled: true, telemetry: None, + protocol_name: grandpa_protocol_name::NAME.into(), }, link, network: net_service, @@ -466,7 +460,7 @@ fn finalize_3_voters_1_full_observer() { // all peers should have stored the justification for the best finalized block #20 for peer_id in 0..4 { - let client = net.lock().peers[peer_id].client().as_full().unwrap(); + let client = net.lock().peers[peer_id].client().as_client(); let justification = crate::aux_schema::best_justification::<_, Block>(&*client).unwrap().unwrap(); @@ -522,6 +516,7 @@ fn transition_3_voters_twice_1_full_observer() { local_role: Role::Authority, observer_enabled: true, telemetry: None, + protocol_name: grandpa_protocol_name::NAME.into(), }, link, network: net_service, @@ -539,7 +534,7 @@ fn transition_3_voters_twice_1_full_observer() { net.lock().block_until_sync(); for (i, peer) in net.lock().peers().iter().enumerate() { - let full_client = peer.client().as_full().expect("only full clients are used in test"); + let full_client = peer.client().as_client(); assert_eq!(full_client.chain_info().best_number, 1, "Peer #{} failed to sync", i); let set: AuthoritySet = @@ -614,7 +609,7 @@ fn transition_3_voters_twice_1_full_observer() { .take_while(|n| future::ready(n.header.number() < &30)) .for_each(move |_| future::ready(())) .map(move |()| { - let full_client = client.as_full().expect("only full clients are used in test"); + let full_client = client.as_client(); let set: AuthoritySet = crate::aux_schema::load_authorities(&*full_client).unwrap(); @@ -835,7 +830,7 @@ fn force_change_to_new_set() { for (i, peer) in net.lock().peers().iter().enumerate() { assert_eq!(peer.client().info().best_number, 26, "Peer #{} failed to sync", i); - let full_client = peer.client().as_full().expect("only full clients are used in test"); + let full_client = peer.client().as_client(); let set: AuthoritySet = crate::aux_schema::load_authorities(&*full_client).unwrap(); @@ -861,7 +856,7 @@ fn allows_reimporting_change_blocks() { let client = net.peer(0).client().clone(); let (mut block_import, ..) = net.make_block_import(client.clone()); - let full_client = client.as_full().unwrap(); + let full_client = client.as_client(); let builder = full_client .new_block_at(&BlockId::Number(0), Default::default(), false) .unwrap(); @@ -908,7 +903,7 @@ fn test_bad_justification() { let client = net.peer(0).client().clone(); let (mut block_import, ..) = net.make_block_import(client.clone()); - let full_client = client.as_full().expect("only full clients are used in test"); + let full_client = client.as_client(); let builder = full_client .new_block_at(&BlockId::Number(0), Default::default(), false) .unwrap(); @@ -980,6 +975,7 @@ fn voter_persists_its_votes() { local_role: Role::Authority, observer_enabled: true, telemetry: None, + protocol_name: grandpa_protocol_name::NAME.into(), }; let set_state = { @@ -1019,6 +1015,7 @@ fn voter_persists_its_votes() { local_role: Role::Authority, observer_enabled: true, telemetry: None, + protocol_name: grandpa_protocol_name::NAME.into(), }, link, network: net_service, @@ -1059,6 +1056,7 @@ fn voter_persists_its_votes() { local_role: Role::Authority, observer_enabled: true, telemetry: None, + protocol_name: grandpa_protocol_name::NAME.into(), }, link, network: net_service, @@ -1148,7 +1146,7 @@ fn voter_persists_its_votes() { .await; let block_30_hash = - net.lock().peer(0).client().as_full().unwrap().hash(30).unwrap().unwrap(); + net.lock().peer(0).client().as_client().hash(30).unwrap().unwrap(); // we restart alice's voter abort.abort(); @@ -1222,6 +1220,7 @@ fn finalize_3_voters_1_light_observer() { local_role: Role::Full, observer_enabled: true, telemetry: None, + protocol_name: grandpa_protocol_name::NAME.into(), }, net.peers[3].data.lock().take().expect("link initialized at startup; qed"), net.peers[3].network_service().clone(), @@ -1268,6 +1267,7 @@ fn voter_catches_up_to_latest_round_when_behind() { local_role: Role::Authority, observer_enabled: true, telemetry: None, + protocol_name: grandpa_protocol_name::NAME.into(), }, link, network: net.lock().peer(peer_id).network_service().clone(), @@ -1385,6 +1385,7 @@ where local_role: Role::Authority, observer_enabled: true, telemetry: None, + protocol_name: grandpa_protocol_name::NAME.into(), }; let network = @@ -1581,7 +1582,7 @@ fn imports_justification_for_regular_blocks_on_import() { let client = net.peer(0).client().clone(); let (mut block_import, ..) = net.make_block_import(client.clone()); - let full_client = client.as_full().expect("only full clients are used in test"); + let full_client = client.as_client(); let builder = full_client .new_block_at(&BlockId::Number(0), Default::default(), false) .unwrap(); @@ -1680,7 +1681,132 @@ fn grandpa_environment_doesnt_send_equivocation_reports_for_itself() { // if we set the equivocation offender to another id for which we don't have // keys it should work - equivocation.identity = Default::default(); + equivocation.identity = TryFrom::try_from(&[1; 32][..]).unwrap(); let equivocation_proof = sp_finality_grandpa::Equivocation::Prevote(equivocation); assert!(environment.report_equivocation(equivocation_proof).is_ok()); } + +#[test] +fn revert_prunes_authority_changes() { + sp_tracing::try_init_simple(); + let runtime = Runtime::new().unwrap(); + + let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; + + type TestBlockBuilder<'a> = + BlockBuilder<'a, Block, PeersFullClient, substrate_test_runtime_client::Backend>; + let edit_block = |builder: TestBlockBuilder| { + let mut block = builder.build().unwrap().block; + add_scheduled_change( + &mut block, + ScheduledChange { next_authorities: make_ids(peers), delay: 0 }, + ); + block + }; + + let api = TestApi::new(make_ids(peers)); + let mut net = GrandpaTestNet::new(api, 3, 0); + runtime.spawn(initialize_grandpa(&mut net, peers)); + + let peer = net.peer(0); + let client = peer.client().as_client(); + + // Test scenario: (X) = auth-change, 24 = revert-point + // + // +---------(27) + // / + // 0---(21)---23---24---25---(28)---30 + // ^ \ + // revert-point +------(29) + + // Construct canonical chain + + // add 20 blocks + peer.push_blocks(20, false); + // at block 21 we add an authority transition + peer.generate_blocks(1, BlockOrigin::File, edit_block); + // add more blocks on top of it (until we have 24) + peer.push_blocks(3, false); + // add more blocks on top of it (until we have 27) + peer.push_blocks(3, false); + // at block 28 we add an authority transition + peer.generate_blocks(1, BlockOrigin::File, edit_block); + // add more blocks on top of it (until we have 30) + peer.push_blocks(2, false); + + // Fork before revert point + + // add more blocks on top of block 23 (until we have 26) + let hash = peer.generate_blocks_at( + BlockId::Number(23), + 3, + BlockOrigin::File, + |builder| { + let mut block = builder.build().unwrap().block; + block.header.digest_mut().push(DigestItem::Other(vec![1])); + block + }, + false, + false, + true, + ForkChoiceStrategy::LongestChain, + ); + // at block 27 of the fork add an authority transition + peer.generate_blocks_at( + BlockId::Hash(hash), + 1, + BlockOrigin::File, + edit_block, + false, + false, + true, + ForkChoiceStrategy::LongestChain, + ); + + // Fork after revert point + + // add more block on top of block 25 (until we have 28) + let hash = peer.generate_blocks_at( + BlockId::Number(25), + 3, + BlockOrigin::File, + |builder| { + let mut block = builder.build().unwrap().block; + block.header.digest_mut().push(DigestItem::Other(vec![2])); + block + }, + false, + false, + true, + ForkChoiceStrategy::LongestChain, + ); + // at block 29 of the fork add an authority transition + peer.generate_blocks_at( + BlockId::Hash(hash), + 1, + BlockOrigin::File, + edit_block, + false, + false, + true, + ForkChoiceStrategy::LongestChain, + ); + + revert(client.clone(), 6).unwrap(); + + let persistent_data: PersistentData = aux_schema::load_persistent( + &*client, + client.info().genesis_hash, + Zero::zero(), + || unreachable!(), + ) + .unwrap(); + let changes_num: Vec<_> = persistent_data + .authority_set + .inner() + .pending_standard_changes + .iter() + .map(|(_, n, _)| *n) + .collect(); + assert_eq!(changes_num, [21, 27]); +} diff --git a/client/finality-grandpa/src/until_imported.rs b/client/finality-grandpa/src/until_imported.rs index deb657726434..6adce0d92020 100644 --- a/client/finality-grandpa/src/until_imported.rs +++ b/client/finality-grandpa/src/until_imported.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -100,7 +100,7 @@ impl Metrics { Ok(Self { global_waiting_messages: register( Gauge::new( - "finality_grandpa_until_imported_waiting_messages_number", + "substrate_finality_grandpa_until_imported_waiting_messages_number", "Number of finality grandpa messages waiting within the until imported queue.", )?, registry, @@ -563,6 +563,7 @@ mod tests { use sc_client_api::BlockImportNotification; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_consensus::BlockOrigin; + use sp_core::crypto::UncheckedFrom; use substrate_test_runtime_client::runtime::{Block, Hash, Header}; #[derive(Clone)] @@ -796,8 +797,8 @@ mod tests { let h3 = make_header(7); let signed_prevote = |header: &Header| finality_grandpa::SignedPrevote { - id: Default::default(), - signature: Default::default(), + id: UncheckedFrom::unchecked_from([1; 32]), + signature: UncheckedFrom::unchecked_from([1; 64]), prevote: finality_grandpa::Prevote { target_hash: header.hash(), target_number: *header.number(), @@ -805,8 +806,8 @@ mod tests { }; let signed_precommit = |header: &Header| finality_grandpa::SignedPrecommit { - id: Default::default(), - signature: Default::default(), + id: UncheckedFrom::unchecked_from([1; 32]), + signature: UncheckedFrom::unchecked_from([1; 64]), precommit: finality_grandpa::Precommit { target_hash: header.hash(), target_number: *header.number(), @@ -844,8 +845,8 @@ mod tests { let h3 = make_header(7); let signed_prevote = |header: &Header| finality_grandpa::SignedPrevote { - id: Default::default(), - signature: Default::default(), + id: UncheckedFrom::unchecked_from([1; 32]), + signature: UncheckedFrom::unchecked_from([1; 64]), prevote: finality_grandpa::Prevote { target_hash: header.hash(), target_number: *header.number(), @@ -853,8 +854,8 @@ mod tests { }; let signed_precommit = |header: &Header| finality_grandpa::SignedPrecommit { - id: Default::default(), - signature: Default::default(), + id: UncheckedFrom::unchecked_from([1; 32]), + signature: UncheckedFrom::unchecked_from([1; 64]), precommit: finality_grandpa::Precommit { target_hash: header.hash(), target_number: *header.number(), diff --git a/client/finality-grandpa/src/voting_rule.rs b/client/finality-grandpa/src/voting_rule.rs index b974afe0d352..749c504a051c 100644 --- a/client/finality-grandpa/src/voting_rule.rs +++ b/client/finality-grandpa/src/voting_rule.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -80,8 +80,14 @@ where } /// A custom voting rule that guarantees that our vote is always behind the best -/// block by at least N blocks. In the best case our vote is exactly N blocks -/// behind the best block. +/// block by at least N blocks, unless the base number is < N blocks behind the +/// best, in which case it votes for the base. +/// +/// In the best case our vote is exactly N blocks +/// behind the best block, but if there is a scenario where either +/// >34% of validators run without this rule or the fork-choice rule +/// can prioritize shorter chains over longer ones, the vote may be +/// closer to the best block than N. #[derive(Clone)] pub struct BeforeBestBlockBy(N); impl VotingRule for BeforeBestBlockBy> @@ -92,7 +98,7 @@ where fn restrict_vote( &self, backend: Arc, - _base: &Block::Header, + base: &Block::Header, best_target: &Block::Header, current_target: &Block::Header, ) -> VotingRuleResult { @@ -102,6 +108,12 @@ where return Box::pin(async { None }) } + // Constrain to the base number, if that's the minimal + // vote that can be placed. + if *base.number() + self.0 > *best_target.number() { + return Box::pin(std::future::ready(Some((base.hash(), *base.number())))) + } + // find the target number restricted by this rule let target_number = best_target.number().saturating_sub(self.0); @@ -393,4 +405,34 @@ mod tests { // only one of the rules is applied. assert_eq!(number, 150); } + + #[test] + fn before_best_by_has_cutoff_at_base() { + let rule = BeforeBestBlockBy(2); + + let mut client = Arc::new(TestClientBuilder::new().build()); + + for _ in 0..5 { + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + + futures::executor::block_on(client.import(BlockOrigin::Own, block)).unwrap(); + } + + let best = client.header(&BlockId::Hash(client.info().best_hash)).unwrap().unwrap(); + let best_number = best.number().clone(); + + for i in 0u32..5 { + let base = client.header(&BlockId::Number(i.into())).unwrap().unwrap(); + let (_, number) = futures::executor::block_on(rule.restrict_vote( + client.clone(), + &base, + &best, + &best, + )) + .unwrap(); + + let expected = std::cmp::max(best_number - 2, *base.number()); + assert_eq!(number, expected, "best = {}, lag = 2, base = {}", best_number, i); + } + } } diff --git a/client/finality-grandpa/src/warp_proof.rs b/client/finality-grandpa/src/warp_proof.rs index 34eaa49cdf36..bdb8e36373de 100644 --- a/client/finality-grandpa/src/warp_proof.rs +++ b/client/finality-grandpa/src/warp_proof.rs @@ -19,8 +19,8 @@ use sp_runtime::codec::{self, Decode, Encode}; use crate::{ - best_justification, find_scheduled_change, AuthoritySetChanges, BlockNumberOps, - GrandpaJustification, SharedAuthoritySet, + best_justification, find_scheduled_change, AuthoritySetChanges, AuthoritySetHardFork, + BlockNumberOps, GrandpaJustification, SharedAuthoritySet, }; use sc_client_api::Backend as ClientBackend; use sc_network::warp_request_handler::{EncodedProof, VerificationResult, WarpSyncProvider}; @@ -31,29 +31,28 @@ use sp_runtime::{ traits::{Block as BlockT, Header as HeaderT, NumberFor, One}, }; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; /// Warp proof processing error. -#[derive(Debug, derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] pub enum Error { /// Decoding error. - #[display(fmt = "Failed to decode block hash: {}.", _0)] - DecodeScale(codec::Error), + #[error("Failed to decode block hash: {0}.")] + DecodeScale(#[from] codec::Error), /// Client backend error. - Client(sp_blockchain::Error), + #[error("{0}")] + Client(#[from] sp_blockchain::Error), /// Invalid request data. - #[from(ignore)] + #[error("{0}")] InvalidRequest(String), /// Invalid warp proof. - #[from(ignore)] + #[error("{0}")] InvalidProof(String), /// Missing header or authority set change data. - #[display(fmt = "Missing required data to be able to answer request.")] + #[error("Missing required data to be able to answer request.")] MissingData, } -impl std::error::Error for Error {} - /// The maximum size in bytes of the `WarpSyncProof`. pub(super) const MAX_WARP_SYNC_PROOF_SIZE: usize = 8 * 1024 * 1024; @@ -194,6 +193,7 @@ impl WarpSyncProof { &self, set_id: SetId, authorities: AuthorityList, + hard_forks: &HashMap<(Block::Hash, NumberFor), (SetId, AuthorityList)>, ) -> Result<(SetId, AuthorityList), Error> where NumberFor: BlockNumberOps, @@ -202,26 +202,34 @@ impl WarpSyncProof { let mut current_authorities = authorities; for (fragment_num, proof) in self.proofs.iter().enumerate() { - proof - .justification - .verify(current_set_id, ¤t_authorities) - .map_err(|err| Error::InvalidProof(err.to_string()))?; - - if proof.justification.target().1 != proof.header.hash() { - return Err(Error::InvalidProof( - "Mismatch between header and justification".to_owned(), - )) - } + let hash = proof.header.hash(); + let number = *proof.header.number(); + + if let Some((set_id, list)) = hard_forks.get(&(hash.clone(), number)) { + current_set_id = *set_id; + current_authorities = list.clone(); + } else { + proof + .justification + .verify(current_set_id, ¤t_authorities) + .map_err(|err| Error::InvalidProof(err.to_string()))?; + + if proof.justification.target().1 != hash { + return Err(Error::InvalidProof( + "Mismatch between header and justification".to_owned(), + )) + } - if let Some(scheduled_change) = find_scheduled_change::(&proof.header) { - current_authorities = scheduled_change.next_authorities; - current_set_id += 1; - } else if fragment_num != self.proofs.len() - 1 || !self.is_finished { - // Only the last fragment of the last proof message is allowed to be missing - // the authority set change. - return Err(Error::InvalidProof( - "Header is missing authority set change digest".to_string(), - )) + if let Some(scheduled_change) = find_scheduled_change::(&proof.header) { + current_authorities = scheduled_change.next_authorities; + current_set_id += 1; + } else if fragment_num != self.proofs.len() - 1 || !self.is_finished { + // Only the last fragment of the last proof message is allowed to be missing the + // authority set change. + return Err(Error::InvalidProof( + "Header is missing authority set change digest".to_string(), + )) + } } } Ok((current_set_id, current_authorities)) @@ -235,6 +243,7 @@ where { backend: Arc, authority_set: SharedAuthoritySet>, + hard_forks: HashMap<(Block::Hash, NumberFor), (SetId, AuthorityList)>, } impl> NetworkProvider @@ -245,8 +254,16 @@ where pub fn new( backend: Arc, authority_set: SharedAuthoritySet>, + hard_forks: Vec>, ) -> Self { - NetworkProvider { backend, authority_set } + NetworkProvider { + backend, + authority_set, + hard_forks: hard_forks + .into_iter() + .map(|fork| (fork.block, (fork.set_id, fork.authorities))) + .collect(), + } } } @@ -283,7 +300,7 @@ where .map(|p| p.header.clone()) .ok_or_else(|| "Empty proof".to_string())?; let (next_set_id, next_authorities) = - proof.verify(set_id, authorities).map_err(Box::new)?; + proof.verify(set_id, authorities, &self.hard_forks).map_err(Box::new)?; if proof.is_finished { Ok(VerificationResult::::Complete(next_set_id, next_authorities, last_header)) } else { @@ -417,7 +434,8 @@ mod tests { WarpSyncProof::generate(&*backend, genesis_hash, &authority_set_changes).unwrap(); // verifying the proof should yield the last set id and authorities - let (new_set_id, new_authorities) = warp_sync_proof.verify(0, genesis_authorities).unwrap(); + let (new_set_id, new_authorities) = + warp_sync_proof.verify(0, genesis_authorities, &Default::default()).unwrap(); let expected_authorities = current_authorities .iter() diff --git a/client/informant/Cargo.toml b/client/informant/Cargo.toml index 88d02f81ad5b..e58763a4ebcc 100644 --- a/client/informant/Cargo.toml +++ b/client/informant/Cargo.toml @@ -3,9 +3,9 @@ name = "sc-informant" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "Substrate informant." -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -14,12 +14,12 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] ansi_term = "0.12.1" -futures = "0.3.9" +futures = "0.3.21" futures-timer = "3.0.1" log = "0.4.8" -parity-util-mem = { version = "0.10.0", default-features = false, features = ["primitive-types"] } +parity-util-mem = { version = "0.11.0", default-features = false, features = ["primitive-types"] } sc-client-api = { version = "4.0.0-dev", path = "../api" } sc-network = { version = "0.10.0-dev", path = "../network" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../transaction-pool/api" } diff --git a/client/informant/src/display.rs b/client/informant/src/display.rs index 1f23856101aa..446ddf47b4ca 100644 --- a/client/informant/src/display.rs +++ b/client/informant/src/display.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,13 +20,9 @@ use crate::OutputFormat; use ansi_term::Colour; use log::info; use sc_client_api::ClientInfo; -use sc_network::{NetworkStatus, SyncState}; +use sc_network::{NetworkStatus, SyncState, WarpSyncPhase, WarpSyncProgress}; use sp_runtime::traits::{Block as BlockT, CheckedDiv, NumberFor, Saturating, Zero}; -use std::{ - convert::{TryFrom, TryInto}, - fmt, - time::Instant, -}; +use std::{fmt, time::Instant}; /// State of the informant display system. /// @@ -34,7 +30,7 @@ use std::{ /// like: /// /// > Syncing 5.4 bps, target=#531028 (4 peers), best: #90683 (0x4ca8…51b8), -/// > finalized #360 (0x6f24…a38b), ⬇ 5.5kiB/s ⬆ 0.9kiB/s +/// > finalized #360 (0x6f24…a38b), ⬇ 5.5kiB/s ⬆ 0.9kiB/s /// /// # Usage /// @@ -97,11 +93,17 @@ impl InformantDisplay { net_status.state_sync, net_status.warp_sync, ) { + ( + _, + _, + _, + Some(WarpSyncProgress { phase: WarpSyncPhase::DownloadingBlocks(n), .. }), + ) => ("⏩", "Block history".into(), format!(", #{}", n)), (_, _, _, Some(warp)) => ( "⏩", "Warping".into(), format!( - ", {}, ({:.2}) Mib", + ", {}, {:.2} Mib", warp.phase, (warp.total_bytes as f32) / (1024f32 * 1024f32) ), @@ -110,7 +112,7 @@ impl InformantDisplay { "⚙️ ", "Downloading state".into(), format!( - ", {}%, ({:.2}) Mib", + ", {}%, {:.2} Mib", state.percentage, (state.size as f32) / (1024f32 * 1024f32) ), diff --git a/client/informant/src/lib.rs b/client/informant/src/lib.rs index f421dbbb7e56..88a500a3a98f 100644 --- a/client/informant/src/lib.rs +++ b/client/informant/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/keystore/Cargo.toml b/client/keystore/Cargo.toml index 17c651a91dec..844110f66886 100644 --- a/client/keystore/Cargo.toml +++ b/client/keystore/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-keystore" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Keystore (and session key management) for ed25519 based chains like Polkadot." documentation = "https://docs.rs/sc-keystore" @@ -16,13 +16,13 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] async-trait = "0.1.50" -derive_more = "0.99.2" -sp-application-crypto = { version = "4.0.0-dev", path = "../../primitives/application-crypto" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore" } +thiserror = "1.0" +sp-application-crypto = { version = "6.0.0", path = "../../primitives/application-crypto" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-keystore = { version = "0.12.0", path = "../../primitives/keystore" } hex = "0.4.0" -parking_lot = "0.11.1" -serde_json = "1.0.68" +parking_lot = "0.12.0" +serde_json = "1.0.79" [dev-dependencies] tempfile = "3.1.0" diff --git a/client/keystore/src/lib.rs b/client/keystore/src/lib.rs index 5e29f691997e..cf94a16f08d8 100644 --- a/client/keystore/src/lib.rs +++ b/client/keystore/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -28,26 +28,31 @@ mod local; pub use local::LocalKeystore; /// Keystore error. -#[derive(Debug, derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] pub enum Error { /// IO error. - Io(io::Error), + #[error(transparent)] + Io(#[from] io::Error), /// JSON error. - Json(serde_json::Error), + #[error(transparent)] + Json(#[from] serde_json::Error), /// Invalid password. - #[display(fmt = "Invalid password")] - InvalidPassword, + #[error( + "Requested public key and public key of the loaded private key do not match. \n + This means either that the keystore password is incorrect or that the private key was stored under a wrong public key." + )] + PublicKeyMismatch, /// Invalid BIP39 phrase - #[display(fmt = "Invalid recovery phrase (BIP39) data")] + #[error("Invalid recovery phrase (BIP39) data")] InvalidPhrase, /// Invalid seed - #[display(fmt = "Invalid seed")] + #[error("Invalid seed")] InvalidSeed, /// Public key type is not supported - #[display(fmt = "Key crypto type is not supported")] + #[error("Key crypto type is not supported")] KeyNotSupported(KeyTypeId), /// Keystore unavailable - #[display(fmt = "Keystore unavailable")] + #[error("Keystore unavailable")] Unavailable, } @@ -58,7 +63,7 @@ impl From for TraitError { fn from(error: Error) -> Self { match error { Error::KeyNotSupported(id) => TraitError::KeyNotSupported(id), - Error::InvalidSeed | Error::InvalidPhrase | Error::InvalidPassword => + Error::InvalidSeed | Error::InvalidPhrase | Error::PublicKeyMismatch => TraitError::ValidationError(error.to_string()), Error::Unavailable => TraitError::Unavailable, Error::Io(e) => TraitError::Other(e.to_string()), @@ -66,13 +71,3 @@ impl From for TraitError { } } } - -impl std::error::Error for Error { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - Error::Io(ref err) => Some(err), - Error::Json(ref err) => Some(err), - _ => None, - } - } -} diff --git a/client/keystore/src/local.rs b/client/keystore/src/local.rs index e5c8ff14af09..9f6f18d0c293 100644 --- a/client/keystore/src/local.rs +++ b/client/keystore/src/local.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,9 @@ use async_trait::async_trait; use parking_lot::RwLock; use sp_application_crypto::{ecdsa, ed25519, sr25519, AppKey, AppPair, IsWrappedBy}; use sp_core::{ - crypto::{CryptoTypePublicPair, ExposeSecret, KeyTypeId, Pair as PairT, Public, SecretString}, + crypto::{ + ByteArray, CryptoTypePublicPair, ExposeSecret, KeyTypeId, Pair as PairT, SecretString, + }, sr25519::{Pair as Sr25519Pair, Public as Sr25519Public}, Encode, }; @@ -189,7 +191,9 @@ impl SyncCryptoStore for LocalKeystore { ) -> std::result::Result>, TraitError> { match key.0 { ed25519::CRYPTO_ID => { - let pub_key = ed25519::Public::from_slice(key.1.as_slice()); + let pub_key = ed25519::Public::from_slice(key.1.as_slice()).map_err(|()| { + TraitError::Other("Corrupted public key - Invalid size".into()) + })?; let key_pair = self .0 .read() @@ -198,7 +202,9 @@ impl SyncCryptoStore for LocalKeystore { key_pair.map(|k| k.sign(msg).encode()).map(Ok).transpose() }, sr25519::CRYPTO_ID => { - let pub_key = sr25519::Public::from_slice(key.1.as_slice()); + let pub_key = sr25519::Public::from_slice(key.1.as_slice()).map_err(|()| { + TraitError::Other("Corrupted public key - Invalid size".into()) + })?; let key_pair = self .0 .read() @@ -207,7 +213,9 @@ impl SyncCryptoStore for LocalKeystore { key_pair.map(|k| k.sign(msg).encode()).map(Ok).transpose() }, ecdsa::CRYPTO_ID => { - let pub_key = ecdsa::Public::from_slice(key.1.as_slice()); + let pub_key = ecdsa::Public::from_slice(key.1.as_slice()).map_err(|()| { + TraitError::Other("Corrupted public key - Invalid size".into()) + })?; let key_pair = self .0 .read() @@ -223,7 +231,11 @@ impl SyncCryptoStore for LocalKeystore { self.0 .read() .raw_public_keys(key_type) - .map(|v| v.into_iter().map(|k| sr25519::Public::from_slice(k.as_slice())).collect()) + .map(|v| { + v.into_iter() + .filter_map(|k| sr25519::Public::from_slice(k.as_slice()).ok()) + .collect() + }) .unwrap_or_default() } @@ -246,7 +258,11 @@ impl SyncCryptoStore for LocalKeystore { self.0 .read() .raw_public_keys(key_type) - .map(|v| v.into_iter().map(|k| ed25519::Public::from_slice(k.as_slice())).collect()) + .map(|v| { + v.into_iter() + .filter_map(|k| ed25519::Public::from_slice(k.as_slice()).ok()) + .collect() + }) .unwrap_or_default() } @@ -269,7 +285,11 @@ impl SyncCryptoStore for LocalKeystore { self.0 .read() .raw_public_keys(key_type) - .map(|v| v.into_iter().map(|k| ecdsa::Public::from_slice(k.as_slice())).collect()) + .map(|v| { + v.into_iter() + .filter_map(|k| ecdsa::Public::from_slice(k.as_slice()).ok()) + .collect() + }) .unwrap_or_default() } @@ -364,8 +384,7 @@ impl KeystoreInner { let path = path.into(); fs::create_dir_all(&path)?; - let instance = Self { path: Some(path), additional: HashMap::new(), password }; - Ok(instance) + Ok(Self { path: Some(path), additional: HashMap::new(), password }) } /// Get the password for this store. @@ -397,10 +416,9 @@ impl KeystoreInner { /// Places it into the file system store, if a path is configured. fn insert_unknown(&self, key_type: KeyTypeId, suri: &str, public: &[u8]) -> Result<()> { if let Some(path) = self.key_file_path(public, key_type) { - let mut file = File::create(path).map_err(Error::Io)?; - serde_json::to_writer(&file, &suri).map_err(Error::Json)?; - file.flush().map_err(Error::Io)?; + Self::write_to_file(path, suri)?; } + Ok(()) } @@ -411,15 +429,29 @@ impl KeystoreInner { fn generate_by_type(&mut self, key_type: KeyTypeId) -> Result { let (pair, phrase, _) = Pair::generate_with_phrase(self.password()); if let Some(path) = self.key_file_path(pair.public().as_slice(), key_type) { - let mut file = File::create(path)?; - serde_json::to_writer(&file, &phrase)?; - file.flush()?; + Self::write_to_file(path, &phrase)?; } else { self.insert_ephemeral_pair(&pair, &phrase, key_type); } + Ok(pair) } + /// Write the given `data` to `file`. + fn write_to_file(file: PathBuf, data: &str) -> Result<()> { + let mut file = File::create(file)?; + + #[cfg(target_family = "unix")] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(fs::Permissions::from_mode(0o600))?; + } + + serde_json::to_writer(&file, data)?; + file.flush()?; + Ok(()) + } + /// Create a new key from seed. /// /// Does not place it into the file system store. @@ -471,7 +503,7 @@ impl KeystoreInner { if &pair.public() == public { Ok(Some(pair)) } else { - Err(Error::InvalidPassword) + Err(Error::PublicKeyMismatch) } } @@ -549,8 +581,9 @@ mod tests { } fn public_keys(&self) -> Result> { - self.raw_public_keys(Public::ID) - .map(|v| v.into_iter().map(|k| Public::from_slice(k.as_slice())).collect()) + self.raw_public_keys(Public::ID).map(|v| { + v.into_iter().filter_map(|k| Public::from_slice(k.as_slice()).ok()).collect() + }) } fn generate(&mut self) -> Result { @@ -735,4 +768,20 @@ mod tests { SyncCryptoStore::sr25519_generate_new(&store, TEST_KEY_TYPE, None).unwrap(); assert_eq!(SyncCryptoStore::sr25519_public_keys(&store, TEST_KEY_TYPE).len(), 2); } + + #[test] + #[cfg(target_family = "unix")] + fn uses_correct_file_permissions_on_unix() { + use std::os::unix::fs::PermissionsExt; + + let temp_dir = TempDir::new().unwrap(); + let store = LocalKeystore::open(temp_dir.path(), None).unwrap(); + + let public = SyncCryptoStore::sr25519_generate_new(&store, TEST_KEY_TYPE, None).unwrap(); + + let path = store.0.read().key_file_path(public.as_ref(), TEST_KEY_TYPE).unwrap(); + let permissions = File::open(path).unwrap().metadata().unwrap().permissions(); + + assert_eq!(0o100600, permissions.mode()); + } } diff --git a/client/light/Cargo.toml b/client/light/Cargo.toml deleted file mode 100644 index b10f7646bf9b..000000000000 --- a/client/light/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -description = "components for a light client" -name = "sc-light" -version = "4.0.0-dev" -license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -authors = ["Parity Technologies "] -edition = "2018" -homepage = "https://substrate.dev" -repository = "https://github.com/paritytech/substrate/" -documentation = "https://docs.rs/sc-light" -readme = "README.md" - -[dependencies] -parking_lot = "0.11.1" -hash-db = "0.15.2" -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sp-externalities = { version = "0.10.0-dev", path = "../../primitives/externalities" } -sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-state-machine = { version = "0.10.0-dev", path = "../../primitives/state-machine" } -sc-client-api = { version = "4.0.0-dev", path = "../api" } -sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } -codec = { package = "parity-scale-codec", version = "2.0.0" } -sc-executor = { version = "0.10.0-dev", path = "../executor" } - -[features] -default = [] diff --git a/client/light/README.md b/client/light/README.md deleted file mode 100644 index 1ba1f155b165..000000000000 --- a/client/light/README.md +++ /dev/null @@ -1,3 +0,0 @@ -Light client components. - -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file diff --git a/client/light/src/backend.rs b/client/light/src/backend.rs deleted file mode 100644 index 3091dce625a3..000000000000 --- a/client/light/src/backend.rs +++ /dev/null @@ -1,578 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Light client backend. Only stores headers and justifications of blocks. -//! Everything else is requested from full nodes on demand. - -use parking_lot::RwLock; -use std::{ - collections::{HashMap, HashSet}, - sync::Arc, -}; - -use codec::{Decode, Encode}; - -use super::blockchain::Blockchain; -use hash_db::Hasher; -use sc_client_api::{ - backend::{ - AuxStore, Backend as ClientBackend, BlockImportOperation, NewBlockState, - PrunableStateChangesTrieStorage, RemoteBackend, - }, - blockchain::{well_known_cache_keys, HeaderBackend as BlockchainHeaderBackend}, - in_mem::check_genesis_storage, - light::Storage as BlockchainStorage, - UsageInfo, -}; -use sp_blockchain::{Error as ClientError, Result as ClientResult}; -use sp_core::{ - offchain::storage::InMemOffchainStorage, - storage::{well_known_keys, ChildInfo}, - ChangesTrieConfiguration, -}; -use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, HashFor, Header, NumberFor, Zero}, - Justification, Justifications, Storage, -}; -use sp_state_machine::{ - Backend as StateBackend, ChangesTrieTransaction, ChildStorageCollection, InMemoryBackend, - IndexOperation, StorageCollection, TrieBackend, -}; - -const IN_MEMORY_EXPECT_PROOF: &str = - "InMemory state backend has Void error type and always succeeds; qed"; - -/// Light client backend. -pub struct Backend { - blockchain: Arc>, - genesis_state: RwLock>>, - import_lock: RwLock<()>, -} - -/// Light block (header and justification) import operation. -pub struct ImportOperation { - header: Option, - cache: HashMap>, - leaf_state: NewBlockState, - aux_ops: Vec<(Vec, Option>)>, - finalized_blocks: Vec>, - set_head: Option>, - storage_update: Option>>, - changes_trie_config_update: Option>, - _phantom: std::marker::PhantomData, -} - -/// Either in-memory genesis state, or locally-unavailable state. -pub enum GenesisOrUnavailableState { - /// Genesis state - storage values are stored in-memory. - Genesis(InMemoryBackend), - /// We know that state exists, but all calls will fail with error, because it - /// isn't locally available. - Unavailable, -} - -impl Backend { - /// Create new light backend. - pub fn new(blockchain: Arc>) -> Self { - Self { blockchain, genesis_state: RwLock::new(None), import_lock: Default::default() } - } - - /// Get shared blockchain reference. - pub fn blockchain(&self) -> &Arc> { - &self.blockchain - } -} - -impl AuxStore for Backend { - fn insert_aux< - 'a, - 'b: 'a, - 'c: 'a, - I: IntoIterator, - D: IntoIterator, - >( - &self, - insert: I, - delete: D, - ) -> ClientResult<()> { - self.blockchain.storage().insert_aux(insert, delete) - } - - fn get_aux(&self, key: &[u8]) -> ClientResult>> { - self.blockchain.storage().get_aux(key) - } -} - -impl ClientBackend for Backend> -where - Block: BlockT, - S: BlockchainStorage, - Block::Hash: Ord, -{ - type BlockImportOperation = ImportOperation; - type Blockchain = Blockchain; - type State = GenesisOrUnavailableState>; - type OffchainStorage = InMemOffchainStorage; - - fn begin_operation(&self) -> ClientResult { - Ok(ImportOperation { - header: None, - cache: Default::default(), - leaf_state: NewBlockState::Normal, - aux_ops: Vec::new(), - finalized_blocks: Vec::new(), - set_head: None, - storage_update: None, - changes_trie_config_update: None, - _phantom: Default::default(), - }) - } - - fn begin_state_operation( - &self, - _operation: &mut Self::BlockImportOperation, - _block: BlockId, - ) -> ClientResult<()> { - Ok(()) - } - - fn commit_operation(&self, mut operation: Self::BlockImportOperation) -> ClientResult<()> { - if !operation.finalized_blocks.is_empty() { - for block in operation.finalized_blocks { - self.blockchain.storage().finalize_header(block)?; - } - } - - if let Some(header) = operation.header { - let is_genesis_import = header.number().is_zero(); - if let Some(new_config) = operation.changes_trie_config_update { - operation - .cache - .insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_config.encode()); - } - self.blockchain.storage().import_header( - header, - operation.cache, - operation.leaf_state, - operation.aux_ops, - )?; - - // when importing genesis block => remember its state - if is_genesis_import { - *self.genesis_state.write() = operation.storage_update.take(); - } - } else { - for (key, maybe_val) in operation.aux_ops { - match maybe_val { - Some(val) => self - .blockchain - .storage() - .insert_aux(&[(&key[..], &val[..])], std::iter::empty())?, - None => - self.blockchain.storage().insert_aux(std::iter::empty(), &[&key[..]])?, - } - } - } - - if let Some(set_head) = operation.set_head { - self.blockchain.storage().set_head(set_head)?; - } - - Ok(()) - } - - fn finalize_block( - &self, - block: BlockId, - _justification: Option, - ) -> ClientResult<()> { - self.blockchain.storage().finalize_header(block) - } - - fn append_justification( - &self, - _block: BlockId, - _justification: Justification, - ) -> ClientResult<()> { - Ok(()) - } - - fn blockchain(&self) -> &Blockchain { - &self.blockchain - } - - fn usage_info(&self) -> Option { - self.blockchain.storage().usage_info() - } - - fn changes_trie_storage(&self) -> Option<&dyn PrunableStateChangesTrieStorage> { - None - } - - fn offchain_storage(&self) -> Option { - None - } - - fn state_at(&self, block: BlockId) -> ClientResult { - let block_number = self.blockchain.expect_block_number_from_id(&block)?; - - // special case for genesis block - if block_number.is_zero() { - if let Some(genesis_state) = self.genesis_state.read().clone() { - return Ok(GenesisOrUnavailableState::Genesis(genesis_state)) - } - } - - // else return unavailable state. We do not return error here, because error - // would mean that we do not know this state at all. But we know that it exists - Ok(GenesisOrUnavailableState::Unavailable) - } - - fn revert( - &self, - _n: NumberFor, - _revert_finalized: bool, - ) -> ClientResult<(NumberFor, HashSet)> { - Err(ClientError::NotAvailableOnLightClient) - } - - fn remove_leaf_block(&self, _hash: &Block::Hash) -> ClientResult<()> { - Err(ClientError::NotAvailableOnLightClient) - } - - fn get_import_lock(&self) -> &RwLock<()> { - &self.import_lock - } -} - -impl RemoteBackend for Backend> -where - Block: BlockT, - S: BlockchainStorage + 'static, - Block::Hash: Ord, -{ - fn is_local_state_available(&self, block: &BlockId) -> bool { - self.genesis_state.read().is_some() && - self.blockchain - .expect_block_number_from_id(block) - .map(|num| num.is_zero()) - .unwrap_or(false) - } - - fn remote_blockchain(&self) -> Arc> { - self.blockchain.clone() - } -} - -impl BlockImportOperation for ImportOperation -where - Block: BlockT, - S: BlockchainStorage, - Block::Hash: Ord, -{ - type State = GenesisOrUnavailableState>; - - fn state(&self) -> ClientResult> { - // None means 'locally-stateless' backend - Ok(None) - } - - fn set_block_data( - &mut self, - header: Block::Header, - _body: Option>, - _indexed_body: Option>>, - _justifications: Option, - state: NewBlockState, - ) -> ClientResult<()> { - self.leaf_state = state; - self.header = Some(header); - Ok(()) - } - - fn update_cache(&mut self, cache: HashMap>) { - self.cache = cache; - } - - fn update_db_storage( - &mut self, - _update: >>::Transaction, - ) -> ClientResult<()> { - // we're not storing anything locally => ignore changes - Ok(()) - } - - fn update_changes_trie( - &mut self, - _update: ChangesTrieTransaction, NumberFor>, - ) -> ClientResult<()> { - // we're not storing anything locally => ignore changes - Ok(()) - } - - fn set_genesis_state(&mut self, input: Storage, commit: bool) -> ClientResult { - check_genesis_storage(&input)?; - - // changes trie configuration - let changes_trie_config = input - .top - .iter() - .find(|(k, _)| &k[..] == well_known_keys::CHANGES_TRIE_CONFIG) - .map(|(_, v)| { - Decode::decode(&mut &v[..]) - .expect("changes trie configuration is encoded properly at genesis") - }); - self.changes_trie_config_update = Some(changes_trie_config); - - // this is only called when genesis block is imported => shouldn't be performance bottleneck - let mut storage: HashMap, _> = HashMap::new(); - storage.insert(None, input.top); - - // create a list of children keys to re-compute roots for - let child_delta = input - .children_default - .iter() - .map(|(_storage_key, storage_child)| (&storage_child.child_info, std::iter::empty())); - - // make sure to persist the child storage - for (_child_key, storage_child) in input.children_default.clone() { - storage.insert(Some(storage_child.child_info), storage_child.data); - } - - let storage_update = InMemoryBackend::from(storage); - let (storage_root, _) = storage_update.full_storage_root(std::iter::empty(), child_delta); - if commit { - self.storage_update = Some(storage_update); - } - - Ok(storage_root) - } - - fn reset_storage(&mut self, _input: Storage) -> ClientResult { - Err(ClientError::NotAvailableOnLightClient) - } - - fn insert_aux(&mut self, ops: I) -> ClientResult<()> - where - I: IntoIterator, Option>)>, - { - self.aux_ops.append(&mut ops.into_iter().collect()); - Ok(()) - } - - fn update_storage( - &mut self, - _update: StorageCollection, - _child_update: ChildStorageCollection, - ) -> ClientResult<()> { - // we're not storing anything locally => ignore changes - Ok(()) - } - - fn mark_finalized( - &mut self, - block: BlockId, - _justifications: Option, - ) -> ClientResult<()> { - self.finalized_blocks.push(block); - Ok(()) - } - - fn mark_head(&mut self, block: BlockId) -> ClientResult<()> { - self.set_head = Some(block); - Ok(()) - } - - fn update_transaction_index( - &mut self, - _index: Vec, - ) -> sp_blockchain::Result<()> { - // noop for the light client - Ok(()) - } -} - -impl std::fmt::Debug for GenesisOrUnavailableState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => state.fmt(f), - GenesisOrUnavailableState::Unavailable => write!(f, "Unavailable"), - } - } -} - -impl StateBackend for GenesisOrUnavailableState -where - H::Out: Ord + codec::Codec, -{ - type Error = ClientError; - type Transaction = as StateBackend>::Transaction; - type TrieBackendStorage = as StateBackend>::TrieBackendStorage; - - fn storage(&self, key: &[u8]) -> ClientResult>> { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => - Ok(state.storage(key).expect(IN_MEMORY_EXPECT_PROOF)), - GenesisOrUnavailableState::Unavailable => Err(ClientError::NotAvailableOnLightClient), - } - } - - fn child_storage(&self, child_info: &ChildInfo, key: &[u8]) -> ClientResult>> { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => - Ok(state.child_storage(child_info, key).expect(IN_MEMORY_EXPECT_PROOF)), - GenesisOrUnavailableState::Unavailable => Err(ClientError::NotAvailableOnLightClient), - } - } - - fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => - Ok(state.next_storage_key(key).expect(IN_MEMORY_EXPECT_PROOF)), - GenesisOrUnavailableState::Unavailable => Err(ClientError::NotAvailableOnLightClient), - } - } - - fn next_child_storage_key( - &self, - child_info: &ChildInfo, - key: &[u8], - ) -> Result>, Self::Error> { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => - Ok(state.next_child_storage_key(child_info, key).expect(IN_MEMORY_EXPECT_PROOF)), - GenesisOrUnavailableState::Unavailable => Err(ClientError::NotAvailableOnLightClient), - } - } - - fn for_keys_with_prefix(&self, prefix: &[u8], action: A) { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => - state.for_keys_with_prefix(prefix, action), - GenesisOrUnavailableState::Unavailable => (), - } - } - - fn for_key_values_with_prefix(&self, prefix: &[u8], action: A) { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => - state.for_key_values_with_prefix(prefix, action), - GenesisOrUnavailableState::Unavailable => (), - } - } - - fn apply_to_key_values_while, Vec) -> bool>( - &self, - child_info: Option<&ChildInfo>, - prefix: Option<&[u8]>, - start_at: Option<&[u8]>, - action: A, - allow_missing: bool, - ) -> ClientResult { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => Ok(state - .apply_to_key_values_while(child_info, prefix, start_at, action, allow_missing) - .expect(IN_MEMORY_EXPECT_PROOF)), - GenesisOrUnavailableState::Unavailable => Err(ClientError::NotAvailableOnLightClient), - } - } - - fn apply_to_keys_while bool>( - &self, - child_info: Option<&ChildInfo>, - prefix: Option<&[u8]>, - action: A, - ) { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => - state.apply_to_keys_while(child_info, prefix, action), - GenesisOrUnavailableState::Unavailable => (), - } - } - - fn for_child_keys_with_prefix( - &self, - child_info: &ChildInfo, - prefix: &[u8], - action: A, - ) { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => - state.for_child_keys_with_prefix(child_info, prefix, action), - GenesisOrUnavailableState::Unavailable => (), - } - } - - fn storage_root<'a>( - &self, - delta: impl Iterator)>, - ) -> (H::Out, Self::Transaction) - where - H::Out: Ord, - { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => state.storage_root(delta), - GenesisOrUnavailableState::Unavailable => Default::default(), - } - } - - fn child_storage_root<'a>( - &self, - child_info: &ChildInfo, - delta: impl Iterator)>, - ) -> (H::Out, bool, Self::Transaction) - where - H::Out: Ord, - { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => { - let (root, is_equal, _) = state.child_storage_root(child_info, delta); - (root, is_equal, Default::default()) - }, - GenesisOrUnavailableState::Unavailable => (H::Out::default(), true, Default::default()), - } - } - - fn pairs(&self) -> Vec<(Vec, Vec)> { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => state.pairs(), - GenesisOrUnavailableState::Unavailable => Vec::new(), - } - } - - fn keys(&self, prefix: &[u8]) -> Vec> { - match *self { - GenesisOrUnavailableState::Genesis(ref state) => state.keys(prefix), - GenesisOrUnavailableState::Unavailable => Vec::new(), - } - } - - fn register_overlay_stats(&self, _stats: &sp_state_machine::StateMachineStats) {} - - fn usage_info(&self) -> sp_state_machine::UsageInfo { - sp_state_machine::UsageInfo::empty() - } - - fn as_trie_backend(&self) -> Option<&TrieBackend> { - match self { - GenesisOrUnavailableState::Genesis(ref state) => state.as_trie_backend(), - GenesisOrUnavailableState::Unavailable => None, - } - } -} diff --git a/client/light/src/blockchain.rs b/client/light/src/blockchain.rs deleted file mode 100644 index e88c72419369..000000000000 --- a/client/light/src/blockchain.rs +++ /dev/null @@ -1,219 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Light client blockchain backend. Only stores headers and justifications of recent -//! blocks. CHT roots are stored for headers of ancient blocks. - -use std::sync::Arc; - -use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}, - Justifications, -}; - -use crate::fetcher::RemoteHeaderRequest; -pub use sc_client_api::{ - backend::{AuxStore, NewBlockState, ProvideChtRoots}, - blockchain::{ - well_known_cache_keys, Backend as BlockchainBackend, BlockStatus, Cache as BlockchainCache, - HeaderBackend as BlockchainHeaderBackend, Info as BlockchainInfo, ProvideCache, - }, - cht, - light::{LocalOrRemote, RemoteBlockchain, Storage}, -}; -use sp_blockchain::{ - CachedHeaderMetadata, Error as ClientError, HeaderMetadata, Result as ClientResult, -}; - -/// Light client blockchain. -pub struct Blockchain { - storage: S, -} - -impl Blockchain { - /// Create new light blockchain backed with given storage. - pub fn new(storage: S) -> Self { - Self { storage } - } - - /// Get storage reference. - pub fn storage(&self) -> &S { - &self.storage - } -} - -impl BlockchainHeaderBackend for Blockchain -where - Block: BlockT, - S: Storage, -{ - fn header(&self, id: BlockId) -> ClientResult> { - match RemoteBlockchain::header(self, id)? { - LocalOrRemote::Local(header) => Ok(Some(header)), - LocalOrRemote::Remote(_) => Err(ClientError::NotAvailableOnLightClient), - LocalOrRemote::Unknown => Ok(None), - } - } - - fn info(&self) -> BlockchainInfo { - self.storage.info() - } - - fn status(&self, id: BlockId) -> ClientResult { - self.storage.status(id) - } - - fn number(&self, hash: Block::Hash) -> ClientResult>> { - self.storage.number(hash) - } - - fn hash( - &self, - number: <::Header as HeaderT>::Number, - ) -> ClientResult> { - self.storage.hash(number) - } -} - -impl HeaderMetadata for Blockchain -where - Block: BlockT, - S: Storage, -{ - type Error = ClientError; - - fn header_metadata( - &self, - hash: Block::Hash, - ) -> Result, Self::Error> { - self.storage.header_metadata(hash) - } - - fn insert_header_metadata(&self, hash: Block::Hash, metadata: CachedHeaderMetadata) { - self.storage.insert_header_metadata(hash, metadata) - } - - fn remove_header_metadata(&self, hash: Block::Hash) { - self.storage.remove_header_metadata(hash) - } -} - -impl BlockchainBackend for Blockchain -where - Block: BlockT, - S: Storage, -{ - fn body(&self, _id: BlockId) -> ClientResult>> { - Err(ClientError::NotAvailableOnLightClient) - } - - fn justifications(&self, _id: BlockId) -> ClientResult> { - Err(ClientError::NotAvailableOnLightClient) - } - - fn last_finalized(&self) -> ClientResult { - self.storage.last_finalized() - } - - fn cache(&self) -> Option>> { - self.storage.cache() - } - - fn leaves(&self) -> ClientResult> { - Err(ClientError::NotAvailableOnLightClient) - } - - fn children(&self, _parent_hash: Block::Hash) -> ClientResult> { - Err(ClientError::NotAvailableOnLightClient) - } - - fn indexed_transaction(&self, _hash: &Block::Hash) -> ClientResult>> { - Err(ClientError::NotAvailableOnLightClient) - } - - fn block_indexed_body( - &self, - _id: BlockId, - ) -> sp_blockchain::Result>>> { - Err(ClientError::NotAvailableOnLightClient) - } -} - -impl, Block: BlockT> ProvideCache for Blockchain { - fn cache(&self) -> Option>> { - self.storage.cache() - } -} - -impl RemoteBlockchain for Blockchain -where - S: Storage, -{ - fn header( - &self, - id: BlockId, - ) -> ClientResult>> { - // first, try to read header from local storage - if let Some(local_header) = self.storage.header(id)? { - return Ok(LocalOrRemote::Local(local_header)) - } - - // we need to know block number to check if it's a part of CHT - let number = match id { - BlockId::Hash(hash) => match self.storage.number(hash)? { - Some(number) => number, - None => return Ok(LocalOrRemote::Unknown), - }, - BlockId::Number(number) => number, - }; - - // if the header is genesis (never pruned), non-canonical, or from future => return - if number.is_zero() || self.storage.status(BlockId::Number(number))? == BlockStatus::Unknown - { - return Ok(LocalOrRemote::Unknown) - } - - Ok(LocalOrRemote::Remote(RemoteHeaderRequest { - cht_root: match self.storage.header_cht_root(cht::size(), number)? { - Some(cht_root) => cht_root, - None => return Ok(LocalOrRemote::Unknown), - }, - block: number, - retry_count: None, - })) - } -} - -impl, Block: BlockT> ProvideChtRoots for Blockchain { - fn header_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result> { - self.storage().header_cht_root(cht_size, block) - } - - fn changes_trie_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> sp_blockchain::Result> { - self.storage().changes_trie_cht_root(cht_size, block) - } -} diff --git a/client/light/src/call_executor.rs b/client/light/src/call_executor.rs deleted file mode 100644 index a0776131e406..000000000000 --- a/client/light/src/call_executor.rs +++ /dev/null @@ -1,206 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Methods that light client could use to execute runtime calls. - -use std::{cell::RefCell, panic::UnwindSafe, result, sync::Arc}; - -use codec::{Decode, Encode}; -use hash_db::Hasher; -use sp_core::{ - convert_hash, - traits::{CodeExecutor, SpawnNamed}, - NativeOrEncoded, -}; -use sp_externalities::Extensions; -use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, Header as HeaderT}, -}; -use sp_state_machine::{ - create_proof_check_backend, execution_proof_check_on_trie_backend, ExecutionManager, - ExecutionStrategy, OverlayedChanges, StorageProof, -}; - -use sp_api::{ProofRecorder, StorageTransactionCache}; - -use sp_blockchain::{Error as ClientError, Result as ClientResult}; - -use sc_client_api::{ - backend::RemoteBackend, call_executor::CallExecutor, light::RemoteCallRequest, -}; -use sc_executor::RuntimeVersion; - -/// Call executor that is able to execute calls only on genesis state. -/// -/// Trying to execute call on non-genesis state leads to error. -pub struct GenesisCallExecutor { - backend: Arc, - local: L, -} - -impl GenesisCallExecutor { - /// Create new genesis call executor. - pub fn new(backend: Arc, local: L) -> Self { - Self { backend, local } - } -} - -impl Clone for GenesisCallExecutor { - fn clone(&self) -> Self { - GenesisCallExecutor { backend: self.backend.clone(), local: self.local.clone() } - } -} - -impl CallExecutor for GenesisCallExecutor -where - Block: BlockT, - B: RemoteBackend, - Local: CallExecutor, -{ - type Error = ClientError; - - type Backend = B; - - fn call( - &self, - id: &BlockId, - method: &str, - call_data: &[u8], - strategy: ExecutionStrategy, - extensions: Option, - ) -> ClientResult> { - if self.backend.is_local_state_available(id) { - self.local.call(id, method, call_data, strategy, extensions) - } else { - Err(ClientError::NotAvailableOnLightClient) - } - } - - fn contextual_call< - EM: Fn( - Result, Self::Error>, - Result, Self::Error>, - ) -> Result, Self::Error>, - R: Encode + Decode + PartialEq, - NC: FnOnce() -> result::Result + UnwindSafe, - >( - &self, - at: &BlockId, - method: &str, - call_data: &[u8], - changes: &RefCell, - _: Option<&RefCell>>, - _manager: ExecutionManager, - native_call: Option, - recorder: &Option>, - extensions: Option, - ) -> ClientResult> - where - ExecutionManager: Clone, - { - // there's no actual way/need to specify native/wasm execution strategy on light node - // => we can safely ignore passed values - - if self.backend.is_local_state_available(at) { - CallExecutor::contextual_call::< - fn( - Result, Local::Error>, - Result, Local::Error>, - ) -> Result, Local::Error>, - _, - NC, - >( - &self.local, - at, - method, - call_data, - changes, - None, - ExecutionManager::NativeWhenPossible, - native_call, - recorder, - extensions, - ) - } else { - Err(ClientError::NotAvailableOnLightClient) - } - } - - fn prove_execution( - &self, - at: &BlockId, - method: &str, - call_data: &[u8], - ) -> ClientResult<(Vec, StorageProof)> { - if self.backend.is_local_state_available(at) { - self.local.prove_execution(at, method, call_data) - } else { - Err(ClientError::NotAvailableOnLightClient) - } - } - - fn runtime_version(&self, id: &BlockId) -> ClientResult { - if self.backend.is_local_state_available(id) { - self.local.runtime_version(id) - } else { - Err(ClientError::NotAvailableOnLightClient) - } - } -} - -/// Check remote contextual execution proof using given backend. -/// -/// Proof should include the method execution proof. -pub fn check_execution_proof( - executor: &E, - spawn_handle: Box, - request: &RemoteCallRequest
, - remote_proof: StorageProof, -) -> ClientResult> -where - Header: HeaderT, - E: CodeExecutor + Clone + 'static, - H: Hasher, - H::Out: Ord + codec::Codec + 'static, -{ - let local_state_root = request.header.state_root(); - let root: H::Out = convert_hash(&local_state_root); - - // prepare execution environment - let mut changes = OverlayedChanges::default(); - let trie_backend = create_proof_check_backend(root, remote_proof)?; - - // TODO: Remove when solved: https://github.com/paritytech/substrate/issues/5047 - let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&trie_backend); - let runtime_code = backend_runtime_code - .runtime_code() - .map_err(|_e| ClientError::RuntimeCodeMissing)?; - - // execute method - execution_proof_check_on_trie_backend::( - &trie_backend, - &mut changes, - executor, - spawn_handle, - &request.method, - &request.call_data, - &runtime_code, - ) - .map_err(Into::into) -} diff --git a/client/light/src/fetcher.rs b/client/light/src/fetcher.rs deleted file mode 100644 index 5740e407a5e8..000000000000 --- a/client/light/src/fetcher.rs +++ /dev/null @@ -1,366 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Light client data fetcher. Fetches requested data from remote full nodes. - -use std::{ - collections::{BTreeMap, HashMap}, - marker::PhantomData, - sync::Arc, -}; - -use codec::{Decode, Encode}; -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; -use sp_blockchain::{Error as ClientError, Result as ClientResult}; -use sp_core::{ - convert_hash, - storage::{ChildInfo, ChildType}, - traits::{CodeExecutor, SpawnNamed}, -}; -use sp_runtime::traits::{ - AtLeast32Bit, Block as BlockT, CheckedConversion, Hash, HashFor, Header as HeaderT, NumberFor, -}; -pub use sp_state_machine::StorageProof; -use sp_state_machine::{ - key_changes_proof_check_with_db, read_child_proof_check, read_proof_check, - ChangesTrieAnchorBlockId, ChangesTrieConfigurationRange, ChangesTrieRootsStorage, - InMemoryChangesTrieStorage, TrieBackend, -}; - -use crate::{blockchain::Blockchain, call_executor::check_execution_proof}; -pub use sc_client_api::{ - cht, - light::{ - ChangesProof, FetchChecker, Fetcher, RemoteBodyRequest, RemoteCallRequest, - RemoteChangesRequest, RemoteHeaderRequest, RemoteReadChildRequest, RemoteReadRequest, - Storage as BlockchainStorage, - }, -}; - -/// Remote data checker. -pub struct LightDataChecker> { - blockchain: Arc>, - executor: E, - spawn_handle: Box, - _marker: PhantomData, -} - -impl> LightDataChecker { - /// Create new light data checker. - pub fn new( - blockchain: Arc>, - executor: E, - spawn_handle: Box, - ) -> Self { - Self { blockchain, executor, spawn_handle, _marker: PhantomData } - } - - /// Check remote changes query proof assuming that CHT-s are of given size. - pub fn check_changes_proof_with_cht_size( - &self, - request: &RemoteChangesRequest, - remote_proof: ChangesProof, - cht_size: NumberFor, - ) -> ClientResult, u32)>> { - // since we need roots of all changes tries for the range begin..max - // => remote node can't use max block greater that one that we have passed - if remote_proof.max_block > request.max_block.0 || - remote_proof.max_block < request.last_block.0 - { - return Err(ClientError::ChangesTrieAccessFailed(format!( - "Invalid max_block used by the remote node: {}. Local: {}..{}..{}", - remote_proof.max_block, - request.first_block.0, - request.last_block.0, - request.max_block.0, - )) - .into()) - } - - // check if remote node has responded with extra changes trie roots proofs - // all changes tries roots must be in range [request.first_block.0; request.tries_roots.0) - let is_extra_first_root = remote_proof - .roots - .keys() - .next() - .map(|first_root| { - *first_root < request.first_block.0 || *first_root >= request.tries_roots.0 - }) - .unwrap_or(false); - let is_extra_last_root = remote_proof - .roots - .keys() - .next_back() - .map(|last_root| *last_root >= request.tries_roots.0) - .unwrap_or(false); - if is_extra_first_root || is_extra_last_root { - return Err(ClientError::ChangesTrieAccessFailed(format!( - "Extra changes tries roots proofs provided by the remote node: [{:?}..{:?}]. Expected in range: [{}; {})", - remote_proof.roots.keys().next(), remote_proof.roots.keys().next_back(), - request.first_block.0, request.tries_roots.0, - )).into()); - } - - // if request has been composed when some required headers were already pruned - // => remote node has sent us CHT-based proof of required changes tries roots - // => check that this proof is correct before proceeding with changes proof - let remote_max_block = remote_proof.max_block; - let remote_roots = remote_proof.roots; - let remote_roots_proof = remote_proof.roots_proof; - let remote_proof = remote_proof.proof; - if !remote_roots.is_empty() { - self.check_changes_tries_proof(cht_size, &remote_roots, remote_roots_proof)?; - } - - // and now check the key changes proof + get the changes - let mut result = Vec::new(); - let proof_storage = InMemoryChangesTrieStorage::with_proof(remote_proof); - for config_range in &request.changes_trie_configs { - let result_range = key_changes_proof_check_with_db::, _>( - ChangesTrieConfigurationRange { - config: config_range - .config - .as_ref() - .ok_or(ClientError::ChangesTriesNotSupported)?, - zero: config_range.zero.0, - end: config_range.end.map(|(n, _)| n), - }, - &RootsStorage { - roots: (request.tries_roots.0, &request.tries_roots.2), - prev_roots: &remote_roots, - }, - &proof_storage, - request.first_block.0, - &ChangesTrieAnchorBlockId { - hash: convert_hash(&request.last_block.1), - number: request.last_block.0, - }, - remote_max_block, - request.storage_key.as_ref(), - &request.key, - ) - .map_err(|err| ClientError::ChangesTrieAccessFailed(err))?; - result.extend(result_range); - } - - Ok(result) - } - - /// Check CHT-based proof for changes tries roots. - pub fn check_changes_tries_proof( - &self, - cht_size: NumberFor, - remote_roots: &BTreeMap, B::Hash>, - remote_roots_proof: StorageProof, - ) -> ClientResult<()> { - // all the checks are sharing the same storage - let storage = remote_roots_proof.into_memory_db(); - - // remote_roots.keys() are sorted => we can use this to group changes tries roots - // that are belongs to the same CHT - let blocks = remote_roots.keys().cloned(); - cht::for_each_cht_group::( - cht_size, - blocks, - |mut storage, _, cht_blocks| { - // get local changes trie CHT root for given CHT - // it should be there, because it is never pruned AND request has been composed - // when required header has been pruned (=> replaced with CHT) - let first_block = cht_blocks - .first() - .cloned() - .expect("for_each_cht_group never calls callback with empty groups"); - let local_cht_root = self - .blockchain - .storage() - .changes_trie_cht_root(cht_size, first_block)? - .ok_or(ClientError::InvalidCHTProof)?; - - // check changes trie root for every block within CHT range - for block in cht_blocks { - // check if the proofs storage contains the root - // normally this happens in when the proving backend is created, but since - // we share the storage for multiple checks, do it here - if !storage.contains(&local_cht_root, EMPTY_PREFIX) { - return Err(ClientError::InvalidCHTProof.into()) - } - - // check proof for single changes trie root - let proving_backend = TrieBackend::new(storage, local_cht_root); - let remote_changes_trie_root = remote_roots[&block]; - cht::check_proof_on_proving_backend::>( - local_cht_root, - block, - remote_changes_trie_root, - &proving_backend, - )?; - - // and return the storage to use in following checks - storage = proving_backend.into_storage(); - } - - Ok(storage) - }, - storage, - ) - } -} - -impl FetchChecker for LightDataChecker -where - Block: BlockT, - E: CodeExecutor + Clone + 'static, - S: BlockchainStorage, -{ - fn check_header_proof( - &self, - request: &RemoteHeaderRequest, - remote_header: Option, - remote_proof: StorageProof, - ) -> ClientResult { - let remote_header = - remote_header.ok_or_else(|| ClientError::from(ClientError::InvalidCHTProof))?; - let remote_header_hash = remote_header.hash(); - cht::check_proof::>( - request.cht_root, - request.block, - remote_header_hash, - remote_proof, - ) - .map(|_| remote_header) - } - - fn check_read_proof( - &self, - request: &RemoteReadRequest, - remote_proof: StorageProof, - ) -> ClientResult, Option>>> { - read_proof_check::, _>( - convert_hash(request.header.state_root()), - remote_proof, - request.keys.iter(), - ) - .map_err(|e| ClientError::from(e)) - } - - fn check_read_child_proof( - &self, - request: &RemoteReadChildRequest, - remote_proof: StorageProof, - ) -> ClientResult, Option>>> { - let child_info = match ChildType::from_prefixed_key(&request.storage_key) { - Some((ChildType::ParentKeyId, storage_key)) => ChildInfo::new_default(storage_key), - None => return Err(ClientError::InvalidChildType), - }; - read_child_proof_check::, _>( - convert_hash(request.header.state_root()), - remote_proof, - &child_info, - request.keys.iter(), - ) - .map_err(|e| ClientError::from(e)) - } - - fn check_execution_proof( - &self, - request: &RemoteCallRequest, - remote_proof: StorageProof, - ) -> ClientResult> { - check_execution_proof::<_, _, HashFor>( - &self.executor, - self.spawn_handle.clone(), - request, - remote_proof, - ) - } - - fn check_changes_proof( - &self, - request: &RemoteChangesRequest, - remote_proof: ChangesProof, - ) -> ClientResult, u32)>> { - self.check_changes_proof_with_cht_size(request, remote_proof, cht::size()) - } - - fn check_body_proof( - &self, - request: &RemoteBodyRequest, - body: Vec, - ) -> ClientResult> { - // TODO: #2621 - let extrinsics_root = - HashFor::::ordered_trie_root(body.iter().map(Encode::encode).collect()); - if *request.header.extrinsics_root() == extrinsics_root { - Ok(body) - } else { - Err(ClientError::ExtrinsicRootInvalid { - received: request.header.extrinsics_root().to_string(), - expected: extrinsics_root.to_string(), - }) - } - } -} - -/// A view of BTreeMap as a changes trie roots storage. -struct RootsStorage<'a, Number: AtLeast32Bit, Hash: 'a> { - roots: (Number, &'a [Hash]), - prev_roots: &'a BTreeMap, -} - -impl<'a, H, Number, Hash> ChangesTrieRootsStorage for RootsStorage<'a, Number, Hash> -where - H: Hasher, - Number: std::fmt::Display - + std::hash::Hash - + Clone - + AtLeast32Bit - + Encode - + Decode - + Send - + Sync - + 'static, - Hash: 'a + Send + Sync + Clone + AsRef<[u8]>, -{ - fn build_anchor( - &self, - _hash: H::Out, - ) -> Result, String> { - Err("build_anchor is only called when building block".into()) - } - - fn root( - &self, - _anchor: &ChangesTrieAnchorBlockId, - block: Number, - ) -> Result, String> { - // we can't ask for roots from parallel forks here => ignore anchor - let root = if block < self.roots.0 { - self.prev_roots.get(&Number::unique_saturated_from(block)).cloned() - } else { - let index: Option = - block.checked_sub(&self.roots.0).and_then(|index| index.checked_into()); - index.and_then(|index| self.roots.1.get(index as usize).cloned()) - }; - - Ok(root.map(|root| { - let mut hasher_root: H::Out = Default::default(); - hasher_root.as_mut().copy_from_slice(root.as_ref()); - hasher_root - })) - } -} diff --git a/client/light/src/lib.rs b/client/light/src/lib.rs deleted file mode 100644 index 0c874326ef2e..000000000000 --- a/client/light/src/lib.rs +++ /dev/null @@ -1,59 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Light client components. - -use sp_core::traits::{CodeExecutor, SpawnNamed}; -use sp_runtime::traits::{Block as BlockT, HashFor}; -use std::sync::Arc; - -pub mod backend; -pub mod blockchain; -pub mod call_executor; -pub mod fetcher; - -pub use backend::*; -pub use blockchain::*; -pub use call_executor::*; -pub use fetcher::*; - -/// Create an instance of fetch data checker. -pub fn new_fetch_checker>( - blockchain: Arc>, - executor: E, - spawn_handle: Box, -) -> LightDataChecker -where - E: CodeExecutor, -{ - LightDataChecker::new(blockchain, executor, spawn_handle) -} - -/// Create an instance of light client blockchain backend. -pub fn new_light_blockchain>(storage: S) -> Arc> { - Arc::new(Blockchain::new(storage)) -} - -/// Create an instance of light client backend. -pub fn new_light_backend(blockchain: Arc>) -> Arc>> -where - B: BlockT, - S: BlockchainStorage, -{ - Arc::new(Backend::new(blockchain)) -} diff --git a/client/network-gossip/Cargo.toml b/client/network-gossip/Cargo.toml index b5fdcfd43430..9854575ab6b0 100644 --- a/client/network-gossip/Cargo.toml +++ b/client/network-gossip/Cargo.toml @@ -4,8 +4,8 @@ name = "sc-network-gossip" version = "0.10.0-dev" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" authors = ["Parity Technologies "] -edition = "2018" -homepage = "https://substrate.dev" +edition = "2021" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sc-network-gossip" readme = "README.md" @@ -15,15 +15,16 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -futures = "0.3.9" +futures = "0.3.21" futures-timer = "3.0.1" -libp2p = { version = "0.39.1", default-features = false } +libp2p = { version = "0.40.0", default-features = false } log = "0.4.8" -lru = "0.6.6" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.9.0", path = "../../utils/prometheus" } +lru = "0.7.5" +ahash = "0.7.6" +prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.10.0-dev", path = "../../utils/prometheus" } sc-network = { version = "0.10.0-dev", path = "../network" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -tracing = "0.1.25" +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } +tracing = "0.1.29" [dev-dependencies] async-std = "1.10.0" diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 70b13983d8bd..2e09e7cc614a 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -39,7 +39,7 @@ use std::{ task::{Context, Poll}, }; -/// Wraps around an implementation of the `Network` crate and provides gossiping capabilities on +/// Wraps around an implementation of the [`Network`] trait and provides gossiping capabilities on /// top of it. pub struct GossipEngine { state_machine: ConsensusGossip, @@ -56,7 +56,7 @@ pub struct GossipEngine { } /// A gossip engine receives messages from the network via the `network_event_stream` and forwards -/// them to upper layers via the `message sinks`. In the scenario where messages have been received +/// them to upper layers via the `message_sinks`. In the scenario where messages have been received /// from the network but a subscribed message sink is not yet ready to receive the messages, the /// messages are buffered. To model this process a gossip engine can be in two states. enum ForwardingState { @@ -303,7 +303,6 @@ mod tests { use sp_runtime::{testing::H256, traits::Block as BlockT}; use std::{ borrow::Cow, - convert::TryInto, sync::{Arc, Mutex}, }; use substrate_test_runtime_client::runtime::Block; diff --git a/client/network-gossip/src/lib.rs b/client/network-gossip/src/lib.rs index 55c2fc820637..4b8370870246 100644 --- a/client/network-gossip/src/lib.rs +++ b/client/network-gossip/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -26,32 +26,32 @@ //! message, assuming it is valid. //! //! Topics are a single 32-byte tag associated with a message, used to group those messages -//! in an opaque way. Consensus code can invoke `broadcast_topic` to attempt to send all messages -//! under a single topic to all peers who don't have them yet, and `send_topic` to -//! send all messages under a single topic to a specific peer. +//! in an opaque way. Consensus code can invoke [`ValidatorContext::broadcast_topic`] to +//! attempt to send all messages under a single topic to all peers who don't have them yet, and +//! [`ValidatorContext::send_topic`] to send all messages under a single topic to a specific peer. //! //! # Usage //! -//! - Implement the `Network` trait, representing the low-level networking primitives. It is already -//! implemented on `sc_network::NetworkService`. -//! - Implement the `Validator` trait. See the section below. +//! - Implement the [`Network`] trait, representing the low-level networking primitives. It is +//! already implemented on `sc_network::NetworkService`. +//! - Implement the [`Validator`] trait. See the section below. //! - Decide on a protocol name. Each gossiping protocol should have a different one. -//! - Build a `GossipEngine` using these three elements. -//! - Use the methods of the `GossipEngine` in order to send out messages and receive incoming +//! - Build a [`GossipEngine`] using these three elements. +//! - Use the methods of the [`GossipEngine`] in order to send out messages and receive incoming //! messages. //! -//! The `GossipEngine` will automatically use `Network::add_set_reserved` and -//! `Network::remove_set_reserved` to maintain a set of peers equal to the set of peers the +//! The [`GossipEngine`] will automatically use [`Network::add_set_reserved`] and +//! [`Network::remove_set_reserved`] to maintain a set of peers equal to the set of peers the //! node is syncing from. See the documentation of `sc-network` for more explanations about the //! concepts of peer sets. //! //! # What is a validator? //! -//! The primary role of a `Validator` is to process incoming messages from peers, and decide +//! The primary role of a [`Validator`] is to process incoming messages from peers, and decide //! whether to discard them or process them. It also decides whether to re-broadcast the message. //! -//! The secondary role of the `Validator` is to check if a message is allowed to be sent to a given -//! peer. All messages, before being sent, will be checked against this filter. +//! The secondary role of the [`Validator`] is to check if a message is allowed to be sent to a +//! given peer. All messages, before being sent, will be checked against this filter. //! This enables the validator to use information it's aware of about connected peers to decide //! whether to send messages to them at any given moment in time - In particular, to wait until //! peers can accept and process the message before sending it. @@ -123,16 +123,7 @@ impl Network for Arc> { } fn remove_set_reserved(&self, who: PeerId, protocol: Cow<'static, str>) { - let addr = - iter::once(multiaddr::Protocol::P2p(who.into())).collect::(); - let result = NetworkService::remove_peers_from_reserved_set( - self, - protocol, - iter::once(addr).collect(), - ); - if let Err(err) = result { - log::error!(target: "gossip", "remove_set_reserved failed: {}", err); - } + NetworkService::remove_peers_from_reserved_set(self, protocol, iter::once(who).collect()); } fn disconnect_peer(&self, who: PeerId, protocol: Cow<'static, str>) { diff --git a/client/network-gossip/src/state_machine.rs b/client/network-gossip/src/state_machine.rs index 920b44d8c1e5..4f06819df64d 100644 --- a/client/network-gossip/src/state_machine.rs +++ b/client/network-gossip/src/state_machine.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,19 +18,13 @@ use crate::{MessageIntent, Network, ValidationResult, Validator, ValidatorContext}; +use ahash::AHashSet; use libp2p::PeerId; use lru::LruCache; use prometheus_endpoint::{register, Counter, PrometheusError, Registry, U64}; use sc_network::ObservedRole; use sp_runtime::traits::{Block as BlockT, Hash, HashFor}; -use std::{ - borrow::Cow, - collections::{HashMap, HashSet}, - iter, - sync::Arc, - time, - time::Instant, -}; +use std::{borrow::Cow, collections::HashMap, iter, sync::Arc, time, time::Instant}; // FIXME: Add additional spam/DoS attack protection: https://github.com/paritytech/substrate/issues/1115 // NOTE: The current value is adjusted based on largest production network deployment (Kusama) and @@ -56,7 +50,7 @@ mod rep { } struct PeerConsensus { - known_messages: HashSet, + known_messages: AHashSet, } /// Topic stream message with sender. @@ -204,7 +198,8 @@ impl ConsensusGossip { ?role, "Registering peer", ); - self.peers.insert(who.clone(), PeerConsensus { known_messages: HashSet::new() }); + self.peers + .insert(who.clone(), PeerConsensus { known_messages: Default::default() }); let validator = self.validator.clone(); let mut context = NetworkContext { gossip: self, network }; @@ -508,14 +503,14 @@ impl Metrics { Ok(Self { registered_messages: register( Counter::new( - "network_gossip_registered_messages_total", + "substrate_network_gossip_registered_messages_total", "Number of registered messages by the gossip service.", )?, registry, )?, expired_messages: register( Counter::new( - "network_gossip_expired_messages_total", + "substrate_network_gossip_expired_messages_total", "Number of expired messages by the gossip service.", )?, registry, diff --git a/client/network-gossip/src/validator.rs b/client/network-gossip/src/validator.rs index 9a2652d03f64..7d60f7b31397 100644 --- a/client/network-gossip/src/validator.rs +++ b/client/network-gossip/src/validator.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index 68b9595ae190..061b7ac7f265 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -4,8 +4,8 @@ name = "sc-network" version = "0.10.0-dev" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" authors = ["Parity Technologies "] -edition = "2018" -homepage = "https://substrate.dev" +edition = "2021" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sc-network" readme = "README.md" @@ -14,68 +14,68 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [build-dependencies] -prost-build = "0.8" +prost-build = "0.9" [dependencies] async-trait = "0.1" -async-std = "1.10.0" bitflags = "1.3.2" cid = "0.6.0" bytes = "1" -codec = { package = "parity-scale-codec", version = "2.0.0", features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", features = [ "derive", ] } -derive_more = "0.99.2" either = "1.5.3" fnv = "1.0.6" fork-tree = { version = "3.0.0", path = "../../utils/fork-tree" } -futures = "0.3.9" +futures = "0.3.21" futures-timer = "3.0.2" asynchronous-codec = "0.5" hex = "0.4.0" -ip_network = "0.4.0" +ip_network = "0.4.1" linked-hash-map = "0.5.4" linked_hash_set = "0.1.3" -lru = "0.6.6" +lru = "0.7.5" log = "0.4.8" -parking_lot = "0.11.1" -pin-project = "1.0.4" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.9.0", path = "../../utils/prometheus" } -prost = "0.8" +parking_lot = "0.12.0" +pin-project = "1.0.10" +prometheus-endpoint = { package = "substrate-prometheus-endpoint", version = "0.10.0-dev", path = "../../utils/prometheus" } +prost = "0.9" rand = "0.7.2" sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" } sc-client-api = { version = "4.0.0-dev", path = "../api" } sc-peerset = { version = "4.0.0-dev", path = "../peerset" } -serde = { version = "1.0.126", features = ["derive"] } -serde_json = "1.0.68" -smallvec = "1.7.0" -sp-arithmetic = { version = "4.0.0-dev", path = "../../primitives/arithmetic" } +serde = { version = "1.0.136", features = ["derive"] } +serde_json = "1.0.79" +smallvec = "1.8.0" +sp-arithmetic = { version = "5.0.0", path = "../../primitives/arithmetic" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-finality-grandpa = { version = "4.0.0-dev", path = "../../primitives/finality-grandpa" } -thiserror = "1" +thiserror = "1.0" unsigned-varint = { version = "0.6.0", features = [ "futures", "asynchronous_codec", ] } void = "1.0.2" -zeroize = "1.4.1" -libp2p = "0.39.1" +# zeroize = "1.5.4" +zeroize = "1.4.3" +libp2p = "0.40.0" [dev-dependencies] assert_matches = "1.3" -libp2p = { version = "0.39.1", default-features = false } +libp2p = { version = "0.40.0", default-features = false } quickcheck = "1.0.3" rand = "0.7.2" sp-test-primitives = { version = "2.0.0", path = "../../primitives/test-primitives" } -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } tempfile = "3.1.0" +async-std = "1.10.0" [features] default = [] diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 7b334175a280..d0de50ef6189 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,19 +20,22 @@ use crate::{ bitswap::Bitswap, config::ProtocolId, discovery::{DiscoveryBehaviour, DiscoveryConfig, DiscoveryOut}, - light_client_requests, peer_info, + peer_info, protocol::{message::Roles, CustomMessageOutcome, NotificationsSink, Protocol}, request_responses, DhtEvent, ObservedRole, }; use bytes::Bytes; use codec::Encode; -use futures::{channel::oneshot, stream::StreamExt}; +use futures::channel::oneshot; use libp2p::{ core::{Multiaddr, PeerId, PublicKey}, identify::IdentifyInfo, kad::record, - swarm::{toggle::Toggle, NetworkBehaviourAction, NetworkBehaviourEventProcess, PollParameters}, + swarm::{ + toggle::Toggle, NetworkBehaviour, NetworkBehaviourAction, NetworkBehaviourEventProcess, + PollParameters, + }, NetworkBehaviour, }; use log::debug; @@ -58,7 +61,7 @@ pub use crate::request_responses::{ /// General behaviour of the network. Combines all protocols together. #[derive(NetworkBehaviour)] -#[behaviour(out_event = "BehaviourOut", poll_method = "poll")] +#[behaviour(out_event = "BehaviourOut", poll_method = "poll", event_process = true)] pub struct Behaviour { /// All the substrate-specific protocols. substrate: Protocol, @@ -76,10 +79,6 @@ pub struct Behaviour { #[behaviour(ignore)] events: VecDeque>, - /// Light client request handling. - #[behaviour(ignore)] - light_client_request_sender: light_client_requests::sender::LightClientRequestSender, - /// Protocol name used to send out block requests via /// [`request_responses::RequestResponsesBehaviour`]. #[behaviour(ignore)] @@ -198,7 +197,6 @@ impl Behaviour { substrate: Protocol, user_agent: String, local_public_key: PublicKey, - light_client_request_sender: light_client_requests::sender::LightClientRequestSender, disco_config: DiscoveryConfig, block_request_protocol_config: request_responses::ProtocolConfig, state_request_protocol_config: request_responses::ProtocolConfig, @@ -233,7 +231,6 @@ impl Behaviour { request_response_protocols.into_iter(), peerset, )?, - light_client_request_sender, events: VecDeque::new(), block_request_protocol_name, state_request_protocol_name, @@ -316,14 +313,6 @@ impl Behaviour { pub fn put_value(&mut self, key: record::Key, value: Vec) { self.discovery.put_value(key, value); } - - /// Issue a light client request. - pub fn light_client_request( - &mut self, - r: light_client_requests::sender::Request, - ) -> Result<(), light_client_requests::sender::SendRequestError> { - self.light_client_request_sender.request(r) - } } fn reported_roles_to_observed_role(roles: Roles) -> ObservedRole { @@ -436,17 +425,11 @@ impl NetworkBehaviourEventProcess> for Behavi CustomMessageOutcome::NotificationsReceived { remote, messages } => { self.events.push_back(BehaviourOut::NotificationsReceived { remote, messages }); }, - CustomMessageOutcome::PeerNewBest(peer_id, number) => { - self.light_client_request_sender.update_best_block(&peer_id, number); - }, - CustomMessageOutcome::SyncConnected(peer_id) => { - self.light_client_request_sender.inject_connected(peer_id); - self.events.push_back(BehaviourOut::SyncConnected(peer_id)) - }, - CustomMessageOutcome::SyncDisconnected(peer_id) => { - self.light_client_request_sender.inject_disconnected(peer_id); - self.events.push_back(BehaviourOut::SyncDisconnected(peer_id)) - }, + CustomMessageOutcome::PeerNewBest(_peer_id, _number) => {}, + CustomMessageOutcome::SyncConnected(peer_id) => + self.events.push_back(BehaviourOut::SyncConnected(peer_id)), + CustomMessageOutcome::SyncDisconnected(peer_id) => + self.events.push_back(BehaviourOut::SyncDisconnected(peer_id)), CustomMessageOutcome::None => {}, } } @@ -532,25 +515,12 @@ impl NetworkBehaviourEventProcess for Behaviour { } impl Behaviour { - fn poll( + fn poll( &mut self, - cx: &mut Context, + _cx: &mut Context, _: &mut impl PollParameters, - ) -> Poll>> { - use light_client_requests::sender::OutEvent; - while let Poll::Ready(Some(event)) = self.light_client_request_sender.poll_next_unpin(cx) { - match event { - OutEvent::SendRequest { target, request, pending_response, protocol_name } => - self.request_responses.send_request( - &target, - &protocol_name, - request, - pending_response, - IfDisconnected::ImmediateError, - ), - } - } - + ) -> Poll, ::ProtocolsHandler>> + { if let Some(event) = self.events.pop_front() { return Poll::Ready(NetworkBehaviourAction::GenerateEvent(event)) } diff --git a/client/network/src/bitswap.rs b/client/network/src/bitswap.rs index 6b53dce62650..e7c37968b5f9 100644 --- a/client/network/src/bitswap.rs +++ b/client/network/src/bitswap.rs @@ -39,8 +39,7 @@ use libp2p::{ UpgradeInfo, }, swarm::{ - IntoProtocolsHandler, NetworkBehaviour, NetworkBehaviourAction, NotifyHandler, - OneShotHandler, PollParameters, ProtocolsHandler, + NetworkBehaviour, NetworkBehaviourAction, NotifyHandler, OneShotHandler, PollParameters, }, }; use log::{debug, error, trace}; @@ -297,12 +296,11 @@ impl NetworkBehaviour for Bitswap { self.ready_blocks.push_back((peer, response)); } - fn poll(&mut self, _ctx: &mut Context, _: &mut impl PollParameters) -> Poll< - NetworkBehaviourAction< - <::Handler as ProtocolsHandler>::InEvent, - Self::OutEvent, - >, - >{ + fn poll( + &mut self, + _ctx: &mut Context, + _: &mut impl PollParameters, + ) -> Poll> { if let Some((peer_id, message)) = self.ready_blocks.pop_front() { return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, @@ -315,21 +313,29 @@ impl NetworkBehaviour for Bitswap { } /// Bitswap protocol error. -#[derive(derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] pub enum BitswapError { /// Protobuf decoding error. - #[display(fmt = "Failed to decode request: {}.", _0)] - DecodeProto(prost::DecodeError), + #[error("Failed to decode request: {0}.")] + DecodeProto(#[from] prost::DecodeError), + /// Protobuf encoding error. - #[display(fmt = "Failed to encode response: {}.", _0)] - EncodeProto(prost::EncodeError), + #[error("Failed to encode response: {0}.")] + EncodeProto(#[from] prost::EncodeError), + /// Client backend error. - Client(sp_blockchain::Error), + #[error(transparent)] + Client(#[from] sp_blockchain::Error), + /// Error parsing CID - BadCid(cid::Error), + #[error(transparent)] + BadCid(#[from] cid::Error), + /// Packet read error. - Read(io::Error), + #[error(transparent)] + Read(#[from] io::Error), + /// Error sending response. - #[display(fmt = "Failed to send response.")] + #[error("Failed to send response.")] SendResponse, } diff --git a/client/network/src/block_request_handler.rs b/client/network/src/block_request_handler.rs index 9411ca71fd00..e1fe9ebf8d06 100644 --- a/client/network/src/block_request_handler.rs +++ b/client/network/src/block_request_handler.rs @@ -15,7 +15,7 @@ // along with Substrate. If not, see . //! Helper for handling (i.e. answering) block requests from a remote peer via the -//! [`crate::request_responses::RequestResponsesBehaviour`]. +//! `crate::request_responses::RequestResponsesBehaviour`. use crate::{ chain::Client, @@ -54,6 +54,10 @@ mod rep { /// Reputation change when a peer sent us the same request multiple times. pub const SAME_REQUEST: Rep = Rep::new_fatal("Same block request multiple times"); + + /// Reputation change when a peer sent us the same "small" request multiple times. + pub const SAME_SMALL_REQUEST: Rep = + Rep::new(-(1 << 10), "same small block request multiple times"); } /// Generates a [`ProtocolConfig`] for the block request protocol, refusing incoming requests. @@ -85,13 +89,14 @@ struct SeenRequestsKey { support_multiple_justifications: bool, } +#[allow(clippy::derive_hash_xor_eq)] impl Hash for SeenRequestsKey { fn hash(&self, state: &mut H) { self.peer.hash(state); self.max_blocks.hash(state); self.direction.hash(state); self.attributes.hash(state); - + self.support_multiple_justifications.hash(state); match self.from { BlockId::Hash(h) => h.hash(state), BlockId::Number(n) => n.hash(state), @@ -199,8 +204,16 @@ impl BlockRequestHandler { Some(SeenRequestsValue::Fulfilled(ref mut requests)) => { *requests = requests.saturating_add(1); + let small_request = attributes + .difference(BlockAttributes::HEADER | BlockAttributes::JUSTIFICATION) + .is_empty(); + if *requests > MAX_NUMBER_OF_SAME_REQUESTS_PER_PEER { - reputation_change = Some(rep::SAME_REQUEST); + reputation_change = Some(if small_request { + rep::SAME_SMALL_REQUEST + } else { + rep::SAME_REQUEST + }); } }, None => { @@ -378,19 +391,20 @@ impl BlockRequestHandler { } } -#[derive(derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] enum HandleRequestError { - #[display(fmt = "Failed to decode request: {}.", _0)] - DecodeProto(prost::DecodeError), - #[display(fmt = "Failed to encode response: {}.", _0)] - EncodeProto(prost::EncodeError), - #[display(fmt = "Failed to decode block hash: {}.", _0)] - DecodeScale(codec::Error), - #[display(fmt = "Missing `BlockRequest::from_block` field.")] + #[error("Failed to decode request: {0}.")] + DecodeProto(#[from] prost::DecodeError), + #[error("Failed to encode response: {0}.")] + EncodeProto(#[from] prost::EncodeError), + #[error("Failed to decode block hash: {0}.")] + DecodeScale(#[from] codec::Error), + #[error("Missing `BlockRequest::from_block` field.")] MissingFromField, - #[display(fmt = "Failed to parse BlockRequest::direction.")] + #[error("Failed to parse BlockRequest::direction.")] ParseDirection, - Client(sp_blockchain::Error), - #[display(fmt = "Failed to send response.")] + #[error(transparent)] + Client(#[from] sp_blockchain::Error), + #[error("Failed to send response.")] SendResponse, } diff --git a/client/network/src/chain.rs b/client/network/src/chain.rs index 7c131dd75370..c66cc2ce1daf 100644 --- a/client/network/src/chain.rs +++ b/client/network/src/chain.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/network/src/config.rs b/client/network/src/config.rs index d08e29ef8589..40aefe9a3ec2 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,7 +23,6 @@ pub use crate::{ chain::Client, - on_demand_layer::{AlwaysBadChecker, OnDemand}, request_responses::{ IncomingRequest, OutgoingResponse, ProtocolConfig as RequestResponseConfig, }, @@ -51,7 +50,6 @@ use sp_runtime::traits::Block as BlockT; use std::{ borrow::Cow, collections::HashMap, - convert::TryFrom, error::Error, fs, future::Future, @@ -83,11 +81,6 @@ pub struct Params { /// Client that contains the blockchain. pub chain: Arc>, - /// The `OnDemand` object acts as a "receiver" for block data requests from the client. - /// If `Some`, the network worker will process these requests and answer them. - /// Normally used only for light clients. - pub on_demand: Option>>, - /// Pool of transactions. /// /// The network worker will fetch transactions from this object in order to propagate them on @@ -155,14 +148,14 @@ pub enum Role { } impl Role { - /// True for `Role::Authority` + /// True for [`Role::Authority`]. pub fn is_authority(&self) -> bool { - matches!(self, Role::Authority { .. }) + matches!(self, Self::Authority { .. }) } - /// True for `Role::Light` + /// True for [`Role::Light`]. pub fn is_light(&self) -> bool { - matches!(self, Role::Light { .. }) + matches!(self, Self::Light { .. }) } } @@ -301,7 +294,7 @@ pub fn parse_addr(mut addr: Multiaddr) -> Result<(PeerId, Multiaddr), ParseErr> /// assert_eq!(addr.peer_id.to_base58(), "QmSk5HQbn6LhUwDiNMseVUjuRYhEtYj4aUZ6WfWoGURpdV"); /// assert_eq!(addr.multiaddr.to_string(), "/ip4/198.51.100.19/tcp/30333"); /// ``` -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] #[serde(try_from = "String", into = "String")] pub struct MultiaddrWithPeerId { /// Address of the node. @@ -329,7 +322,7 @@ impl FromStr for MultiaddrWithPeerId { fn from_str(s: &str) -> Result { let (peer_id, multiaddr) = parse_str_addr(s)?; - Ok(MultiaddrWithPeerId { peer_id, multiaddr }) + Ok(Self { peer_id, multiaddr }) } } @@ -383,8 +376,8 @@ impl From for ParseErr { } } -#[derive(Clone, Debug, Eq, PartialEq)] /// Sync operation mode. +#[derive(Clone, Debug, Eq, PartialEq)] pub enum SyncMode { /// Full block download and verification. Full, @@ -399,6 +392,18 @@ pub enum SyncMode { Warp, } +impl SyncMode { + /// Returns if `self` is [`Self::Warp`]. + pub fn is_warp(&self) -> bool { + matches!(self, Self::Warp) + } + + /// Returns if `self` is [`Self::Fast`]. + pub fn is_fast(&self) -> bool { + matches!(self, Self::Fast { .. }) + } +} + impl Default for SyncMode { fn default() -> Self { Self::Full @@ -422,6 +427,11 @@ pub struct NetworkConfiguration { pub request_response_protocols: Vec, /// Configuration for the default set of nodes used for block syncing and transactions. pub default_peers_set: SetConfig, + /// Number of substreams to reserve for full nodes for block syncing and transactions. + /// Any other slot will be dedicated to light nodes. + /// + /// This value is implicitly capped to `default_set.out_peers + default_set.in_peers`. + pub default_peers_set_num_full: u32, /// Configuration for extra sets of nodes. pub extra_sets: Vec, /// Client identifier. Sent over the wire for debugging purposes. @@ -479,6 +489,7 @@ impl NetworkConfiguration { node_key: NodeKeyConfig, net_config_path: Option, ) -> Self { + let default_peers_set = SetConfig::default(); Self { net_config_path, listen_addresses: Vec::new(), @@ -486,7 +497,8 @@ impl NetworkConfiguration { boot_nodes: Vec::new(), node_key, request_response_protocols: Vec::new(), - default_peers_set: Default::default(), + default_peers_set_num_full: default_peers_set.in_peers + default_peers_set.out_peers, + default_peers_set, extra_sets: Vec::new(), client_version: client_version.into(), node_name: node_name.into(), @@ -567,7 +579,7 @@ pub struct NonDefaultSetConfig { /// considered established once this protocol is open. /// /// > **Note**: This field isn't present for the default set, as this is handled internally - /// > by the networking code. + /// > by the networking code. pub notifications_protocol: Cow<'static, str>, /// If the remote reports that it doesn't support the protocol indicated in the /// `notifications_protocol` field, then each of these fallback names will be tried one by @@ -609,6 +621,13 @@ impl NonDefaultSetConfig { pub fn add_reserved(&mut self, peer: MultiaddrWithPeerId) { self.set_config.reserved_nodes.push(peer); } + + /// Add a list of protocol names used for backward compatibility. + /// + /// See the explanations in [`NonDefaultSetConfig::fallback_names`]. + pub fn add_fallback_names(&mut self, fallback_names: Vec>) { + self.fallback_names.extend(fallback_names); + } } /// Configuration for the transport layer. diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs index 431de50c0f19..cf75e2dcb4c6 100644 --- a/client/network/src/discovery.rs +++ b/client/network/src/discovery.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -67,8 +67,8 @@ use libp2p::{ mdns::{Mdns, MdnsConfig, MdnsEvent}, multiaddr::Protocol, swarm::{ - protocols_handler::multi::IntoMultiHandler, IntoProtocolsHandler, NetworkBehaviour, - NetworkBehaviourAction, PollParameters, ProtocolsHandler, + protocols_handler::multi::IntoMultiHandler, DialError, IntoProtocolsHandler, + NetworkBehaviour, NetworkBehaviourAction, PollParameters, ProtocolsHandler, }, }; use log::{debug, error, info, trace, warn}; @@ -107,7 +107,7 @@ impl DiscoveryConfig { /// Create a default configuration with the given public key. pub fn new(local_public_key: PublicKey) -> Self { Self { - local_peer_id: local_public_key.into_peer_id(), + local_peer_id: local_public_key.to_peer_id(), permanent_addresses: Vec::new(), dht_random_walk: true, allow_private_ipv4: true, @@ -428,6 +428,29 @@ impl DiscoveryBehaviour { }; ip.is_global() } + + fn new_handler_with_replacement( + &mut self, + pid: ProtocolId, + handler: KademliaHandlerProto, + ) -> ::ProtocolsHandler { + let mut handlers: HashMap<_, _> = self + .kademlias + .iter_mut() + .map(|(p, k)| (p.clone(), NetworkBehaviour::new_handler(k))) + .collect(); + + if let Some(h) = handlers.get_mut(&pid) { + *h = handler + } + + IntoMultiHandler::try_from_iter(handlers).expect( + "There can be at most one handler per `ProtocolId` and protocol names contain the \ + `ProtocolId` so no two protocol names in `self.kademlias` can be equal which is the \ + only error `try_from_iter` can return, therefore this call is guaranteed to succeed; \ + qed", + ) + } } /// Event generated by the `DiscoveryBehaviour`. @@ -512,14 +535,10 @@ impl NetworkBehaviour for DiscoveryBehaviour { list_to_filter.extend(self.mdns.addresses_of_peer(peer_id)); if !self.allow_private_ipv4 { - list_to_filter.retain(|addr| { - if let Some(Protocol::Ip4(addr)) = addr.iter().next() { - if addr.is_private() { - return false - } - } - - true + list_to_filter.retain(|addr| match addr.iter().next() { + Some(Protocol::Ip4(addr)) if !IpNetwork::from(addr).is_global() => false, + Some(Protocol::Ip6(addr)) if !IpNetwork::from(addr).is_global() => false, + _ => true, }); } @@ -531,15 +550,34 @@ impl NetworkBehaviour for DiscoveryBehaviour { list } + fn inject_address_change( + &mut self, + peer_id: &PeerId, + connection_id: &ConnectionId, + old: &ConnectedPoint, + new: &ConnectedPoint, + ) { + for k in self.kademlias.values_mut() { + NetworkBehaviour::inject_address_change(k, peer_id, connection_id, old, new); + } + } + fn inject_connection_established( &mut self, peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint, + failed_addresses: Option<&Vec>, ) { self.num_connections += 1; for k in self.kademlias.values_mut() { - NetworkBehaviour::inject_connection_established(k, peer_id, conn, endpoint) + NetworkBehaviour::inject_connection_established( + k, + peer_id, + conn, + endpoint, + failed_addresses, + ) } } @@ -551,14 +589,13 @@ impl NetworkBehaviour for DiscoveryBehaviour { fn inject_connection_closed( &mut self, - peer_id: &PeerId, - conn: &ConnectionId, - endpoint: &ConnectedPoint, + _peer_id: &PeerId, + _conn: &ConnectionId, + _endpoint: &ConnectedPoint, + _handler: ::Handler, ) { self.num_connections -= 1; - for k in self.kademlias.values_mut() { - NetworkBehaviour::inject_connection_closed(k, peer_id, conn, endpoint) - } + // NetworkBehaviour::inject_connection_closed on Kademlia does nothing. } fn inject_disconnected(&mut self, peer_id: &PeerId) { @@ -567,20 +604,25 @@ impl NetworkBehaviour for DiscoveryBehaviour { } } - fn inject_addr_reach_failure( + fn inject_dial_failure( &mut self, - peer_id: Option<&PeerId>, - addr: &Multiaddr, - error: &dyn std::error::Error, + peer_id: Option, + _: Self::ProtocolsHandler, + error: &DialError, ) { if let Some(peer_id) = peer_id { - if let Some(list) = self.ephemeral_addresses.get_mut(peer_id) { - list.retain(|a| a != addr); + if let DialError::Transport(errors) = error { + if let Some(list) = self.ephemeral_addresses.get_mut(&peer_id) { + for (addr, _error) in errors { + list.retain(|a| a != addr); + } + } } } for k in self.kademlias.values_mut() { - NetworkBehaviour::inject_addr_reach_failure(k, peer_id, addr, error) + let handler = k.new_handler(); + NetworkBehaviour::inject_dial_failure(k, peer_id, handler, error); } } @@ -603,14 +645,16 @@ impl NetworkBehaviour for DiscoveryBehaviour { fn inject_new_external_addr(&mut self, addr: &Multiaddr) { let new_addr = addr.clone().with(Protocol::P2p(self.local_peer_id.into())); - // NOTE: we might re-discover the same address multiple times - // in which case we just want to refrain from logging. - if self.known_external_addresses.insert(new_addr.clone()) { - info!( - target: "sub-libp2p", - "🔍 Discovered new external address for our node: {}", - new_addr, - ); + if self.can_add_to_dht(addr) { + // NOTE: we might re-discover the same address multiple times + // in which case we just want to refrain from logging. + if self.known_external_addresses.insert(new_addr.clone()) { + info!( + target: "sub-libp2p", + "🔍 Discovered new external address for our node: {}", + new_addr, + ); + } } for k in self.kademlias.values_mut() { @@ -633,12 +677,6 @@ impl NetworkBehaviour for DiscoveryBehaviour { } } - fn inject_dial_failure(&mut self, peer_id: &PeerId) { - for k in self.kademlias.values_mut() { - NetworkBehaviour::inject_dial_failure(k, peer_id) - } - } - fn inject_new_listener(&mut self, id: ListenerId) { for k in self.kademlias.values_mut() { NetworkBehaviour::inject_new_listener(k, id) @@ -651,6 +689,10 @@ impl NetworkBehaviour for DiscoveryBehaviour { } } + fn inject_listen_failure(&mut self, _: &Multiaddr, _: &Multiaddr, _: Self::ProtocolsHandler) { + // NetworkBehaviour::inject_listen_failure on Kademlia does nothing. + } + fn inject_listener_error(&mut self, id: ListenerId, err: &(dyn std::error::Error + 'static)) { for k in self.kademlias.values_mut() { NetworkBehaviour::inject_listener_error(k, id, err) @@ -667,12 +709,7 @@ impl NetworkBehaviour for DiscoveryBehaviour { &mut self, cx: &mut Context, params: &mut impl PollParameters, - ) -> Poll< - NetworkBehaviourAction< - <::Handler as ProtocolsHandler>::InEvent, - Self::OutEvent, - >, - >{ + ) -> Poll> { // Immediately process the content of `discovered`. if let Some(ev) = self.pending_events.pop_front() { return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)) @@ -733,6 +770,10 @@ impl NetworkBehaviour for DiscoveryBehaviour { let ev = DiscoveryOut::Discovered(peer); return Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)) }, + KademliaEvent::InboundPutRecordRequest { .. } | + KademliaEvent::InboundAddProviderRequest { .. } => { + debug_assert!(false, "We don't use kad filtering at the moment"); + }, KademliaEvent::PendingRoutablePeer { .. } | KademliaEvent::InboundRequestServed { .. } => { // We are not interested in this event at the moment. @@ -849,10 +890,20 @@ impl NetworkBehaviour for DiscoveryBehaviour { warn!(target: "sub-libp2p", "Libp2p => Unhandled Kademlia event: {:?}", e) }, }, - NetworkBehaviourAction::DialAddress { address } => - return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - NetworkBehaviourAction::DialPeer { peer_id, condition } => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + NetworkBehaviourAction::DialAddress { address, handler } => { + let pid = pid.clone(); + let handler = self.new_handler_with_replacement(pid, handler); + return Poll::Ready(NetworkBehaviourAction::DialAddress { address, handler }) + }, + NetworkBehaviourAction::DialPeer { peer_id, condition, handler } => { + let pid = pid.clone(); + let handler = self.new_handler_with_replacement(pid, handler); + return Poll::Ready(NetworkBehaviourAction::DialPeer { + peer_id, + condition, + handler, + }) + }, NetworkBehaviourAction::NotifyHandler { peer_id, handler, event } => return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, @@ -890,10 +941,12 @@ impl NetworkBehaviour for DiscoveryBehaviour { }, MdnsEvent::Expired(_) => {}, }, - NetworkBehaviourAction::DialAddress { address } => - return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - NetworkBehaviourAction::DialPeer { peer_id, condition } => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + NetworkBehaviourAction::DialAddress { .. } => { + unreachable!("mDNS never dials!"); + }, + NetworkBehaviourAction::DialPeer { .. } => { + unreachable!("mDNS never dials!"); + }, NetworkBehaviourAction::NotifyHandler { event, .. } => match event {}, /* `event` is an enum with no variant */ NetworkBehaviourAction::ReportObservedAddr { address, score } => return Poll::Ready(NetworkBehaviourAction::ReportObservedAddr { @@ -942,7 +995,7 @@ impl MdnsWrapper { &mut self, cx: &mut Context<'_>, params: &mut impl PollParameters, - ) -> Poll> { + ) -> Poll::ProtocolsHandler>> { loop { match self { Self::Instantiating(fut) => @@ -1009,13 +1062,13 @@ mod tests { config.finish() }; - let mut swarm = Swarm::new(transport, behaviour, keypair.public().into_peer_id()); + let mut swarm = Swarm::new(transport, behaviour, keypair.public().to_peer_id()); let listen_addr: Multiaddr = format!("/memory/{}", rand::random::()).parse().unwrap(); if i == 0 { first_swarm_peer_id_and_addr = - Some((keypair.public().into_peer_id(), listen_addr.clone())) + Some((keypair.public().to_peer_id(), listen_addr.clone())) } swarm.listen_on(listen_addr.clone()).unwrap(); diff --git a/client/network/src/error.rs b/client/network/src/error.rs index b8a31def7dc6..716235193a80 100644 --- a/client/network/src/error.rs +++ b/client/network/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -27,18 +27,18 @@ use std::{borrow::Cow, fmt}; pub type Result = std::result::Result; /// Error type for the network. -#[derive(derive_more::Display, derive_more::From)] +#[derive(thiserror::Error)] pub enum Error { /// Io error - Io(std::io::Error), + #[error(transparent)] + Io(#[from] std::io::Error), + /// Client error - Client(Box), + #[error(transparent)] + Client(#[from] Box), /// The same bootnode (based on address) is registered with two different peer ids. - #[display( - fmt = "The same bootnode (`{}`) is registered with two different peer ids: `{}` and `{}`", - address, - first_id, - second_id + #[error( + "The same bootnode (`{address}`) is registered with two different peer ids: `{first_id}` and `{second_id}`" )] DuplicateBootnode { /// The address of the bootnode. @@ -49,11 +49,11 @@ pub enum Error { second_id: PeerId, }, /// Prometheus metrics error. - Prometheus(prometheus_endpoint::PrometheusError), + #[error(transparent)] + Prometheus(#[from] prometheus_endpoint::PrometheusError), /// The network addresses are invalid because they don't match the transport. - #[display( - fmt = "The following addresses are invalid because they don't match the transport: {:?}", - addresses + #[error( + "The following addresses are invalid because they don't match the transport: {addresses:?}" )] AddressesForAnotherTransport { /// Transport used. @@ -62,7 +62,7 @@ pub enum Error { addresses: Vec, }, /// The same request-response protocol has been registered multiple times. - #[display(fmt = "Request-response protocol registered multiple times: {}", protocol)] + #[error("Request-response protocol registered multiple times: {protocol}")] DuplicateRequestResponseProtocol { /// Name of the protocol registered multiple times. protocol: Cow<'static, str>, @@ -75,16 +75,3 @@ impl fmt::Debug for Error { fmt::Display::fmt(self, f) } } - -impl std::error::Error for Error { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - Self::Io(ref err) => Some(err), - Self::Client(ref err) => Some(err), - Self::Prometheus(ref err) => Some(err), - Self::DuplicateBootnode { .. } | - Self::AddressesForAnotherTransport { .. } | - Self::DuplicateRequestResponseProtocol { .. } => None, - } - } -} diff --git a/client/network/src/gossip.rs b/client/network/src/gossip.rs deleted file mode 100644 index 0bc46b2164bc..000000000000 --- a/client/network/src/gossip.rs +++ /dev/null @@ -1,229 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Helper for sending rate-limited gossip messages. -//! -//! # Context -//! -//! The [`NetworkService`] struct provides a way to send notifications to a certain peer through -//! the [`NetworkService::notification_sender`] method. This method is quite low level and isn't -//! expected to be used directly. -//! -//! The [`QueuedSender`] struct provided by this module is built on top of -//! [`NetworkService::notification_sender`] and provides a cleaner way to send notifications. -//! -//! # Behaviour -//! -//! An instance of [`QueuedSender`] is specific to a certain combination of `PeerId` and -//! protocol name. It maintains a buffer of messages waiting to be sent out. The user of this API -//! is able to manipulate that queue, adding or removing obsolete messages. -//! -//! Creating a [`QueuedSender`] also returns a opaque `Future` whose responsibility it to -//! drain that queue and actually send the messages. If the substream with the given combination -//! of peer and protocol is closed, the queue is silently discarded. It is the role of the user -//! to track which peers we are connected to. -//! -//! In normal situations, messages sent through a [`QueuedSender`] will arrive in the same -//! order as they have been sent. -//! It is possible, in the situation of disconnects and reconnects, that messages arrive in a -//! different order. See also . -//! However, if multiple instances of [`QueuedSender`] exist for the same peer and protocol, or -//! if some other code uses the [`NetworkService`] to send notifications to this combination or -//! peer and protocol, then the notifications will be interleaved in an unpredictable way. -//! - -use crate::{ExHashT, NetworkService}; - -use async_std::sync::{Mutex, MutexGuard}; -use futures::prelude::*; -use futures::channel::mpsc::{channel, Receiver, Sender}; -use libp2p::PeerId; -use sp_runtime::traits::Block as BlockT; -use std::{ - borrow::Cow, - collections::VecDeque, - fmt, - sync::Arc, -}; - -#[cfg(test)] -mod tests; - -/// Notifications sender for a specific combination of network service, peer, and protocol. -pub struct QueuedSender { - /// Shared between the user-facing [`QueuedSender`] and the background future. - shared_message_queue: SharedMessageQueue, - /// Used to notify the background future to check for new messages in the message queue. - notify_background_future: Sender<()>, - /// Maximum number of elements in [`QueuedSender::shared_message_queue`]. - queue_size_limit: usize, -} - -impl QueuedSender { - /// Returns a new [`QueuedSender`] containing a queue of message for this specific - /// combination of peer and protocol. - /// - /// In addition to the [`QueuedSender`], also returns a `Future` whose role is to drive - /// the messages sending forward. - pub fn new( - service: Arc>, - peer_id: PeerId, - protocol: Cow<'static, str>, - queue_size_limit: usize, - messages_encode: F - ) -> (Self, impl Future + Send + 'static) - where - M: Send + 'static, - B: BlockT + 'static, - H: ExHashT, - F: Fn(M) -> Vec + Send + 'static, - { - let (notify_background_future, wait_for_sender) = channel(0); - - let shared_message_queue = Arc::new(Mutex::new( - VecDeque::with_capacity(queue_size_limit), - )); - - let background_future = create_background_future( - wait_for_sender, - service, - peer_id, - protocol, - shared_message_queue.clone(), - messages_encode - ); - - let sender = Self { - shared_message_queue, - notify_background_future, - queue_size_limit, - }; - - (sender, background_future) - } - - /// Locks the queue of messages towards this peer. - /// - /// The returned `Future` is expected to be ready quite quickly. - pub async fn lock_queue<'a>(&'a mut self) -> QueueGuard<'a, M> { - QueueGuard { - message_queue: self.shared_message_queue.lock().await, - queue_size_limit: self.queue_size_limit, - notify_background_future: &mut self.notify_background_future, - } - } - - /// Pushes a message to the queue, or discards it if the queue is full. - /// - /// The returned `Future` is expected to be ready quite quickly. - pub async fn queue_or_discard(&mut self, message: M) - where - M: Send + 'static - { - self.lock_queue().await.push_or_discard(message); - } -} - -impl fmt::Debug for QueuedSender { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("QueuedSender").finish() - } -} - -/// Locked queue of messages to the given peer. -/// -/// As long as this struct exists, the background future is asleep and the owner of the -/// [`QueueGuard`] is in total control of the message queue. Messages can only ever be sent out on -/// the network after the [`QueueGuard`] is dropped. -#[must_use] -pub struct QueueGuard<'a, M> { - message_queue: MutexGuard<'a, MessageQueue>, - /// Same as [`QueuedSender::queue_size_limit`]. - queue_size_limit: usize, - notify_background_future: &'a mut Sender<()>, -} - -impl<'a, M: Send + 'static> QueueGuard<'a, M> { - /// Pushes a message to the queue, or discards it if the queue is full. - /// - /// The message will only start being sent out after the [`QueueGuard`] is dropped. - pub fn push_or_discard(&mut self, message: M) { - if self.message_queue.len() < self.queue_size_limit { - self.message_queue.push_back(message); - } - } - - /// Calls `filter` for each message in the queue, and removes the ones for which `false` is - /// returned. - /// - /// > **Note**: The parameter of `filter` is a `&M` and not a `&mut M` (which would be - /// > better) because the underlying implementation relies on `VecDeque::retain`. - pub fn retain(&mut self, filter: impl FnMut(&M) -> bool) { - self.message_queue.retain(filter); - } -} - -impl<'a, M> Drop for QueueGuard<'a, M> { - fn drop(&mut self) { - // Notify background future to check for new messages in the message queue. - let _ = self.notify_background_future.try_send(()); - } -} - -type MessageQueue = VecDeque; - -/// [`MessageQueue`] shared between [`QueuedSender`] and background future. -type SharedMessageQueue = Arc>>; - -async fn create_background_future Vec>( - mut wait_for_sender: Receiver<()>, - service: Arc>, - peer_id: PeerId, - protocol: Cow<'static, str>, - shared_message_queue: SharedMessageQueue, - messages_encode: F, -) { - loop { - if wait_for_sender.next().await.is_none() { - return - } - - loop { - let mut queue_guard = shared_message_queue.lock().await; - let next_message = match queue_guard.pop_front() { - Some(msg) => msg, - None => break, - }; - drop(queue_guard); - - // Starting from below, we try to send the message. If an error happens when sending, - // the only sane option we have is to silently discard the message. - let sender = match service.notification_sender(peer_id.clone(), protocol.clone()) { - Ok(s) => s, - Err(_) => continue, - }; - - let ready = match sender.ready().await { - Ok(r) => r, - Err(_) => continue, - }; - - let _ = ready.send(messages_encode(next_message)); - } - } -} diff --git a/client/network/src/gossip/tests.rs b/client/network/src/gossip/tests.rs deleted file mode 100644 index 88c4160bc506..000000000000 --- a/client/network/src/gossip/tests.rs +++ /dev/null @@ -1,250 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use crate::block_request_handler::BlockRequestHandler; -use crate::state_request_handler::StateRequestHandler; -use crate::light_client_requests::handler::LightClientRequestHandler; -use crate::gossip::QueuedSender; -use crate::{config, Event, NetworkService, NetworkWorker}; - -use futures::prelude::*; -use sp_runtime::traits::{Block as BlockT, Header as _}; -use std::{borrow::Cow, sync::Arc, time::Duration}; -use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt as _}; - -type TestNetworkService = NetworkService< - substrate_test_runtime_client::runtime::Block, - substrate_test_runtime_client::runtime::Hash, ->; - -/// Builds a full node to be used for testing. Returns the node service and its associated events -/// stream. -/// -/// > **Note**: We return the events stream in order to not possibly lose events between the -/// > construction of the service and the moment the events stream is grabbed. -fn build_test_full_node(network_config: config::NetworkConfiguration) - -> (Arc, impl Stream) -{ - let client = Arc::new( - TestClientBuilder::with_default_backend() - .build_with_longest_chain() - .0, - ); - - #[derive(Clone)] - struct PassThroughVerifier(bool); - - #[async_trait::async_trait] - impl sc_consensus::Verifier for PassThroughVerifier { - async fn verify( - &mut self, - mut block: sp_consensus::BlockImportParams, - ) -> Result< - ( - sc_consensus::BlockImportParams, - Option)>>, - ), - String, - > { - let maybe_keys = block.header - .digest() - .log(|l| { - l.try_as_raw(sp_runtime::generic::OpaqueDigestItemId::Consensus(b"aura")) - .or_else(|| { - l.try_as_raw(sp_runtime::generic::OpaqueDigestItemId::Consensus(b"babe")) - }) - }) - .map(|blob| { - vec![( - sp_blockchain::well_known_cache_keys::AUTHORITIES, - blob.to_vec(), - )] - }); - - block.finalized = self.0; - block.fork_choice = Some(sc_consensus::ForkChoiceStrategy::LongestChain); - Ok((block, maybe_keys)) - } - } - - let import_queue = Box::new(sc_consensus::BasicQueue::new( - PassThroughVerifier(false), - Box::new(client.clone()), - None, - &sp_core::testing::TaskExecutor::new(), - None, - )); - - let protocol_id = config::ProtocolId::from("/test-protocol-name"); - - let block_request_protocol_config = { - let (handler, protocol_config) = BlockRequestHandler::new( - &protocol_id, - client.clone(), - 50, - ); - async_std::task::spawn(handler.run().boxed()); - protocol_config - }; - - let state_request_protocol_config = { - let (handler, protocol_config) = StateRequestHandler::new( - &protocol_id, - client.clone(), - 50, - ); - async_std::task::spawn(handler.run().boxed()); - protocol_config - }; - - let light_client_request_protocol_config = { - let (handler, protocol_config) = LightClientRequestHandler::new( - &protocol_id, - client.clone(), - ); - async_std::task::spawn(handler.run().boxed()); - protocol_config - }; - - let worker = NetworkWorker::new(config::Params { - role: config::Role::Full, - executor: None, - transactions_handler_executor: Box::new(|task| { async_std::task::spawn(task); }), - network_config, - chain: client.clone(), - on_demand: None, - transaction_pool: Arc::new(crate::config::EmptyTransactionPool), - protocol_id, - import_queue, - block_announce_validator: Box::new( - sp_consensus::block_validation::DefaultBlockAnnounceValidator, - ), - metrics_registry: None, - block_request_protocol_config, - state_request_protocol_config, - light_client_request_protocol_config, - warp_sync: None, - }) - .unwrap(); - - let service = worker.service().clone(); - let event_stream = service.event_stream("test"); - - async_std::task::spawn(async move { - futures::pin_mut!(worker); - let _ = worker.await; - }); - - (service, event_stream) -} - -const PROTOCOL_NAME: Cow<'static, str> = Cow::Borrowed("/foo"); - -/// Builds two nodes and their associated events stream. -/// The nodes are connected together and have the `PROTOCOL_NAME` protocol registered. -fn build_nodes_one_proto() - -> (Arc, impl Stream, Arc, impl Stream) -{ - let listen_addr = config::build_multiaddr![Memory(rand::random::())]; - - let (node1, events_stream1) = build_test_full_node(config::NetworkConfiguration { - extra_sets: vec![ - config::NonDefaultSetConfig { - notifications_protocol: PROTOCOL_NAME, - fallback_names: Vec::new(), - max_notification_size: 1024 * 1024, - set_config: Default::default() - } - ], - listen_addresses: vec![listen_addr.clone()], - transport: config::TransportConfig::MemoryOnly, - .. config::NetworkConfiguration::new_local() - }); - - let (node2, events_stream2) = build_test_full_node(config::NetworkConfiguration { - listen_addresses: vec![], - extra_sets: vec![ - config::NonDefaultSetConfig { - notifications_protocol: PROTOCOL_NAME, - fallback_names: Vec::new(), - max_notification_size: 1024 * 1024, - set_config: config::SetConfig { - reserved_nodes: vec![config::MultiaddrWithPeerId { - multiaddr: listen_addr, - peer_id: node1.local_peer_id().clone(), - }], - .. Default::default() - }, - } - ], - transport: config::TransportConfig::MemoryOnly, - .. config::NetworkConfiguration::new_local() - }); - - (node1, events_stream1, node2, events_stream2) -} - -#[test] -fn basic_works() { - const NUM_NOTIFS: usize = 256; - - let (node1, mut events_stream1, node2, mut events_stream2) = build_nodes_one_proto(); - let node2_id = node2.local_peer_id().clone(); - - let receiver = async_std::task::spawn(async move { - let mut received_notifications = 0; - - while received_notifications < NUM_NOTIFS { - match events_stream2.next().await.unwrap() { - Event::NotificationStreamClosed { .. } => panic!(), - Event::NotificationsReceived { messages, .. } => { - for message in messages { - assert_eq!(message.0, PROTOCOL_NAME); - assert_eq!(message.1, &b"message"[..]); - received_notifications += 1; - } - } - _ => {} - }; - - if rand::random::() < 2 { - async_std::task::sleep(Duration::from_millis(rand::random::() % 750)).await; - } - } - }); - - async_std::task::block_on(async move { - let (mut sender, bg_future) = - QueuedSender::new(node1, node2_id, PROTOCOL_NAME, NUM_NOTIFS, |msg| msg); - async_std::task::spawn(bg_future); - - // Wait for the `NotificationStreamOpened`. - loop { - match events_stream1.next().await.unwrap() { - Event::NotificationStreamOpened { .. } => break, - _ => {} - }; - } - - for _ in 0..NUM_NOTIFS { - sender.queue_or_discard(b"message".to_vec()).await; - } - - receiver.await; - }); -} diff --git a/client/network/src/lib.rs b/client/network/src/lib.rs index 51bc370265ef..d9f5b3de1bb1 100644 --- a/client/network/src/lib.rs +++ b/client/network/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -103,8 +103,8 @@ //! protocol ID. //! //! > **Note**: It is possible for the same connection to be used for multiple chains. For example, -//! > one can use both the `/dot/sync/2` and `/sub/sync/2` protocols on the same -//! > connection, provided that the remote supports them. +//! > one can use both the `/dot/sync/2` and `/sub/sync/2` protocols on the same +//! > connection, provided that the remote supports them. //! //! Substrate uses the following standard libp2p protocols: //! @@ -247,7 +247,6 @@ mod behaviour; mod chain; mod discovery; -mod on_demand_layer; mod peer_info; mod protocol; mod request_responses; @@ -274,8 +273,9 @@ pub use protocol::{ PeerInfo, }; pub use service::{ - IfDisconnected, NetworkService, NetworkWorker, NotificationSender, NotificationSenderReady, - OutboundFailure, RequestFailure, + DecodingError, IfDisconnected, KademliaKey, Keypair, NetworkService, NetworkWorker, + NotificationSender, NotificationSenderReady, OutboundFailure, PublicKey, RequestFailure, + Signature, SigningError, }; pub use sc_peerset::ReputationChange; @@ -328,5 +328,5 @@ pub struct NetworkStatus { /// State sync in progress. pub state_sync: Option, /// Warp sync in progress. - pub warp_sync: Option, + pub warp_sync: Option>, } diff --git a/client/network/src/light_client_requests.rs b/client/network/src/light_client_requests.rs index e18b783f219b..c77416003f82 100644 --- a/client/network/src/light_client_requests.rs +++ b/client/network/src/light_client_requests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,8 +20,6 @@ /// For incoming light client requests. pub mod handler; -/// For outgoing light client requests. -pub mod sender; use crate::{config::ProtocolId, request_responses::ProtocolConfig}; @@ -47,269 +45,3 @@ pub fn generate_protocol_config(protocol_id: &ProtocolId) -> ProtocolConfig { inbound_queue: None, } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::{config::ProtocolId, request_responses::IncomingRequest}; - - use assert_matches::assert_matches; - use futures::{ - channel::oneshot, - executor::{block_on, LocalPool}, - prelude::*, - task::Spawn, - }; - use libp2p::PeerId; - use sc_client_api::{ - light::{ - self, ChangesProof, RemoteBodyRequest, RemoteCallRequest, RemoteChangesRequest, - RemoteHeaderRequest, RemoteReadRequest, - }, - FetchChecker, RemoteReadChildRequest, StorageProof, - }; - use sp_blockchain::Error as ClientError; - use sp_core::storage::ChildInfo; - use sp_runtime::{ - generic::Header, - traits::{BlakeTwo256, Block as BlockT, NumberFor}, - }; - use std::{collections::HashMap, sync::Arc}; - - pub struct DummyFetchChecker { - pub ok: bool, - pub _mark: std::marker::PhantomData, - } - - impl FetchChecker for DummyFetchChecker { - fn check_header_proof( - &self, - _request: &RemoteHeaderRequest, - header: Option, - _remote_proof: StorageProof, - ) -> Result { - match self.ok { - true if header.is_some() => Ok(header.unwrap()), - _ => Err(ClientError::Backend("Test error".into())), - } - } - - fn check_read_proof( - &self, - request: &RemoteReadRequest, - _: StorageProof, - ) -> Result, Option>>, ClientError> { - match self.ok { - true => Ok(request.keys.iter().cloned().map(|k| (k, Some(vec![42]))).collect()), - false => Err(ClientError::Backend("Test error".into())), - } - } - - fn check_read_child_proof( - &self, - request: &RemoteReadChildRequest, - _: StorageProof, - ) -> Result, Option>>, ClientError> { - match self.ok { - true => Ok(request.keys.iter().cloned().map(|k| (k, Some(vec![42]))).collect()), - false => Err(ClientError::Backend("Test error".into())), - } - } - - fn check_execution_proof( - &self, - _: &RemoteCallRequest, - _: StorageProof, - ) -> Result, ClientError> { - match self.ok { - true => Ok(vec![42]), - false => Err(ClientError::Backend("Test error".into())), - } - } - - fn check_changes_proof( - &self, - _: &RemoteChangesRequest, - _: ChangesProof, - ) -> Result, u32)>, ClientError> { - match self.ok { - true => Ok(vec![(100u32.into(), 2)]), - false => Err(ClientError::Backend("Test error".into())), - } - } - - fn check_body_proof( - &self, - _: &RemoteBodyRequest, - body: Vec, - ) -> Result, ClientError> { - match self.ok { - true => Ok(body), - false => Err(ClientError::Backend("Test error".into())), - } - } - } - - pub fn protocol_id() -> ProtocolId { - ProtocolId::from("test") - } - - pub fn peerset() -> (sc_peerset::Peerset, sc_peerset::PeersetHandle) { - let cfg = sc_peerset::SetConfig { - in_peers: 128, - out_peers: 128, - bootnodes: Default::default(), - reserved_only: false, - reserved_nodes: Default::default(), - }; - sc_peerset::Peerset::from_config(sc_peerset::PeersetConfig { sets: vec![cfg] }) - } - - pub fn dummy_header() -> sp_test_primitives::Header { - sp_test_primitives::Header { - parent_hash: Default::default(), - number: 0, - state_root: Default::default(), - extrinsics_root: Default::default(), - digest: Default::default(), - } - } - - type Block = - sp_runtime::generic::Block, substrate_test_runtime::Extrinsic>; - - fn send_receive(request: sender::Request, pool: &LocalPool) { - let client = Arc::new(substrate_test_runtime_client::new()); - let (handler, protocol_config) = - handler::LightClientRequestHandler::new(&protocol_id(), client); - pool.spawner().spawn_obj(handler.run().boxed().into()).unwrap(); - - let (_peer_set, peer_set_handle) = peerset(); - let mut sender = sender::LightClientRequestSender::::new( - &protocol_id(), - Arc::new(crate::light_client_requests::tests::DummyFetchChecker { - ok: true, - _mark: std::marker::PhantomData, - }), - peer_set_handle, - ); - sender.inject_connected(PeerId::random()); - - sender.request(request).unwrap(); - let sender::OutEvent::SendRequest { pending_response, request, .. } = - block_on(sender.next()).unwrap(); - let (tx, rx) = oneshot::channel(); - block_on(protocol_config.inbound_queue.unwrap().send(IncomingRequest { - peer: PeerId::random(), - payload: request, - pending_response: tx, - })) - .unwrap(); - pool.spawner() - .spawn_obj( - async move { - pending_response.send(Ok(rx.await.unwrap().result.unwrap())).unwrap(); - } - .boxed() - .into(), - ) - .unwrap(); - - pool.spawner() - .spawn_obj(sender.for_each(|_| future::ready(())).boxed().into()) - .unwrap(); - } - - #[test] - fn send_receive_call() { - let chan = oneshot::channel(); - let request = light::RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: None, - }; - - let mut pool = LocalPool::new(); - send_receive(sender::Request::Call { request, sender: chan.0 }, &pool); - assert_eq!(vec![42], pool.run_until(chan.1).unwrap().unwrap()); - // ^--- from `DummyFetchChecker::check_execution_proof` - } - - #[test] - fn send_receive_read() { - let chan = oneshot::channel(); - let request = light::RemoteReadRequest { - header: dummy_header(), - block: Default::default(), - keys: vec![b":key".to_vec()], - retry_count: None, - }; - let mut pool = LocalPool::new(); - send_receive(sender::Request::Read { request, sender: chan.0 }, &pool); - assert_eq!( - Some(vec![42]), - pool.run_until(chan.1).unwrap().unwrap().remove(&b":key"[..]).unwrap() - ); - // ^--- from `DummyFetchChecker::check_read_proof` - } - - #[test] - fn send_receive_read_child() { - let chan = oneshot::channel(); - let child_info = ChildInfo::new_default(&b":child_storage:default:sub"[..]); - let request = light::RemoteReadChildRequest { - header: dummy_header(), - block: Default::default(), - storage_key: child_info.prefixed_storage_key(), - keys: vec![b":key".to_vec()], - retry_count: None, - }; - let mut pool = LocalPool::new(); - send_receive(sender::Request::ReadChild { request, sender: chan.0 }, &pool); - assert_eq!( - Some(vec![42]), - pool.run_until(chan.1).unwrap().unwrap().remove(&b":key"[..]).unwrap() - ); - // ^--- from `DummyFetchChecker::check_read_child_proof` - } - - #[test] - fn send_receive_header() { - sp_tracing::try_init_simple(); - let chan = oneshot::channel(); - let request = light::RemoteHeaderRequest { - cht_root: Default::default(), - block: 1, - retry_count: None, - }; - let mut pool = LocalPool::new(); - send_receive(sender::Request::Header { request, sender: chan.0 }, &pool); - // The remote does not know block 1: - assert_matches!(pool.run_until(chan.1).unwrap(), Err(ClientError::RemoteFetchFailed)); - } - - #[test] - fn send_receive_changes() { - let chan = oneshot::channel(); - let request = light::RemoteChangesRequest { - changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { - zero: (0, Default::default()), - end: None, - config: Some(sp_core::ChangesTrieConfiguration::new(4, 2)), - }], - first_block: (1, Default::default()), - last_block: (100, Default::default()), - max_block: (100, Default::default()), - tries_roots: (1, Default::default(), Vec::new()), - key: Vec::new(), - storage_key: None, - retry_count: None, - }; - let mut pool = LocalPool::new(); - send_receive(sender::Request::Changes { request, sender: chan.0 }, &pool); - assert_eq!(vec![(100, 2)], pool.run_until(chan.1).unwrap().unwrap()); - // ^--- from `DummyFetchChecker::check_changes_proof` - } -} diff --git a/client/network/src/light_client_requests/handler.rs b/client/network/src/light_client_requests/handler.rs index 43504edddd73..fb258304f2e8 100644 --- a/client/network/src/light_client_requests/handler.rs +++ b/client/network/src/light_client_requests/handler.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,7 +19,7 @@ //! Helper for incoming light client requests. //! //! Handle (i.e. answer) incoming light client requests from a remote peer received via -//! [`crate::request_responses::RequestResponsesBehaviour`] with +//! `crate::request_responses::RequestResponsesBehaviour` with //! [`LightClientRequestHandler`](handler::LightClientRequestHandler). use crate::{ @@ -32,17 +32,14 @@ use codec::{self, Decode, Encode}; use futures::{channel::mpsc, prelude::*}; use log::{debug, trace}; use prost::Message; -use sc_client_api::{light, StorageProof}; +use sc_client_api::StorageProof; use sc_peerset::ReputationChange; use sp_core::{ hexdisplay::HexDisplay, - storage::{ChildInfo, ChildType, PrefixedStorageKey, StorageKey}, + storage::{ChildInfo, ChildType, PrefixedStorageKey}, }; -use sp_runtime::{ - generic::BlockId, - traits::{Block, Zero}, -}; -use std::{collections::BTreeMap, sync::Arc}; +use sp_runtime::{generic::BlockId, traits::Block}; +use std::sync::Arc; const LOG_TARGET: &str = "light-client-request-handler"; @@ -137,12 +134,12 @@ impl LightClientRequestHandler { self.on_remote_call_request(&peer, r)?, Some(schema::v1::light::request::Request::RemoteReadRequest(r)) => self.on_remote_read_request(&peer, r)?, - Some(schema::v1::light::request::Request::RemoteHeaderRequest(r)) => - self.on_remote_header_request(&peer, r)?, + Some(schema::v1::light::request::Request::RemoteHeaderRequest(_r)) => + return Err(HandleRequestError::BadRequest("Not supported.")), Some(schema::v1::light::request::Request::RemoteReadChildRequest(r)) => self.on_remote_read_child_request(&peer, r)?, - Some(schema::v1::light::request::Request::RemoteChangesRequest(r)) => - self.on_remote_changes_request(&peer, r)?, + Some(schema::v1::light::request::Request::RemoteChangesRequest(_r)) => + return Err(HandleRequestError::BadRequest("Not supported.")), None => return Err(HandleRequestError::BadRequest("Remote request without request data.")), }; @@ -285,122 +282,22 @@ impl LightClientRequestHandler { Ok(schema::v1::light::Response { response: Some(response) }) } - - fn on_remote_header_request( - &mut self, - peer: &PeerId, - request: &schema::v1::light::RemoteHeaderRequest, - ) -> Result { - trace!("Remote header proof request from {} ({:?}).", peer, request.block); - - let block = Decode::decode(&mut request.block.as_ref())?; - let (header, proof) = match self.client.header_proof(&BlockId::Number(block)) { - Ok((header, proof)) => (header.encode(), proof), - Err(error) => { - trace!( - "Remote header proof request from {} ({:?}) failed with: {}.", - peer, - request.block, - error - ); - (Default::default(), StorageProof::empty()) - }, - }; - - let response = { - let r = schema::v1::light::RemoteHeaderResponse { header, proof: proof.encode() }; - schema::v1::light::response::Response::RemoteHeaderResponse(r) - }; - - Ok(schema::v1::light::Response { response: Some(response) }) - } - - fn on_remote_changes_request( - &mut self, - peer: &PeerId, - request: &schema::v1::light::RemoteChangesRequest, - ) -> Result { - trace!( - "Remote changes proof request from {} for key {} ({:?}..{:?}).", - peer, - if !request.storage_key.is_empty() { - format!( - "{} : {}", - HexDisplay::from(&request.storage_key), - HexDisplay::from(&request.key) - ) - } else { - HexDisplay::from(&request.key).to_string() - }, - request.first, - request.last, - ); - - let first = Decode::decode(&mut request.first.as_ref())?; - let last = Decode::decode(&mut request.last.as_ref())?; - let min = Decode::decode(&mut request.min.as_ref())?; - let max = Decode::decode(&mut request.max.as_ref())?; - let key = StorageKey(request.key.clone()); - let storage_key = if request.storage_key.is_empty() { - None - } else { - Some(PrefixedStorageKey::new_ref(&request.storage_key)) - }; - - let proof = - match self.client.key_changes_proof(first, last, min, max, storage_key, &key) { - Ok(proof) => proof, - Err(error) => { - trace!( - "Remote changes proof request from {} for key {} ({:?}..{:?}) failed with: {}.", - peer, - format!("{} : {}", HexDisplay::from(&request.storage_key), HexDisplay::from(&key.0)), - request.first, - request.last, - error, - ); - - light::ChangesProof:: { - max_block: Zero::zero(), - proof: Vec::new(), - roots: BTreeMap::new(), - roots_proof: StorageProof::empty(), - } - }, - }; - - let response = { - let r = schema::v1::light::RemoteChangesResponse { - max: proof.max_block.encode(), - proof: proof.proof, - roots: proof - .roots - .into_iter() - .map(|(k, v)| schema::v1::light::Pair { fst: k.encode(), snd: v.encode() }) - .collect(), - roots_proof: proof.roots_proof.encode(), - }; - schema::v1::light::response::Response::RemoteChangesResponse(r) - }; - - Ok(schema::v1::light::Response { response: Some(response) }) - } } -#[derive(derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] enum HandleRequestError { - #[display(fmt = "Failed to decode request: {}.", _0)] - DecodeProto(prost::DecodeError), - #[display(fmt = "Failed to encode response: {}.", _0)] - EncodeProto(prost::EncodeError), - #[display(fmt = "Failed to send response.")] + #[error("Failed to decode request: {0}.")] + DecodeProto(#[from] prost::DecodeError), + #[error("Failed to encode response: {0}.")] + EncodeProto(#[from] prost::EncodeError), + #[error("Failed to send response.")] SendResponse, /// A bad request has been received. - #[display(fmt = "bad request: {}", _0)] + #[error("bad request: {0}")] BadRequest(&'static str), /// Encoding or decoding of some data failed. - #[display(fmt = "codec error: {}", _0)] - Codec(codec::Error), + #[error("codec error: {0}")] + Codec(#[from] codec::Error), } fn fmt_keys(first: Option<&Vec>, last: Option<&Vec>) -> String { diff --git a/client/network/src/light_client_requests/sender.rs b/client/network/src/light_client_requests/sender.rs deleted file mode 100644 index 284db827594b..000000000000 --- a/client/network/src/light_client_requests/sender.rs +++ /dev/null @@ -1,1294 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Helper for outgoing light client requests. -//! -//! Call [`LightClientRequestSender::request`](sender::LightClientRequestSender::request) -//! to send out light client requests. It will: -//! -//! 1. Build the request. -//! -//! 2. Forward the request to [`crate::request_responses::RequestResponsesBehaviour`] via -//! [`OutEvent::SendRequest`](sender::OutEvent::SendRequest). -//! -//! 3. Wait for the response and forward the response via the [`futures::channel::oneshot::Sender`] -//! provided earlier with [`LightClientRequestSender::request`](sender::LightClientRequestSender:: -//! request). - -use crate::{ - config::ProtocolId, - protocol::message::BlockAttributes, - request_responses::{OutboundFailure, RequestFailure}, - schema, PeerId, -}; -use codec::{self, Decode, Encode}; -use futures::{channel::oneshot, future::BoxFuture, prelude::*, stream::FuturesUnordered}; -use prost::Message; -use sc_client_api::light::{self, RemoteBodyRequest}; -use sc_peerset::ReputationChange; -use sp_blockchain::Error as ClientError; -use sp_runtime::traits::{Block, Header, NumberFor}; -use std::{ - collections::{BTreeMap, HashMap, VecDeque}, - pin::Pin, - sync::Arc, - task::{Context, Poll}, -}; - -mod rep { - use super::*; - - /// Reputation change for a peer when a request timed out. - pub const TIMEOUT: ReputationChange = - ReputationChange::new(-(1 << 8), "light client request timeout"); - /// Reputation change for a peer when a request is refused. - pub const REFUSED: ReputationChange = - ReputationChange::new(-(1 << 8), "light client request refused"); -} - -/// Configuration options for [`LightClientRequestSender`]. -#[derive(Debug, Clone)] -struct Config { - max_pending_requests: usize, - light_protocol: String, - block_protocol: String, -} - -impl Config { - /// Create a new [`LightClientRequestSender`] configuration. - pub fn new(id: &ProtocolId) -> Self { - Self { - max_pending_requests: 128, - light_protocol: super::generate_protocol_name(id), - block_protocol: crate::block_request_handler::generate_protocol_name(id), - } - } -} - -/// State machine helping to send out light client requests. -pub struct LightClientRequestSender { - /// This behaviour's configuration. - config: Config, - /// Verifies that received responses are correct. - checker: Arc>, - /// Peer information (addresses, their best block, etc.) - peers: HashMap>, - /// Pending (local) requests. - pending_requests: VecDeque>, - /// Requests on their way to remote peers. - sent_requests: FuturesUnordered< - BoxFuture< - 'static, - (SentRequest, Result, RequestFailure>, oneshot::Canceled>), - >, - >, - /// Handle to use for reporting misbehaviour of peers. - peerset: sc_peerset::PeersetHandle, -} - -/// Augments a pending light client request with metadata. -#[derive(Debug)] -struct PendingRequest { - /// Remaining attempts. - attempts_left: usize, - /// The actual request. - request: Request, -} - -impl PendingRequest { - fn new(req: Request) -> Self { - Self { - // Number of retries + one for the initial attempt. - attempts_left: req.retries() + 1, - request: req, - } - } - - fn into_sent(self, peer_id: PeerId) -> SentRequest { - SentRequest { attempts_left: self.attempts_left, request: self.request, peer: peer_id } - } -} - -/// Augments a light client request with metadata that is currently being send to a remote. -#[derive(Debug)] -struct SentRequest { - /// Remaining attempts. - attempts_left: usize, - /// The actual request. - request: Request, - /// The peer that the request is send to. - peer: PeerId, -} - -impl SentRequest { - fn into_pending(self) -> PendingRequest { - PendingRequest { attempts_left: self.attempts_left, request: self.request } - } -} - -impl Unpin for LightClientRequestSender {} - -impl LightClientRequestSender -where - B: Block, -{ - /// Construct a new light client handler. - pub fn new( - id: &ProtocolId, - checker: Arc>, - peerset: sc_peerset::PeersetHandle, - ) -> Self { - Self { - config: Config::new(id), - checker, - peers: Default::default(), - pending_requests: Default::default(), - sent_requests: Default::default(), - peerset, - } - } - - /// We rely on external information about peers best blocks as we lack the - /// means to determine it ourselves. - pub fn update_best_block(&mut self, peer: &PeerId, num: NumberFor) { - if let Some(info) = self.peers.get_mut(peer) { - log::trace!("new best block for {:?}: {:?}", peer, num); - info.best_block = Some(num) - } - } - - /// Issue a new light client request. - pub fn request(&mut self, req: Request) -> Result<(), SendRequestError> { - if self.pending_requests.len() >= self.config.max_pending_requests { - return Err(SendRequestError::TooManyRequests) - } - self.pending_requests.push_back(PendingRequest::new(req)); - Ok(()) - } - - /// Remove the given peer. - /// - /// In-flight requests to the given peer might fail and be retried. See - /// [`::poll_next`]. - fn remove_peer(&mut self, peer: PeerId) { - self.peers.remove(&peer); - } - - /// Process a local request's response from remote. - /// - /// If successful, this will give us the actual, checked data we should be - /// sending back to the client, otherwise an error. - fn on_response( - &mut self, - peer: PeerId, - request: &Request, - response: Response, - ) -> Result, Error> { - log::trace!("response from {}", peer); - match response { - Response::Light(r) => self.on_response_light(request, r), - Response::Block(r) => self.on_response_block(request, r), - } - } - - fn on_response_light( - &mut self, - request: &Request, - response: schema::v1::light::Response, - ) -> Result, Error> { - use schema::v1::light::response::Response; - match response.response { - Some(Response::RemoteCallResponse(response)) => { - if let Request::Call { request, .. } = request { - let proof = Decode::decode(&mut response.proof.as_ref())?; - let reply = self.checker.check_execution_proof(request, proof)?; - Ok(Reply::VecU8(reply)) - } else { - Err(Error::UnexpectedResponse) - } - }, - Some(Response::RemoteReadResponse(response)) => match request { - Request::Read { request, .. } => { - let proof = Decode::decode(&mut response.proof.as_ref())?; - let reply = self.checker.check_read_proof(&request, proof)?; - Ok(Reply::MapVecU8OptVecU8(reply)) - }, - Request::ReadChild { request, .. } => { - let proof = Decode::decode(&mut response.proof.as_ref())?; - let reply = self.checker.check_read_child_proof(&request, proof)?; - Ok(Reply::MapVecU8OptVecU8(reply)) - }, - _ => Err(Error::UnexpectedResponse), - }, - Some(Response::RemoteChangesResponse(response)) => { - if let Request::Changes { request, .. } = request { - let max_block = Decode::decode(&mut response.max.as_ref())?; - let roots_proof = Decode::decode(&mut response.roots_proof.as_ref())?; - let roots = { - let mut r = BTreeMap::new(); - for pair in response.roots { - let k = Decode::decode(&mut pair.fst.as_ref())?; - let v = Decode::decode(&mut pair.snd.as_ref())?; - r.insert(k, v); - } - r - }; - let reply = self.checker.check_changes_proof( - &request, - light::ChangesProof { - max_block, - proof: response.proof, - roots, - roots_proof, - }, - )?; - Ok(Reply::VecNumberU32(reply)) - } else { - Err(Error::UnexpectedResponse) - } - }, - Some(Response::RemoteHeaderResponse(response)) => { - if let Request::Header { request, .. } = request { - let header = if response.header.is_empty() { - None - } else { - Some(Decode::decode(&mut response.header.as_ref())?) - }; - let proof = Decode::decode(&mut response.proof.as_ref())?; - let reply = self.checker.check_header_proof(&request, header, proof)?; - Ok(Reply::Header(reply)) - } else { - Err(Error::UnexpectedResponse) - } - }, - None => Err(Error::UnexpectedResponse), - } - } - - fn on_response_block( - &mut self, - request: &Request, - response: schema::v1::BlockResponse, - ) -> Result, Error> { - let request = if let Request::Body { request, .. } = &request { - request - } else { - return Err(Error::UnexpectedResponse) - }; - - let body: Vec<_> = match response.blocks.into_iter().next() { - Some(b) => b.body, - None => return Err(Error::UnexpectedResponse), - }; - - let body = body - .into_iter() - .map(|extrinsic| B::Extrinsic::decode(&mut &extrinsic[..])) - .collect::>()?; - - let body = self.checker.check_body_proof(&request, body)?; - Ok(Reply::Extrinsics(body)) - } - - /// Signal that the node is connected to the given peer. - pub fn inject_connected(&mut self, peer: PeerId) { - let prev_entry = self.peers.insert(peer, Default::default()); - debug_assert!( - prev_entry.is_none(), - "Expect `inject_connected` to be called for disconnected peer.", - ); - } - - /// Signal that the node disconnected from the given peer. - pub fn inject_disconnected(&mut self, peer: PeerId) { - self.remove_peer(peer) - } -} - -impl Stream for LightClientRequestSender { - type Item = OutEvent; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - // If we have received responses to previously sent requests, check them and pass them on. - while let Poll::Ready(Some((sent_request, request_result))) = - self.sent_requests.poll_next_unpin(cx) - { - if let Some(info) = self.peers.get_mut(&sent_request.peer) { - if info.status != PeerStatus::Busy { - // If we get here, something is wrong with our internal handling of peer status - // information. At any time, a single peer processes at most one request from - // us. A malicious peer should not be able to get us here. It is our own fault - // and must be fixed! - panic!("unexpected peer status {:?} for {}", info.status, sent_request.peer); - } - - info.status = PeerStatus::Idle; // Make peer available again. - } - - let request_result = match request_result { - Ok(r) => r, - Err(oneshot::Canceled) => { - log::debug!("Oneshot for request to peer {} was canceled.", sent_request.peer); - self.remove_peer(sent_request.peer); - self.peerset.report_peer( - sent_request.peer, - ReputationChange::new_fatal("no response from peer"), - ); - self.pending_requests.push_back(sent_request.into_pending()); - continue - }, - }; - - let decoded_request_result = request_result.map(|response| { - if sent_request.request.is_block_request() { - schema::v1::BlockResponse::decode(&response[..]).map(|r| Response::Block(r)) - } else { - schema::v1::light::Response::decode(&response[..]).map(|r| Response::Light(r)) - } - }); - - let response = match decoded_request_result { - Ok(Ok(response)) => response, - Ok(Err(e)) => { - log::debug!( - "Failed to decode response from peer {}: {:?}.", - sent_request.peer, - e - ); - self.remove_peer(sent_request.peer); - self.peerset.report_peer( - sent_request.peer, - ReputationChange::new_fatal("invalid response from peer"), - ); - self.pending_requests.push_back(sent_request.into_pending()); - continue - }, - Err(e) => { - log::debug!("Request to peer {} failed with {:?}.", sent_request.peer, e); - - match e { - RequestFailure::NotConnected => { - self.remove_peer(sent_request.peer); - self.pending_requests.push_back(sent_request.into_pending()); - }, - RequestFailure::UnknownProtocol => { - debug_assert!( - false, - "Light client and block request protocol should be known when \ - sending requests.", - ); - }, - RequestFailure::Refused => { - self.remove_peer(sent_request.peer); - self.peerset.report_peer(sent_request.peer, rep::REFUSED); - self.pending_requests.push_back(sent_request.into_pending()); - }, - RequestFailure::Obsolete => { - debug_assert!( - false, - "Can not receive `RequestFailure::Obsolete` after dropping the \ - response receiver.", - ); - self.pending_requests.push_back(sent_request.into_pending()); - }, - RequestFailure::Network(OutboundFailure::Timeout) => { - self.remove_peer(sent_request.peer); - self.peerset.report_peer(sent_request.peer, rep::TIMEOUT); - self.pending_requests.push_back(sent_request.into_pending()); - }, - RequestFailure::Network(OutboundFailure::UnsupportedProtocols) => { - self.remove_peer(sent_request.peer); - self.peerset.report_peer( - sent_request.peer, - ReputationChange::new_fatal( - "peer does not support light client or block request protocol", - ), - ); - self.pending_requests.push_back(sent_request.into_pending()); - }, - RequestFailure::Network(OutboundFailure::DialFailure) => { - self.remove_peer(sent_request.peer); - self.peerset.report_peer( - sent_request.peer, - ReputationChange::new_fatal("failed to dial peer"), - ); - self.pending_requests.push_back(sent_request.into_pending()); - }, - RequestFailure::Network(OutboundFailure::ConnectionClosed) => { - self.remove_peer(sent_request.peer); - self.peerset.report_peer( - sent_request.peer, - ReputationChange::new_fatal("connection to peer closed"), - ); - self.pending_requests.push_back(sent_request.into_pending()); - }, - } - - continue - }, - }; - - match self.on_response(sent_request.peer, &sent_request.request, response) { - Ok(reply) => sent_request.request.return_reply(Ok(reply)), - Err(Error::UnexpectedResponse) => { - log::debug!("Unexpected response from peer {}.", sent_request.peer); - self.remove_peer(sent_request.peer); - self.peerset.report_peer( - sent_request.peer, - ReputationChange::new_fatal("unexpected response from peer"), - ); - self.pending_requests.push_back(sent_request.into_pending()); - }, - Err(other) => { - log::debug!( - "error handling response from peer {}: {}", - sent_request.peer, - other - ); - self.remove_peer(sent_request.peer); - self.peerset.report_peer( - sent_request.peer, - ReputationChange::new_fatal("invalid response from peer"), - ); - self.pending_requests.push_back(sent_request.into_pending()) - }, - } - } - - // If we have a pending request to send, try to find an available peer and send it. - while let Some(mut pending_request) = self.pending_requests.pop_front() { - if pending_request.attempts_left == 0 { - pending_request.request.return_reply(Err(ClientError::RemoteFetchFailed)); - continue - } - - let protocol = if pending_request.request.is_block_request() { - self.config.block_protocol.clone() - } else { - self.config.light_protocol.clone() - }; - - // Out of all idle peers, find one who's best block is high enough, choose any idle peer - // if none exists. - let mut peer = None; - for (peer_id, peer_info) in self.peers.iter_mut() { - if peer_info.status == PeerStatus::Idle { - match peer_info.best_block { - Some(n) if n >= pending_request.request.required_block() => { - peer = Some((*peer_id, peer_info)); - break - }, - _ => peer = Some((*peer_id, peer_info)), - } - } - } - - // Break in case there is no idle peer. - let (peer_id, peer_info) = match peer { - Some((peer_id, peer_info)) => (peer_id, peer_info), - None => { - self.pending_requests.push_front(pending_request); - log::debug!("No peer available to send request to."); - - break - }, - }; - - let request_bytes = match pending_request.request.serialize_request() { - Ok(bytes) => bytes, - Err(error) => { - log::debug!("failed to serialize request: {}", error); - pending_request.request.return_reply(Err(ClientError::RemoteFetchFailed)); - continue - }, - }; - - let (tx, rx) = oneshot::channel(); - - peer_info.status = PeerStatus::Busy; - - pending_request.attempts_left -= 1; - - self.sent_requests - .push(async move { (pending_request.into_sent(peer_id), rx.await) }.boxed()); - - return Poll::Ready(Some(OutEvent::SendRequest { - target: peer_id, - request: request_bytes, - pending_response: tx, - protocol_name: protocol, - })) - } - - Poll::Pending - } -} - -/// Events returned by [`LightClientRequestSender`]. -#[derive(Debug)] -pub enum OutEvent { - /// Emit a request to be send out on the network e.g. via [`crate::request_responses`]. - SendRequest { - /// The remote peer to send the request to. - target: PeerId, - /// The encoded request. - request: Vec, - /// The [`oneshot::Sender`] channel to pass the response to. - pending_response: oneshot::Sender, RequestFailure>>, - /// The name of the protocol to use to send the request. - protocol_name: String, - }, -} - -/// Incoming response from remote. -#[derive(Debug, Clone)] -pub enum Response { - /// Incoming light response from remote. - Light(schema::v1::light::Response), - /// Incoming block response from remote. - Block(schema::v1::BlockResponse), -} - -/// Error returned by [`LightClientRequestSender::request`]. -#[derive(Debug, derive_more::Display, derive_more::From)] -pub enum SendRequestError { - /// There are currently too many pending request. - #[display(fmt = "too many pending requests")] - TooManyRequests, -} - -/// Error type to propagate errors internally. -#[derive(Debug, derive_more::Display, derive_more::From)] -enum Error { - /// The response type does not correspond to the issued request. - #[display(fmt = "unexpected response")] - UnexpectedResponse, - /// Encoding or decoding of some data failed. - #[display(fmt = "codec error: {}", _0)] - Codec(codec::Error), - /// The chain client errored. - #[display(fmt = "client error: {}", _0)] - Client(ClientError), -} - -/// The data to send back to the light client over the oneshot channel. -// It is unified here in order to be able to return it as a function -// result instead of delivering it to the client as a side effect of -// response processing. -#[derive(Debug)] -enum Reply { - VecU8(Vec), - VecNumberU32(Vec<(::Number, u32)>), - MapVecU8OptVecU8(HashMap, Option>>), - Header(B::Header), - Extrinsics(Vec), -} - -/// Information we have about some peer. -#[derive(Debug)] -struct PeerInfo { - best_block: Option>, - status: PeerStatus, -} - -impl Default for PeerInfo { - fn default() -> Self { - PeerInfo { best_block: None, status: PeerStatus::Idle } - } -} - -/// A peer is either idle or busy processing a request from us. -#[derive(Debug, Clone, PartialEq, Eq)] -enum PeerStatus { - /// The peer is available. - Idle, - /// We wait for the peer to return us a response for the given request ID. - Busy, -} - -/// The possible light client requests we support. -/// -/// The associated `oneshot::Sender` will be used to convey the result of -/// their request back to them (cf. `Reply`). -// This is modeled after light_dispatch.rs's `RequestData` which is not -// used because we currently only support a subset of those. -#[derive(Debug)] -pub enum Request { - /// Remote body request. - Body { - /// Request. - request: RemoteBodyRequest, - /// [`oneshot::Sender`] to return response. - sender: oneshot::Sender, ClientError>>, - }, - /// Remote header request. - Header { - /// Request. - request: light::RemoteHeaderRequest, - /// [`oneshot::Sender`] to return response. - sender: oneshot::Sender>, - }, - /// Remote read request. - Read { - /// Request. - request: light::RemoteReadRequest, - /// [`oneshot::Sender`] to return response. - sender: oneshot::Sender, Option>>, ClientError>>, - }, - /// Remote read child request. - ReadChild { - /// Request. - request: light::RemoteReadChildRequest, - /// [`oneshot::Sender`] to return response. - sender: oneshot::Sender, Option>>, ClientError>>, - }, - /// Remote call request. - Call { - /// Request. - request: light::RemoteCallRequest, - /// [`oneshot::Sender`] to return response. - sender: oneshot::Sender, ClientError>>, - }, - /// Remote changes request. - Changes { - /// Request. - request: light::RemoteChangesRequest, - /// [`oneshot::Sender`] to return response. - sender: oneshot::Sender, u32)>, ClientError>>, - }, -} - -impl Request { - fn is_block_request(&self) -> bool { - matches!(self, Request::Body { .. }) - } - - fn required_block(&self) -> NumberFor { - match self { - Request::Body { request, .. } => *request.header.number(), - Request::Header { request, .. } => request.block, - Request::Read { request, .. } => *request.header.number(), - Request::ReadChild { request, .. } => *request.header.number(), - Request::Call { request, .. } => *request.header.number(), - Request::Changes { request, .. } => request.max_block.0, - } - } - - fn retries(&self) -> usize { - let rc = match self { - Request::Body { request, .. } => request.retry_count, - Request::Header { request, .. } => request.retry_count, - Request::Read { request, .. } => request.retry_count, - Request::ReadChild { request, .. } => request.retry_count, - Request::Call { request, .. } => request.retry_count, - Request::Changes { request, .. } => request.retry_count, - }; - rc.unwrap_or(0) - } - - fn serialize_request(&self) -> Result, prost::EncodeError> { - let request = match self { - Request::Body { request, .. } => { - let rq = schema::v1::BlockRequest { - fields: BlockAttributes::BODY.to_be_u32(), - from_block: Some(schema::v1::block_request::FromBlock::Hash( - request.header.hash().encode(), - )), - to_block: Default::default(), - direction: schema::v1::Direction::Ascending as i32, - max_blocks: 1, - support_multiple_justifications: true, - }; - - let mut buf = Vec::with_capacity(rq.encoded_len()); - rq.encode(&mut buf)?; - return Ok(buf) - }, - Request::Header { request, .. } => { - let r = schema::v1::light::RemoteHeaderRequest { block: request.block.encode() }; - schema::v1::light::request::Request::RemoteHeaderRequest(r) - }, - Request::Read { request, .. } => { - let r = schema::v1::light::RemoteReadRequest { - block: request.block.encode(), - keys: request.keys.clone(), - }; - schema::v1::light::request::Request::RemoteReadRequest(r) - }, - Request::ReadChild { request, .. } => { - let r = schema::v1::light::RemoteReadChildRequest { - block: request.block.encode(), - storage_key: request.storage_key.clone().into_inner(), - keys: request.keys.clone(), - }; - schema::v1::light::request::Request::RemoteReadChildRequest(r) - }, - Request::Call { request, .. } => { - let r = schema::v1::light::RemoteCallRequest { - block: request.block.encode(), - method: request.method.clone(), - data: request.call_data.clone(), - }; - schema::v1::light::request::Request::RemoteCallRequest(r) - }, - Request::Changes { request, .. } => { - let r = schema::v1::light::RemoteChangesRequest { - first: request.first_block.1.encode(), - last: request.last_block.1.encode(), - min: request.tries_roots.1.encode(), - max: request.max_block.1.encode(), - storage_key: request - .storage_key - .clone() - .map(|s| s.into_inner()) - .unwrap_or_default(), - key: request.key.clone(), - }; - schema::v1::light::request::Request::RemoteChangesRequest(r) - }, - }; - - let rq = schema::v1::light::Request { request: Some(request) }; - let mut buf = Vec::with_capacity(rq.encoded_len()); - rq.encode(&mut buf)?; - Ok(buf) - } - - fn return_reply(self, result: Result, ClientError>) { - fn send(item: T, sender: oneshot::Sender) { - let _ = sender.send(item); // It is okay if the other end already hung up. - } - match self { - Request::Body { request, sender } => match result { - Err(e) => send(Err(e), sender), - Ok(Reply::Extrinsics(x)) => send(Ok(x), sender), - reply => log::error!("invalid reply for body request: {:?}, {:?}", reply, request), - }, - Request::Header { request, sender } => match result { - Err(e) => send(Err(e), sender), - Ok(Reply::Header(x)) => send(Ok(x), sender), - reply => { - log::error!("invalid reply for header request: {:?}, {:?}", reply, request) - }, - }, - Request::Read { request, sender } => match result { - Err(e) => send(Err(e), sender), - Ok(Reply::MapVecU8OptVecU8(x)) => send(Ok(x), sender), - reply => log::error!("invalid reply for read request: {:?}, {:?}", reply, request), - }, - Request::ReadChild { request, sender } => match result { - Err(e) => send(Err(e), sender), - Ok(Reply::MapVecU8OptVecU8(x)) => send(Ok(x), sender), - reply => { - log::error!("invalid reply for read child request: {:?}, {:?}", reply, request) - }, - }, - Request::Call { request, sender } => match result { - Err(e) => send(Err(e), sender), - Ok(Reply::VecU8(x)) => send(Ok(x), sender), - reply => log::error!("invalid reply for call request: {:?}, {:?}", reply, request), - }, - Request::Changes { request, sender } => match result { - Err(e) => send(Err(e), sender), - Ok(Reply::VecNumberU32(x)) => send(Ok(x), sender), - reply => { - log::error!("invalid reply for changes request: {:?}, {:?}", reply, request) - }, - }, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - light_client_requests::tests::{dummy_header, peerset, protocol_id, DummyFetchChecker}, - request_responses::OutboundFailure, - }; - - use assert_matches::assert_matches; - use futures::{channel::oneshot, executor::block_on, poll}; - use sc_client_api::StorageProof; - use sp_core::storage::ChildInfo; - use sp_runtime::{generic::Header, traits::BlakeTwo256}; - use std::{collections::HashSet, iter::FromIterator}; - - fn empty_proof() -> Vec { - StorageProof::empty().encode() - } - - #[test] - fn removes_peer_if_told() { - let peer = PeerId::random(); - let (_peer_set, peer_set_handle) = peerset(); - let mut sender = LightClientRequestSender::::new( - &protocol_id(), - Arc::new(DummyFetchChecker { ok: true, _mark: std::marker::PhantomData }), - peer_set_handle, - ); - - sender.inject_connected(peer); - assert_eq!(1, sender.peers.len()); - - sender.inject_disconnected(peer); - assert_eq!(0, sender.peers.len()); - } - - type Block = - sp_runtime::generic::Block, substrate_test_runtime::Extrinsic>; - - #[test] - fn body_request_fields_encoded_properly() { - let (sender, _receiver) = oneshot::channel(); - let request = Request::::Body { - request: RemoteBodyRequest { header: dummy_header(), retry_count: None }, - sender, - }; - let serialized_request = request.serialize_request().unwrap(); - let deserialized_request = - schema::v1::BlockRequest::decode(&serialized_request[..]).unwrap(); - assert!(BlockAttributes::from_be_u32(deserialized_request.fields) - .unwrap() - .contains(BlockAttributes::BODY)); - } - - #[test] - fn disconnects_from_peer_if_request_times_out() { - let peer0 = PeerId::random(); - let peer1 = PeerId::random(); - - let (_peer_set, peer_set_handle) = peerset(); - let mut sender = LightClientRequestSender::::new( - &protocol_id(), - Arc::new(crate::light_client_requests::tests::DummyFetchChecker { - ok: true, - _mark: std::marker::PhantomData, - }), - peer_set_handle, - ); - - sender.inject_connected(peer0); - sender.inject_connected(peer1); - - assert_eq!( - HashSet::from_iter(&[peer0.clone(), peer1.clone()]), - sender.peers.keys().collect::>(), - "Expect knowledge of two peers." - ); - - assert!(sender.pending_requests.is_empty(), "Expect no pending request."); - assert!(sender.sent_requests.is_empty(), "Expect no sent request."); - - // Issue a request! - let chan = oneshot::channel(); - let request = light::RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: Some(1), - }; - sender.request(Request::Call { request, sender: chan.0 }).unwrap(); - assert_eq!(1, sender.pending_requests.len(), "Expect one pending request."); - - let OutEvent::SendRequest { target, pending_response, .. } = - block_on(sender.next()).unwrap(); - assert!(target == peer0 || target == peer1, "Expect request to originate from known peer."); - - // And we should have one busy peer. - assert!({ - let (idle, busy): (Vec<_>, Vec<_>) = - sender.peers.iter().partition(|(_, info)| info.status == PeerStatus::Idle); - idle.len() == 1 && - busy.len() == 1 && (idle[0].0 == &peer0 || busy[0].0 == &peer0) && - (idle[0].0 == &peer1 || busy[0].0 == &peer1) - }); - - assert_eq!(0, sender.pending_requests.len(), "Expect no pending request."); - assert_eq!(1, sender.sent_requests.len(), "Expect one request to be sent."); - - // Report first attempt as timed out. - pending_response - .send(Err(RequestFailure::Network(OutboundFailure::Timeout))) - .unwrap(); - - // Expect a new request to be issued. - let OutEvent::SendRequest { pending_response, .. } = block_on(sender.next()).unwrap(); - - assert_eq!(1, sender.peers.len(), "Expect peer to be removed."); - assert_eq!(0, sender.pending_requests.len(), "Expect no request to be pending."); - assert_eq!(1, sender.sent_requests.len(), "Expect new request to be issued."); - - // Report second attempt as timed out. - pending_response - .send(Err(RequestFailure::Network(OutboundFailure::Timeout))) - .unwrap(); - assert_matches!( - block_on(async { poll!(sender.next()) }), - Poll::Pending, - "Expect sender to not issue another attempt.", - ); - assert_matches!( - block_on(chan.1).unwrap(), - Err(ClientError::RemoteFetchFailed), - "Expect request failure to be reported.", - ); - assert_eq!(0, sender.peers.len(), "Expect no peer to be left"); - assert_eq!(0, sender.pending_requests.len(), "Expect no request to be pending."); - assert_eq!(0, sender.sent_requests.len(), "Expect no other request to be in progress."); - } - - #[test] - fn disconnects_from_peer_on_incorrect_response() { - let peer = PeerId::random(); - - let (_peer_set, peer_set_handle) = peerset(); - let mut sender = LightClientRequestSender::::new( - &protocol_id(), - Arc::new(crate::light_client_requests::tests::DummyFetchChecker { - ok: false, - // ^--- Making sure the response data check fails. - _mark: std::marker::PhantomData, - }), - peer_set_handle, - ); - - sender.inject_connected(peer); - assert_eq!(1, sender.peers.len(), "Expect one peer."); - - let chan = oneshot::channel(); - let request = light::RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: Some(1), - }; - sender.request(Request::Call { request, sender: chan.0 }).unwrap(); - - assert_eq!(1, sender.pending_requests.len(), "Expect one pending request."); - assert_eq!(0, sender.sent_requests.len(), "Expect zero sent requests."); - - let OutEvent::SendRequest { pending_response, .. } = block_on(sender.next()).unwrap(); - assert_eq!(0, sender.pending_requests.len(), "Expect zero pending requests."); - assert_eq!(1, sender.sent_requests.len(), "Expect one sent request."); - - let response = { - let r = schema::v1::light::RemoteCallResponse { proof: empty_proof() }; - let response = schema::v1::light::Response { - response: Some(schema::v1::light::response::Response::RemoteCallResponse(r)), - }; - let mut data = Vec::new(); - response.encode(&mut data).unwrap(); - data - }; - - pending_response.send(Ok(response)).unwrap(); - - assert_matches!( - block_on(async { poll!(sender.next()) }), - Poll::Pending, - "Expect sender to not issue another attempt, given that there is no peer left.", - ); - - assert!(sender.peers.is_empty(), "Expect no peers to be left."); - assert_eq!(1, sender.pending_requests.len(), "Expect request to be pending again."); - assert_eq!(0, sender.sent_requests.len(), "Expect no request to be sent."); - } - - #[test] - fn disconnects_from_peer_on_wrong_response_type() { - let peer = PeerId::random(); - let (_peer_set, peer_set_handle) = peerset(); - let mut sender = LightClientRequestSender::::new( - &protocol_id(), - Arc::new(crate::light_client_requests::tests::DummyFetchChecker { - ok: true, - _mark: std::marker::PhantomData, - }), - peer_set_handle, - ); - - sender.inject_connected(peer); - assert_eq!(1, sender.peers.len(), "Expect one peer."); - - let chan = oneshot::channel(); - let request = light::RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: Some(1), - }; - sender.request(Request::Call { request, sender: chan.0 }).unwrap(); - - assert_eq!(1, sender.pending_requests.len()); - assert_eq!(0, sender.sent_requests.len()); - let OutEvent::SendRequest { pending_response, .. } = block_on(sender.next()).unwrap(); - assert_eq!(0, sender.pending_requests.len(), "Expect zero pending requests."); - assert_eq!(1, sender.sent_requests.len(), "Expect one sent request."); - - let response = { - let r = schema::v1::light::RemoteReadResponse { proof: empty_proof() }; // Not a RemoteCallResponse! - let response = schema::v1::light::Response { - response: Some(schema::v1::light::response::Response::RemoteReadResponse(r)), - }; - let mut data = Vec::new(); - response.encode(&mut data).unwrap(); - data - }; - - pending_response.send(Ok(response)).unwrap(); - assert_matches!( - block_on(async { poll!(sender.next()) }), - Poll::Pending, - "Expect sender to not issue another attempt, given that there is no peer left.", - ); - - assert!(sender.peers.is_empty(), "Expect no peers to be left."); - assert_eq!(1, sender.pending_requests.len(), "Expect request to be pending again."); - assert_eq!(0, sender.sent_requests.len(), "Expect no request to be sent."); - } - - #[test] - fn receives_remote_failure_after_retry_count_failures() { - let peers = (0..4).map(|_| PeerId::random()).collect::>(); - - let (_peer_set, peer_set_handle) = peerset(); - let mut sender = LightClientRequestSender::::new( - &protocol_id(), - Arc::new(crate::light_client_requests::tests::DummyFetchChecker { - ok: false, - // ^--- Making sure the response data check fails. - _mark: std::marker::PhantomData, - }), - peer_set_handle, - ); - - for peer in &peers { - sender.inject_connected(*peer); - } - assert_eq!(4, sender.peers.len(), "Expect four peers."); - - let mut chan = oneshot::channel(); - let request = light::RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: Some(3), // Attempt up to three retries. - }; - sender.request(Request::Call { request, sender: chan.0 }).unwrap(); - - assert_eq!(1, sender.pending_requests.len()); - assert_eq!(0, sender.sent_requests.len()); - let mut pending_response = match block_on(sender.next()).unwrap() { - OutEvent::SendRequest { pending_response, .. } => Some(pending_response), - }; - assert_eq!(0, sender.pending_requests.len(), "Expect zero pending requests."); - assert_eq!(1, sender.sent_requests.len(), "Expect one sent request."); - - for (i, _peer) in peers.iter().enumerate() { - // Construct an invalid response - let response = { - let r = schema::v1::light::RemoteCallResponse { proof: empty_proof() }; - let response = schema::v1::light::Response { - response: Some(schema::v1::light::response::Response::RemoteCallResponse(r)), - }; - let mut data = Vec::new(); - response.encode(&mut data).unwrap(); - data - }; - pending_response.take().unwrap().send(Ok(response)).unwrap(); - - if i < 3 { - pending_response = match block_on(sender.next()).unwrap() { - OutEvent::SendRequest { pending_response, .. } => Some(pending_response), - }; - assert_matches!(chan.1.try_recv(), Ok(None)) - } else { - // Last peer and last attempt. - assert_matches!( - block_on(async { poll!(sender.next()) }), - Poll::Pending, - "Expect sender to not issue another attempt, given that there is no peer left.", - ); - assert_matches!(chan.1.try_recv(), Ok(Some(Err(ClientError::RemoteFetchFailed)))) - } - } - } - - fn issue_request(request: Request) { - let peer = PeerId::random(); - - let (_peer_set, peer_set_handle) = peerset(); - let mut sender = LightClientRequestSender::::new( - &protocol_id(), - Arc::new(crate::light_client_requests::tests::DummyFetchChecker { - ok: true, - _mark: std::marker::PhantomData, - }), - peer_set_handle, - ); - - sender.inject_connected(peer); - assert_eq!(1, sender.peers.len(), "Expect one peer."); - - let response = match request { - Request::Body { .. } => unimplemented!(), - Request::Header { .. } => { - let r = schema::v1::light::RemoteHeaderResponse { - header: dummy_header().encode(), - proof: empty_proof(), - }; - schema::v1::light::Response { - response: Some(schema::v1::light::response::Response::RemoteHeaderResponse(r)), - } - }, - Request::Read { .. } => { - let r = schema::v1::light::RemoteReadResponse { proof: empty_proof() }; - schema::v1::light::Response { - response: Some(schema::v1::light::response::Response::RemoteReadResponse(r)), - } - }, - Request::ReadChild { .. } => { - let r = schema::v1::light::RemoteReadResponse { proof: empty_proof() }; - schema::v1::light::Response { - response: Some(schema::v1::light::response::Response::RemoteReadResponse(r)), - } - }, - Request::Call { .. } => { - let r = schema::v1::light::RemoteCallResponse { proof: empty_proof() }; - schema::v1::light::Response { - response: Some(schema::v1::light::response::Response::RemoteCallResponse(r)), - } - }, - Request::Changes { .. } => { - let r = schema::v1::light::RemoteChangesResponse { - max: std::iter::repeat(1).take(32).collect(), - proof: Vec::new(), - roots: Vec::new(), - roots_proof: empty_proof(), - }; - schema::v1::light::Response { - response: Some(schema::v1::light::response::Response::RemoteChangesResponse(r)), - } - }, - }; - - let response = { - let mut data = Vec::new(); - response.encode(&mut data).unwrap(); - data - }; - - sender.request(request).unwrap(); - - assert_eq!(1, sender.pending_requests.len()); - assert_eq!(0, sender.sent_requests.len()); - let OutEvent::SendRequest { pending_response, .. } = block_on(sender.next()).unwrap(); - assert_eq!(0, sender.pending_requests.len()); - assert_eq!(1, sender.sent_requests.len()); - - pending_response.send(Ok(response)).unwrap(); - assert_matches!( - block_on(async { poll!(sender.next()) }), - Poll::Pending, - "Expect sender to not issue another attempt, given that there is no peer left.", - ); - - assert_eq!(0, sender.pending_requests.len()); - assert_eq!(0, sender.sent_requests.len()) - } - - #[test] - fn receives_remote_call_response() { - let mut chan = oneshot::channel(); - let request = light::RemoteCallRequest { - block: Default::default(), - header: dummy_header(), - method: "test".into(), - call_data: vec![], - retry_count: None, - }; - issue_request(Request::Call { request, sender: chan.0 }); - assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) - } - - #[test] - fn receives_remote_read_response() { - let mut chan = oneshot::channel(); - let request = light::RemoteReadRequest { - header: dummy_header(), - block: Default::default(), - keys: vec![b":key".to_vec()], - retry_count: None, - }; - issue_request(Request::Read { request, sender: chan.0 }); - assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) - } - - #[test] - fn receives_remote_read_child_response() { - let mut chan = oneshot::channel(); - let child_info = ChildInfo::new_default(&b":child_storage:default:sub"[..]); - let request = light::RemoteReadChildRequest { - header: dummy_header(), - block: Default::default(), - storage_key: child_info.prefixed_storage_key(), - keys: vec![b":key".to_vec()], - retry_count: None, - }; - issue_request(Request::ReadChild { request, sender: chan.0 }); - assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) - } - - #[test] - fn receives_remote_header_response() { - let mut chan = oneshot::channel(); - let request = light::RemoteHeaderRequest { - cht_root: Default::default(), - block: 1, - retry_count: None, - }; - issue_request(Request::Header { request, sender: chan.0 }); - assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) - } - - #[test] - fn receives_remote_changes_response() { - let mut chan = oneshot::channel(); - let request = light::RemoteChangesRequest { - changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { - zero: (0, Default::default()), - end: None, - config: Some(sp_core::ChangesTrieConfiguration::new(4, 2)), - }], - first_block: (1, Default::default()), - last_block: (100, Default::default()), - max_block: (100, Default::default()), - tries_roots: (1, Default::default(), Vec::new()), - key: Vec::new(), - storage_key: None, - retry_count: None, - }; - issue_request(Request::Changes { request, sender: chan.0 }); - assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) - } -} diff --git a/client/network/src/network_state.rs b/client/network/src/network_state.rs index 6f5f031bf35d..a5e2fbef421d 100644 --- a/client/network/src/network_state.rs +++ b/client/network/src/network_state.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/network/src/on_demand_layer.rs b/client/network/src/on_demand_layer.rs deleted file mode 100644 index eaeb0bee98f2..000000000000 --- a/client/network/src/on_demand_layer.rs +++ /dev/null @@ -1,241 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! On-demand requests service. - -use crate::light_client_requests; - -use futures::{channel::oneshot, prelude::*}; -use parking_lot::Mutex; -use sc_client_api::{ - ChangesProof, FetchChecker, Fetcher, RemoteBodyRequest, RemoteCallRequest, - RemoteChangesRequest, RemoteHeaderRequest, RemoteReadChildRequest, RemoteReadRequest, - StorageProof, -}; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; -use sp_blockchain::Error as ClientError; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; -use std::{ - collections::HashMap, - pin::Pin, - sync::Arc, - task::{Context, Poll}, -}; - -/// Implements the `Fetcher` trait of the client. Makes it possible for the light client to perform -/// network requests for some state. -/// -/// This implementation stores all the requests in a queue. The network, in parallel, is then -/// responsible for pulling elements out of that queue and fulfilling them. -pub struct OnDemand { - /// Objects that checks whether what has been retrieved is correct. - checker: Arc>, - - /// Queue of requests. Set to `Some` at initialization, then extracted by the network. - /// - /// Note that a better alternative would be to use a MPMC queue here, and add a `poll` method - /// from the `OnDemand`. However there exists no popular implementation of MPMC channels in - /// asynchronous Rust at the moment - requests_queue: - Mutex>>>, - - /// Sending side of `requests_queue`. - requests_send: TracingUnboundedSender>, -} - -#[derive(Debug, thiserror::Error)] -#[error("AlwaysBadChecker")] -struct ErrorAlwaysBadChecker; - -impl Into for ErrorAlwaysBadChecker { - fn into(self) -> ClientError { - ClientError::Application(Box::new(self)) - } -} - -/// Dummy implementation of `FetchChecker` that always assumes that responses are bad. -/// -/// Considering that it is the responsibility of the client to build the fetcher, it can use this -/// implementation if it knows that it will never perform any request. -#[derive(Default, Clone)] -pub struct AlwaysBadChecker; - -impl FetchChecker for AlwaysBadChecker { - fn check_header_proof( - &self, - _request: &RemoteHeaderRequest, - _remote_header: Option, - _remote_proof: StorageProof, - ) -> Result { - Err(ErrorAlwaysBadChecker.into()) - } - - fn check_read_proof( - &self, - _request: &RemoteReadRequest, - _remote_proof: StorageProof, - ) -> Result, Option>>, ClientError> { - Err(ErrorAlwaysBadChecker.into()) - } - - fn check_read_child_proof( - &self, - _request: &RemoteReadChildRequest, - _remote_proof: StorageProof, - ) -> Result, Option>>, ClientError> { - Err(ErrorAlwaysBadChecker.into()) - } - - fn check_execution_proof( - &self, - _request: &RemoteCallRequest, - _remote_proof: StorageProof, - ) -> Result, ClientError> { - Err(ErrorAlwaysBadChecker.into()) - } - - fn check_changes_proof( - &self, - _request: &RemoteChangesRequest, - _remote_proof: ChangesProof, - ) -> Result, u32)>, ClientError> { - Err(ErrorAlwaysBadChecker.into()) - } - - fn check_body_proof( - &self, - _request: &RemoteBodyRequest, - _body: Vec, - ) -> Result, ClientError> { - Err(ErrorAlwaysBadChecker.into()) - } -} - -impl OnDemand -where - B::Header: HeaderT, -{ - /// Creates new on-demand service. - pub fn new(checker: Arc>) -> Self { - let (requests_send, requests_queue) = tracing_unbounded("mpsc_ondemand"); - let requests_queue = Mutex::new(Some(requests_queue)); - - Self { checker, requests_queue, requests_send } - } - - /// Get checker reference. - pub fn checker(&self) -> &Arc> { - &self.checker - } - - /// Extracts the queue of requests. - /// - /// Whenever one of the methods of the `Fetcher` trait is called, an element is pushed on this - /// channel. - /// - /// If this function returns `None`, that means that the receiver has already been extracted in - /// the past, and therefore that something already handles the requests. - pub(crate) fn extract_receiver( - &self, - ) -> Option>> { - self.requests_queue.lock().take() - } -} - -impl Fetcher for OnDemand -where - B: BlockT, - B::Header: HeaderT, -{ - type RemoteHeaderResult = RemoteResponse; - type RemoteReadResult = RemoteResponse, Option>>>; - type RemoteCallResult = RemoteResponse>; - type RemoteChangesResult = RemoteResponse, u32)>>; - type RemoteBodyResult = RemoteResponse>; - - fn remote_header(&self, request: RemoteHeaderRequest) -> Self::RemoteHeaderResult { - let (sender, receiver) = oneshot::channel(); - let _ = self - .requests_send - .unbounded_send(light_client_requests::sender::Request::Header { request, sender }); - RemoteResponse { receiver } - } - - fn remote_read(&self, request: RemoteReadRequest) -> Self::RemoteReadResult { - let (sender, receiver) = oneshot::channel(); - let _ = self - .requests_send - .unbounded_send(light_client_requests::sender::Request::Read { request, sender }); - RemoteResponse { receiver } - } - - fn remote_read_child( - &self, - request: RemoteReadChildRequest, - ) -> Self::RemoteReadResult { - let (sender, receiver) = oneshot::channel(); - let _ = self - .requests_send - .unbounded_send(light_client_requests::sender::Request::ReadChild { request, sender }); - RemoteResponse { receiver } - } - - fn remote_call(&self, request: RemoteCallRequest) -> Self::RemoteCallResult { - let (sender, receiver) = oneshot::channel(); - let _ = self - .requests_send - .unbounded_send(light_client_requests::sender::Request::Call { request, sender }); - RemoteResponse { receiver } - } - - fn remote_changes( - &self, - request: RemoteChangesRequest, - ) -> Self::RemoteChangesResult { - let (sender, receiver) = oneshot::channel(); - let _ = self - .requests_send - .unbounded_send(light_client_requests::sender::Request::Changes { request, sender }); - RemoteResponse { receiver } - } - - fn remote_body(&self, request: RemoteBodyRequest) -> Self::RemoteBodyResult { - let (sender, receiver) = oneshot::channel(); - let _ = self - .requests_send - .unbounded_send(light_client_requests::sender::Request::Body { request, sender }); - RemoteResponse { receiver } - } -} - -/// Future for an on-demand remote call response. -pub struct RemoteResponse { - receiver: oneshot::Receiver>, -} - -impl Future for RemoteResponse { - type Output = Result; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { - match self.receiver.poll_unpin(cx) { - Poll::Ready(Ok(res)) => Poll::Ready(res), - Poll::Ready(Err(_)) => Poll::Ready(Err(ClientError::RemoteFetchCancelled)), - Poll::Pending => Poll::Pending, - } - } -} diff --git a/client/network/src/peer_info.rs b/client/network/src/peer_info.rs index 141cc59247d1..378c258820ff 100644 --- a/client/network/src/peer_info.rs +++ b/client/network/src/peer_info.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -186,6 +186,17 @@ impl NetworkBehaviour for PeerInfoBehaviour { list } + fn inject_address_change( + &mut self, + peer_id: &PeerId, + conn: &ConnectionId, + old: &ConnectedPoint, + new: &ConnectedPoint, + ) { + self.ping.inject_address_change(peer_id, conn, old, new); + self.identify.inject_address_change(peer_id, conn, old, new); + } + fn inject_connected(&mut self, peer_id: &PeerId) { self.ping.inject_connected(peer_id); self.identify.inject_connected(peer_id); @@ -196,9 +207,12 @@ impl NetworkBehaviour for PeerInfoBehaviour { peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint, + failed_addresses: Option<&Vec>, ) { - self.ping.inject_connection_established(peer_id, conn, endpoint); - self.identify.inject_connection_established(peer_id, conn, endpoint); + self.ping + .inject_connection_established(peer_id, conn, endpoint, failed_addresses); + self.identify + .inject_connection_established(peer_id, conn, endpoint, failed_addresses); match self.nodes_info.entry(*peer_id) { Entry::Vacant(e) => { e.insert(NodeInfo::new(endpoint.clone())); @@ -220,9 +234,12 @@ impl NetworkBehaviour for PeerInfoBehaviour { peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint, + handler: ::Handler, ) { - self.ping.inject_connection_closed(peer_id, conn, endpoint); - self.identify.inject_connection_closed(peer_id, conn, endpoint); + let (ping_handler, identity_handler) = handler.into_inner(); + self.identify + .inject_connection_closed(peer_id, conn, endpoint, identity_handler); + self.ping.inject_connection_closed(peer_id, conn, endpoint, ping_handler); if let Some(entry) = self.nodes_info.get_mut(peer_id) { entry.endpoints.retain(|ep| ep != endpoint) @@ -256,19 +273,15 @@ impl NetworkBehaviour for PeerInfoBehaviour { } } - fn inject_addr_reach_failure( + fn inject_dial_failure( &mut self, - peer_id: Option<&PeerId>, - addr: &Multiaddr, - error: &dyn std::error::Error, + peer_id: Option, + handler: Self::ProtocolsHandler, + error: &libp2p::swarm::DialError, ) { - self.ping.inject_addr_reach_failure(peer_id, addr, error); - self.identify.inject_addr_reach_failure(peer_id, addr, error); - } - - fn inject_dial_failure(&mut self, peer_id: &PeerId) { - self.ping.inject_dial_failure(peer_id); - self.identify.inject_dial_failure(peer_id); + let (ping_handler, identity_handler) = handler.into_inner(); + self.identify.inject_dial_failure(peer_id, identity_handler, error); + self.ping.inject_dial_failure(peer_id, ping_handler, error); } fn inject_new_listener(&mut self, id: ListenerId) { @@ -296,6 +309,18 @@ impl NetworkBehaviour for PeerInfoBehaviour { self.identify.inject_expired_external_addr(addr); } + fn inject_listen_failure( + &mut self, + local_addr: &Multiaddr, + send_back_addr: &Multiaddr, + handler: Self::ProtocolsHandler, + ) { + let (ping_handler, identity_handler) = handler.into_inner(); + self.identify + .inject_listen_failure(local_addr, send_back_addr, identity_handler); + self.ping.inject_listen_failure(local_addr, send_back_addr, ping_handler); + } + fn inject_listener_error(&mut self, id: ListenerId, err: &(dyn error::Error + 'static)) { self.ping.inject_listener_error(id, err); self.identify.inject_listener_error(id, err); @@ -309,13 +334,8 @@ impl NetworkBehaviour for PeerInfoBehaviour { fn poll( &mut self, cx: &mut Context, - params: &mut impl PollParameters - ) -> Poll< - NetworkBehaviourAction< - <::Handler as ProtocolsHandler>::InEvent, - Self::OutEvent - > - >{ + params: &mut impl PollParameters, + ) -> Poll> { loop { match self.ping.poll(cx, params) { Poll::Pending => break, @@ -324,10 +344,20 @@ impl NetworkBehaviour for PeerInfoBehaviour { self.handle_ping_report(&peer, rtt) } }, - Poll::Ready(NetworkBehaviourAction::DialAddress { address }) => - return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }) => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + Poll::Ready(NetworkBehaviourAction::DialAddress { address, handler }) => { + let handler = + IntoProtocolsHandler::select(handler, self.identify.new_handler()); + return Poll::Ready(NetworkBehaviourAction::DialAddress { address, handler }) + }, + Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition, handler }) => { + let handler = + IntoProtocolsHandler::select(handler, self.identify.new_handler()); + return Poll::Ready(NetworkBehaviourAction::DialPeer { + peer_id, + condition, + handler, + }) + }, Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, handler, event }) => return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, @@ -362,10 +392,18 @@ impl NetworkBehaviour for PeerInfoBehaviour { IdentifyEvent::Pushed { .. } => {}, IdentifyEvent::Sent { .. } => {}, }, - Poll::Ready(NetworkBehaviourAction::DialAddress { address }) => - return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }) => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + Poll::Ready(NetworkBehaviourAction::DialAddress { address, handler }) => { + let handler = IntoProtocolsHandler::select(self.ping.new_handler(), handler); + return Poll::Ready(NetworkBehaviourAction::DialAddress { address, handler }) + }, + Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition, handler }) => { + let handler = IntoProtocolsHandler::select(self.ping.new_handler(), handler); + return Poll::Ready(NetworkBehaviourAction::DialPeer { + peer_id, + condition, + handler, + }) + }, Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, handler, event }) => return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index e22d96f32aeb..80694210e77d 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -60,7 +60,6 @@ use sp_runtime::{ use std::{ borrow::Cow, collections::{HashMap, HashSet, VecDeque}, - convert::TryFrom as _, io, iter, num::NonZeroUsize, pin::Pin, @@ -132,21 +131,22 @@ impl Metrics { fn register(r: &Registry) -> Result { Ok(Self { peers: { - let g = Gauge::new("sync_peers", "Number of peers we sync with")?; + let g = Gauge::new("substrate_sync_peers", "Number of peers we sync with")?; register(g, r)? }, queued_blocks: { - let g = Gauge::new("sync_queued_blocks", "Number of blocks in import queue")?; + let g = + Gauge::new("substrate_sync_queued_blocks", "Number of blocks in import queue")?; register(g, r)? }, fork_targets: { - let g = Gauge::new("sync_fork_targets", "Number of fork sync targets")?; + let g = Gauge::new("substrate_sync_fork_targets", "Number of fork sync targets")?; register(g, r)? }, justifications: { let g = GaugeVec::new( Opts::new( - "sync_extra_justifications", + "substrate_sync_extra_justifications", "Number of extra justifications requests", ), &["status"], @@ -165,13 +165,19 @@ pub struct Protocol { pending_messages: VecDeque>, config: ProtocolConfig, genesis_hash: B::Hash, + /// State machine that handles the list of in-progress requests. Only full node peers are + /// registered. sync: ChainSync, - // All connected peers + // All connected peers. Contains both full and light node peers. peers: HashMap>, chain: Arc>, /// List of nodes for which we perform additional logging because they are important for the /// user. important_peers: HashSet, + /// Value that was passed as part of the configuration. Used to cap the number of full nodes. + default_peers_set_num_full: usize, + /// Number of slots to allocate to light nodes. + default_peers_set_num_light: usize, /// Used to report reputation changes. peerset_handle: sc_peerset::PeersetHandle, /// Handles opening the unique substream and sending and receiving raw messages. @@ -427,6 +433,12 @@ impl Protocol { genesis_hash: info.genesis_hash, sync, important_peers, + default_peers_set_num_full: network_config.default_peers_set_num_full as usize, + default_peers_set_num_light: { + let total = network_config.default_peers_set.out_peers + + network_config.default_peers_set.in_peers; + total.saturating_sub(network_config.default_peers_set_num_full) as usize + }, peerset_handle: peerset_handle.clone(), behaviour, notification_protocols: network_config @@ -452,12 +464,6 @@ impl Protocol { self.behaviour.open_peers() } - /// Returns the list of all the peers that the peerset currently requests us to be connected - /// to on the default set. - pub fn requested_peers(&self) -> impl Iterator { - self.behaviour.requested_peers(HARDCODED_PEERSETS_SYNC) - } - /// Returns the number of discovered nodes that we keep in memory. pub fn num_discovered_peers(&self) -> usize { self.behaviour.num_discovered_peers() @@ -483,7 +489,7 @@ impl Protocol { /// Returns the number of peers we're connected to. pub fn num_connected_peers(&self) -> usize { - self.peers.values().count() + self.peers.len() } /// Returns the number of peers we're connected to and that are being queried. @@ -710,8 +716,7 @@ impl Protocol { match self.sync.on_state_data(&peer_id, response) { Ok(sync::OnStateData::Import(origin, block)) => CustomMessageOutcome::BlockImport(origin, vec![block]), - Ok(sync::OnStateData::Request(peer, req)) => - prepare_state_request::(&mut self.peers, peer, req), + Ok(sync::OnStateData::Continue) => CustomMessageOutcome::None, Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id, HARDCODED_PEERSETS_SYNC); self.peerset_handle.report_peer(id, repu); @@ -728,10 +733,7 @@ impl Protocol { response: crate::warp_request_handler::EncodedProof, ) -> CustomMessageOutcome { match self.sync.on_warp_sync_data(&peer_id, response) { - Ok(sync::OnWarpSyncData::WarpProofRequest(peer, req)) => - prepare_warp_sync_request::(&mut self.peers, peer, req), - Ok(sync::OnWarpSyncData::StateRequest(peer, req)) => - prepare_state_request::(&mut self.peers, peer, req), + Ok(()) => CustomMessageOutcome::None, Err(sync::BadPeer(id, repu)) => { self.behaviour.disconnect_peer(&id, HARDCODED_PEERSETS_SYNC); self.peerset_handle.report_peer(id, repu); @@ -811,6 +813,21 @@ impl Protocol { } } + if status.roles.is_full() && self.sync.num_peers() >= self.default_peers_set_num_full { + debug!(target: "sync", "Too many full nodes, rejecting {}", who); + self.behaviour.disconnect_peer(&who, HARDCODED_PEERSETS_SYNC); + return Err(()) + } + + if status.roles.is_light() && + (self.peers.len() - self.sync.num_peers()) >= self.default_peers_set_num_light + { + // Make sure that not all slots are occupied by light clients. + debug!(target: "sync", "Too many light nodes, rejecting {}", who); + self.behaviour.disconnect_peer(&who, HARDCODED_PEERSETS_SYNC); + return Err(()) + } + let peer = Peer { info: PeerInfo { roles: status.roles, @@ -862,7 +879,7 @@ impl Protocol { return }, Err(e) => { - warn!("Error reading block header {}: {:?}", hash, e); + warn!("Error reading block header {}: {}", hash, e); return }, }; @@ -1106,7 +1123,7 @@ impl Protocol { /// Removes a `PeerId` from the list of reserved peers for syncing purposes. pub fn remove_reserved_peer(&self, peer: PeerId) { - self.peerset_handle.remove_reserved_peer(HARDCODED_PEERSETS_SYNC, peer.clone()); + self.peerset_handle.remove_reserved_peer(HARDCODED_PEERSETS_SYNC, peer); } /// Returns the list of reserved peers. @@ -1116,12 +1133,26 @@ impl Protocol { /// Adds a `PeerId` to the list of reserved peers for syncing purposes. pub fn add_reserved_peer(&self, peer: PeerId) { - self.peerset_handle.add_reserved_peer(HARDCODED_PEERSETS_SYNC, peer.clone()); + self.peerset_handle.add_reserved_peer(HARDCODED_PEERSETS_SYNC, peer); } /// Sets the list of reserved peers for syncing purposes. pub fn set_reserved_peers(&self, peers: HashSet) { - self.peerset_handle.set_reserved_peers(HARDCODED_PEERSETS_SYNC, peers.clone()); + self.peerset_handle.set_reserved_peers(HARDCODED_PEERSETS_SYNC, peers); + } + + /// Sets the list of reserved peers for the given protocol/peerset. + pub fn set_reserved_peerset_peers(&self, protocol: Cow<'static, str>, peers: HashSet) { + if let Some(index) = self.notification_protocols.iter().position(|p| *p == protocol) { + self.peerset_handle + .set_reserved_peers(sc_peerset::SetId::from(index + NUM_HARDCODED_PEERSETS), peers); + } else { + error!( + target: "sub-libp2p", + "set_reserved_peerset_peers with unknown protocol: {}", + protocol + ); + } } /// Removes a `PeerId` from the list of reserved peers. @@ -1352,8 +1383,10 @@ impl NetworkBehaviour for Protocol { peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint, + failed_addresses: Option<&Vec>, ) { - self.behaviour.inject_connection_established(peer_id, conn, endpoint) + self.behaviour + .inject_connection_established(peer_id, conn, endpoint, failed_addresses) } fn inject_connection_closed( @@ -1361,8 +1394,9 @@ impl NetworkBehaviour for Protocol { peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint, + handler: ::Handler, ) { - self.behaviour.inject_connection_closed(peer_id, conn, endpoint) + self.behaviour.inject_connection_closed(peer_id, conn, endpoint, handler) } fn inject_connected(&mut self, peer_id: &PeerId) { @@ -1386,12 +1420,7 @@ impl NetworkBehaviour for Protocol { &mut self, cx: &mut std::task::Context, params: &mut impl PollParameters, - ) -> Poll< - NetworkBehaviourAction< - <::Handler as ProtocolsHandler>::InEvent, - Self::OutEvent - > - >{ + ) -> Poll> { if let Some(message) = self.pending_messages.pop_front() { return Poll::Ready(NetworkBehaviourAction::GenerateEvent(message)) } @@ -1552,10 +1581,10 @@ impl NetworkBehaviour for Protocol { let event = match self.behaviour.poll(cx, params) { Poll::Pending => return Poll::Pending, Poll::Ready(NetworkBehaviourAction::GenerateEvent(ev)) => ev, - Poll::Ready(NetworkBehaviourAction::DialAddress { address }) => - return Poll::Ready(NetworkBehaviourAction::DialAddress { address }), - Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }) => - return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition }), + Poll::Ready(NetworkBehaviourAction::DialAddress { address, handler }) => + return Poll::Ready(NetworkBehaviourAction::DialAddress { address, handler }), + Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition, handler }) => + return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition, handler }), Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, handler, event }) => return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, @@ -1636,7 +1665,7 @@ impl NetworkBehaviour for Protocol { } } else { match ( - message::Roles::decode_all(&received_handshake[..]), + message::Roles::decode_all(&mut &received_handshake[..]), self.peers.get(&peer_id), ) { (Ok(roles), _) => CustomMessageOutcome::NotificationStreamOpened { @@ -1768,17 +1797,13 @@ impl NetworkBehaviour for Protocol { Poll::Pending } - fn inject_addr_reach_failure( + fn inject_dial_failure( &mut self, - peer_id: Option<&PeerId>, - addr: &Multiaddr, - error: &dyn std::error::Error, + peer_id: Option, + handler: Self::ProtocolsHandler, + error: &libp2p::swarm::DialError, ) { - self.behaviour.inject_addr_reach_failure(peer_id, addr, error) - } - - fn inject_dial_failure(&mut self, peer_id: &PeerId) { - self.behaviour.inject_dial_failure(peer_id) + self.behaviour.inject_dial_failure(peer_id, handler, error); } fn inject_new_listener(&mut self, id: ListenerId) { diff --git a/client/network/src/protocol/event.rs b/client/network/src/protocol/event.rs index e0b35647c753..26c954496060 100644 --- a/client/network/src/protocol/event.rs +++ b/client/network/src/protocol/event.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/network/src/protocol/message.rs b/client/network/src/protocol/message.rs index 001f6cbd7e45..3fb57b1c824a 100644 --- a/client/network/src/protocol/message.rs +++ b/client/network/src/protocol/message.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/network/src/protocol/notifications.rs b/client/network/src/protocol/notifications.rs index e489970e987c..bf183abf160c 100644 --- a/client/network/src/protocol/notifications.rs +++ b/client/network/src/protocol/notifications.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index da2967d6f26e..b47216473970 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -26,7 +26,8 @@ use futures::prelude::*; use libp2p::{ core::{connection::ConnectionId, ConnectedPoint, Multiaddr, PeerId}, swarm::{ - DialPeerCondition, NetworkBehaviour, NetworkBehaviourAction, NotifyHandler, PollParameters, + DialError, DialPeerCondition, IntoProtocolsHandler, NetworkBehaviour, + NetworkBehaviourAction, NotifyHandler, PollParameters, }, }; use log::{error, trace, warn}; @@ -38,7 +39,7 @@ use std::{ borrow::Cow, cmp, collections::{hash_map::Entry, VecDeque}, - error, mem, + mem, pin::Pin, str, sync::Arc, @@ -132,7 +133,7 @@ pub struct Notifications { next_incoming_index: sc_peerset::IncomingIndex, /// Events to produce from `poll()`. - events: VecDeque>, + events: VecDeque>, } /// Configuration for a notifications protocol. @@ -250,16 +251,6 @@ impl PeerState { _ => None, } } - - /// True if that node has been requested by the PSM. - fn is_requested(&self) -> bool { - matches!( - self, - Self::PendingRequest { .. } | - Self::Requested | Self::DisabledPendingEnable { .. } | - Self::Enabled { .. } - ) - } } /// State of the handler of a single connection visible from this state machine. @@ -417,7 +408,7 @@ impl Notifications { /// Returns true if we have an open substream to the given peer. pub fn is_open(&self, peer_id: &PeerId, set_id: sc_peerset::SetId) -> bool { - self.peers.get(&(peer_id.clone(), set_id)).map(|p| p.is_open()).unwrap_or(false) + self.peers.get(&(*peer_id, set_id)).map(|p| p.is_open()).unwrap_or(false) } /// Disconnects the given peer if we are connected to it. @@ -559,17 +550,6 @@ impl Notifications { } } - /// Returns the list of all the peers that the peerset currently requests us to be connected to. - pub fn requested_peers<'a>( - &'a self, - set_id: sc_peerset::SetId, - ) -> impl Iterator + 'a { - self.peers - .iter() - .filter(move |((_, set), state)| *set == set_id && state.is_requested()) - .map(|((id, _), _)| id) - } - /// Returns the list of reserved peers. pub fn reserved_peers<'a>( &'a self, @@ -628,6 +608,7 @@ impl Notifications { /// Function that is called when the peerset wants us to connect to a peer. fn peerset_report_connect(&mut self, peer_id: PeerId, set_id: sc_peerset::SetId) { // If `PeerId` is unknown to us, insert an entry, start dialing, and return early. + let handler = self.new_handler(); let mut occ_entry = match self.peers.entry((peer_id, set_id)) { Entry::Occupied(entry) => entry, Entry::Vacant(entry) => { @@ -643,6 +624,7 @@ impl Notifications { self.events.push_back(NetworkBehaviourAction::DialPeer { peer_id: entry.key().0.clone(), condition: DialPeerCondition::Disconnected, + handler, }); entry.insert(PeerState::Requested); return @@ -679,6 +661,7 @@ impl Notifications { self.events.push_back(NetworkBehaviourAction::DialPeer { peer_id: occ_entry.key().0.clone(), condition: DialPeerCondition::Disconnected, + handler, }); *occ_entry.into_mut() = PeerState::Requested; }, @@ -712,7 +695,7 @@ impl Notifications { timer: delay_id, timer_deadline: *backoff, }; - } + }, // Disabled => Enabled PeerState::Disabled { mut connections, backoff_until } => { @@ -1094,6 +1077,7 @@ impl NetworkBehaviour for Notifications { peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint, + _failed_addresses: Option<&Vec>, ) { for set_id in (0..self.notif_protocols.len()).map(sc_peerset::SetId::from) { match self.peers.entry((*peer_id, set_id)).or_insert(PeerState::Poisoned) { @@ -1152,6 +1136,7 @@ impl NetworkBehaviour for Notifications { peer_id: &PeerId, conn: &ConnectionId, _endpoint: &ConnectedPoint, + _handler: ::Handler, ) { for set_id in (0..self.notif_protocols.len()).map(sc_peerset::SetId::from) { let mut entry = if let Entry::Occupied(entry) = self.peers.entry((*peer_id, set_id)) { @@ -1411,70 +1396,74 @@ impl NetworkBehaviour for Notifications { fn inject_disconnected(&mut self, _peer_id: &PeerId) {} - fn inject_addr_reach_failure( + fn inject_dial_failure( &mut self, - peer_id: Option<&PeerId>, - addr: &Multiaddr, - error: &dyn error::Error, + peer_id: Option, + _: Self::ProtocolsHandler, + error: &DialError, ) { - trace!(target: "sub-libp2p", "Libp2p => Reach failure for {:?} through {:?}: {:?}", peer_id, addr, error); - } - - fn inject_dial_failure(&mut self, peer_id: &PeerId) { - trace!(target: "sub-libp2p", "Libp2p => Dial failure for {:?}", peer_id); + if let DialError::Transport(errors) = error { + for (addr, error) in errors.iter() { + trace!(target: "sub-libp2p", "Libp2p => Reach failure for {:?} through {:?}: {:?}", peer_id, addr, error); + } + } - for set_id in (0..self.notif_protocols.len()).map(sc_peerset::SetId::from) { - if let Entry::Occupied(mut entry) = self.peers.entry((peer_id.clone(), set_id)) { - match mem::replace(entry.get_mut(), PeerState::Poisoned) { - // The peer is not in our list. - st @ PeerState::Backoff { .. } => { - *entry.into_mut() = st; - }, + if let Some(peer_id) = peer_id { + trace!(target: "sub-libp2p", "Libp2p => Dial failure for {:?}", peer_id); - // "Basic" situation: we failed to reach a peer that the peerset requested. - st @ PeerState::Requested | st @ PeerState::PendingRequest { .. } => { - trace!(target: "sub-libp2p", "PSM <= Dropped({}, {:?})", peer_id, set_id); - self.peerset.dropped(set_id, *peer_id, DropReason::Unknown); + for set_id in (0..self.notif_protocols.len()).map(sc_peerset::SetId::from) { + if let Entry::Occupied(mut entry) = self.peers.entry((peer_id.clone(), set_id)) { + match mem::replace(entry.get_mut(), PeerState::Poisoned) { + // The peer is not in our list. + st @ PeerState::Backoff { .. } => { + *entry.into_mut() = st; + }, - let now = Instant::now(); - let ban_duration = match st { - PeerState::PendingRequest { timer_deadline, .. } - if timer_deadline > now => - cmp::max(timer_deadline - now, Duration::from_secs(5)), - _ => Duration::from_secs(5), - }; + // "Basic" situation: we failed to reach a peer that the peerset requested. + st @ PeerState::Requested | st @ PeerState::PendingRequest { .. } => { + trace!(target: "sub-libp2p", "PSM <= Dropped({}, {:?})", peer_id, set_id); + self.peerset.dropped(set_id, peer_id, DropReason::Unknown); - let delay_id = self.next_delay_id; - self.next_delay_id.0 += 1; - let delay = futures_timer::Delay::new(ban_duration); - let peer_id = *peer_id; - self.delays.push( - async move { - delay.await; - (delay_id, peer_id, set_id) - } - .boxed(), - ); - - *entry.into_mut() = PeerState::Backoff { - timer: delay_id, - timer_deadline: now + ban_duration, - }; - }, + let now = Instant::now(); + let ban_duration = match st { + PeerState::PendingRequest { timer_deadline, .. } + if timer_deadline > now => + cmp::max(timer_deadline - now, Duration::from_secs(5)), + _ => Duration::from_secs(5), + }; - // We can still get dial failures even if we are already connected to the peer, - // as an extra diagnostic for an earlier attempt. - st @ PeerState::Disabled { .. } | - st @ PeerState::Enabled { .. } | - st @ PeerState::DisabledPendingEnable { .. } | - st @ PeerState::Incoming { .. } => { - *entry.into_mut() = st; - }, + let delay_id = self.next_delay_id; + self.next_delay_id.0 += 1; + let delay = futures_timer::Delay::new(ban_duration); + let peer_id = peer_id; + self.delays.push( + async move { + delay.await; + (delay_id, peer_id, set_id) + } + .boxed(), + ); - PeerState::Poisoned => { - error!(target: "sub-libp2p", "State of {:?} is poisoned", peer_id); - debug_assert!(false); - }, + *entry.into_mut() = PeerState::Backoff { + timer: delay_id, + timer_deadline: now + ban_duration, + }; + }, + + // We can still get dial failures even if we are already connected to the + // peer, as an extra diagnostic for an earlier attempt. + st @ PeerState::Disabled { .. } | + st @ PeerState::Enabled { .. } | + st @ PeerState::DisabledPendingEnable { .. } | + st @ PeerState::Incoming { .. } => { + *entry.into_mut() = st; + }, + + PeerState::Poisoned => { + error!(target: "sub-libp2p", "State of {:?} is poisoned", peer_id); + debug_assert!(false); + }, + } } } } @@ -1777,7 +1766,7 @@ impl NetworkBehaviour for Notifications { "Handler({}, {:?}) => CloseResult({:?})", source, connection, set_id); - match self.peers.get_mut(&(source.clone(), set_id)) { + match self.peers.get_mut(&(source, set_id)) { // Move the connection from `Closing` to `Closed`. Some(PeerState::Incoming { connections, .. }) | Some(PeerState::DisabledPendingEnable { connections, .. }) | @@ -2000,7 +1989,7 @@ impl NetworkBehaviour for Notifications { &mut self, cx: &mut Context, _params: &mut impl PollParameters, - ) -> Poll> { + ) -> Poll> { if let Some(event) = self.events.pop_front() { return Poll::Ready(event) } @@ -2032,6 +2021,8 @@ impl NetworkBehaviour for Notifications { while let Poll::Ready(Some((delay_id, peer_id, set_id))) = Pin::new(&mut self.delays).poll_next(cx) { + let handler = self.new_handler(); + let peer_state = match self.peers.get_mut(&(peer_id, set_id)) { Some(s) => s, // We intentionally never remove elements from `delays`, and it may @@ -2051,6 +2042,7 @@ impl NetworkBehaviour for Notifications { self.events.push_back(NetworkBehaviourAction::DialPeer { peer_id, condition: DialPeerCondition::Disconnected, + handler, }); *peer_state = PeerState::Requested; }, @@ -2085,7 +2077,7 @@ impl NetworkBehaviour for Notifications { .boxed(), ); } - } + }, // We intentionally never remove elements from `delays`, and it may // thus contain obsolete entries. This is a normal situation. diff --git a/client/network/src/protocol/notifications/handler.rs b/client/network/src/protocol/notifications/handler.rs index a0c49fa592b2..91225f54203a 100644 --- a/client/network/src/protocol/notifications/handler.rs +++ b/client/network/src/protocol/notifications/handler.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -365,10 +365,12 @@ struct NotificationsSinkInner { /// Sender to use in asynchronous contexts. Uses an asynchronous mutex. async_channel: FuturesMutex>, /// Sender to use in synchronous contexts. Uses a synchronous mutex. + /// Contains `None` if the channel was full at some point, in which case the channel will + /// be closed in the near future anyway. /// This channel has a large capacity and is meant to be used in contexts where /// back-pressure cannot be properly exerted. /// It will be removed in a future version. - sync_channel: Mutex>, + sync_channel: Mutex>>, } /// Message emitted through the [`NotificationsSink`] and processed by the background task @@ -400,14 +402,20 @@ impl NotificationsSink { /// This method will be removed in a future version. pub fn send_sync_notification<'a>(&'a self, message: impl Into>) { let mut lock = self.inner.sync_channel.lock(); - let result = - lock.try_send(NotificationsSinkMessage::Notification { message: message.into() }); - - if result.is_err() { - // Cloning the `mpsc::Sender` guarantees the allocation of an extra spot in the - // buffer, and therefore `try_send` will succeed. - let _result2 = lock.clone().try_send(NotificationsSinkMessage::ForceClose); - debug_assert!(_result2.map(|()| true).unwrap_or_else(|err| err.is_disconnected())); + + if let Some(tx) = lock.as_mut() { + let result = + tx.try_send(NotificationsSinkMessage::Notification { message: message.into() }); + + if result.is_err() { + // Cloning the `mpsc::Sender` guarantees the allocation of an extra spot in the + // buffer, and therefore `try_send` will succeed. + let _result2 = tx.clone().try_send(NotificationsSinkMessage::ForceClose); + debug_assert!(_result2.map(|()| true).unwrap_or_else(|err| err.is_disconnected())); + + // Destroy the sender in order to not send more `ForceClose` messages. + *lock = None; + } } } @@ -449,9 +457,9 @@ impl<'a> Ready<'a> { } /// Error specific to the collection of protocols. -#[derive(Debug, derive_more::Display, derive_more::Error)] +#[derive(Debug, thiserror::Error)] pub enum NotifsHandlerError { - /// Channel of synchronous notifications is full. + #[error("Channel of synchronous notifications is full.")] SyncNotificationsClogged, } @@ -554,7 +562,7 @@ impl ProtocolsHandler for NotifsHandler { inner: Arc::new(NotificationsSinkInner { peer_id: self.peer_id, async_channel: FuturesMutex::new(async_tx), - sync_channel: Mutex::new(sync_tx), + sync_channel: Mutex::new(Some(sync_tx)), }), }; diff --git a/client/network/src/protocol/notifications/tests.rs b/client/network/src/protocol/notifications/tests.rs index 0b3ffc01a4b8..73058598a1e3 100644 --- a/client/network/src/protocol/notifications/tests.rs +++ b/client/network/src/protocol/notifications/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -29,7 +29,7 @@ use libp2p::{ }, identity, noise, swarm::{ - IntoProtocolsHandler, NetworkBehaviour, NetworkBehaviourAction, PollParameters, + DialError, IntoProtocolsHandler, NetworkBehaviour, NetworkBehaviourAction, PollParameters, ProtocolsHandler, Swarm, SwarmEvent, }, yamux, Multiaddr, PeerId, Transport, @@ -68,7 +68,7 @@ fn build_nodes() -> (Swarm, Swarm) { in_peers: 25, out_peers: 25, bootnodes: if index == 0 { - keypairs.iter().skip(1).map(|keypair| keypair.public().into_peer_id()).collect() + keypairs.iter().skip(1).map(|keypair| keypair.public().to_peer_id()).collect() } else { vec![] }, @@ -92,7 +92,7 @@ fn build_nodes() -> (Swarm, Swarm) { .enumerate() .filter_map(|(n, a)| { if n != index { - Some((keypairs[n].public().into_peer_id(), a.clone())) + Some((keypairs[n].public().to_peer_id(), a.clone())) } else { None } @@ -100,7 +100,7 @@ fn build_nodes() -> (Swarm, Swarm) { .collect(), }; - let mut swarm = Swarm::new(transport, behaviour, keypairs[index].public().into_peer_id()); + let mut swarm = Swarm::new(transport, behaviour, keypairs[index].public().to_peer_id()); swarm.listen_on(addrs[index].clone()).unwrap(); out.push(swarm); } @@ -163,8 +163,10 @@ impl NetworkBehaviour for CustomProtoWithAddr { peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint, + failed_addresses: Option<&Vec>, ) { - self.inner.inject_connection_established(peer_id, conn, endpoint) + self.inner + .inject_connection_established(peer_id, conn, endpoint, failed_addresses) } fn inject_connection_closed( @@ -172,8 +174,9 @@ impl NetworkBehaviour for CustomProtoWithAddr { peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint, + handler: ::Handler, ) { - self.inner.inject_connection_closed(peer_id, conn, endpoint) + self.inner.inject_connection_closed(peer_id, conn, endpoint, handler) } fn inject_event( @@ -188,27 +191,18 @@ impl NetworkBehaviour for CustomProtoWithAddr { fn poll( &mut self, cx: &mut Context, - params: &mut impl PollParameters - ) -> Poll< - NetworkBehaviourAction< - <::Handler as ProtocolsHandler>::InEvent, - Self::OutEvent - > - >{ + params: &mut impl PollParameters, + ) -> Poll> { self.inner.poll(cx, params) } - fn inject_addr_reach_failure( + fn inject_dial_failure( &mut self, - peer_id: Option<&PeerId>, - addr: &Multiaddr, - error: &dyn std::error::Error, + peer_id: Option, + handler: Self::ProtocolsHandler, + error: &DialError, ) { - self.inner.inject_addr_reach_failure(peer_id, addr, error) - } - - fn inject_dial_failure(&mut self, peer_id: &PeerId) { - self.inner.inject_dial_failure(peer_id) + self.inner.inject_dial_failure(peer_id, handler, error) } fn inject_new_listener(&mut self, id: ListenerId) { diff --git a/client/network/src/protocol/notifications/upgrade.rs b/client/network/src/protocol/notifications/upgrade.rs index 196b4f44f81f..c273361acabd 100644 --- a/client/network/src/protocol/notifications/upgrade.rs +++ b/client/network/src/protocol/notifications/upgrade.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/network/src/protocol/notifications/upgrade/collec.rs b/client/network/src/protocol/notifications/upgrade/collec.rs index 2462d2becf4b..db9850c8da74 100644 --- a/client/network/src/protocol/notifications/upgrade/collec.rs +++ b/client/network/src/protocol/notifications/upgrade/collec.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/network/src/protocol/notifications/upgrade/notifications.rs b/client/network/src/protocol/notifications/upgrade/notifications.rs index 997a1ccf1dec..3fbb59d399a0 100644 --- a/client/network/src/protocol/notifications/upgrade/notifications.rs +++ b/client/network/src/protocol/notifications/upgrade/notifications.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -41,7 +41,7 @@ use libp2p::core::{upgrade, InboundUpgrade, OutboundUpgrade, UpgradeInfo}; use log::{error, warn}; use std::{ borrow::Cow, - convert::{Infallible, TryFrom as _}, + convert::Infallible, io, mem, pin::Pin, task::{Context, Poll}, @@ -457,13 +457,14 @@ where } /// Error generated by sending on a notifications out substream. -#[derive(Debug, derive_more::From, derive_more::Display)] +#[derive(Debug, thiserror::Error)] pub enum NotificationsHandshakeError { /// I/O error on the substream. - Io(io::Error), + #[error(transparent)] + Io(#[from] io::Error), /// Initial message or handshake was too large. - #[display(fmt = "Initial message or handshake was too large: {}", requested)] + #[error("Initial message or handshake was too large: {requested}")] TooLarge { /// Size requested by the remote. requested: usize, @@ -472,7 +473,8 @@ pub enum NotificationsHandshakeError { }, /// Error while decoding the variable-length integer. - VarintDecode(unsigned_varint::decode::Error), + #[error(transparent)] + VarintDecode(#[from] unsigned_varint::decode::Error), } impl From for NotificationsHandshakeError { @@ -489,10 +491,11 @@ impl From for NotificationsHandshakeError { } /// Error generated by sending on a notifications out substream. -#[derive(Debug, derive_more::From, derive_more::Display)] +#[derive(Debug, thiserror::Error)] pub enum NotificationsOutError { /// I/O error on the substream. - Io(io::Error), + #[error(transparent)] + Io(#[from] io::Error), } #[cfg(test)] diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 07f5f76fce7f..749366f6c165 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -141,12 +141,12 @@ mod rep { pub const BAD_RESPONSE: Rep = Rep::new(-(1 << 12), "Incomplete response"); } -enum PendingRequests { +enum AllowedRequests { Some(HashSet), All, } -impl PendingRequests { +impl AllowedRequests { fn add(&mut self, id: &PeerId) { if let Self::Some(ref mut set) = self { set.insert(*id); @@ -174,14 +174,24 @@ impl PendingRequests { Self::All => false, } } + + fn clear(&mut self) { + std::mem::take(self); + } } -impl Default for PendingRequests { +impl Default for AllowedRequests { fn default() -> Self { Self::Some(HashSet::default()) } } +struct GapSync { + blocks: BlockCollection, + best_queued_number: NumberFor, + target: NumberFor, +} + /// The main data structure which contains all the state for a chains /// active syncing strategy. pub struct ChainSync { @@ -205,7 +215,7 @@ pub struct ChainSync { /// Fork sync targets. fork_targets: HashMap>, /// A set of peers for which there might be potential block requests - pending_requests: PendingRequests, + allowed_requests: AllowedRequests, /// A type to check incoming block announcements. block_announce_validator: Box + Send>, /// Maximum number of peers to ask the same blocks in parallel. @@ -226,6 +236,8 @@ pub struct ChainSync { /// Enable importing existing blocks. This is used used after the state download to /// catch up to the latest state while re-importing blocks. import_existing: bool, + /// Gap download process. + gap_sync: Option>, } /// All the data we have about a Peer that we are trying to sync with @@ -298,6 +310,8 @@ pub enum PeerSyncState { DownloadingState, /// Downloading warp proof. DownloadingWarpProof, + /// Actively downloading block history after warp sync. + DownloadingGap(NumberFor), } impl PeerSyncState { @@ -326,7 +340,7 @@ pub struct StateDownloadProgress { /// Reported warp sync phase. #[derive(Clone, Eq, PartialEq, Debug)] -pub enum WarpSyncPhase { +pub enum WarpSyncPhase { /// Waiting for peers to connect. AwaitingPeers, /// Downloading and verifying grandpa warp proofs. @@ -335,24 +349,27 @@ pub enum WarpSyncPhase { DownloadingState, /// Importing state. ImportingState, + /// Downloading block history. + DownloadingBlocks(NumberFor), } -impl fmt::Display for WarpSyncPhase { +impl fmt::Display for WarpSyncPhase { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Self::AwaitingPeers => write!(f, "Waiting for peers"), Self::DownloadingWarpProofs => write!(f, "Downloading finality proofs"), Self::DownloadingState => write!(f, "Downloading state"), Self::ImportingState => write!(f, "Importing state"), + Self::DownloadingBlocks(n) => write!(f, "Downloading block history (#{})", n), } } } /// Reported warp sync progress. #[derive(Clone, Eq, PartialEq, Debug)] -pub struct WarpSyncProgress { +pub struct WarpSyncProgress { /// Estimated download percentage. - pub phase: WarpSyncPhase, + pub phase: WarpSyncPhase, /// Total bytes downloaded so far. pub total_bytes: u64, } @@ -371,7 +388,7 @@ pub struct Status { /// State sync status in progress, if any. pub state_sync: Option, /// Warp sync in progress, if any. - pub warp_sync: Option, + pub warp_sync: Option>, } /// A peer did not behave as expected and should be reported. @@ -413,16 +430,7 @@ pub enum OnStateData { /// The block and state that should be imported. Import(BlockOrigin, IncomingBlock), /// A new state request needs to be made to the given peer. - Request(PeerId, StateRequest), -} - -/// Result of [`ChainSync::on_warp_sync_data`]. -#[derive(Debug)] -pub enum OnWarpSyncData { - /// Warp proof request is issued. - WarpProofRequest(PeerId, warp::WarpProofRequest), - /// A new state request needs to be made to the given peer. - StateRequest(PeerId, StateRequest), + Continue, } /// Result of [`ChainSync::poll_block_announce_validation`]. @@ -545,7 +553,7 @@ impl ChainSync { mode, queue_blocks: Default::default(), fork_targets: Default::default(), - pending_requests: Default::default(), + allowed_requests: Default::default(), block_announce_validator, max_parallel_downloads, downloaded_blocks: 0, @@ -555,6 +563,7 @@ impl ChainSync { warp_sync: None, warp_sync_provider, import_existing: false, + gap_sync: None, }; sync.reset_sync_start_point()?; Ok(sync) @@ -608,10 +617,14 @@ impl ChainSync { SyncState::Idle }; - let warp_sync_progress = match (&self.warp_sync, &self.mode) { - (None, SyncMode::Warp) => + let warp_sync_progress = match (&self.warp_sync, &self.mode, &self.gap_sync) { + (_, _, Some(gap_sync)) => Some(WarpSyncProgress { + phase: WarpSyncPhase::DownloadingBlocks(gap_sync.best_queued_number), + total_bytes: 0, + }), + (None, SyncMode::Warp, _) => Some(WarpSyncProgress { phase: WarpSyncPhase::AwaitingPeers, total_bytes: 0 }), - (Some(sync), _) => Some(sync.progress()), + (Some(sync), _, _) => Some(sync.progress()), _ => None, }; @@ -639,6 +652,11 @@ impl ChainSync { self.downloaded_blocks } + /// Returns the current number of peers stored within this state machine. + pub fn num_peers(&self) -> usize { + self.peers.len() + } + /// Handle a new connected peer. /// /// Call this method whenever we connect to a new peer. @@ -651,7 +669,7 @@ impl ChainSync { // There is nothing sync can get from the node that has no blockchain data. match self.block_status(&best_hash) { Err(e) => { - debug!(target:"sync", "Error reading blockchain: {:?}", e); + debug!(target:"sync", "Error reading blockchain: {}", e); Err(BadPeer(who, rep::BLOCKCHAIN_READ_ERROR)) }, Ok(BlockStatus::KnownBad) => { @@ -686,17 +704,6 @@ impl ChainSync { return Ok(None) } - if let SyncMode::Warp = &self.mode { - if self.peers.len() >= MIN_PEERS_TO_START_WARP_SYNC && self.warp_sync.is_none() - { - log::debug!(target: "sync", "Starting warp state sync."); - if let Some(provider) = &self.warp_sync_provider { - self.warp_sync = - Some(WarpSync::new(self.client.clone(), provider.clone())); - } - } - } - // If we are at genesis, just start downloading. let (state, req) = if self.best_queued_number.is_zero() { debug!( @@ -727,7 +734,7 @@ impl ChainSync { ) }; - self.pending_requests.add(&who); + self.allowed_requests.add(&who); self.peers.insert( who, PeerSync { @@ -739,6 +746,17 @@ impl ChainSync { }, ); + if let SyncMode::Warp = &self.mode { + if self.peers.len() >= MIN_PEERS_TO_START_WARP_SYNC && self.warp_sync.is_none() + { + log::debug!(target: "sync", "Starting warp state sync."); + if let Some(provider) = &self.warp_sync_provider { + self.warp_sync = + Some(WarpSync::new(self.client.clone(), provider.clone())); + } + } + } + Ok(req) }, Ok(BlockStatus::Queued) | @@ -760,7 +778,7 @@ impl ChainSync { state: PeerSyncState::Available, }, ); - self.pending_requests.add(&who); + self.allowed_requests.add(&who); Ok(None) }, } @@ -827,7 +845,7 @@ impl ChainSync { peer.best_number = number; peer.best_hash = *hash; } - self.pending_requests.add(peer_id); + self.allowed_requests.add(peer_id); } } @@ -869,10 +887,13 @@ impl ChainSync { /// Get an iterator over all block requests of all peers. pub fn block_requests(&mut self) -> impl Iterator)> + '_ { - if self.pending_requests.is_empty() || self.state_sync.is_some() || self.warp_sync.is_some() + if self.allowed_requests.is_empty() || + self.state_sync.is_some() || + self.mode == SyncMode::Warp { return Either::Left(std::iter::empty()) } + if self.queue_blocks.len() > MAX_IMPORTING_BLOCKS { trace!(target: "sync", "Too many blocks in the queue."); return Either::Left(std::iter::empty()) @@ -886,10 +907,11 @@ impl ChainSync { let best_queued = self.best_queued_number; let client = &self.client; let queue = &self.queue_blocks; - let pending_requests = self.pending_requests.take(); + let allowed_requests = self.allowed_requests.take(); let max_parallel = if major_sync { 1 } else { self.max_parallel_downloads }; + let gap_sync = &mut self.gap_sync; let iter = self.peers.iter_mut().filter_map(move |(id, peer)| { - if !peer.state.is_available() || !pending_requests.contains(id) { + if !peer.state.is_available() || !allowed_requests.contains(id) { return None } @@ -947,6 +969,26 @@ impl ChainSync { trace!(target: "sync", "Downloading fork {:?} from {}", hash, id); peer.state = PeerSyncState::DownloadingStale(hash); Some((id, req)) + } else if let Some((range, req)) = gap_sync.as_mut().and_then(|sync| { + peer_gap_block_request( + id, + peer, + &mut sync.blocks, + attrs, + sync.target, + sync.best_queued_number, + ) + }) { + peer.state = PeerSyncState::DownloadingGap(range.start); + trace!( + target: "sync", + "New gap block request for {}, (best:{}, common:{}) {:?}", + id, + peer.best_number, + peer.common_number, + req, + ); + Some((id, req)) } else { None } @@ -956,7 +998,12 @@ impl ChainSync { /// Get a state request, if any. pub fn state_request(&mut self) -> Option<(PeerId, StateRequest)> { - if self.peers.iter().any(|(_, peer)| peer.state == PeerSyncState::DownloadingState) { + if self.allowed_requests.is_empty() { + return None + } + if (self.state_sync.is_some() || self.warp_sync.is_some()) && + self.peers.iter().any(|(_, peer)| peer.state == PeerSyncState::DownloadingState) + { // Only one pending state request is allowed. return None } @@ -964,11 +1011,13 @@ impl ChainSync { if sync.is_complete() { return None } + for (id, peer) in self.peers.iter_mut() { if peer.state.is_available() && peer.common_number >= sync.target_block_num() { - trace!(target: "sync", "New StateRequest for {}", id); peer.state = PeerSyncState::DownloadingState; let request = sync.next_request(); + trace!(target: "sync", "New StateRequest for {}: {:?}", id, request); + self.allowed_requests.clear(); return Some((*id, request)) } } @@ -982,8 +1031,9 @@ impl ChainSync { { for (id, peer) in self.peers.iter_mut() { if peer.state.is_available() && peer.best_number >= target { - trace!(target: "sync", "New StateRequest for {}", id); + trace!(target: "sync", "New StateRequest for {}: {:?}", id, request); peer.state = PeerSyncState::DownloadingState; + self.allowed_requests.clear(); return Some((*id, request)) } } @@ -994,16 +1044,14 @@ impl ChainSync { /// Get a warp sync request, if any. pub fn warp_sync_request(&mut self) -> Option<(PeerId, WarpProofRequest)> { - if self - .peers - .iter() - .any(|(_, peer)| peer.state == PeerSyncState::DownloadingWarpProof) - { - // Only one pending state request is allowed. - return None - } if let Some(sync) = &self.warp_sync { - if sync.is_complete() { + if self.allowed_requests.is_empty() || + sync.is_complete() || + self.peers + .iter() + .any(|(_, peer)| peer.state == PeerSyncState::DownloadingWarpProof) + { + // Only one pending state request is allowed. return None } if let Some(request) = sync.next_warp_poof_request() { @@ -1016,6 +1064,7 @@ impl ChainSync { if peer.state.is_available() && peer.best_number >= median { trace!(target: "sync", "New WarpProofRequest for {}", id); peer.state = PeerSyncState::DownloadingWarpProof; + self.allowed_requests.clear(); return Some((*id, request)) } } @@ -1039,6 +1088,7 @@ impl ChainSync { response: BlockResponse, ) -> Result, BadPeer> { self.downloaded_blocks += response.blocks.len(); + let mut gap = false; let new_blocks: Vec> = if let Some(peer) = self.peers.get_mut(who) { let mut blocks = response.blocks; if request @@ -1048,7 +1098,7 @@ impl ChainSync { trace!(target: "sync", "Reversing incoming block list"); blocks.reverse() } - self.pending_requests.add(who); + self.allowed_requests.add(who); if let Some(request) = request { match &mut peer.state { PeerSyncState::DownloadingNew(_) => { @@ -1061,6 +1111,47 @@ impl ChainSync { } self.drain_blocks() }, + PeerSyncState::DownloadingGap(_) => { + peer.state = PeerSyncState::Available; + if let Some(gap_sync) = &mut self.gap_sync { + gap_sync.blocks.clear_peer_download(who); + if let Some(start_block) = + validate_blocks::(&blocks, who, Some(request))? + { + gap_sync.blocks.insert(start_block, blocks, who.clone()); + } + gap = true; + let blocks: Vec<_> = gap_sync + .blocks + .drain(gap_sync.best_queued_number + One::one()) + .into_iter() + .map(|block_data| { + let justifications = block_data.block.justifications.or( + legacy_justification_mapping( + block_data.block.justification, + ), + ); + IncomingBlock { + hash: block_data.block.hash, + header: block_data.block.header, + body: block_data.block.body, + indexed_body: block_data.block.indexed_body, + justifications, + origin: block_data.origin, + allow_missing_state: true, + import_existing: self.import_existing, + skip_execution: true, + state: None, + } + }) + .collect(); + debug!(target: "sync", "Drained {} gap blocks from {}", blocks.len(), gap_sync.best_queued_number); + blocks + } else { + debug!(target: "sync", "Unexpected gap block response from {}", who); + return Err(BadPeer(who.clone(), rep::NO_BLOCK)) + } + }, PeerSyncState::DownloadingStale(_) => { peer.state = PeerSyncState::Available; if blocks.is_empty() { @@ -1112,7 +1203,7 @@ impl ChainSync { (_, Err(e)) => { info!( target: "sync", - "❌ Error answering legitimate blockchain query: {:?}", + "❌ Error answering legitimate blockchain query: {}", e, ); return Err(BadPeer(*who, rep::BLOCKCHAIN_READ_ERROR)) @@ -1212,7 +1303,7 @@ impl ChainSync { return Err(BadPeer(*who, rep::NOT_REQUESTED)) }; - Ok(self.validate_and_queue_blocks(new_blocks)) + Ok(self.validate_and_queue_blocks(new_blocks, gap)) } /// Handle a response from the remote to a state request that we made. @@ -1223,6 +1314,12 @@ impl ChainSync { who: &PeerId, response: StateResponse, ) -> Result, BadPeer> { + if let Some(peer) = self.peers.get_mut(&who) { + if let PeerSyncState::DownloadingState = peer.state { + peer.state = PeerSyncState::Available; + self.allowed_requests.set_all(); + } + } let import_result = if let Some(sync) = &mut self.state_sync { debug!( target: "sync", @@ -1261,11 +1358,10 @@ impl ChainSync { skip_execution: self.skip_execution(), state: Some(state), }; - debug!(target: "sync", "State sync is complete. Import is queued"); + debug!(target: "sync", "State download is complete. Import is queued"); Ok(OnStateData::Import(origin, block)) }, - state::ImportResult::Continue(request) => - Ok(OnStateData::Request(who.clone(), request)), + state::ImportResult::Continue => Ok(OnStateData::Continue), state::ImportResult::BadResponse => { debug!(target: "sync", "Bad state data received from {}", who); Err(BadPeer(*who, rep::BAD_BLOCK)) @@ -1280,7 +1376,13 @@ impl ChainSync { &mut self, who: &PeerId, response: warp::EncodedProof, - ) -> Result, BadPeer> { + ) -> Result<(), BadPeer> { + if let Some(peer) = self.peers.get_mut(&who) { + if let PeerSyncState::DownloadingWarpProof = peer.state { + peer.state = PeerSyncState::Available; + self.allowed_requests.set_all(); + } + } let import_result = if let Some(sync) = &mut self.warp_sync { debug!( target: "sync", @@ -1295,10 +1397,7 @@ impl ChainSync { }; match import_result { - warp::WarpProofImportResult::StateRequest(request) => - Ok(OnWarpSyncData::StateRequest(*who, request)), - warp::WarpProofImportResult::WarpProofRequest(request) => - Ok(OnWarpSyncData::WarpProofRequest(*who, request)), + warp::WarpProofImportResult::Success => Ok(()), warp::WarpProofImportResult::BadResponse => { debug!(target: "sync", "Bad proof data received from {}", who); Err(BadPeer(*who, rep::BAD_BLOCK)) @@ -1309,6 +1408,7 @@ impl ChainSync { fn validate_and_queue_blocks( &mut self, mut new_blocks: Vec>, + gap: bool, ) -> OnBlockData { let orig_len = new_blocks.len(); new_blocks.retain(|b| !self.queue_blocks.contains(&b.hash)); @@ -1320,7 +1420,7 @@ impl ChainSync { ); } - let origin = if self.status().state != SyncState::Downloading { + let origin = if !gap && self.status().state != SyncState::Downloading { BlockOrigin::NetworkBroadcast } else { BlockOrigin::NetworkInitialSync @@ -1361,7 +1461,7 @@ impl ChainSync { return Ok(OnBlockJustification::Nothing) }; - self.pending_requests.add(&who); + self.allowed_requests.add(&who); if let PeerSyncState::DownloadingJustification(hash) = peer.state { peer.state = PeerSyncState::Available; @@ -1494,6 +1594,15 @@ impl ChainSync { self.mode = SyncMode::Full; output.extend(self.restart()); } + let gap_sync_complete = + self.gap_sync.as_ref().map_or(false, |s| s.target == number); + if gap_sync_complete { + info!( + target: "sync", + "Block history download is complete." + ); + self.gap_sync = None; + } }, Err(BlockImportError::IncompleteHeader(who)) => if let Some(peer) = who { @@ -1533,7 +1642,7 @@ impl ChainSync { trace!(target: "sync", "Obsolete block {:?}", hash); }, e @ Err(BlockImportError::UnknownParent) | e @ Err(BlockImportError::Other(_)) => { - warn!(target: "sync", "💔 Error importing block {:?}: {:?}", hash, e); + warn!(target: "sync", "💔 Error importing block {:?}: {}", hash, e.unwrap_err()); self.state_sync = None; self.warp_sync = None; output.extend(self.restart()); @@ -1542,7 +1651,7 @@ impl ChainSync { }; } - self.pending_requests.set_all(); + self.allowed_requests.set_all(); output.into_iter() } @@ -1552,7 +1661,7 @@ impl ChainSync { let finalization_result = if success { Ok((hash, number)) } else { Err(()) }; self.extra_justifications .try_finalize_root((hash, number), finalization_result, true); - self.pending_requests.set_all(); + self.allowed_requests.set_all(); } /// Notify about finalization of the given block. @@ -1579,6 +1688,7 @@ impl ChainSync { ); self.state_sync = Some(StateSync::new(self.client.clone(), header, *skip_proofs)); + self.allowed_requests.set_all(); } } } @@ -1587,7 +1697,7 @@ impl ChainSync { if let Err(err) = r { warn!( target: "sync", - "💔 Error cleaning up pending extra justification data requests: {:?}", + "💔 Error cleaning up pending extra justification data requests: {}", err, ); } @@ -1601,6 +1711,11 @@ impl ChainSync { if self.fork_targets.remove(&hash).is_some() { trace!(target: "sync", "Completed fork sync {:?}", hash); } + if let Some(gap_sync) = &mut self.gap_sync { + if number > gap_sync.best_queued_number && number <= gap_sync.target { + gap_sync.best_queued_number = number; + } + } if number > self.best_queued_number { self.best_queued_number = number; self.best_queued_hash = *hash; @@ -1624,7 +1739,7 @@ impl ChainSync { peer.common_number = new_common_number; } } - self.pending_requests.set_all(); + self.allowed_requests.set_all(); } /// Checks if there is a slot for a block announce validation. @@ -1893,7 +2008,7 @@ impl ChainSync { peer.update_common_number(number - One::one()); } } - self.pending_requests.add(&who); + self.allowed_requests.add(&who); // known block case if known || self.is_already_downloading(&hash) { @@ -1954,16 +2069,19 @@ impl ChainSync { /// import, so this functions checks for such blocks and returns them. pub fn peer_disconnected(&mut self, who: &PeerId) -> Option> { self.blocks.clear_peer_download(who); + if let Some(gap_sync) = &mut self.gap_sync { + gap_sync.blocks.clear_peer_download(who) + } self.peers.remove(who); self.extra_justifications.peer_disconnected(who); - self.pending_requests.set_all(); + self.allowed_requests.set_all(); self.fork_targets.retain(|_, target| { target.peers.remove(who); !target.peers.is_empty() }); let blocks = self.drain_blocks(); if !blocks.is_empty() { - Some(self.validate_and_queue_blocks(blocks)) + Some(self.validate_and_queue_blocks(blocks, false)) } else { None } @@ -1977,9 +2095,9 @@ impl ChainSync { ) -> impl Iterator), BadPeer>> + 'a { self.blocks.clear(); if let Err(e) = self.reset_sync_start_point() { - warn!(target: "sync", "💔 Unable to restart sync. :{:?}", e); + warn!(target: "sync", "💔 Unable to restart sync: {}", e); } - self.pending_requests.set_all(); + self.allowed_requests.set_all(); debug!(target:"sync", "Restarted with {} ({})", self.best_queued_number, self.best_queued_hash); let old_peers = std::mem::take(&mut self.peers); @@ -2043,6 +2161,14 @@ impl ChainSync { } } } + if let Some((start, end)) = info.block_gap { + debug!(target: "sync", "Starting gap sync #{} - #{}", start, end); + self.gap_sync = Some(GapSync { + best_queued_number: start - One::one(), + target: end, + blocks: BlockCollection::new(), + }); + } trace!(target: "sync", "Restarted sync at #{} ({:?})", self.best_queued_number, self.best_queued_hash); Ok(()) } @@ -2069,7 +2195,6 @@ impl ChainSync { /// Return some key metrics. pub(crate) fn metrics(&self) -> Metrics { - use std::convert::TryInto; Metrics { queued_blocks: self.queue_blocks.len().try_into().unwrap_or(std::u32::MAX), fork_targets: self.fork_targets.len().try_into().unwrap_or(std::u32::MAX), @@ -2195,7 +2320,11 @@ fn handle_ancestor_search_state( } assert!(right >= left); let middle = left + (right - left) / two; - Some((AncestorSearchState::BinarySearch(left, right), middle)) + if middle == curr_block_num { + None + } else { + Some((AncestorSearchState::BinarySearch(left, right), middle)) + } }, } } @@ -2250,6 +2379,39 @@ fn peer_block_request( Some((range, request)) } +/// Get a new block request for the peer if any. +fn peer_gap_block_request( + id: &PeerId, + peer: &PeerSync, + blocks: &mut BlockCollection, + attrs: message::BlockAttributes, + target: NumberFor, + common_number: NumberFor, +) -> Option<(Range>, BlockRequest)> { + let range = blocks.needed_blocks( + id.clone(), + MAX_BLOCKS_TO_REQUEST, + std::cmp::min(peer.best_number, target), + common_number, + 1, + MAX_DOWNLOAD_AHEAD, + )?; + + // The end is not part of the range. + let last = range.end.saturating_sub(One::one()); + let from = message::FromBlock::Number(last); + + let request = message::generic::BlockRequest { + id: 0, + fields: attrs.clone(), + from, + to: None, + direction: message::Direction::Descending, + max: Some((range.end - range.start).saturated_into::()), + }; + Some((range, request)) +} + /// Get pending fork sync targets for a peer. fn fork_sync_request( id: &PeerId, @@ -2274,7 +2436,11 @@ fn fork_sync_request( if !r.peers.contains(id) { continue } - if r.number <= best_num { + // Download the fork only if it is behind or not too far ahead our tip of the chain + // Otherwise it should be downloaded in full sync mode. + if r.number <= best_num || + (r.number - best_num).saturated_into::() < MAX_BLOCKS_TO_REQUEST as u32 + { let parent_status = r.parent_hash.as_ref().map_or(BlockStatus::Unknown, check_block); let count = if parent_status == BlockStatus::Unknown { (r.number - finalized).saturated_into::() // up to the last finalized block @@ -2294,6 +2460,8 @@ fn fork_sync_request( max: Some(count), }, )) + } else { + trace!(target: "sync", "Fork too far in the future: {:?} (#{})", hash, r.number); } } None @@ -2404,8 +2572,10 @@ fn validate_blocks( } if let (Some(header), Some(body)) = (&b.header, &b.body) { let expected = *header.extrinsics_root(); - let got = - HashFor::::ordered_trie_root(body.iter().map(Encode::encode).collect()); + let got = HashFor::::ordered_trie_root( + body.iter().map(Encode::encode).collect(), + sp_runtime::StateVersion::V0, + ); if expected != got { debug!( target:"sync", @@ -3032,4 +3202,49 @@ mod test { sync.peer_disconnected(&peer_id1); assert!(sync.fork_targets.len() == 0); } + + #[test] + fn can_import_response_with_missing_blocks() { + sp_tracing::try_init_simple(); + let mut client2 = Arc::new(TestClientBuilder::new().build()); + let blocks = (0..4).map(|_| build_block(&mut client2, None, false)).collect::>(); + + let empty_client = Arc::new(TestClientBuilder::new().build()); + + let mut sync = ChainSync::new( + SyncMode::Full, + empty_client.clone(), + Box::new(DefaultBlockAnnounceValidator), + 1, + None, + ) + .unwrap(); + + let peer_id1 = PeerId::random(); + let best_block = blocks[3].clone(); + sync.new_peer(peer_id1.clone(), best_block.hash(), *best_block.header().number()) + .unwrap(); + + sync.peers.get_mut(&peer_id1).unwrap().state = PeerSyncState::Available; + sync.peers.get_mut(&peer_id1).unwrap().common_number = 0; + + // Request all missing blocks and respond only with some. + let request = + get_block_request(&mut sync, FromBlock::Hash(best_block.hash()), 4, &peer_id1); + let response = + create_block_response(vec![blocks[3].clone(), blocks[2].clone(), blocks[1].clone()]); + sync.on_block_data(&peer_id1, Some(request.clone()), response).unwrap(); + assert_eq!(sync.best_queued_number, 0); + + // Request should only contain the missing block. + let request = get_block_request(&mut sync, FromBlock::Number(1), 1, &peer_id1); + let response = create_block_response(vec![blocks[0].clone()]); + sync.on_block_data(&peer_id1, Some(request), response).unwrap(); + assert_eq!(sync.best_queued_number, 4); + } + #[test] + fn ancestor_search_repeat() { + let state = AncestorSearchState::::BinarySearch(1, 3); + assert!(handle_ancestor_search_state(&state, 2, true).is_none()); + } } diff --git a/client/network/src/protocol/sync/blocks.rs b/client/network/src/protocol/sync/blocks.rs index 30ba7ffafeff..43b70d17d8ad 100644 --- a/client/network/src/protocol/sync/blocks.rs +++ b/client/network/src/protocol/sync/blocks.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -203,7 +203,7 @@ impl BlockCollection { { *downloading -= 1; false - } + }, Some(&mut BlockRangeState::Downloading { .. }) => true, _ => false, }; diff --git a/client/network/src/protocol/sync/extra_requests.rs b/client/network/src/protocol/sync/extra_requests.rs index 226762b9658d..d0bfebab6601 100644 --- a/client/network/src/protocol/sync/extra_requests.rs +++ b/client/network/src/protocol/sync/extra_requests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -108,7 +108,7 @@ impl ExtraRequests { // ignore the `Revert` error. }, Err(err) => { - debug!(target: "sync", "Failed to insert request {:?} into tree: {:?}", request, err); + debug!(target: "sync", "Failed to insert request {:?} into tree: {}", request, err); }, _ => (), } @@ -173,9 +173,7 @@ impl ExtraRequests { } if best_finalized_number > self.best_seen_finalized_number { - // normally we'll receive finality notifications for every block => finalize would be - // enough but if many blocks are finalized at once, some notifications may be omitted - // => let's use finalize_with_ancestors here + // we receive finality notification only for the finalized branch head. match self.tree.finalize_with_ancestors( best_finalized_hash, best_finalized_number, @@ -255,7 +253,6 @@ impl ExtraRequests { /// Get some key metrics. pub(crate) fn metrics(&self) -> Metrics { - use std::convert::TryInto; Metrics { pending_requests: self.pending_requests.len().try_into().unwrap_or(std::u32::MAX), active_requests: self.active_requests.len().try_into().unwrap_or(std::u32::MAX), diff --git a/client/network/src/protocol/sync/state.rs b/client/network/src/protocol/sync/state.rs index d2e4463f9891..0df862a48333 100644 --- a/client/network/src/protocol/sync/state.rs +++ b/client/network/src/protocol/sync/state.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,9 +23,11 @@ use crate::{ }; use codec::{Decode, Encode}; use log::debug; -use sc_client_api::StorageProof; +use sc_client_api::CompactProof; +use smallvec::SmallVec; +use sp_core::storage::well_known_keys; use sp_runtime::traits::{Block as BlockT, Header, NumberFor}; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; /// State sync support. @@ -35,8 +37,8 @@ pub struct StateSync { target_block: B::Hash, target_header: B::Header, target_root: B::Hash, - last_key: Vec, - state: Vec<(Vec, Vec)>, + last_key: SmallVec<[Vec; 2]>, + state: HashMap, (Vec<(Vec, Vec)>, Vec>)>, complete: bool, client: Arc>, imported_bytes: u64, @@ -47,8 +49,8 @@ pub struct StateSync { pub enum ImportResult { /// State is complete and ready for import. Import(B::Hash, B::Header, ImportedState), - /// Continue dowloading. - Continue(StateRequest), + /// Continue downloading. + Continue, /// Bad state chunk. BadResponse, } @@ -61,8 +63,8 @@ impl StateSync { target_block: target.hash(), target_root: target.state_root().clone(), target_header: target, - last_key: Vec::default(), - state: Vec::default(), + last_key: SmallVec::default(), + state: HashMap::default(), complete: false, imported_bytes: 0, skip_proof, @@ -71,7 +73,7 @@ impl StateSync { /// Validate and import a state reponse. pub fn import(&mut self, response: StateResponse) -> ImportResult { - if response.entries.is_empty() && response.proof.is_empty() && !response.complete { + if response.entries.is_empty() && response.proof.is_empty() { debug!(target: "sync", "Bad state response"); return ImportResult::BadResponse } @@ -82,59 +84,138 @@ impl StateSync { let complete = if !self.skip_proof { debug!(target: "sync", "Importing state from {} trie nodes", response.proof.len()); let proof_size = response.proof.len() as u64; - let proof = match StorageProof::decode(&mut response.proof.as_ref()) { + let proof = match CompactProof::decode(&mut response.proof.as_ref()) { Ok(proof) => proof, Err(e) => { debug!(target: "sync", "Error decoding proof: {:?}", e); return ImportResult::BadResponse }, }; - let (values, complete) = - match self.client.verify_range_proof(self.target_root, proof, &self.last_key) { - Err(e) => { - debug!(target: "sync", "StateResponse failed proof verification: {:?}", e); - return ImportResult::BadResponse - }, - Ok(values) => values, - }; + let (values, completed) = match self.client.verify_range_proof( + self.target_root, + proof, + self.last_key.as_slice(), + ) { + Err(e) => { + debug!( + target: "sync", + "StateResponse failed proof verification: {}", + e, + ); + return ImportResult::BadResponse + }, + Ok(values) => values, + }; debug!(target: "sync", "Imported with {} keys", values.len()); - if let Some(last) = values.last().map(|(k, _)| k) { - self.last_key = last.clone(); - } + let complete = completed == 0; + if !complete && !values.update_last_key(completed, &mut self.last_key) { + debug!(target: "sync", "Error updating key cursor, depth: {}", completed); + }; - for (key, value) in values { - self.imported_bytes += key.len() as u64; - self.state.push((key, value)) + for values in values.0 { + let key_values = if values.state_root.is_empty() { + // Read child trie roots. + values + .key_values + .into_iter() + .filter(|key_value| { + if well_known_keys::is_child_storage_key(key_value.0.as_slice()) { + self.state + .entry(key_value.1.clone()) + .or_default() + .1 + .push(key_value.0.clone()); + false + } else { + true + } + }) + .collect() + } else { + values.key_values + }; + let mut entry = self.state.entry(values.state_root).or_default(); + if entry.0.len() > 0 && entry.1.len() > 1 { + // Already imported child_trie with same root. + // Warning this will not work with parallel download. + } else { + if entry.0.is_empty() { + for (key, _value) in key_values.iter() { + self.imported_bytes += key.len() as u64; + } + + entry.0 = key_values; + } else { + for (key, value) in key_values { + self.imported_bytes += key.len() as u64; + entry.0.push((key, value)) + } + } + } } self.imported_bytes += proof_size; complete } else { - debug!( - target: "sync", - "Importing state from {:?} to {:?}", - response.entries.last().map(|e| sp_core::hexdisplay::HexDisplay::from(&e.key)), - response.entries.first().map(|e| sp_core::hexdisplay::HexDisplay::from(&e.key)), - ); - - if let Some(e) = response.entries.last() { - self.last_key = e.key.clone(); + let mut complete = true; + // if the trie is a child trie and one of its parent trie is empty, + // the parent cursor stays valid. + // Empty parent trie content only happens when all the response content + // is part of a single child trie. + if self.last_key.len() == 2 && response.entries[0].entries.len() == 0 { + // Do not remove the parent trie position. + self.last_key.pop(); + } else { + self.last_key.clear(); } - for StateEntry { key, value } in response.entries { - self.imported_bytes += (key.len() + value.len()) as u64; - self.state.push((key, value)) + for state in response.entries { + debug!( + target: "sync", + "Importing state from {:?} to {:?}", + state.entries.last().map(|e| sp_core::hexdisplay::HexDisplay::from(&e.key)), + state.entries.first().map(|e| sp_core::hexdisplay::HexDisplay::from(&e.key)), + ); + + if !state.complete { + if let Some(e) = state.entries.last() { + self.last_key.push(e.key.clone()); + } + complete = false; + } + let is_top = state.state_root.is_empty(); + let entry = self.state.entry(state.state_root).or_default(); + if entry.0.len() > 0 && entry.1.len() > 1 { + // Already imported child trie with same root. + } else { + let mut child_roots = Vec::new(); + for StateEntry { key, value } in state.entries { + // Skip all child key root (will be recalculated on import). + if is_top && well_known_keys::is_child_storage_key(key.as_slice()) { + child_roots.push((value, key)); + } else { + self.imported_bytes += key.len() as u64; + entry.0.push((key, value)) + } + } + for (root, storage_key) in child_roots { + self.state.entry(root).or_default().1.push(storage_key); + } + } } - response.complete + complete }; if complete { self.complete = true; ImportResult::Import( self.target_block, self.target_header.clone(), - ImportedState { block: self.target_block, state: std::mem::take(&mut self.state) }, + ImportedState { + block: self.target_block.clone(), + state: std::mem::take(&mut self.state).into(), + }, ) } else { - ImportResult::Continue(self.next_request()) + ImportResult::Continue } } @@ -142,7 +223,7 @@ impl StateSync { pub fn next_request(&self) -> StateRequest { StateRequest { block: self.target_block.encode(), - start: self.last_key.clone(), + start: self.last_key.clone().into_vec(), no_proof: self.skip_proof, } } @@ -164,7 +245,8 @@ impl StateSync { /// Returns state sync estimated progress. pub fn progress(&self) -> StateDownloadProgress { - let percent_done = (*self.last_key.get(0).unwrap_or(&0u8) as u32) * 100 / 256; + let cursor = *self.last_key.get(0).and_then(|last| last.get(0)).unwrap_or(&0u8); + let percent_done = cursor as u32 * 100 / 256; StateDownloadProgress { percentage: percent_done, size: self.imported_bytes } } } diff --git a/client/network/src/protocol/sync/warp.rs b/client/network/src/protocol/sync/warp.rs index 32bd5cb9ed79..f12deb2dbb43 100644 --- a/client/network/src/protocol/sync/warp.rs +++ b/client/network/src/protocol/sync/warp.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -37,11 +37,9 @@ enum Phase { } /// Import warp proof result. -pub enum WarpProofImportResult { - /// Start downloading state data. - StateRequest(StateRequest), - /// Continue dowloading warp sync proofs. - WarpProofRequest(WarpProofRequest), +pub enum WarpProofImportResult { + /// Import was successful. + Success, /// Bad proof. BadResponse, } @@ -69,7 +67,7 @@ impl WarpSync { Self { client, warp_sync_provider, phase, total_proof_bytes: 0 } } - /// Validate and import a state reponse. + /// Validate and import a state response. pub fn import_state(&mut self, response: StateResponse) -> ImportResult { match &mut self.phase { Phase::WarpProof { .. } => { @@ -80,21 +78,17 @@ impl WarpSync { } } - /// Validate and import a warp proof reponse. - pub fn import_warp_proof(&mut self, response: EncodedProof) -> WarpProofImportResult { + /// Validate and import a warp proof response. + pub fn import_warp_proof(&mut self, response: EncodedProof) -> WarpProofImportResult { match &mut self.phase { Phase::State(_) => { log::debug!(target: "sync", "Unexpected warp proof response"); WarpProofImportResult::BadResponse }, Phase::WarpProof { set_id, authorities, last_hash } => { - match self.warp_sync_provider.verify( - &response, - *set_id, - std::mem::take(authorities), - ) { + match self.warp_sync_provider.verify(&response, *set_id, authorities.clone()) { Err(e) => { - log::debug!(target: "sync", "Bad warp proof response: {:?}", e); + log::debug!(target: "sync", "Bad warp proof response: {}", e); return WarpProofImportResult::BadResponse }, Ok(VerificationResult::Partial(new_set_id, new_authorities, new_last_hash)) => { @@ -103,17 +97,14 @@ impl WarpSync { *authorities = new_authorities; *last_hash = new_last_hash.clone(); self.total_proof_bytes += response.0.len() as u64; - WarpProofImportResult::WarpProofRequest(WarpProofRequest { - begin: new_last_hash, - }) + WarpProofImportResult::Success }, Ok(VerificationResult::Complete(new_set_id, _, header)) => { log::debug!(target: "sync", "Verified complete proof, set_id={:?}", new_set_id); self.total_proof_bytes += response.0.len() as u64; let state_sync = StateSync::new(self.client.clone(), header, false); - let request = state_sync.next_request(); self.phase = Phase::State(state_sync); - WarpProofImportResult::StateRequest(request) + WarpProofImportResult::Success }, } }, @@ -161,7 +152,7 @@ impl WarpSync { } /// Returns state sync estimated progress (percentage, bytes) - pub fn progress(&self) -> WarpSyncProgress { + pub fn progress(&self) -> WarpSyncProgress { match &self.phase { Phase::WarpProof { .. } => WarpSyncProgress { phase: WarpSyncPhase::DownloadingWarpProofs, diff --git a/client/network/src/request_responses.rs b/client/network/src/request_responses.rs index 0908d7510e35..4613a15af936 100644 --- a/client/network/src/request_responses.rs +++ b/client/network/src/request_responses.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -45,18 +45,17 @@ use libp2p::{ ConnectedPoint, Multiaddr, PeerId, }, request_response::{ - ProtocolSupport, RequestResponse, RequestResponseCodec, RequestResponseConfig, - RequestResponseEvent, RequestResponseMessage, ResponseChannel, + handler::RequestResponseHandler, ProtocolSupport, RequestResponse, RequestResponseCodec, + RequestResponseConfig, RequestResponseEvent, RequestResponseMessage, ResponseChannel, }, swarm::{ - protocols_handler::multi::MultiHandler, NetworkBehaviour, NetworkBehaviourAction, - PollParameters, ProtocolsHandler, + protocols_handler::multi::MultiHandler, IntoProtocolsHandler, NetworkBehaviour, + NetworkBehaviourAction, PollParameters, ProtocolsHandler, }, }; use std::{ borrow::Cow, collections::{hash_map::Entry, HashMap}, - convert::TryFrom as _, io, iter, pin::Pin, task::{Context, Poll}, @@ -377,6 +376,27 @@ impl RequestResponsesBehaviour { }; } } + + fn new_handler_with_replacement( + &mut self, + protocol: String, + handler: RequestResponseHandler, + ) -> ::ProtocolsHandler { + let mut handlers: HashMap<_, _> = self + .protocols + .iter_mut() + .map(|(p, (r, _))| (p.to_string(), NetworkBehaviour::new_handler(r))) + .collect(); + + if let Some(h) = handlers.get_mut(&protocol) { + *h = handler + } + + MultiHandler::try_from_iter(handlers).expect( + "Protocols are in a HashMap and there can be at most one handler per protocol name, \ + which is the only possible error; qed", + ) + } } impl NetworkBehaviour for RequestResponsesBehaviour { @@ -405,9 +425,16 @@ impl NetworkBehaviour for RequestResponsesBehaviour { peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint, + failed_addresses: Option<&Vec>, ) { for (p, _) in self.protocols.values_mut() { - NetworkBehaviour::inject_connection_established(p, peer_id, conn, endpoint) + NetworkBehaviour::inject_connection_established( + p, + peer_id, + conn, + endpoint, + failed_addresses, + ) } } @@ -422,9 +449,11 @@ impl NetworkBehaviour for RequestResponsesBehaviour { peer_id: &PeerId, conn: &ConnectionId, endpoint: &ConnectedPoint, + _handler: ::Handler, ) { for (p, _) in self.protocols.values_mut() { - NetworkBehaviour::inject_connection_closed(p, peer_id, conn, endpoint) + let handler = p.new_handler(); + NetworkBehaviour::inject_connection_closed(p, peer_id, conn, endpoint, handler); } } @@ -434,17 +463,6 @@ impl NetworkBehaviour for RequestResponsesBehaviour { } } - fn inject_addr_reach_failure( - &mut self, - peer_id: Option<&PeerId>, - addr: &Multiaddr, - error: &dyn std::error::Error, - ) { - for (p, _) in self.protocols.values_mut() { - NetworkBehaviour::inject_addr_reach_failure(p, peer_id, addr, error) - } - } - fn inject_event( &mut self, peer_id: PeerId, @@ -478,9 +496,15 @@ impl NetworkBehaviour for RequestResponsesBehaviour { } } - fn inject_dial_failure(&mut self, peer_id: &PeerId) { + fn inject_dial_failure( + &mut self, + peer_id: Option, + _: Self::ProtocolsHandler, + error: &libp2p::swarm::DialError, + ) { for (p, _) in self.protocols.values_mut() { - NetworkBehaviour::inject_dial_failure(p, peer_id) + let handler = p.new_handler(); + NetworkBehaviour::inject_dial_failure(p, peer_id, handler, error) } } @@ -512,12 +536,7 @@ impl NetworkBehaviour for RequestResponsesBehaviour { &mut self, cx: &mut Context, params: &mut impl PollParameters, - ) -> Poll< - NetworkBehaviourAction< - ::InEvent, - Self::OutEvent, - >, - > { + ) -> Poll> { 'poll_all: loop { if let Some(message_request) = self.message_request.take() { // Now we can can poll `MessageRequest` until we get the reputation @@ -658,17 +677,26 @@ impl NetworkBehaviour for RequestResponsesBehaviour { // Other events generated by the underlying behaviour are transparently // passed through. - NetworkBehaviourAction::DialAddress { address } => { + NetworkBehaviourAction::DialAddress { address, handler } => { log::error!( "The request-response isn't supposed to start dialing peers" ); - return Poll::Ready(NetworkBehaviourAction::DialAddress { address }) + let protocol = protocol.to_string(); + let handler = self.new_handler_with_replacement(protocol, handler); + return Poll::Ready(NetworkBehaviourAction::DialAddress { + address, + handler, + }) }, - NetworkBehaviourAction::DialPeer { peer_id, condition } => + NetworkBehaviourAction::DialPeer { peer_id, condition, handler } => { + let protocol = protocol.to_string(); + let handler = self.new_handler_with_replacement(protocol, handler); return Poll::Ready(NetworkBehaviourAction::DialPeer { peer_id, condition, - }), + handler, + }) + }, NetworkBehaviourAction::NotifyHandler { peer_id, handler, event } => return Poll::Ready(NetworkBehaviourAction::NotifyHandler { peer_id, @@ -855,34 +883,35 @@ impl NetworkBehaviour for RequestResponsesBehaviour { } /// Error when registering a protocol. -#[derive(Debug, derive_more::Display, derive_more::Error)] +#[derive(Debug, thiserror::Error)] pub enum RegisterError { /// A protocol has been specified multiple times. - DuplicateProtocol(#[error(ignore)] Cow<'static, str>), + #[error("{0}")] + DuplicateProtocol(Cow<'static, str>), } /// Error in a request. -#[derive(Debug, derive_more::Display, derive_more::Error)] +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] pub enum RequestFailure { - /// We are not currently connected to the requested peer. + #[error("We are not currently connected to the requested peer.")] NotConnected, - /// Given protocol hasn't been registered. + #[error("Given protocol hasn't been registered.")] UnknownProtocol, - /// Remote has closed the substream before answering, thereby signaling that it considers the - /// request as valid, but refused to answer it. + #[error("Remote has closed the substream before answering, thereby signaling that it considers the request as valid, but refused to answer it.")] Refused, - /// The remote replied, but the local node is no longer interested in the response. + #[error("The remote replied, but the local node is no longer interested in the response.")] Obsolete, /// Problem on the network. - #[display(fmt = "Problem on the network: {}", _0)] + #[error("Problem on the network: {0}")] Network(OutboundFailure), } /// Error when processing a request sent by a remote. -#[derive(Debug, derive_more::Display, derive_more::Error)] +#[derive(Debug, thiserror::Error)] pub enum ResponseFailure { /// Problem on the network. - #[display(fmt = "Problem on the network: {}", _0)] + #[error("Problem on the network: {0}")] Network(InboundFailure), } @@ -1061,7 +1090,7 @@ mod tests { let behaviour = RequestResponsesBehaviour::new(list, handle).unwrap(); - let mut swarm = Swarm::new(transport, behaviour, keypair.public().into_peer_id()); + let mut swarm = Swarm::new(transport, behaviour, keypair.public().to_peer_id()); let listen_addr: Multiaddr = format!("/memory/{}", rand::random::()).parse().unwrap(); swarm.listen_on(listen_addr.clone()).unwrap(); diff --git a/client/network/src/schema.rs b/client/network/src/schema.rs index d4572fca7594..032db5f1733c 100644 --- a/client/network/src/schema.rs +++ b/client/network/src/schema.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/network/src/schema/api.v1.proto b/client/network/src/schema/api.v1.proto index c5333c7dcdbf..b51137d1d51d 100644 --- a/client/network/src/schema/api.v1.proto +++ b/client/network/src/schema/api.v1.proto @@ -74,22 +74,32 @@ message BlockData { message StateRequest { // Block header hash. bytes block = 1; - // Start from this key. Equivalent to if omitted. - bytes start = 2; // optional + // Start from this key. + // Multiple keys used for nested state start. + repeated bytes start = 2; // optional // if 'true' indicates that response should contain raw key-values, rather than proof. bool no_proof = 3; } message StateResponse { - // A collection of keys-values. Only populated if `no_proof` is `true` - repeated StateEntry entries = 1; + // A collection of keys-values states. Only populated if `no_proof` is `true` + repeated KeyValueStateEntry entries = 1; // If `no_proof` is false in request, this contains proof nodes. bytes proof = 2; +} + +// A key value state. +message KeyValueStateEntry { + // Root of for this level, empty length bytes + // if top level. + bytes state_root = 1; + // A collection of keys-values. + repeated StateEntry entries = 2; // Set to true when there are no more keys to return. bool complete = 3; } -// A key-value pair +// A key-value pair. message StateEntry { bytes key = 1; bytes value = 2; diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 23f9c614d906..e89be325fa48 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -33,11 +33,9 @@ use crate::{ config::{parse_str_addr, Params, TransportConfig}, discovery::DiscoveryConfig, error::Error, - light_client_requests, network_state::{ NetworkState, NotConnectedPeer as NetworkStateNotConnectedPeer, Peer as NetworkStatePeer, }, - on_demand_layer::AlwaysBadChecker, protocol::{ self, event::Event, @@ -56,12 +54,11 @@ use libp2p::{ either::EitherError, upgrade, ConnectedPoint, Executor, }, - kad::record, multiaddr, - ping::handler::PingFailure, + ping::Failure as PingFailure, swarm::{ - protocols_handler::NodeHandlerWrapperError, AddressScore, NetworkBehaviour, SwarmBuilder, - SwarmEvent, + protocols_handler::NodeHandlerWrapperError, AddressScore, DialError, NetworkBehaviour, + SwarmBuilder, SwarmEvent, }, Multiaddr, PeerId, }; @@ -76,7 +73,6 @@ use std::{ borrow::Cow, cmp, collections::{HashMap, HashSet}, - convert::TryFrom as _, fs, iter, marker::PhantomData, num::NonZeroUsize, @@ -95,9 +91,19 @@ pub use behaviour::{ mod metrics; mod out_events; +mod signature; #[cfg(test)] mod tests; +pub use libp2p::{ + identity::{ + error::{DecodingError, SigningError}, + Keypair, PublicKey, + }, + kad::record::Key as KademliaKey, +}; +pub use signature::*; + /// Substrate network service. Handles network IO and manages connectivity. pub struct NetworkService { /// Number of peers we're connected to. @@ -108,6 +114,8 @@ pub struct NetworkService { is_major_syncing: Arc, /// Local copy of the `PeerId` of the local node. local_peer_id: PeerId, + /// The `KeyPair` that defines the `PeerId` of the local node. + local_identity: Keypair, /// Bandwidth logging system. Can be queried to know the average bandwidth consumed. bandwidth: Arc, /// Peerset manager (PSM); manages the reputation of nodes and indicates the network which @@ -178,10 +186,10 @@ impl NetworkWorker { // Private and public keys configuration. let local_identity = params.network_config.node_key.clone().into_keypair()?; let local_public = local_identity.public(); - let local_peer_id = local_public.clone().into_peer_id(); + let local_peer_id = local_public.clone().to_peer_id(); info!( target: "sub-libp2p", - "🏷 Local node identity is: {}", + "🏷 Local node identity is: {}", local_peer_id.to_base58(), ); @@ -238,12 +246,6 @@ impl NetworkWorker { } })?; - let checker = params - .on_demand - .as_ref() - .map(|od| od.checker().clone()) - .unwrap_or_else(|| Arc::new(AlwaysBadChecker)); - let num_connected = Arc::new(AtomicUsize::new(0)); let is_major_syncing = Arc::new(AtomicBool::new(false)); @@ -255,14 +257,6 @@ impl NetworkWorker { params.network_config.client_version, params.network_config.node_name ); - let light_client_request_sender = { - light_client_requests::sender::LightClientRequestSender::new( - ¶ms.protocol_id, - checker, - peerset_handle.clone(), - ) - }; - let discovery_config = { let mut config = DiscoveryConfig::new(local_public.clone()); config.with_permanent_addresses(known_addresses); @@ -334,7 +328,7 @@ impl NetworkWorker { }; transport::build_transport( - local_identity, + local_identity.clone(), config_mem, params.network_config.yamux_window_size, yamux_maximum_buffer_size, @@ -347,7 +341,6 @@ impl NetworkWorker { protocol, user_agent, local_public, - light_client_request_sender, discovery_config, params.block_request_protocol_config, params.state_request_protocol_config, @@ -423,6 +416,7 @@ impl NetworkWorker { is_major_syncing: is_major_syncing.clone(), peerset: peerset_handle, local_peer_id, + local_identity, to_worker, peers_notifications_sinks: peers_notifications_sinks.clone(), notifications_sizes_metric: metrics @@ -447,7 +441,6 @@ impl NetworkWorker { service, import_queue: params.import_queue, from_service, - light_client_rqs: params.on_demand.and_then(|od| od.extract_receiver()), event_streams: out_events::OutChannels::new(params.metrics_registry.as_ref())?, peers_notifications_sinks, tx_handler_controller, @@ -685,6 +678,14 @@ impl NetworkService { &self.local_peer_id } + /// Signs the message with the `KeyPair` that defined the local `PeerId`. + pub fn sign_with_local_identity( + &self, + msg: impl AsRef<[u8]>, + ) -> Result { + Signature::sign_message(msg.as_ref(), &self.local_identity) + } + /// Set authorized peers. /// /// Need a better solution to manage authorized peers, but now just use reserved peers for @@ -728,8 +729,7 @@ impl NetworkService { /// > preventing the message from being delivered. /// /// The protocol must have been registered with - /// [`NetworkConfiguration::notifications_protocols`](crate::config::NetworkConfiguration:: - /// notifications_protocols). + /// `crate::config::NetworkConfiguration::notifications_protocols`. pub fn write_notification( &self, target: PeerId, @@ -793,8 +793,7 @@ impl NetworkService { /// in which case enqueued notifications will be lost. /// /// The protocol must have been registered with - /// [`NetworkConfiguration::notifications_protocols`](crate::config::NetworkConfiguration:: - /// notifications_protocols). + /// `crate::config::NetworkConfiguration::notifications_protocols`. /// /// # Usage /// @@ -835,8 +834,7 @@ impl NetworkService { /// if buffer is full /// /// - /// See also the [`gossip`](crate::gossip) module for a higher-level way to send - /// notifications. + /// See also the `sc-network-gossip` crate for a higher-level way to send notifications. pub fn notification_sender( &self, target: PeerId, @@ -1045,7 +1043,7 @@ impl NetworkService { /// /// This will generate either a `ValueFound` or a `ValueNotFound` event and pass it as an /// item on the [`NetworkWorker`] stream. - pub fn get_value(&self, key: &record::Key) { + pub fn get_value(&self, key: &KademliaKey) { let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::GetValue(key.clone())); } @@ -1053,7 +1051,7 @@ impl NetworkService { /// /// This will generate either a `ValuePut` or a `ValuePutFailed` event and pass it as an /// item on the [`NetworkWorker`] stream. - pub fn put_value(&self, key: record::Key, value: Vec) { + pub fn put_value(&self, key: KademliaKey, value: Vec) { let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::PutValue(key, value)); } @@ -1092,61 +1090,95 @@ impl NetworkService { let _ = self.to_worker.unbounded_send(ServiceToWorkerMsg::RemoveReserved(peer_id)); } - /// Add peers to a peer set. + /// Sets the reserved set of a protocol to the given set of peers. /// /// Each `Multiaddr` must end with a `/p2p/` component containing the `PeerId`. It can also /// consist of only `/p2p/`. /// + /// The node will start establishing/accepting connections and substreams to/from peers in this + /// set, if it doesn't have any substream open with them yet. + /// + /// Note however, if a call to this function results in less peers on the reserved set, they + /// will not necessarily get disconnected (depending on available free slots in the peer set). + /// If you want to also disconnect those removed peers, you will have to call + /// `remove_from_peers_set` on those in addition to updating the reserved set. You can omit + /// this step if the peer set is in reserved only mode. + /// /// Returns an `Err` if one of the given addresses is invalid or contains an /// invalid peer ID (which includes the local peer ID). - pub fn add_peers_to_reserved_set( + pub fn set_reserved_peers( &self, protocol: Cow<'static, str>, peers: HashSet, ) -> Result<(), String> { - let peers = self.split_multiaddr_and_peer_id(peers)?; + let peers_addrs = self.split_multiaddr_and_peer_id(peers)?; - for (peer_id, addr) in peers.into_iter() { + let mut peers: HashSet = HashSet::with_capacity(peers_addrs.len()); + + for (peer_id, addr) in peers_addrs.into_iter() { // Make sure the local peer ID is never added to the PSM. if peer_id == self.local_peer_id { return Err("Local peer ID cannot be added as a reserved peer.".to_string()) } + peers.insert(peer_id); + if !addr.is_empty() { let _ = self .to_worker - .unbounded_send(ServiceToWorkerMsg::AddKnownAddress(peer_id.clone(), addr)); + .unbounded_send(ServiceToWorkerMsg::AddKnownAddress(peer_id, addr)); } - let _ = self - .to_worker - .unbounded_send(ServiceToWorkerMsg::AddSetReserved(protocol.clone(), peer_id)); } + let _ = self + .to_worker + .unbounded_send(ServiceToWorkerMsg::SetPeersetReserved(protocol, peers)); + Ok(()) } - /// Remove peers from a peer set. + /// Add peers to a peer set. /// - /// Each `Multiaddr` must end with a `/p2p/` component containing the `PeerId`. + /// Each `Multiaddr` must end with a `/p2p/` component containing the `PeerId`. It can also + /// consist of only `/p2p/`. /// /// Returns an `Err` if one of the given addresses is invalid or contains an /// invalid peer ID (which includes the local peer ID). - // NOTE: technically, this function only needs `Vec`, but we use `Multiaddr` here for - // convenience. - pub fn remove_peers_from_reserved_set( + pub fn add_peers_to_reserved_set( &self, protocol: Cow<'static, str>, peers: HashSet, ) -> Result<(), String> { let peers = self.split_multiaddr_and_peer_id(peers)?; - for (peer_id, _) in peers.into_iter() { + + for (peer_id, addr) in peers.into_iter() { + // Make sure the local peer ID is never added to the PSM. + if peer_id == self.local_peer_id { + return Err("Local peer ID cannot be added as a reserved peer.".to_string()) + } + + if !addr.is_empty() { + let _ = self + .to_worker + .unbounded_send(ServiceToWorkerMsg::AddKnownAddress(peer_id.clone(), addr)); + } let _ = self .to_worker - .unbounded_send(ServiceToWorkerMsg::RemoveSetReserved(protocol.clone(), peer_id)); + .unbounded_send(ServiceToWorkerMsg::AddSetReserved(protocol.clone(), peer_id)); } + Ok(()) } + /// Remove peers from a peer set. + pub fn remove_peers_from_reserved_set(&self, protocol: Cow<'static, str>, peers: Vec) { + for peer_id in peers.into_iter() { + let _ = self + .to_worker + .unbounded_send(ServiceToWorkerMsg::RemoveSetReserved(protocol.clone(), peer_id)); + } + } + /// Configure an explicit fork sync request. /// Note that this function should not be used for recent blocks. /// Sync should be able to download all the recent forks normally. @@ -1195,25 +1227,12 @@ impl NetworkService { /// Remove peers from a peer set. /// /// If we currently have an open substream with this peer, it will soon be closed. - /// - /// Each `Multiaddr` must end with a `/p2p/` component containing the `PeerId`. - /// - /// Returns an `Err` if one of the given addresses is invalid or contains an - /// invalid peer ID (which includes the local peer ID). - // NOTE: technically, this function only needs `Vec`, but we use `Multiaddr` here for - // convenience. - pub fn remove_from_peers_set( - &self, - protocol: Cow<'static, str>, - peers: HashSet, - ) -> Result<(), String> { - let peers = self.split_multiaddr_and_peer_id(peers)?; - for (peer_id, _) in peers.into_iter() { + pub fn remove_from_peers_set(&self, protocol: Cow<'static, str>, peers: Vec) { + for peer_id in peers.into_iter() { let _ = self .to_worker .unbounded_send(ServiceToWorkerMsg::RemoveFromPeersSet(protocol.clone(), peer_id)); } - Ok(()) } /// Returns the number of peers we're connected to. @@ -1371,7 +1390,7 @@ impl<'a> NotificationSenderReady<'a> { } /// Error returned by [`NetworkService::send_notification`]. -#[derive(Debug, derive_more::Display, derive_more::Error)] +#[derive(Debug, thiserror::Error)] pub enum NotificationSenderError { /// The notification receiver has been closed, usually because the underlying connection /// closed. @@ -1379,8 +1398,10 @@ pub enum NotificationSenderError { /// Some of the notifications most recently sent may not have been received. However, /// the peer may still be connected and a new `NotificationSender` for the same /// protocol obtained from [`NetworkService::notification_sender`]. + #[error("The notification receiver has been closed")] Closed, /// Protocol name hasn't been registered. + #[error("Protocol name hasn't been registered")] BadProtocol, } @@ -1393,13 +1414,14 @@ enum ServiceToWorkerMsg { RequestJustification(B::Hash, NumberFor), ClearJustificationRequests, AnnounceBlock(B::Hash, Option>), - GetValue(record::Key), - PutValue(record::Key, Vec), + GetValue(KademliaKey), + PutValue(KademliaKey, Vec), AddKnownAddress(PeerId, Multiaddr), SetReservedOnly(bool), AddReserved(PeerId), RemoveReserved(PeerId), SetReserved(HashSet), + SetPeersetReserved(Cow<'static, str>, HashSet), AddSetReserved(Cow<'static, str>, PeerId), RemoveSetReserved(Cow<'static, str>, PeerId), AddToPeersSet(Cow<'static, str>, PeerId), @@ -1442,8 +1464,6 @@ pub struct NetworkWorker { import_queue: Box>, /// Messages from the [`NetworkService`] that must be processed. from_service: TracingUnboundedReceiver>, - /// Receiver for queries from the light client that must be processed. - light_client_rqs: Option>>, /// Senders for events that happen on the network. event_streams: out_events::OutChannels, /// Prometheus network metrics. @@ -1467,23 +1487,6 @@ impl Future for NetworkWorker { this.import_queue .poll_actions(cx, &mut NetworkLink { protocol: &mut this.network_service }); - // Check for new incoming light client requests. - if let Some(light_client_rqs) = this.light_client_rqs.as_mut() { - while let Poll::Ready(Some(rq)) = light_client_rqs.poll_next_unpin(cx) { - let result = this.network_service.behaviour_mut().light_client_request(rq); - match result { - Ok(()) => {}, - Err(light_client_requests::sender::SendRequestError::TooManyRequests) => { - warn!("Couldn't start light client request: too many pending requests"); - }, - } - - if let Some(metrics) = this.metrics.as_ref() { - metrics.issued_light_requests.inc(); - } - } - } - // At the time of writing of this comment, due to a high volume of messages, the network // worker sometimes takes a long time to process the loop below. When that happens, the // rest of the polling is frozen. In order to avoid negative side-effects caused by this @@ -1541,6 +1544,11 @@ impl Future for NetworkWorker { .behaviour_mut() .user_protocol_mut() .set_reserved_peers(peers), + ServiceToWorkerMsg::SetPeersetReserved(protocol, peers) => this + .network_service + .behaviour_mut() + .user_protocol_mut() + .set_reserved_peerset_peers(protocol, peers), ServiceToWorkerMsg::AddReserved(peer_id) => this .network_service .behaviour_mut() @@ -1855,8 +1863,13 @@ impl Future for NetworkWorker { peer_id, endpoint, num_established, + concurrent_dial_errors, }) => { - debug!(target: "sub-libp2p", "Libp2p => Connected({:?})", peer_id); + if let Some(errors) = concurrent_dial_errors { + debug!(target: "sub-libp2p", "Libp2p => Connected({:?}) with errors: {:?}", peer_id, errors); + } else { + debug!(target: "sub-libp2p", "Libp2p => Connected({:?})", peer_id); + } if let Some(metrics) = this.metrics.as_ref() { let direction = match endpoint { @@ -1924,37 +1937,41 @@ impl Future for NetworkWorker { metrics.listeners_local_addresses.dec(); } }, - Poll::Ready(SwarmEvent::UnreachableAddr { peer_id, address, error, .. }) => { - trace!( - target: "sub-libp2p", - "Libp2p => Failed to reach {:?} through {:?}: {}", - peer_id, address, error, - ); + Poll::Ready(SwarmEvent::OutgoingConnectionError { peer_id, error }) => { + if let Some(peer_id) = peer_id { + trace!( + target: "sub-libp2p", + "Libp2p => Failed to reach {:?}: {}", + peer_id, error, + ); - if this.boot_node_ids.contains(&peer_id) { - if let PendingConnectionError::InvalidPeerId = error { - error!( - "💔 The bootnode you want to connect to at `{}` provided a different peer ID than the one you expect: `{}`.", - address, peer_id, - ); + if this.boot_node_ids.contains(&peer_id) { + if let DialError::InvalidPeerId = error { + error!( + "💔 The bootnode you want to connect provided a different peer ID than the one you expect: `{}`.", + peer_id, + ); + } } } if let Some(metrics) = this.metrics.as_ref() { - match error { - PendingConnectionError::ConnectionLimit(_) => metrics - .pending_connections_errors_total - .with_label_values(&["limit-reached"]) - .inc(), - PendingConnectionError::InvalidPeerId => metrics - .pending_connections_errors_total - .with_label_values(&["invalid-peer-id"]) - .inc(), - PendingConnectionError::Transport(_) | - PendingConnectionError::IO(_) => metrics + let reason = match error { + DialError::ConnectionLimit(_) => Some("limit-reached"), + DialError::InvalidPeerId => Some("invalid-peer-id"), + DialError::Transport(_) | DialError::ConnectionIo(_) => + Some("transport-error"), + DialError::Banned | + DialError::LocalPeerId | + DialError::NoAddresses | + DialError::DialPeerConditionFalse(_) | + DialError::Aborted => None, // ignore them + }; + if let Some(reason) = reason { + metrics .pending_connections_errors_total - .with_label_values(&["transport-error"]) - .inc(), + .with_label_values(&[reason]) + .inc(); } } }, @@ -1980,16 +1997,19 @@ impl Future for NetworkWorker { ); if let Some(metrics) = this.metrics.as_ref() { let reason = match error { - PendingConnectionError::ConnectionLimit(_) => "limit-reached", - PendingConnectionError::InvalidPeerId => "invalid-peer-id", + PendingConnectionError::ConnectionLimit(_) => Some("limit-reached"), + PendingConnectionError::InvalidPeerId => Some("invalid-peer-id"), PendingConnectionError::Transport(_) | - PendingConnectionError::IO(_) => "transport-error", + PendingConnectionError::IO(_) => Some("transport-error"), + PendingConnectionError::Aborted => None, // ignore it }; - metrics - .incoming_connections_errors_total - .with_label_values(&[reason]) - .inc(); + if let Some(reason) = reason { + metrics + .incoming_connections_errors_total + .with_label_values(&[reason]) + .inc(); + } } }, Poll::Ready(SwarmEvent::BannedPeer { peer_id, endpoint }) => { @@ -2005,8 +2025,6 @@ impl Future for NetworkWorker { .inc(); } }, - Poll::Ready(SwarmEvent::UnknownPeerUnreachableAddr { address, error }) => - trace!(target: "sub-libp2p", "Libp2p => UnknownPeerUnreachableAddr({}): {}", address, error), Poll::Ready(SwarmEvent::ListenerClosed { reason, addresses, .. }) => { if let Some(metrics) = this.metrics.as_ref() { metrics.listeners_local_addresses.sub(addresses.len() as u64); @@ -2089,10 +2107,6 @@ impl Future for NetworkWorker { .peerset_num_discovered .set(this.network_service.behaviour_mut().user_protocol().num_discovered_peers() as u64); - metrics.peerset_num_requested.set( - this.network_service.behaviour_mut().user_protocol().requested_peers().count() - as u64, - ); metrics.pending_connections.set( Swarm::network_info(&this.network_service).connection_counters().num_pending() as u64, diff --git a/client/network/src/service/metrics.rs b/client/network/src/service/metrics.rs index e33cd4b194d6..ad30b1b093ff 100644 --- a/client/network/src/service/metrics.rs +++ b/client/network/src/service/metrics.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -69,7 +69,6 @@ pub struct Metrics { pub notifications_streams_closed_total: CounterVec, pub notifications_streams_opened_total: CounterVec, pub peerset_num_discovered: Gauge, - pub peerset_num_requested: Gauge, pub pending_connections: Gauge, pub pending_connections_errors_total: CounterVec, pub requests_in_failure_total: CounterVec, @@ -84,54 +83,54 @@ impl Metrics { // This list is ordered alphabetically connections_closed_total: prometheus::register(CounterVec::new( Opts::new( - "sub_libp2p_connections_closed_total", + "substrate_sub_libp2p_connections_closed_total", "Total number of connections closed, by direction and reason" ), &["direction", "reason"] )?, registry)?, connections_opened_total: prometheus::register(CounterVec::new( Opts::new( - "sub_libp2p_connections_opened_total", + "substrate_sub_libp2p_connections_opened_total", "Total number of connections opened by direction" ), &["direction"] )?, registry)?, distinct_peers_connections_closed_total: prometheus::register(Counter::new( - "sub_libp2p_distinct_peers_connections_closed_total", + "substrate_sub_libp2p_distinct_peers_connections_closed_total", "Total number of connections closed with distinct peers" )?, registry)?, distinct_peers_connections_opened_total: prometheus::register(Counter::new( - "sub_libp2p_distinct_peers_connections_opened_total", + "substrate_sub_libp2p_distinct_peers_connections_opened_total", "Total number of connections opened with distinct peers" )?, registry)?, import_queue_blocks_submitted: prometheus::register(Counter::new( - "import_queue_blocks_submitted", + "substrate_import_queue_blocks_submitted", "Number of blocks submitted to the import queue.", )?, registry)?, import_queue_justifications_submitted: prometheus::register(Counter::new( - "import_queue_justifications_submitted", + "substrate_import_queue_justifications_submitted", "Number of justifications submitted to the import queue.", )?, registry)?, incoming_connections_errors_total: prometheus::register(CounterVec::new( Opts::new( - "sub_libp2p_incoming_connections_handshake_errors_total", + "substrate_sub_libp2p_incoming_connections_handshake_errors_total", "Total number of incoming connections that have failed during the \ initial handshake" ), &["reason"] )?, registry)?, incoming_connections_total: prometheus::register(Counter::new( - "sub_libp2p_incoming_connections_total", + "substrate_sub_libp2p_incoming_connections_total", "Total number of incoming connections on the listening sockets" )?, registry)?, issued_light_requests: prometheus::register(Counter::new( - "issued_light_requests", + "substrate_issued_light_requests", "Number of light client requests that our node has issued.", )?, registry)?, kademlia_query_duration: prometheus::register(HistogramVec::new( HistogramOpts { common_opts: Opts::new( - "sub_libp2p_kademlia_query_duration", + "substrate_sub_libp2p_kademlia_query_duration", "Duration of Kademlia queries per query type" ), buckets: prometheus::exponential_buckets(0.5, 2.0, 10) @@ -141,43 +140,44 @@ impl Metrics { )?, registry)?, kademlia_random_queries_total: prometheus::register(CounterVec::new( Opts::new( - "sub_libp2p_kademlia_random_queries_total", + "substrate_sub_libp2p_kademlia_random_queries_total", "Number of random Kademlia queries started" ), &["protocol"] )?, registry)?, kademlia_records_count: prometheus::register(GaugeVec::new( Opts::new( - "sub_libp2p_kademlia_records_count", + "substrate_sub_libp2p_kademlia_records_count", "Number of records in the Kademlia records store" ), &["protocol"] )?, registry)?, kademlia_records_sizes_total: prometheus::register(GaugeVec::new( Opts::new( - "sub_libp2p_kademlia_records_sizes_total", + "substrate_sub_libp2p_kademlia_records_sizes_total", "Total size of all the records in the Kademlia records store" ), &["protocol"] )?, registry)?, kbuckets_num_nodes: prometheus::register(GaugeVec::new( Opts::new( - "sub_libp2p_kbuckets_num_nodes", + "substrate_sub_libp2p_kbuckets_num_nodes", "Number of nodes per kbucket per Kademlia instance" ), &["protocol", "lower_ilog2_bucket_bound"] )?, registry)?, listeners_local_addresses: prometheus::register(Gauge::new( - "sub_libp2p_listeners_local_addresses", "Number of local addresses we're listening on" + "substrate_sub_libp2p_listeners_local_addresses", + "Number of local addresses we're listening on" )?, registry)?, listeners_errors_total: prometheus::register(Counter::new( - "sub_libp2p_listeners_errors_total", + "substrate_sub_libp2p_listeners_errors_total", "Total number of non-fatal errors reported by a listener" )?, registry)?, notifications_sizes: prometheus::register(HistogramVec::new( HistogramOpts { common_opts: Opts::new( - "sub_libp2p_notifications_sizes", + "substrate_sub_libp2p_notifications_sizes", "Sizes of the notifications send to and received from all nodes" ), buckets: prometheus::exponential_buckets(64.0, 4.0, 8) @@ -187,38 +187,36 @@ impl Metrics { )?, registry)?, notifications_streams_closed_total: prometheus::register(CounterVec::new( Opts::new( - "sub_libp2p_notifications_streams_closed_total", + "substrate_sub_libp2p_notifications_streams_closed_total", "Total number of notification substreams that have been closed" ), &["protocol"] )?, registry)?, notifications_streams_opened_total: prometheus::register(CounterVec::new( Opts::new( - "sub_libp2p_notifications_streams_opened_total", + "substrate_sub_libp2p_notifications_streams_opened_total", "Total number of notification substreams that have been opened" ), &["protocol"] )?, registry)?, peerset_num_discovered: prometheus::register(Gauge::new( - "sub_libp2p_peerset_num_discovered", "Number of nodes stored in the peerset manager", - )?, registry)?, - peerset_num_requested: prometheus::register(Gauge::new( - "sub_libp2p_peerset_num_requested", "Number of nodes that the peerset manager wants us to be connected to", + "substrate_sub_libp2p_peerset_num_discovered", + "Number of nodes stored in the peerset manager", )?, registry)?, pending_connections: prometheus::register(Gauge::new( - "sub_libp2p_pending_connections", + "substrate_sub_libp2p_pending_connections", "Number of connections in the process of being established", )?, registry)?, pending_connections_errors_total: prometheus::register(CounterVec::new( Opts::new( - "sub_libp2p_pending_connections_errors_total", + "substrate_sub_libp2p_pending_connections_errors_total", "Total number of pending connection errors" ), &["reason"] )?, registry)?, requests_in_failure_total: prometheus::register(CounterVec::new( Opts::new( - "sub_libp2p_requests_in_failure_total", + "substrate_sub_libp2p_requests_in_failure_total", "Total number of incoming requests that the node has failed to answer" ), &["protocol", "reason"] @@ -226,7 +224,7 @@ impl Metrics { requests_in_success_total: prometheus::register(HistogramVec::new( HistogramOpts { common_opts: Opts::new( - "sub_libp2p_requests_in_success_total", + "substrate_sub_libp2p_requests_in_success_total", "For successful incoming requests, time between receiving the request and \ starting to send the response" ), @@ -237,7 +235,7 @@ impl Metrics { )?, registry)?, requests_out_failure_total: prometheus::register(CounterVec::new( Opts::new( - "sub_libp2p_requests_out_failure_total", + "substrate_sub_libp2p_requests_out_failure_total", "Total number of requests that have failed" ), &["protocol", "reason"] @@ -245,7 +243,7 @@ impl Metrics { requests_out_success_total: prometheus::register(HistogramVec::new( HistogramOpts { common_opts: Opts::new( - "sub_libp2p_requests_out_success_total", + "substrate_sub_libp2p_requests_out_success_total", "For successful outgoing requests, time between a request's start and finish" ), buckets: prometheus::exponential_buckets(0.001, 2.0, 16) @@ -267,7 +265,7 @@ impl BandwidthCounters { fn register(registry: &Registry, sinks: Arc) -> Result<(), PrometheusError> { prometheus::register( SourcedCounter::new( - &Opts::new("sub_libp2p_network_bytes_total", "Total bandwidth usage") + &Opts::new("substrate_sub_libp2p_network_bytes_total", "Total bandwidth usage") .variable_label("direction"), BandwidthCounters(sinks), )?, @@ -298,7 +296,7 @@ impl MajorSyncingGauge { prometheus::register( SourcedGauge::new( &Opts::new( - "sub_libp2p_is_major_syncing", + "substrate_sub_libp2p_is_major_syncing", "Whether the node is performing a major sync or not.", ), MajorSyncingGauge(value), @@ -328,7 +326,7 @@ impl NumConnectedGauge { fn register(registry: &Registry, value: Arc) -> Result<(), PrometheusError> { prometheus::register( SourcedGauge::new( - &Opts::new("sub_libp2p_peers_count", "Number of connected peers"), + &Opts::new("substrate_sub_libp2p_peers_count", "Number of connected peers"), NumConnectedGauge(value), )?, registry, diff --git a/client/network/src/service/out_events.rs b/client/network/src/service/out_events.rs index 2d6241278005..3bff5a16fd0c 100644 --- a/client/network/src/service/out_events.rs +++ b/client/network/src/service/out_events.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -37,7 +37,7 @@ use futures::{channel::mpsc, prelude::*, ready, stream::FusedStream}; use parking_lot::Mutex; use prometheus_endpoint::{register, CounterVec, GaugeVec, Opts, PrometheusError, Registry, U64}; use std::{ - convert::TryFrom as _, + cell::RefCell, fmt, pin::Pin, sync::Arc, @@ -187,12 +187,29 @@ struct Metrics { num_channels: GaugeVec, } +thread_local! { + static LABEL_BUFFER: RefCell = RefCell::new(String::new()); +} + +fn format_label(prefix: &str, protocol: &str, callback: impl FnOnce(&str)) { + LABEL_BUFFER.with(|label_buffer| { + let mut label_buffer = label_buffer.borrow_mut(); + label_buffer.clear(); + label_buffer.reserve(prefix.len() + protocol.len() + 2); + label_buffer.push_str(prefix); + label_buffer.push_str("\""); + label_buffer.push_str(protocol); + label_buffer.push_str("\""); + callback(&label_buffer); + }); +} + impl Metrics { fn register(registry: &Registry) -> Result { Ok(Self { events_total: register(CounterVec::new( Opts::new( - "sub_libp2p_out_events_events_total", + "substrate_sub_libp2p_out_events_events_total", "Number of broadcast network events that have been sent or received across all \ channels" ), @@ -200,7 +217,7 @@ impl Metrics { )?, registry)?, notifications_sizes: register(CounterVec::new( Opts::new( - "sub_libp2p_out_events_notifications_sizes", + "substrate_sub_libp2p_out_events_notifications_sizes", "Size of notification events that have been sent or received across all \ channels" ), @@ -208,7 +225,7 @@ impl Metrics { )?, registry)?, num_channels: register(GaugeVec::new( Opts::new( - "sub_libp2p_out_events_num_channels", + "substrate_sub_libp2p_out_events_num_channels", "Number of internal active channels that broadcast network events", ), &["name"] @@ -232,20 +249,26 @@ impl Metrics { .inc_by(num); }, Event::NotificationStreamOpened { protocol, .. } => { - self.events_total - .with_label_values(&[&format!("notif-open-{:?}", protocol), "sent", name]) - .inc_by(num); + format_label("notif-open-", &protocol, |protocol_label| { + self.events_total + .with_label_values(&[protocol_label, "sent", name]) + .inc_by(num); + }); }, Event::NotificationStreamClosed { protocol, .. } => { - self.events_total - .with_label_values(&[&format!("notif-closed-{:?}", protocol), "sent", name]) - .inc_by(num); + format_label("notif-closed-", &protocol, |protocol_label| { + self.events_total + .with_label_values(&[protocol_label, "sent", name]) + .inc_by(num); + }); }, Event::NotificationsReceived { messages, .. } => for (protocol, message) in messages { - self.events_total - .with_label_values(&[&format!("notif-{:?}", protocol), "sent", name]) - .inc_by(num); + format_label("notif-", &protocol, |protocol_label| { + self.events_total + .with_label_values(&[protocol_label, "sent", name]) + .inc_by(num); + }); self.notifications_sizes.with_label_values(&[protocol, "sent", name]).inc_by( num.saturating_mul(u64::try_from(message.len()).unwrap_or(u64::MAX)), ); @@ -267,20 +290,22 @@ impl Metrics { .inc(); }, Event::NotificationStreamOpened { protocol, .. } => { - self.events_total - .with_label_values(&[&format!("notif-open-{:?}", protocol), "received", name]) - .inc(); + format_label("notif-open-", &protocol, |protocol_label| { + self.events_total.with_label_values(&[protocol_label, "received", name]).inc(); + }); }, Event::NotificationStreamClosed { protocol, .. } => { - self.events_total - .with_label_values(&[&format!("notif-closed-{:?}", protocol), "received", name]) - .inc(); + format_label("notif-closed-", &protocol, |protocol_label| { + self.events_total.with_label_values(&[protocol_label, "received", name]).inc(); + }); }, Event::NotificationsReceived { messages, .. } => for (protocol, message) in messages { - self.events_total - .with_label_values(&[&format!("notif-{:?}", protocol), "received", name]) - .inc(); + format_label("notif-", &protocol, |protocol_label| { + self.events_total + .with_label_values(&[protocol_label, "received", name]) + .inc(); + }); self.notifications_sizes .with_label_values(&[&protocol, "received", name]) .inc_by(u64::try_from(message.len()).unwrap_or(u64::MAX)); diff --git a/client/network/src/service/signature.rs b/client/network/src/service/signature.rs new file mode 100644 index 000000000000..d21d28a3007b --- /dev/null +++ b/client/network/src/service/signature.rs @@ -0,0 +1,50 @@ +// This file is part of Substrate. +// +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// +// If you read this, you are very thorough, congratulations. + +use super::*; + +/// A result of signing a message with a network identity. Since `PeerId` is potentially a hash of a +/// `PublicKey`, you need to reveal the `PublicKey` next to the signature, so the verifier can check +/// if the signature was made by the entity that controls a given `PeerId`. +pub struct Signature { + /// The public key derived from the network identity that signed the message. + pub public_key: PublicKey, + /// The actual signature made for the message signed. + pub bytes: Vec, +} + +impl Signature { + /// Create a signature for a message with a given network identity. + pub fn sign_message( + message: impl AsRef<[u8]>, + keypair: &Keypair, + ) -> Result { + let public_key = keypair.public(); + let bytes = keypair.sign(message.as_ref())?; + Ok(Self { public_key, bytes }) + } + + /// Verify whether the signature was made for the given message by the entity that controls the + /// given `PeerId`. + pub fn verify(&self, message: impl AsRef<[u8]>, peer_id: &PeerId) -> bool { + *peer_id == self.public_key.to_peer_id() && + self.public_key.verify(message.as_ref(), &self.bytes) + } +} diff --git a/client/network/src/service/tests.rs b/client/network/src/service/tests.rs index 69b172d07edf..03d647eade17 100644 --- a/client/network/src/service/tests.rs +++ b/client/network/src/service/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -116,7 +116,6 @@ fn build_test_full_node( }), network_config: config, chain: client.clone(), - on_demand: None, transaction_pool: Arc::new(crate::config::EmptyTransactionPool), protocol_id, import_queue, @@ -187,7 +186,6 @@ fn build_nodes_one_proto() -> ( (node1, events_stream1, node2, events_stream2) } -#[ignore] #[test] fn notifications_state_consistent() { // Runs two nodes and ensures that events are propagated out of the API in a consistent @@ -273,38 +271,38 @@ fn notifications_state_consistent() { match next_event { future::Either::Left(Event::NotificationStreamOpened { remote, protocol, .. - }) => { - something_happened = true; - assert!(!node1_to_node2_open); - node1_to_node2_open = true; - assert_eq!(remote, *node2.local_peer_id()); - assert_eq!(protocol, PROTOCOL_NAME); - }, + }) => + if protocol == PROTOCOL_NAME { + something_happened = true; + assert!(!node1_to_node2_open); + node1_to_node2_open = true; + assert_eq!(remote, *node2.local_peer_id()); + }, future::Either::Right(Event::NotificationStreamOpened { remote, protocol, .. - }) => { - something_happened = true; - assert!(!node2_to_node1_open); - node2_to_node1_open = true; - assert_eq!(remote, *node1.local_peer_id()); - assert_eq!(protocol, PROTOCOL_NAME); - }, + }) => + if protocol == PROTOCOL_NAME { + something_happened = true; + assert!(!node2_to_node1_open); + node2_to_node1_open = true; + assert_eq!(remote, *node1.local_peer_id()); + }, future::Either::Left(Event::NotificationStreamClosed { remote, protocol, .. - }) => { - assert!(node1_to_node2_open); - node1_to_node2_open = false; - assert_eq!(remote, *node2.local_peer_id()); - assert_eq!(protocol, PROTOCOL_NAME); - }, + }) => + if protocol == PROTOCOL_NAME { + assert!(node1_to_node2_open); + node1_to_node2_open = false; + assert_eq!(remote, *node2.local_peer_id()); + }, future::Either::Right(Event::NotificationStreamClosed { remote, protocol, .. - }) => { - assert!(node2_to_node1_open); - node2_to_node1_open = false; - assert_eq!(remote, *node1.local_peer_id()); - assert_eq!(protocol, PROTOCOL_NAME); - }, + }) => + if protocol == PROTOCOL_NAME { + assert!(node2_to_node1_open); + node2_to_node1_open = false; + assert_eq!(remote, *node1.local_peer_id()); + }, future::Either::Left(Event::NotificationsReceived { remote, .. }) => { assert!(node1_to_node2_open); assert_eq!(remote, *node2.local_peer_id()); @@ -530,7 +528,7 @@ fn fallback_name_working() { { assert_eq!(negotiated_fallback, Some(PROTOCOL_NAME)); break - } + }, _ => {}, }; } diff --git a/client/network/src/state_request_handler.rs b/client/network/src/state_request_handler.rs index b4e5320ebfda..10a77061a031 100644 --- a/client/network/src/state_request_handler.rs +++ b/client/network/src/state_request_handler.rs @@ -15,13 +15,13 @@ // along with Substrate. If not, see . //! Helper for handling (i.e. answering) state requests from a remote peer via the -//! [`crate::request_responses::RequestResponsesBehaviour`]. +//! `crate::request_responses::RequestResponsesBehaviour`. use crate::{ chain::Client, config::ProtocolId, request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig}, - schema::v1::{StateEntry, StateRequest, StateResponse}, + schema::v1::{KeyValueStateEntry, StateEntry, StateRequest, StateResponse}, PeerId, ReputationChange, }; use codec::{Decode, Encode}; @@ -66,7 +66,7 @@ fn generate_protocol_name(protocol_id: &ProtocolId) -> String { let mut s = String::new(); s.push_str("/"); s.push_str(protocol_id.as_ref()); - s.push_str("/state/1"); + s.push_str("/state/2"); s } @@ -75,9 +75,10 @@ fn generate_protocol_name(protocol_id: &ProtocolId) -> String { struct SeenRequestsKey { peer: PeerId, block: B::Hash, - start: Vec, + start: Vec>, } +#[allow(clippy::derive_hash_xor_eq)] impl Hash for SeenRequestsKey { fn hash(&self, state: &mut H) { self.peer.hash(state); @@ -168,10 +169,10 @@ impl StateRequestHandler { trace!( target: LOG_TARGET, - "Handling state request from {}: Block {:?}, Starting at {:?}, no_proof={}", + "Handling state request from {}: Block {:?}, Starting at {:x?}, no_proof={}", peer, request.block, - sp_core::hexdisplay::HexDisplay::from(&request.start), + &request.start, request.no_proof, ); @@ -179,36 +180,45 @@ impl StateRequestHandler { let mut response = StateResponse::default(); if !request.no_proof { - let (proof, count) = self.client.read_proof_collection( + let (proof, _count) = self.client.read_proof_collection( &BlockId::hash(block), - &request.start, + request.start.as_slice(), MAX_RESPONSE_BYTES, )?; response.proof = proof.encode(); - if count == 0 { - response.complete = true; - } } else { let entries = self.client.storage_collection( &BlockId::hash(block), - &request.start, + request.start.as_slice(), MAX_RESPONSE_BYTES, )?; - response.entries = - entries.into_iter().map(|(key, value)| StateEntry { key, value }).collect(); - if response.entries.is_empty() { - response.complete = true; - } + response.entries = entries + .into_iter() + .map(|(state, complete)| KeyValueStateEntry { + state_root: state.state_root, + entries: state + .key_values + .into_iter() + .map(|(key, value)| StateEntry { key, value }) + .collect(), + complete, + }) + .collect(); } trace!( target: LOG_TARGET, - "StateResponse contains {} keys, {}, proof nodes, complete={}, from {:?} to {:?}", + "StateResponse contains {} keys, {}, proof nodes, from {:?} to {:?}", response.entries.len(), response.proof.len(), - response.complete, - response.entries.first().map(|e| sp_core::hexdisplay::HexDisplay::from(&e.key)), - response.entries.last().map(|e| sp_core::hexdisplay::HexDisplay::from(&e.key)), + response.entries.get(0).and_then(|top| top + .entries + .first() + .map(|e| sp_core::hexdisplay::HexDisplay::from(&e.key))), + response.entries.get(0).and_then(|top| top + .entries + .last() + .map(|e| sp_core::hexdisplay::HexDisplay::from(&e.key))), ); if let Some(value) = self.seen_requests.get_mut(&key) { // If this is the first time we have processed this request, we need to change @@ -231,15 +241,20 @@ impl StateRequestHandler { } } -#[derive(derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] enum HandleRequestError { - #[display(fmt = "Failed to decode request: {}.", _0)] - DecodeProto(prost::DecodeError), - #[display(fmt = "Failed to encode response: {}.", _0)] - EncodeProto(prost::EncodeError), - #[display(fmt = "Failed to decode block hash: {}.", _0)] - InvalidHash(codec::Error), - Client(sp_blockchain::Error), - #[display(fmt = "Failed to send response.")] + #[error("Failed to decode request: {0}.")] + DecodeProto(#[from] prost::DecodeError), + + #[error("Failed to encode response: {0}.")] + EncodeProto(#[from] prost::EncodeError), + + #[error("Failed to decode block hash: {0}.")] + InvalidHash(#[from] codec::Error), + + #[error(transparent)] + Client(#[from] sp_blockchain::Error), + + #[error("Failed to send response.")] SendResponse, } diff --git a/client/network/src/transactions.rs b/client/network/src/transactions.rs index 82e7e8fe1714..c09c6b88dab8 100644 --- a/client/network/src/transactions.rs +++ b/client/network/src/transactions.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -92,10 +92,10 @@ struct Metrics { impl Metrics { fn register(r: &Registry) -> Result { - Ok(Metrics { + Ok(Self { propagated_transactions: register( Counter::new( - "sync_propagated_transactions", + "substrate_sync_propagated_transactions", "Number of transactions propagated to at least one peer", )?, r, @@ -133,7 +133,7 @@ pub struct TransactionsHandlerPrototype { impl TransactionsHandlerPrototype { /// Create a new instance. pub fn new(protocol_id: ProtocolId) -> Self { - TransactionsHandlerPrototype { + Self { protocol_name: Cow::from({ let mut proto = String::new(); proto.push_str("/"); @@ -317,15 +317,10 @@ impl TransactionsHandler { } }, Event::SyncDisconnected { remote } => { - let addr = iter::once(multiaddr::Protocol::P2p(remote.into())) - .collect::(); - let result = self.service.remove_peers_from_reserved_set( + self.service.remove_peers_from_reserved_set( self.protocol_name.clone(), - iter::once(addr).collect(), + iter::once(remote).collect(), ); - if let Err(err) = result { - log::error!(target: "sync", "Removing reserved peer failed: {}", err); - } }, Event::NotificationStreamOpened { remote, protocol, role, .. } @@ -341,13 +336,13 @@ impl TransactionsHandler { }, ); debug_assert!(_was_in.is_none()); - } + }, Event::NotificationStreamClosed { remote, protocol } if protocol == self.protocol_name => { let _peer = self.peers.remove(&remote); debug_assert!(_peer.is_some()); - } + }, Event::NotificationsReceived { remote, messages } => { for (protocol, message) in messages { @@ -401,7 +396,7 @@ impl TransactionsHandler { let hash = self.transaction_pool.hash_of(&t); peer.known_transactions.insert(hash.clone()); - self.service.report_peer(who.clone(), rep::ANY_TRANSACTION); + self.service.report_peer(who, rep::ANY_TRANSACTION); match self.pending_transactions_peers.entry(hash.clone()) { Entry::Vacant(entry) => { @@ -409,10 +404,10 @@ impl TransactionsHandler { validation: self.transaction_pool.import(t), tx_hash: hash, }); - entry.insert(vec![who.clone()]); + entry.insert(vec![who]); }, Entry::Occupied(mut entry) => { - entry.get_mut().push(who.clone()); + entry.get_mut().push(who); }, } } @@ -468,11 +463,8 @@ impl TransactionsHandler { propagated_to.entry(hash).or_default().push(who.to_base58()); } trace!(target: "sync", "Sending {} transactions to {}", to_send.len(), who); - self.service.write_notification( - who.clone(), - self.protocol_name.clone(), - to_send.encode(), - ); + self.service + .write_notification(*who, self.protocol_name.clone(), to_send.encode()); } } diff --git a/client/network/src/transport.rs b/client/network/src/transport.rs index 3f977a21b116..9a8e080234f3 100644 --- a/client/network/src/transport.rs +++ b/client/network/src/transport.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/network/src/utils.rs b/client/network/src/utils.rs index b23b7e0c101e..d0e61a0d0475 100644 --- a/client/network/src/utils.rs +++ b/client/network/src/utils.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/network/src/warp_request_handler.rs b/client/network/src/warp_request_handler.rs index 2ab95bb3853b..4c839825ff5e 100644 --- a/client/network/src/warp_request_handler.rs +++ b/client/network/src/warp_request_handler.rs @@ -23,10 +23,11 @@ use futures::{ stream::StreamExt, }; use log::debug; -use sp_finality_grandpa::{AuthorityList, SetId}; use sp_runtime::traits::Block as BlockT; use std::{sync::Arc, time::Duration}; +pub use sp_finality_grandpa::{AuthorityList, SetId}; + /// Scale-encoded warp sync proof response. pub struct EncodedProof(pub Vec); @@ -55,7 +56,7 @@ pub trait WarpSyncProvider: Send + Sync { &self, start: B::Hash, ) -> Result>; - /// Verify warp proof agains current set of authorities. + /// Verify warp proof against current set of authorities. fn verify( &self, proof: &EncodedProof, @@ -148,18 +149,23 @@ impl RequestHandler { } } -#[derive(Debug, derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] enum HandleRequestError { - #[display(fmt = "Failed to decode request: {}.", _0)] - DecodeProto(prost::DecodeError), - #[display(fmt = "Failed to encode response: {}.", _0)] - EncodeProto(prost::EncodeError), - #[display(fmt = "Failed to decode block hash: {}.", _0)] - DecodeScale(codec::Error), - Client(sp_blockchain::Error), - #[from(ignore)] - #[display(fmt = "Invalid request {}.", _0)] - InvalidRequest(Box), - #[display(fmt = "Failed to send response.")] + #[error("Failed to decode request: {0}.")] + DecodeProto(#[from] prost::DecodeError), + + #[error("Failed to encode response: {0}.")] + EncodeProto(#[from] prost::EncodeError), + + #[error("Failed to decode block hash: {0}.")] + DecodeScale(#[from] codec::Error), + + #[error(transparent)] + Client(#[from] sp_blockchain::Error), + + #[error("Invalid request {0}.")] + InvalidRequest(#[from] Box), + + #[error("Failed to send response.")] SendResponse, } diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml index b4c3a74607f6..b9505f97d14c 100644 --- a/client/network/test/Cargo.toml +++ b/client/network/test/Cargo.toml @@ -4,9 +4,9 @@ name = "sc-network-test" version = "0.8.0" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" publish = false -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] @@ -16,21 +16,21 @@ targets = ["x86_64-unknown-linux-gnu"] async-std = "1.10.0" sc-network = { version = "0.10.0-dev", path = "../" } log = "0.4.8" -parking_lot = "0.11.1" -futures = "0.3.9" +parking_lot = "0.12.0" +futures = "0.3.21" futures-timer = "3.0.1" rand = "0.7.2" -libp2p = { version = "0.39.1", default-features = false } +libp2p = { version = "0.40.0", default-features = false } sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } sc-consensus = { version = "0.10.0-dev", path = "../../consensus/common" } sc-client-api = { version = "4.0.0-dev", path = "../../api" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } sc-block-builder = { version = "0.10.0-dev", path = "../../block-builder" } sp-consensus-babe = { version = "0.10.0-dev", path = "../../../primitives/consensus/babe" } substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } substrate-test-runtime = { version = "2.0.0", path = "../../../test-utils/runtime" } -sp-tracing = { version = "4.0.0-dev", path = "../../../primitives/tracing" } +sp-tracing = { version = "5.0.0", path = "../../../primitives/tracing" } sc-service = { version = "0.10.0-dev", default-features = false, features = ["test-helpers"], path = "../../service" } async-trait = "0.1.50" diff --git a/client/network/test/src/block_import.rs b/client/network/test/src/block_import.rs index 7a4c4f6c8308..a2bd5276c31d 100644 --- a/client/network/test/src/block_import.rs +++ b/client/network/test/src/block_import.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/network/test/src/lib.rs b/client/network/test/src/lib.rs index bb49cef8c642..552879f35d93 100644 --- a/client/network/test/src/lib.rs +++ b/client/network/test/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -28,8 +28,10 @@ use std::{ pin::Pin, sync::Arc, task::{Context as FutureContext, Poll}, + time::Duration, }; +use async_std::future::timeout; use futures::{future::BoxFuture, prelude::*}; use libp2p::{build_multiaddr, PeerId}; use log::trace; @@ -46,14 +48,14 @@ use sc_consensus::{ }; pub use sc_network::config::EmptyTransactionPool; use sc_network::{ - block_request_handler::{self, BlockRequestHandler}, + block_request_handler::BlockRequestHandler, config::{ MultiaddrWithPeerId, NetworkConfiguration, NonDefaultSetConfig, NonReservedPeerMode, ProtocolConfig, ProtocolId, Role, SyncMode, TransportConfig, }, - light_client_requests::{self, handler::LightClientRequestHandler}, - state_request_handler::{self, StateRequestHandler}, - Multiaddr, NetworkService, NetworkWorker, + light_client_requests::handler::LightClientRequestHandler, + state_request_handler::StateRequestHandler, + warp_request_handler, Multiaddr, NetworkService, NetworkWorker, }; use sc_service::client::Client; use sp_blockchain::{ @@ -66,6 +68,7 @@ use sp_consensus::{ }; use sp_core::H256; use sp_runtime::{ + codec::{Decode, Encode}, generic::{BlockId, OpaqueDigestItemId}, traits::{Block as BlockT, Header as HeaderT, NumberFor}, Justification, Justifications, @@ -83,7 +86,6 @@ type AuthorityId = sp_consensus_babe::AuthorityId; #[derive(Clone)] pub struct PassThroughVerifier { finalized: bool, - fork_choice: ForkChoiceStrategy, } impl PassThroughVerifier { @@ -91,15 +93,7 @@ impl PassThroughVerifier { /// /// Every verified block will use `finalized` for the `BlockImportParams`. pub fn new(finalized: bool) -> Self { - Self { finalized, fork_choice: ForkChoiceStrategy::LongestChain } - } - - /// Create a new instance. - /// - /// Every verified block will use `finalized` for the `BlockImportParams` and - /// the given [`ForkChoiceStrategy`]. - pub fn new_with_fork_choice(finalized: bool, fork_choice: ForkChoiceStrategy) -> Self { - Self { finalized, fork_choice } + Self { finalized } } } @@ -118,8 +112,10 @@ impl Verifier for PassThroughVerifier { .or_else(|| l.try_as_raw(OpaqueDigestItemId::Consensus(b"babe"))) }) .map(|blob| vec![(well_known_cache_keys::AUTHORITIES, blob.to_vec())]); + if block.fork_choice.is_none() { + block.fork_choice = Some(ForkChoiceStrategy::LongestChain); + }; block.finalized = self.finalized; - block.fork_choice = Some(self.fork_choice.clone()); Ok((block, maybe_keys)) } } @@ -130,25 +126,20 @@ pub type PeersFullClient = Client< Block, substrate_test_runtime_client::runtime::RuntimeApi, >; -pub type PeersLightClient = Client< - substrate_test_runtime_client::LightBackend, - substrate_test_runtime_client::LightExecutor, - Block, - substrate_test_runtime_client::runtime::RuntimeApi, ->; #[derive(Clone)] -pub enum PeersClient { - Full(Arc, Arc), - Light(Arc, Arc), +pub struct PeersClient { + client: Arc, + backend: Arc, } impl PeersClient { - pub fn as_full(&self) -> Option> { - match *self { - PeersClient::Full(ref client, _) => Some(client.clone()), - _ => None, - } + pub fn as_client(&self) -> Arc { + self.client.clone() + } + + pub fn as_backend(&self) -> Arc { + self.backend.clone() } pub fn as_block_import(&self) -> BlockImportAdapter { @@ -156,27 +147,18 @@ impl PeersClient { } pub fn get_aux(&self, key: &[u8]) -> ClientResult>> { - match *self { - PeersClient::Full(ref client, _) => client.get_aux(key), - PeersClient::Light(ref client, _) => client.get_aux(key), - } + self.client.get_aux(key) } pub fn info(&self) -> BlockchainInfo { - match *self { - PeersClient::Full(ref client, _) => client.chain_info(), - PeersClient::Light(ref client, _) => client.chain_info(), - } + self.client.info() } pub fn header( &self, block: &BlockId, ) -> ClientResult::Header>> { - match *self { - PeersClient::Full(ref client, _) => client.header(block), - PeersClient::Light(ref client, _) => client.header(block), - } + self.client.header(block) } pub fn has_state_at(&self, block: &BlockId) -> bool { @@ -184,33 +166,19 @@ impl PeersClient { Some(header) => header, None => return false, }; - match self { - PeersClient::Full(_client, backend) => - backend.have_state_at(&header.hash(), *header.number()), - PeersClient::Light(_client, backend) => - backend.have_state_at(&header.hash(), *header.number()), - } + self.backend.have_state_at(&header.hash(), *header.number()) } pub fn justifications(&self, block: &BlockId) -> ClientResult> { - match *self { - PeersClient::Full(ref client, _) => client.justifications(block), - PeersClient::Light(ref client, _) => client.justifications(block), - } + self.client.justifications(block) } pub fn finality_notification_stream(&self) -> FinalityNotifications { - match *self { - PeersClient::Full(ref client, _) => client.finality_notification_stream(), - PeersClient::Light(ref client, _) => client.finality_notification_stream(), - } + self.client.finality_notification_stream() } pub fn import_notification_stream(&self) -> ImportNotifications { - match *self { - PeersClient::Full(ref client, _) => client.import_notification_stream(), - PeersClient::Light(ref client, _) => client.import_notification_stream(), - } + self.client.import_notification_stream() } pub fn finalize_block( @@ -219,12 +187,7 @@ impl PeersClient { justification: Option, notify: bool, ) -> ClientResult<()> { - match *self { - PeersClient::Full(ref client, ref _backend) => - client.finalize_block(id, justification, notify), - PeersClient::Light(ref client, ref _backend) => - client.finalize_block(id, justification, notify), - } + self.client.finalize_block(id, justification, notify) } } @@ -237,10 +200,7 @@ impl BlockImport for PeersClient { &mut self, block: BlockCheckParams, ) -> Result { - match self { - PeersClient::Full(client, _) => client.check_block(block).await, - PeersClient::Light(client, _) => client.check_block(block).await, - } + self.client.check_block(block).await } async fn import_block( @@ -248,12 +208,7 @@ impl BlockImport for PeersClient { block: BlockImportParams, cache: HashMap>, ) -> Result { - match self { - PeersClient::Full(client, _) => - client.import_block(block.clear_storage_changes_and_mutate(), cache).await, - PeersClient::Light(client, _) => - client.import_block(block.clear_storage_changes_and_mutate(), cache).await, - } + self.client.import_block(block.clear_storage_changes_and_mutate(), cache).await } } @@ -347,12 +302,39 @@ where false, true, true, + ForkChoiceStrategy::LongestChain, + ) + } + + /// Add blocks to the peer -- edit the block before adding and use custom fork choice rule. + pub fn generate_blocks_with_fork_choice( + &mut self, + count: usize, + origin: BlockOrigin, + edit_block: F, + fork_choice: ForkChoiceStrategy, + ) -> H256 + where + F: FnMut( + BlockBuilder, + ) -> Block, + { + let best_hash = self.client.info().best_hash; + self.generate_blocks_at( + BlockId::Hash(best_hash), + count, + origin, + edit_block, + false, + true, + true, + fork_choice, ) } /// Add blocks to the peer -- edit the block before adding. The chain will /// start at the given block iD. - fn generate_blocks_at( + pub fn generate_blocks_at( &mut self, at: BlockId, count: usize, @@ -361,14 +343,14 @@ where headers_only: bool, inform_sync_about_new_best_block: bool, announce_block: bool, + fork_choice: ForkChoiceStrategy, ) -> H256 where F: FnMut( BlockBuilder, ) -> Block, { - let full_client = - self.client.as_full().expect("blocks could only be generated by full clients"); + let full_client = self.client.as_client(); let mut at = full_client.header(&at).unwrap().unwrap().hash(); for _ in 0..count { let builder = @@ -385,6 +367,7 @@ where let header = block.header.clone(); let mut import_block = BlockImportParams::new(origin, header.clone()); import_block.body = if headers_only { None } else { Some(block.extrinsics) }; + import_block.fork_choice = Some(fork_choice); let (import_block, cache) = futures::executor::block_on(self.verifier.verify(import_block)).unwrap(); let cache = if let Some(cache) = cache { @@ -481,6 +464,7 @@ where headers_only, inform_sync_about_new_best_block, announce_block, + ForkChoiceStrategy::LongestChain, ) } else { self.generate_blocks_at( @@ -491,6 +475,7 @@ where headers_only, inform_sync_about_new_best_block, announce_block, + ForkChoiceStrategy::LongestChain, ) } } @@ -650,6 +635,33 @@ impl VerifierAdapter { } } +struct TestWarpSyncProvider(Arc>); + +impl warp_request_handler::WarpSyncProvider for TestWarpSyncProvider { + fn generate( + &self, + _start: B::Hash, + ) -> Result> { + let info = self.0.info(); + let best_header = self.0.header(BlockId::hash(info.best_hash)).unwrap().unwrap(); + Ok(warp_request_handler::EncodedProof(best_header.encode())) + } + fn verify( + &self, + proof: &warp_request_handler::EncodedProof, + _set_id: warp_request_handler::SetId, + _authorities: warp_request_handler::AuthorityList, + ) -> Result, Box> + { + let warp_request_handler::EncodedProof(encoded) = proof; + let header = B::Header::decode(&mut encoded.as_slice()).unwrap(); + Ok(warp_request_handler::VerificationResult::Complete(0, Default::default(), header)) + } + fn current_authorities(&self) -> warp_request_handler::AuthorityList { + Default::default() + } +} + /// Configuration for a full peer. #[derive(Default)] pub struct FullPeerConfig { @@ -667,6 +679,8 @@ pub struct FullPeerConfig { pub is_authority: bool, /// Syncing mode pub sync_mode: SyncMode, + /// Extra genesis storage. + pub extra_storage: Option, /// Enable transaction indexing. pub storage_chain: bool, } @@ -735,18 +749,23 @@ where (Some(keep_blocks), false) => TestClientBuilder::with_pruning_window(keep_blocks), (None, false) => TestClientBuilder::with_default_backend(), }; - if matches!(config.sync_mode, SyncMode::Fast { .. }) { + if let Some(storage) = config.extra_storage { + let genesis_extra_storage = test_client_builder.genesis_init_mut().extra_storage(); + *genesis_extra_storage = storage; + } + + if matches!(config.sync_mode, SyncMode::Fast { .. } | SyncMode::Warp) { test_client_builder = test_client_builder.set_no_genesis(); } let backend = test_client_builder.backend(); let (c, longest_chain) = test_client_builder.build_with_longest_chain(); let client = Arc::new(c); - let (block_import, justification_import, data) = - self.make_block_import(PeersClient::Full(client.clone(), backend.clone())); + let (block_import, justification_import, data) = self + .make_block_import(PeersClient { client: client.clone(), backend: backend.clone() }); let verifier = self.make_verifier( - PeersClient::Full(client.clone(), backend.clone()), + PeersClient { client: client.clone(), backend: backend.clone() }, &Default::default(), &data, ); @@ -814,6 +833,15 @@ where protocol_config }; + let warp_sync = Arc::new(TestWarpSyncProvider(client.clone())); + + let warp_protocol_config = { + let (handler, protocol_config) = + warp_request_handler::RequestHandler::new(protocol_id.clone(), warp_sync.clone()); + self.spawn_task(handler.run().boxed()); + protocol_config + }; + let network = NetworkWorker::new(sc_network::config::Params { role: if config.is_authority { Role::Authority } else { Role::Full }, executor: None, @@ -822,7 +850,6 @@ where }), network_config, chain: client.clone(), - on_demand: None, transaction_pool: Arc::new(EmptyTransactionPool), protocol_id, import_queue, @@ -833,7 +860,7 @@ where block_request_protocol_config, state_request_protocol_config, light_client_request_protocol_config, - warp_sync: None, + warp_sync: Some((warp_sync, warp_protocol_config)), }) .unwrap(); @@ -853,7 +880,7 @@ where peers.push(Peer { data, - client: PeersClient::Full(client.clone(), backend.clone()), + client: PeersClient { client: client.clone(), backend: backend.clone() }, select_chain: Some(longest_chain), backend: Some(backend), imported_blocks_stream, @@ -866,94 +893,6 @@ where }); } - /// Add a light peer. - fn add_light_peer(&mut self) { - let (c, backend) = substrate_test_runtime_client::new_light(); - let client = Arc::new(c); - let (block_import, justification_import, data) = - self.make_block_import(PeersClient::Light(client.clone(), backend.clone())); - - let verifier = self.make_verifier( - PeersClient::Light(client.clone(), backend.clone()), - &Default::default(), - &data, - ); - let verifier = VerifierAdapter::new(verifier); - - let import_queue = Box::new(BasicQueue::new( - verifier.clone(), - Box::new(block_import.clone()), - justification_import, - &sp_core::testing::TaskExecutor::new(), - None, - )); - - let listen_addr = build_multiaddr![Memory(rand::random::())]; - - let mut network_config = - NetworkConfiguration::new("test-node", "test-client", Default::default(), None); - network_config.transport = TransportConfig::MemoryOnly; - network_config.listen_addresses = vec![listen_addr.clone()]; - network_config.allow_non_globals_in_dht = true; - - let protocol_id = ProtocolId::from("test-protocol-name"); - - let block_request_protocol_config = - block_request_handler::generate_protocol_config(&protocol_id); - let state_request_protocol_config = - state_request_handler::generate_protocol_config(&protocol_id); - - let light_client_request_protocol_config = - light_client_requests::generate_protocol_config(&protocol_id); - - let network = NetworkWorker::new(sc_network::config::Params { - role: Role::Light, - executor: None, - transactions_handler_executor: Box::new(|task| { - async_std::task::spawn(task); - }), - network_config, - chain: client.clone(), - on_demand: None, - transaction_pool: Arc::new(EmptyTransactionPool), - protocol_id, - import_queue, - block_announce_validator: Box::new(DefaultBlockAnnounceValidator), - metrics_registry: None, - block_request_protocol_config, - state_request_protocol_config, - light_client_request_protocol_config, - warp_sync: None, - }) - .unwrap(); - - self.mut_peers(|peers| { - for peer in peers.iter_mut() { - peer.network.add_known_address( - network.service().local_peer_id().clone(), - listen_addr.clone(), - ); - } - - let imported_blocks_stream = Box::pin(client.import_notification_stream().fuse()); - let finality_notification_stream = - Box::pin(client.finality_notification_stream().fuse()); - - peers.push(Peer { - data, - verifier, - select_chain: None, - backend: None, - block_import, - client: PeersClient::Light(client, backend), - imported_blocks_stream, - finality_notification_stream, - network, - listen_addr, - }); - }); - } - /// Used to spawn background tasks, e.g. the block request protocol handler. fn spawn_task(&self, f: BoxFuture<'static, ()>) { async_std::task::spawn(f); @@ -1017,10 +956,13 @@ where /// Blocks the current thread until we are sync'ed. /// /// Calls `poll_until_sync` repeatedly. + /// (If we've not synced within 10 mins then panic rather than hang.) fn block_until_sync(&mut self) { - futures::executor::block_on(futures::future::poll_fn::<(), _>(|cx| { - self.poll_until_sync(cx) - })); + futures::executor::block_on(timeout( + Duration::from_secs(10 * 60), + futures::future::poll_fn::<(), _>(|cx| self.poll_until_sync(cx)), + )) + .expect("sync didn't happen within 10 mins"); } /// Blocks the current thread until there are no pending packets. @@ -1058,14 +1000,10 @@ where peer.network.service().announce_block(notification.hash, None); } - // We poll `finality_notification_stream`, but we only take the last event. - let mut last = None; - while let Poll::Ready(Some(item)) = + // We poll `finality_notification_stream`. + while let Poll::Ready(Some(notification)) = peer.finality_notification_stream.as_mut().poll_next(cx) { - last = Some(item); - } - if let Some(notification) = last { peer.network.on_block_finalized(notification.hash, notification.header); } } @@ -1075,14 +1013,6 @@ where pub struct TestNet { peers: Vec>, - fork_choice: ForkChoiceStrategy, -} - -impl TestNet { - /// Create a `TestNet` that used the given fork choice rule. - pub fn with_fork_choice(fork_choice: ForkChoiceStrategy) -> Self { - Self { peers: Vec::new(), fork_choice } - } } impl TestNetFactory for TestNet { @@ -1092,7 +1022,7 @@ impl TestNetFactory for TestNet { /// Create new test network with peers and given config. fn from_config(_config: &ProtocolConfig) -> Self { - TestNet { peers: Vec::new(), fork_choice: ForkChoiceStrategy::LongestChain } + TestNet { peers: Vec::new() } } fn make_verifier( @@ -1101,7 +1031,7 @@ impl TestNetFactory for TestNet { _config: &ProtocolConfig, _peer_data: &(), ) -> Self::Verifier { - PassThroughVerifier::new_with_fork_choice(false, self.fork_choice.clone()) + PassThroughVerifier::new(false) } fn make_block_import( diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index f413b705e52c..84a5c2ca13fa 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,7 +20,6 @@ use super::*; use futures::{executor::block_on, Future}; use sp_consensus::{block_validation::Validation, BlockOrigin}; use sp_runtime::Justifications; -use std::time::Duration; use substrate_test_runtime::Header; fn test_ancestor_search_when_common_is(n: usize) { @@ -391,35 +390,6 @@ fn own_blocks_are_announced() { (net.peers()[2].blockchain_canon_equals(peer0)); } -#[test] -fn blocks_are_not_announced_by_light_nodes() { - sp_tracing::try_init_simple(); - let mut net = TestNet::new(0); - - // full peer0 is connected to light peer - // light peer1 is connected to full peer2 - net.add_full_peer(); - net.add_light_peer(); - - // Sync between 0 and 1. - net.peer(0).push_blocks(1, false); - assert_eq!(net.peer(0).client.info().best_number, 1); - net.block_until_sync(); - assert_eq!(net.peer(1).client.info().best_number, 1); - - // Add another node and remove node 0. - net.add_full_peer(); - net.peers.remove(0); - - // Poll for a few seconds and make sure 1 and 2 (now 0 and 1) don't sync together. - let mut delay = futures_timer::Delay::new(Duration::from_secs(5)); - block_on(futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - Pin::new(&mut delay).poll(cx) - })); - assert_eq!(net.peer(1).client.info().best_number, 0); -} - #[test] fn can_sync_small_non_best_forks() { sp_tracing::try_init_simple(); @@ -484,69 +454,35 @@ fn can_sync_small_non_best_forks() { } #[test] -fn can_not_sync_from_light_peer() { +fn can_sync_forks_ahead_of_the_best_chain() { sp_tracing::try_init_simple(); - - // given the network with 1 full nodes (#0) and 1 light node (#1) - let mut net = TestNet::new(1); - net.add_light_peer(); - - // generate some blocks on #0 + let mut net = TestNet::new(2); net.peer(0).push_blocks(1, false); + net.peer(1).push_blocks(1, false); - // and let the light client sync from this node - net.block_until_sync(); - - // ensure #0 && #1 have the same best block - let full0_info = net.peer(0).client.info(); - let light_info = net.peer(1).client.info(); - assert_eq!(full0_info.best_number, 1); - assert_eq!(light_info.best_number, 1); - assert_eq!(light_info.best_hash, full0_info.best_hash); - - // add new full client (#2) && remove #0 - net.add_full_peer(); - net.peers.remove(0); + net.block_until_connected(); + // Peer 0 is on 2-block fork which is announced with is_best=false + let fork_hash = net.peer(0).generate_blocks_with_fork_choice( + 2, + BlockOrigin::Own, + |builder| builder.build().unwrap().block, + ForkChoiceStrategy::Custom(false), + ); + // Peer 1 is on 1-block fork + net.peer(1).push_blocks(1, false); + assert!(net.peer(0).client().header(&BlockId::Hash(fork_hash)).unwrap().is_some()); + assert_eq!(net.peer(0).client().info().best_number, 1); + assert_eq!(net.peer(1).client().info().best_number, 2); - // ensure that the #2 (now #1) fails to sync block #1 even after 5 seconds - let mut test_finished = futures_timer::Delay::new(Duration::from_secs(5)); + // after announcing, peer 1 downloads the block. block_on(futures::future::poll_fn::<(), _>(|cx| { net.poll(cx); - Pin::new(&mut test_finished).poll(cx) - })); -} - -#[test] -fn light_peer_imports_header_from_announce() { - sp_tracing::try_init_simple(); - - fn import_with_announce(net: &mut TestNet, hash: H256) { - net.peer(0).announce_block(hash, None); - - block_on(futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(1).client().header(&BlockId::Hash(hash)).unwrap().is_some() { - Poll::Ready(()) - } else { - Poll::Pending - } - })); - } - - // given the network with 1 full nodes (#0) and 1 light node (#1) - let mut net = TestNet::new(1); - net.add_light_peer(); - // let them connect to each other - net.block_until_sync(); - - // check that NEW block is imported from announce message - let new_hash = net.peer(0).push_blocks(1, false); - import_with_announce(&mut net, new_hash); - - // check that KNOWN STALE block is imported from announce message - let known_stale_hash = net.peer(0).push_blocks_at(BlockId::Number(0), 1, true); - import_with_announce(&mut net, known_stale_hash); + if net.peer(1).client().header(&BlockId::Hash(fork_hash)).unwrap().is_none() { + return Poll::Pending + } + Poll::Ready(()) + })); } #[test] @@ -774,7 +710,7 @@ impl BlockAnnounceValidator for FailingBlockAnnounceValidator { #[test] fn sync_blocks_when_block_announce_validator_says_it_is_new_best() { sp_tracing::try_init_simple(); - let mut net = TestNet::with_fork_choice(ForkChoiceStrategy::Custom(false)); + let mut net = TestNet::new(0); net.add_full_peer_with_config(Default::default()); net.add_full_peer_with_config(Default::default()); net.add_full_peer_with_config(FullPeerConfig { @@ -784,16 +720,17 @@ fn sync_blocks_when_block_announce_validator_says_it_is_new_best() { net.block_until_connected(); - let block_hash = net.peer(0).push_blocks(1, false); + // Add blocks but don't set them as best + let block_hash = net.peer(0).generate_blocks_with_fork_choice( + 1, + BlockOrigin::Own, + |builder| builder.build().unwrap().block, + ForkChoiceStrategy::Custom(false), + ); while !net.peer(2).has_block(&block_hash) { net.block_until_idle(); } - - // Peer1 should not have the block, because peer 0 did not reported the block - // as new best. However, peer2 has a special block announcement validator - // that flags all blocks as `is_new_best` and thus, it should have synced the blocks. - assert!(!net.peer(1).has_block(&block_hash)); } /// Waits for some time until the validation is successfull. @@ -817,7 +754,7 @@ impl BlockAnnounceValidator for DeferredBlockAnnounceValidator { #[test] fn wait_until_deferred_block_announce_validation_is_ready() { sp_tracing::try_init_simple(); - let mut net = TestNet::with_fork_choice(ForkChoiceStrategy::Custom(false)); + let mut net = TestNet::new(0); net.add_full_peer_with_config(Default::default()); net.add_full_peer_with_config(FullPeerConfig { block_announce_validator: Some(Box::new(NewBestBlockAnnounceValidator)), @@ -826,7 +763,13 @@ fn wait_until_deferred_block_announce_validation_is_ready() { net.block_until_connected(); - let block_hash = net.peer(0).push_blocks(1, true); + // Add blocks but don't set them as best + let block_hash = net.peer(0).generate_blocks_with_fork_choice( + 1, + BlockOrigin::Own, + |builder| builder.build().unwrap().block, + ForkChoiceStrategy::Custom(false), + ); while !net.peer(1).has_block(&block_hash) { net.block_until_idle(); @@ -943,7 +886,10 @@ fn block_announce_data_is_propagated() { // Wait until peer 1 is connected to both nodes. block_on(futures::future::poll_fn::<(), _>(|cx| { net.poll(cx); - if net.peer(1).num_peers() == 2 { + if net.peer(1).num_peers() == 2 && + net.peer(0).num_peers() == 1 && + net.peer(2).num_peers() == 1 + { Poll::Ready(()) } else { Poll::Pending @@ -1110,11 +1056,44 @@ fn syncs_state() { sp_tracing::try_init_simple(); for skip_proofs in &[false, true] { let mut net = TestNet::new(0); - net.add_full_peer_with_config(Default::default()); - net.add_full_peer_with_config(FullPeerConfig { - sync_mode: SyncMode::Fast { skip_proofs: *skip_proofs, storage_chain_mode: false }, - ..Default::default() - }); + let mut genesis_storage: sp_core::storage::Storage = Default::default(); + genesis_storage.top.insert(b"additional_key".to_vec(), vec![1]); + let mut child_data: std::collections::BTreeMap, Vec> = Default::default(); + for i in 0u8..16 { + child_data.insert(vec![i; 5], vec![i; 33]); + } + let child1 = sp_core::storage::StorageChild { + data: child_data.clone(), + child_info: sp_core::storage::ChildInfo::new_default(b"child1"), + }; + let child3 = sp_core::storage::StorageChild { + data: child_data.clone(), + child_info: sp_core::storage::ChildInfo::new_default(b"child3"), + }; + for i in 22u8..33 { + child_data.insert(vec![i; 5], vec![i; 33]); + } + let child2 = sp_core::storage::StorageChild { + data: child_data.clone(), + child_info: sp_core::storage::ChildInfo::new_default(b"child2"), + }; + genesis_storage + .children_default + .insert(child1.child_info.storage_key().to_vec(), child1); + genesis_storage + .children_default + .insert(child2.child_info.storage_key().to_vec(), child2); + genesis_storage + .children_default + .insert(child3.child_info.storage_key().to_vec(), child3); + let mut config_one = FullPeerConfig::default(); + config_one.extra_storage = Some(genesis_storage.clone()); + net.add_full_peer_with_config(config_one); + let mut config_two = FullPeerConfig::default(); + config_two.extra_storage = Some(genesis_storage); + config_two.sync_mode = + SyncMode::Fast { skip_proofs: *skip_proofs, storage_chain_mode: false }; + net.add_full_peer_with_config(config_two); net.peer(0).push_blocks(64, false); // Wait for peer 1 to sync header chain. net.block_until_sync(); @@ -1172,21 +1151,20 @@ fn syncs_indexed_blocks() { false, true, true, + ForkChoiceStrategy::LongestChain, ); let indexed_key = sp_runtime::traits::BlakeTwo256::hash(&42u64.to_le_bytes()); assert!(net .peer(0) .client() - .as_full() - .unwrap() + .as_client() .indexed_transaction(&indexed_key) .unwrap() .is_some()); assert!(net .peer(1) .client() - .as_full() - .unwrap() + .as_client() .indexed_transaction(&indexed_key) .unwrap() .is_none()); @@ -1195,13 +1173,44 @@ fn syncs_indexed_blocks() { assert!(net .peer(1) .client() - .as_full() - .unwrap() + .as_client() .indexed_transaction(&indexed_key) .unwrap() .is_some()); } +#[test] +fn warp_sync() { + sp_tracing::try_init_simple(); + let mut net = TestNet::new(0); + // Create 3 synced peers and 1 peer trying to warp sync. + net.add_full_peer_with_config(Default::default()); + net.add_full_peer_with_config(Default::default()); + net.add_full_peer_with_config(Default::default()); + net.add_full_peer_with_config(FullPeerConfig { + sync_mode: SyncMode::Warp, + ..Default::default() + }); + let gap_end = net.peer(0).push_blocks(63, false); + net.peer(0).push_blocks(1, false); + net.peer(1).push_blocks(64, false); + net.peer(2).push_blocks(64, false); + // Wait for peer 1 to sync state. + net.block_until_sync(); + assert!(!net.peer(3).client().has_state_at(&BlockId::Number(1))); + assert!(net.peer(3).client().has_state_at(&BlockId::Number(64))); + + // Wait for peer 1 download block history + block_on(futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); + if net.peer(3).has_block(&gap_end) { + Poll::Ready(()) + } else { + Poll::Pending + } + })); +} + #[test] fn syncs_huge_blocks() { use sp_core::storage::well_known_keys::HEAP_PAGES; diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index 1a4523977dd0..d221b15b2f54 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -4,8 +4,8 @@ name = "sc-offchain" version = "4.0.0-dev" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" authors = ["Parity Technologies "] -edition = "2018" -homepage = "https://substrate.dev" +edition = "2021" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,38 +13,38 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -bytes = "1.0" -codec = { package = "parity-scale-codec", version = "2.0.0", features = ["derive"] } +bytes = "1.1" +codec = { package = "parity-scale-codec", version = "3.0.0", features = ["derive"] } hex = "0.4" fnv = "1.0.6" -futures = "0.3.9" -futures-timer = "3.0.1" -log = "0.4.8" -num_cpus = "1.10" -parking_lot = "0.11.1" +futures = "0.3.21" +futures-timer = "3.0.2" +num_cpus = "1.13" +parking_lot = "0.12.0" rand = "0.7.2" sc-client-api = { version = "4.0.0-dev", path = "../api" } sc-network = { version = "0.10.0-dev", path = "../network" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } sp-offchain = { version = "4.0.0-dev", path = "../../primitives/offchain" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } sc-utils = { version = "4.0.0-dev", path = "../utils" } threadpool = "1.7" +hyper = { version = "0.14.16", features = ["stream", "http2"] } cid = "0.5" -ipfs = { git = "https://github.com/rs-ipfs/rust-ipfs" } -tokio = { version = "1.10", default-features = false } - -[target.'cfg(not(target_os = "unknown"))'.dependencies] -hyper = "0.14.11" +ipfs = { git = "https://github.com/rs-ipfs/rust-ipfs", branch="master" } hyper-rustls = "0.22.1" +once_cell = "1.8" +tracing = "0.1.29" +log = "0.4.8" +tokio = "1.17.0" [dev-dependencies] sc-client-db = { version = "0.10.0-dev", default-features = true, path = "../db" } sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" } sc-transaction-pool = { version = "4.0.0-dev", path = "../transaction-pool" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../transaction-pool/api" } -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } lazy_static = "1.4.0" diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs index aa5b1f31f51b..58aa89ce1f4c 100644 --- a/client/offchain/src/api.rs +++ b/client/offchain/src/api.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,10 +16,11 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use std::{collections::HashSet, convert::TryFrom, str::FromStr, sync::Arc, thread::sleep}; +use std::{collections::HashSet, str::FromStr, sync::Arc, thread::sleep}; use crate::NetworkProvider; use codec::{Decode, Encode}; +use futures::Future; pub use http::SharedClient; use sc_network::{Multiaddr, PeerId}; use sp_core::{ @@ -44,10 +45,11 @@ mod http_dummy; mod timestamp; fn unavailable_yet(name: &str) -> R { - log::error!( - target: "sc_offchain", + tracing::error!( + target: super::LOG_TARGET, "The {:?} API is not available for offchain workers yet. Follow \ - https://github.com/paritytech/substrate/issues/1458 for details", name + https://github.com/paritytech/substrate/issues/1458 for details", + name ); Default::default() } @@ -82,9 +84,12 @@ impl Db { impl offchain::DbExternalities for Db { fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { - log::debug!( - target: "sc_offchain", - "{:?}: Write: {:?} <= {:?}", kind, hex::encode(key), hex::encode(value) + tracing::debug!( + target: "offchain-worker::storage", + ?kind, + key = ?hex::encode(key), + value = ?hex::encode(value), + "Write", ); match kind { StorageKind::PERSISTENT => self.persistent.set(STORAGE_PREFIX, key, value), @@ -93,9 +98,11 @@ impl offchain::DbExternalities for Db { } fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { - log::debug!( - target: "sc_offchain", - "{:?}: Clear: {:?}", kind, hex::encode(key) + tracing::debug!( + target: "offchain-worker::storage", + ?kind, + key = ?hex::encode(key), + "Clear", ); match kind { StorageKind::PERSISTENT => self.persistent.remove(STORAGE_PREFIX, key), @@ -110,13 +117,13 @@ impl offchain::DbExternalities for Db { old_value: Option<&[u8]>, new_value: &[u8], ) -> bool { - log::debug!( - target: "sc_offchain", - "{:?}: CAS: {:?} <= {:?} vs {:?}", - kind, - hex::encode(key), - hex::encode(new_value), - old_value.as_ref().map(hex::encode), + tracing::debug!( + target: "offchain-worker::storage", + ?kind, + key = ?hex::encode(key), + new_value = ?hex::encode(new_value), + old_value = ?old_value.as_ref().map(hex::encode), + "CAS", ); match kind { StorageKind::PERSISTENT => @@ -130,12 +137,12 @@ impl offchain::DbExternalities for Db { StorageKind::PERSISTENT => self.persistent.get(STORAGE_PREFIX, key), StorageKind::LOCAL => unavailable_yet(LOCAL_DB), }; - log::debug!( - target: "sc_offchain", - "{:?}: Read: {:?} => {:?}", - kind, - hex::encode(key), - result.as_ref().map(hex::encode) + tracing::debug!( + target: "offchain-worker::storage", + ?kind, + key = ?hex::encode(key), + result = ?result.as_ref().map(hex::encode), + "Read", ); result } @@ -320,10 +327,10 @@ impl AsyncApi { pub fn new( network_provider: Arc, is_validator: bool, - shared_client: SharedClient, + shared_http_client: SharedClient, ipfs_node: ::ipfs::Ipfs ) -> (Api, Self) { - let (http_api, http_worker) = http::http(shared_client); + let (http_api, http_worker) = http::http(shared_http_client); let (ipfs_api, ipfs_worker) = ipfs::ipfs(ipfs_node); @@ -335,7 +342,10 @@ impl AsyncApi { } /// Run a processing task for the API + // pub async fn process(self) -> impl Future { pub async fn process(mut self) { + // let http self.http.expect("`process` is only called once; qed") + // pub async fn process(mut self) { let http = self.http.take().expect("Take invoked only once."); let ipfs = self.ipfs.take().expect("Take invoked only once."); futures::join!(http, ipfs); @@ -354,7 +364,7 @@ mod tests { }; // use ipfs::TestTypes; - struct TestNetwork(); + pub(super) struct TestNetwork(); impl NetworkProvider for TestNetwork { fn set_authorized_peers(&self, _peers: HashSet) { diff --git a/client/offchain/src/api/http.rs b/client/offchain/src/api/http.rs index ce9fb298d1b0..012de78c5f64 100644 --- a/client/offchain/src/api/http.rs +++ b/client/offchain/src/api/http.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -33,11 +33,10 @@ use fnv::FnvHashMap; use futures::{channel::mpsc, future, prelude::*}; use hyper::{client, Body, Client as HyperClient}; use hyper_rustls::HttpsConnector; -use log::error; +use once_cell::sync::Lazy; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_core::offchain::{HttpError, HttpRequestId, HttpRequestStatus, Timestamp}; use std::{ - convert::TryFrom, fmt, io::Read as _, pin::Pin, @@ -45,13 +44,17 @@ use std::{ task::{Context, Poll}, }; +const LOG_TARGET: &str = "offchain-worker::http"; + /// Wrapper struct used for keeping the hyper_rustls client running. #[derive(Clone)] -pub struct SharedClient(Arc, Body>>); +pub struct SharedClient(Arc, Body>>>); impl SharedClient { pub fn new() -> Self { - Self(Arc::new(HyperClient::builder().build(HttpsConnector::with_native_roots()))) + Self(Arc::new(Lazy::new(|| { + HyperClient::builder().build(HttpsConnector::with_native_roots()) + }))) } } @@ -143,13 +146,24 @@ impl HttpApi { match self.next_id.0.checked_add(1) { Some(new_id) => self.next_id.0 = new_id, None => { - error!("Overflow in offchain worker HTTP request ID assignment"); + tracing::error!( + target: LOG_TARGET, + "Overflow in offchain worker HTTP request ID assignment" + ); return Err(()) }, }; self.requests .insert(new_id, HttpApiRequest::NotDispatched(request, body_sender)); + tracing::error!( + target: LOG_TARGET, + id = %new_id.0, + %method, + %uri, + "Requested started", + ); + Ok(new_id) } @@ -165,11 +179,14 @@ impl HttpApi { _ => return Err(()), }; - let name = hyper::header::HeaderName::try_from(name).map_err(drop)?; - let value = hyper::header::HeaderValue::try_from(value).map_err(drop)?; + let header_name = hyper::header::HeaderName::try_from(name).map_err(drop)?; + let header_value = hyper::header::HeaderValue::try_from(value).map_err(drop)?; // Note that we're always appending headers and never replacing old values. // We assume here that the user knows what they're doing. - request.headers_mut().append(name, value); + request.headers_mut().append(header_name, header_value); + + tracing::debug!(target: LOG_TARGET, id = %request_id.0, %name, %value, "Added header to request"); + Ok(()) } @@ -204,7 +221,7 @@ impl HttpApi { sender.send_data(hyper::body::Bytes::from(chunk.to_owned())), ) .map_err(|_| { - error!("HTTP sender refused data despite being ready"); + tracing::error!(target: "offchain-worker::http", "HTTP sender refused data despite being ready"); HttpError::IoError }) }; @@ -212,6 +229,7 @@ impl HttpApi { loop { request = match request { HttpApiRequest::NotDispatched(request, sender) => { + tracing::debug!(target: LOG_TARGET, id = %request_id.0, "Added new body chunk"); // If the request is not dispatched yet, dispatch it and loop again. let _ = self .to_worker @@ -222,14 +240,20 @@ impl HttpApi { HttpApiRequest::Dispatched(Some(mut sender)) => { if !chunk.is_empty() { match poll_sender(&mut sender) { - Err(HttpError::IoError) => return Err(HttpError::IoError), + Err(HttpError::IoError) => { + tracing::debug!(target: LOG_TARGET, id = %request_id.0, "Encountered io error while trying to add new chunk to body"); + return Err(HttpError::IoError) + }, other => { + tracing::debug!(target: LOG_TARGET, id = %request_id.0, res = ?other, "Added chunk to body"); self.requests .insert(request_id, HttpApiRequest::Dispatched(Some(sender))); return other }, } } else { + tracing::debug!(target: LOG_TARGET, id = %request_id.0, "Finished writing body"); + // Writing an empty body is a hint that we should stop writing. Dropping // the sender. self.requests.insert(request_id, HttpApiRequest::Dispatched(None)); @@ -247,14 +271,20 @@ impl HttpApi { .as_mut() .expect("Can only enter this match branch if Some; qed"), ) { - Err(HttpError::IoError) => return Err(HttpError::IoError), + Err(HttpError::IoError) => { + tracing::debug!(target: LOG_TARGET, id = %request_id.0, "Encountered io error while trying to add new chunk to body"); + return Err(HttpError::IoError) + }, other => { + tracing::debug!(target: LOG_TARGET, id = %request_id.0, res = ?other, "Added chunk to body"); self.requests .insert(request_id, HttpApiRequest::Response(response)); return other }, } } else { + tracing::debug!(target: LOG_TARGET, id = %request_id.0, "Finished writing body"); + // Writing an empty body is a hint that we should stop writing. Dropping // the sender. self.requests.insert( @@ -268,13 +298,18 @@ impl HttpApi { } }, - HttpApiRequest::Fail(_) => - // If the request has already failed, return without putting back the request - // in the list. - return Err(HttpError::IoError), + HttpApiRequest::Fail(error) => { + tracing::debug!(target: LOG_TARGET, id = %request_id.0, ?error, "Request failed"); + + // If the request has already failed, return without putting back the request + // in the list. + return Err(HttpError::IoError) + }, v @ HttpApiRequest::Dispatched(None) | v @ HttpApiRequest::Response(HttpApiRequestRp { sending_body: None, .. }) => { + tracing::debug!(target: LOG_TARGET, id = %request_id.0, "Body sending already finished"); + // We have already finished sending this body. self.requests.insert(request_id, v); return Err(HttpError::Invalid) @@ -347,8 +382,19 @@ impl HttpApi { // Requests in "fail" mode are purged before returning. debug_assert_eq!(output.len(), ids.len()); for n in (0..ids.len()).rev() { - if let HttpRequestStatus::IoError = output[n] { - self.requests.remove(&ids[n]); + match output[n] { + HttpRequestStatus::IoError => { + self.requests.remove(&ids[n]); + }, + HttpRequestStatus::Invalid => { + tracing::debug!(target: LOG_TARGET, id = %ids[n].0, "Unknown request"); + }, + HttpRequestStatus::DeadlineReached => { + tracing::debug!(target: LOG_TARGET, id = %ids[n].0, "Deadline reached"); + }, + HttpRequestStatus::Finished(_) => { + tracing::debug!(target: LOG_TARGET, id = %ids[n].0, "Request finished"); + }, } } return output @@ -385,20 +431,25 @@ impl HttpApi { ); }, None => {}, // can happen if we detected an IO error when sending the body - _ => error!("State mismatch between the API and worker"), + _ => { + tracing::error!(target: "offchain-worker::http", "State mismatch between the API and worker") + }, } }, Some(WorkerToApi::Fail { id, error }) => match self.requests.remove(&id) { Some(HttpApiRequest::Dispatched(_)) => { + tracing::debug!(target: LOG_TARGET, id = %id.0, ?error, "Request failed"); self.requests.insert(id, HttpApiRequest::Fail(error)); }, None => {}, // can happen if we detected an IO error when sending the body - _ => error!("State mismatch between the API and worker"), + _ => { + tracing::error!(target: "offchain-worker::http", "State mismatch between the API and worker") + }, }, None => { - error!("Worker has crashed"); + tracing::error!(target: "offchain-worker::http", "Worker has crashed"); return ids.iter().map(|_| HttpRequestStatus::IoError).collect() }, } @@ -471,7 +522,7 @@ impl HttpApi { }, Err(err) => { // This code should never be reached unless there's a logic error somewhere. - error!("Failed to read from current read chunk: {:?}", err); + tracing::error!(target: "offchain-worker::http", "Failed to read from current read chunk: {:?}", err); return Err(HttpError::IoError) }, } @@ -567,7 +618,7 @@ pub struct HttpWorker { /// Used to receive messages from the `HttpApi`. from_api: TracingUnboundedReceiver, /// The engine that runs HTTP requests. - http_client: Arc, Body>>, + http_client: Arc, Body>>>, /// HTTP requests that are being worked on by the engine. requests: Vec<(HttpRequestId, HttpWorkerRequest)>, } @@ -697,12 +748,15 @@ impl fmt::Debug for HttpWorkerRequest { #[cfg(test)] mod tests { - use super::{http, SharedClient}; + use super::{ + super::{tests::TestNetwork, AsyncApi}, + *, + }; use crate::api::timestamp; use core::convert::Infallible; - use futures::future; + use futures::{future, StreamExt}; use lazy_static::lazy_static; - use sp_core::offchain::{Duration, HttpError, HttpRequestId, HttpRequestStatus}; + use sp_core::offchain::{Duration, Externalities, HttpError, HttpRequestId, HttpRequestStatus}; // Using lazy_static to avoid spawning lots of different SharedClients, // as spawning a SharedClient is CPU-intensive and opens lots of fds. @@ -713,7 +767,10 @@ mod tests { // Returns an `HttpApi` whose worker is ran in the background, and a `SocketAddr` to an HTTP // server that runs in the background as well. macro_rules! build_api_server { - () => {{ + () => { + build_api_server!(hyper::Response::new(hyper::Body::from("Hello World!"))) + }; + ( $response:expr ) => {{ let hyper_client = SHARED_CLIENT.clone(); let (api, worker) = http(hyper_client.clone()); @@ -725,10 +782,12 @@ mod tests { let server = hyper::Server::bind(&"127.0.0.1:0".parse().unwrap()).serve( hyper::service::make_service_fn(|_| async move { Ok::<_, Infallible>(hyper::service::service_fn( - move |_req| async move { - Ok::<_, Infallible>(hyper::Response::new(hyper::Body::from( - "Hello World!", - ))) + move |req: hyper::Request| async move { + // Wait until the complete request was received and processed, + // otherwise the tests are flaky. + let _ = req.into_body().collect::>().await; + + Ok::<_, Infallible>($response) }, )) }), @@ -766,6 +825,33 @@ mod tests { assert_eq!(&buf[..n], b"Hello World!"); } + #[test] + fn basic_http2_localhost() { + let deadline = timestamp::now().add(Duration::from_millis(10_000)); + + // Performs an HTTP query to a background HTTP server. + + let (mut api, addr) = build_api_server!(hyper::Response::builder() + .version(hyper::Version::HTTP_2) + .body(hyper::Body::from("Hello World!")) + .unwrap()); + + let id = api.request_start("POST", &format!("http://{}", addr)).unwrap(); + api.request_write_body(id, &[], Some(deadline)).unwrap(); + + match api.response_wait(&[id], Some(deadline))[0] { + HttpRequestStatus::Finished(200) => {}, + v => panic!("Connecting to localhost failed: {:?}", v), + } + + let headers = api.response_headers(id); + assert!(headers.iter().any(|(h, _)| h.eq_ignore_ascii_case(b"Date"))); + + let mut buf = vec![0; 2048]; + let n = api.response_read_body(id, &mut buf, Some(deadline)).unwrap(); + assert_eq!(&buf[..n], b"Hello World!"); + } + #[test] fn request_start_invalid_call() { let (mut api, addr) = build_api_server!(); @@ -1002,4 +1088,37 @@ mod tests { } } } + + #[test] + fn shared_http_client_is_only_initialized_on_access() { + let shared_client = SharedClient::new(); + + { + let mock = Arc::new(TestNetwork()); + let (mut api, async_api) = AsyncApi::new(mock, false, shared_client.clone()); + api.timestamp(); + + futures::executor::block_on(async move { + assert!(futures::poll!(async_api.process()).is_pending()); + }); + } + + // Check that the http client wasn't initialized, because it wasn't used. + assert!(Lazy::into_value(Arc::try_unwrap(shared_client.0).unwrap()).is_err()); + + let shared_client = SharedClient::new(); + + { + let mock = Arc::new(TestNetwork()); + let (mut api, async_api) = AsyncApi::new(mock, false, shared_client.clone()); + let id = api.http_request_start("lol", "nope", &[]).unwrap(); + api.http_request_write_body(id, &[], None).unwrap(); + futures::executor::block_on(async move { + assert!(futures::poll!(async_api.process()).is_pending()); + }); + } + + // Check that the http client initialized, because it was used. + assert!(Lazy::into_value(Arc::try_unwrap(shared_client.0).unwrap()).is_ok()); + } } diff --git a/client/offchain/src/api/ipfs.rs b/client/offchain/src/api/ipfs.rs index 39658de53de5..56e8eb6e3e2f 100644 --- a/client/offchain/src/api/ipfs.rs +++ b/client/offchain/src/api/ipfs.rs @@ -355,7 +355,7 @@ impl From for IpfsResponse { IpfsResponse::FindPeer(addrs) }, IpfsNativeResponse::Identity(pk, addrs) => { - let pk = pk.into_peer_id().as_ref().to_bytes(); + let pk = pk.to_peer_id().as_ref().to_bytes(); let addrs = addrs.into_iter().map(|addr| OpaqueMultiaddr(addr.to_string().into_bytes()) ).collect(); diff --git a/client/offchain/src/api/timestamp.rs b/client/offchain/src/api/timestamp.rs index f1c8c004a019..4b3f5efddf27 100644 --- a/client/offchain/src/api/timestamp.rs +++ b/client/offchain/src/api/timestamp.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,10 +19,7 @@ //! Helper methods dedicated to timestamps. use sp_core::offchain::Timestamp; -use std::{ - convert::TryInto, - time::{Duration, SystemTime}, -}; +use std::time::{Duration, SystemTime}; /// Returns the current time as a `Timestamp`. pub fn now() -> Timestamp { diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index cc645f53a385..f63f9291f22f 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -41,7 +41,6 @@ use futures::{ future::{ready, Future}, prelude::*, }; -use log::{debug, warn}; use parking_lot::Mutex; use sc_network::{ExHashT, NetworkService, NetworkStateInfo, PeerId}; use sp_api::{ApiExt, ProvideRuntimeApi}; @@ -57,6 +56,8 @@ mod api; pub use api::Db as OffchainDb; pub use sp_offchain::{OffchainWorkerApi, STORAGE_PREFIX}; +const LOG_TARGET: &str = "offchain-worker"; + /// NetworkProvider provides [`OffchainWorkers`] with all necessary hooks into the /// underlying Substrate networking. pub trait NetworkProvider: NetworkStateInfo { @@ -81,19 +82,32 @@ where } } +/// Options for [`OffchainWorkers`] +pub struct OffchainWorkerOptions { + /// Enable http requests from offchain workers? + /// + /// If not enabled, any http request will panic. + pub enable_http_requests: bool, +} + /// An offchain workers manager. pub struct OffchainWorkers { client: Arc, ipfs_node: ipfs::Ipfs, _block: PhantomData, thread_pool: Mutex, - shared_client: api::SharedClient, + shared_http_client: api::SharedClient, + enable_http: bool, } impl OffchainWorkers { - /// Creates new `OffchainWorkers`. + /// Creates new [`OffchainWorkers`]. pub fn new(client: Arc, ipfs_rt: Arc>) -> Self { - let shared_client = api::SharedClient::new(); + Self::new_with_options(client, ipfs_rt, OffchainWorkerOptions { enable_http_requests: true }) + } + + /// Creates new [`OffchainWorkers`] using the given `options`. + pub fn new_with_options(client: Arc, ipfs_rt: Arc>, options: OffchainWorkerOptions) -> Self { let (ipfs_node, node_info) = std::thread::spawn(move || { let ipfs_rt = ipfs_rt.lock(); @@ -112,7 +126,7 @@ impl OffchainWorkers { log::info!( "IPFS: node started with PeerId {} and addresses {:?}", - node_info.0.into_peer_id(), node_info.1 + node_info.0.to_peer_id(), node_info.1 ); Self { @@ -123,7 +137,8 @@ impl OffchainWorkers { "offchain-worker".into(), num_cpus::get(), )), - shared_client, + shared_http_client: api::SharedClient::new(), + enable_http: options.enable_http_requests, } } } @@ -158,23 +173,37 @@ where err => { let help = "Consider turning off offchain workers if they are not part of your runtime."; - log::error!("Unsupported Offchain Worker API version: {:?}. {}.", err, help); + tracing::error!( + target: LOG_TARGET, + "Unsupported Offchain Worker API version: {:?}. {}.", + err, + help + ); 0 }, }; - debug!("Checking offchain workers at {:?}: version:{}", at, version); - if version > 0 { + tracing::debug!( + target: LOG_TARGET, + "Checking offchain workers at {:?}: version:{}", + at, + version + ); + let process = (version > 0).then(|| { let (api, runner) = - api::AsyncApi::new(network_provider, is_validator, self.shared_client.clone(), self.ipfs_node.clone()); - debug!("Spawning offchain workers at {:?}", at); + api::AsyncApi::new(network_provider, is_validator, self.shared_http_client.clone(), self.ipfs_node.clone()); + tracing::debug!(target: LOG_TARGET, "Spawning offchain workers at {:?}", at); let header = header.clone(); let client = self.client.clone(); + + let mut capabilities = offchain::Capabilities::all(); + + capabilities.set(offchain::Capabilities::HTTP, self.enable_http); self.spawn_worker(move || { let runtime = client.runtime_api(); let api = Box::new(api); - debug!("Running offchain workers at {:?}", at); - let context = - ExecutionContext::OffchainCall(Some((api, offchain::Capabilities::all()))); + tracing::debug!(target: LOG_TARGET, "Running offchain workers at {:?}", at); + + let context = ExecutionContext::OffchainCall(Some((api, capabilities))); let run = if version == 2 { runtime.offchain_worker_with_context(&at, context, &header) } else { @@ -186,12 +215,20 @@ where ) }; if let Err(e) = run { - log::error!("Error running offchain workers at {:?}: {:?}", at, e); + tracing::error!( + target: LOG_TARGET, + "Error running offchain workers at {:?}: {}", + at, + e + ); } }); - futures::future::Either::Left(runner.process()) - } else { - futures::future::Either::Right(futures::future::ready(())) + + runner.process() + }); + + async move { + futures::future::OptionFuture::from(process).await; } } @@ -228,13 +265,14 @@ pub async fn notification_future( if n.is_new_best { spawner.spawn( "offchain-on-block", + Some("offchain-worker"), offchain .on_block_imported(&n.header, network_provider.clone(), is_validator) .boxed(), ); } else { - log::debug!( - target: "sc_offchain", + tracing::debug!( + target: LOG_TARGET, "Skipping offchain workers for non-canon block: {:?}", n.header, ) diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index 5962620d6e06..b292f300ba35 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -5,7 +5,7 @@ license = "GPL-3.0-or-later WITH Classpath-exception-2.0" name = "sc-peerset" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sc-peerset" readme = "README.md" @@ -13,13 +13,12 @@ readme = "README.md" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] - [dependencies] -futures = "0.3.9" -libp2p = { version = "0.39.1", default-features = false } +futures = "0.3.21" +libp2p = { version = "0.40.0", default-features = false } sc-utils = { version = "4.0.0-dev", path = "../utils"} log = "0.4.8" -serde_json = "1.0.68" +serde_json = "1.0.79" wasm-timer = "0.2" [dev-dependencies] diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 0775354befee..859319fab132 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -79,13 +79,13 @@ pub struct SetId(usize); impl SetId { pub const fn from(id: usize) -> Self { - SetId(id) + Self(id) } } impl From for SetId { fn from(id: usize) -> Self { - SetId(id) + Self(id) } } @@ -107,12 +107,12 @@ pub struct ReputationChange { impl ReputationChange { /// New reputation change with given delta and reason. pub const fn new(value: i32, reason: &'static str) -> ReputationChange { - ReputationChange { value, reason } + Self { value, reason } } /// New reputation change that forces minimum possible reputation. pub const fn new_fatal(reason: &'static str) -> ReputationChange { - ReputationChange { value: i32::MIN, reason } + Self { value: i32::MIN, reason } } } @@ -129,7 +129,7 @@ impl PeersetHandle { /// Has no effect if the node was already a reserved peer. /// /// > **Note**: Keep in mind that the networking has to know an address for this node, - /// > otherwise it will not be able to connect to it. + /// > otherwise it will not be able to connect to it. pub fn add_reserved_peer(&self, set_id: SetId, peer_id: PeerId) { let _ = self.tx.unbounded_send(Action::AddReservedPeer(set_id, peer_id)); } @@ -208,8 +208,8 @@ pub enum Message { pub struct IncomingIndex(pub u64); impl From for IncomingIndex { - fn from(val: u64) -> IncomingIndex { - IncomingIndex(val) + fn from(val: u64) -> Self { + Self(val) } } @@ -232,7 +232,7 @@ pub struct SetConfig { /// List of bootstrap nodes to initialize the set with. /// /// > **Note**: Keep in mind that the networking has to know an address for these nodes, - /// > otherwise it will not be able to connect to them. + /// > otherwise it will not be able to connect to them. pub bootnodes: Vec, /// Lists of nodes we should always be connected to. @@ -274,7 +274,7 @@ pub struct Peerset { impl Peerset { /// Builds a new peerset from the given configuration. - pub fn from_config(config: PeersetConfig) -> (Peerset, PeersetHandle) { + pub fn from_config(config: PeersetConfig) -> (Self, PeersetHandle) { let (tx, rx) = tracing_unbounded("mpsc_peerset_messages"); let handle = PeersetHandle { tx: tx.clone() }; @@ -282,7 +282,7 @@ impl Peerset { let mut peerset = { let now = Instant::now(); - Peerset { + Self { data: peersstate::PeersState::new(config.sets.iter().map(|set| { peersstate::SetConfig { in_peers: set.in_peers, out_peers: set.out_peers } })), @@ -322,7 +322,7 @@ impl Peerset { } fn on_add_reserved_peer(&mut self, set_id: SetId, peer_id: PeerId) { - let newly_inserted = self.reserved_nodes[set_id.0].0.insert(peer_id.clone()); + let newly_inserted = self.reserved_nodes[set_id.0].0.insert(peer_id); if !newly_inserted { return } @@ -422,8 +422,7 @@ impl Peerset { match self.data.peer(set_id.0, &peer_id) { peersstate::Peer::Connected(peer) => { - self.message_queue - .push_back(Message::Drop { set_id, peer_id: peer.peer_id().clone() }); + self.message_queue.push_back(Message::Drop { set_id, peer_id: *peer.peer_id() }); peer.disconnect().forget_peer(); }, peersstate::Peer::NotConnected(peer) => { @@ -819,8 +818,8 @@ mod tests { }; let (peerset, handle) = Peerset::from_config(config); - handle.add_reserved_peer(SetId::from(0), reserved_peer.clone()); - handle.add_reserved_peer(SetId::from(0), reserved_peer2.clone()); + handle.add_reserved_peer(SetId::from(0), reserved_peer); + handle.add_reserved_peer(SetId::from(0), reserved_peer2); assert_messages( peerset, @@ -845,22 +844,22 @@ mod tests { sets: vec![SetConfig { in_peers: 2, out_peers: 1, - bootnodes: vec![bootnode.clone()], + bootnodes: vec![bootnode], reserved_nodes: Default::default(), reserved_only: false, }], }; let (mut peerset, _handle) = Peerset::from_config(config); - peerset.incoming(SetId::from(0), incoming.clone(), ii); - peerset.incoming(SetId::from(0), incoming.clone(), ii4); - peerset.incoming(SetId::from(0), incoming2.clone(), ii2); - peerset.incoming(SetId::from(0), incoming3.clone(), ii3); + peerset.incoming(SetId::from(0), incoming, ii); + peerset.incoming(SetId::from(0), incoming, ii4); + peerset.incoming(SetId::from(0), incoming2, ii2); + peerset.incoming(SetId::from(0), incoming3, ii3); assert_messages( peerset, vec![ - Message::Connect { set_id: SetId::from(0), peer_id: bootnode.clone() }, + Message::Connect { set_id: SetId::from(0), peer_id: bootnode }, Message::Accept(ii), Message::Accept(ii2), Message::Reject(ii3), @@ -883,7 +882,7 @@ mod tests { }; let (mut peerset, _) = Peerset::from_config(config); - peerset.incoming(SetId::from(0), incoming.clone(), ii); + peerset.incoming(SetId::from(0), incoming, ii); assert_messages(peerset, vec![Message::Reject(ii)]); } @@ -897,15 +896,15 @@ mod tests { sets: vec![SetConfig { in_peers: 0, out_peers: 2, - bootnodes: vec![bootnode.clone()], + bootnodes: vec![bootnode], reserved_nodes: Default::default(), reserved_only: false, }], }; let (mut peerset, _handle) = Peerset::from_config(config); - peerset.add_to_peers_set(SetId::from(0), discovered.clone()); - peerset.add_to_peers_set(SetId::from(0), discovered.clone()); + peerset.add_to_peers_set(SetId::from(0), discovered); + peerset.add_to_peers_set(SetId::from(0), discovered); peerset.add_to_peers_set(SetId::from(0), discovered2); assert_messages( @@ -931,7 +930,7 @@ mod tests { // We ban a node by setting its reputation under the threshold. let peer_id = PeerId::random(); - handle.report_peer(peer_id.clone(), ReputationChange::new(BANNED_THRESHOLD - 1, "")); + handle.report_peer(peer_id, ReputationChange::new(BANNED_THRESHOLD - 1, "")); let fut = futures::future::poll_fn(move |cx| { // We need one polling for the message to be processed. @@ -974,7 +973,7 @@ mod tests { // We ban a node by setting its reputation under the threshold. let peer_id = PeerId::random(); - handle.report_peer(peer_id.clone(), ReputationChange::new(BANNED_THRESHOLD - 1, "")); + handle.report_peer(peer_id, ReputationChange::new(BANNED_THRESHOLD - 1, "")); let fut = futures::future::poll_fn(move |cx| { // We need one polling for the message to be processed. diff --git a/client/peerset/src/peersstate.rs b/client/peerset/src/peersstate.rs index 7717620eae3a..ca22cac32408 100644 --- a/client/peerset/src/peersstate.rs +++ b/client/peerset/src/peersstate.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -25,8 +25,8 @@ //! slots. //! //! > Note: This module is purely dedicated to managing slots and reputations. Features such as -//! > for example connecting to some nodes in priority should be added outside of this -//! > module, rather than inside. +//! > for example connecting to some nodes in priority should be added outside of this +//! > module, rather than inside. use libp2p::PeerId; use log::error; @@ -50,8 +50,8 @@ pub struct PeersState { /// List of nodes that we know about. /// /// > **Note**: This list should really be ordered by decreasing reputation, so that we can - /// easily select the best node to connect to. As a first draft, however, we don't - /// sort, to make the logic easier. + /// > easily select the best node to connect to. As a first draft, however, we don't sort, to + /// > make the logic easier. nodes: HashMap, /// Configuration of each set. The size of this `Vec` is never modified. @@ -105,8 +105,8 @@ struct Node { } impl Node { - fn new(num_sets: usize) -> Node { - Node { sets: (0..num_sets).map(|_| MembershipState::NotMember).collect(), reputation: 0 } + fn new(num_sets: usize) -> Self { + Self { sets: (0..num_sets).map(|_| MembershipState::NotMember).collect(), reputation: 0 } } } @@ -128,21 +128,24 @@ enum MembershipState { } impl MembershipState { - /// Returns `true` for `In` and `Out`. + /// Returns `true` for [`MembershipState::In`] and [`MembershipState::Out`]. fn is_connected(self) -> bool { match self { - MembershipState::NotMember => false, - MembershipState::In => true, - MembershipState::Out => true, - MembershipState::NotConnected { .. } => false, + Self::In | Self::Out => true, + Self::NotMember | Self::NotConnected { .. } => false, } } + + /// Returns `true` for [`MembershipState::NotConnected`]. + fn is_not_connected(self) -> bool { + matches!(self, Self::NotConnected { .. }) + } } impl PeersState { - /// Builds a new empty `PeersState`. + /// Builds a new empty [`PeersState`]. pub fn new(sets: impl IntoIterator) -> Self { - PeersState { + Self { nodes: HashMap::new(), sets: sets .into_iter() @@ -242,12 +245,7 @@ impl PeersState { let outcome = self .nodes .iter_mut() - .filter(|(_, Node { sets, .. })| match sets[set] { - MembershipState::NotMember => false, - MembershipState::In => false, - MembershipState::Out => false, - MembershipState::NotConnected { .. } => true, - }) + .filter(|(_, Node { sets, .. })| sets[set].is_not_connected()) .fold(None::<(&PeerId, &mut Node)>, |mut cur_node, to_try| { if let Some(cur_node) = cur_node.take() { if cur_node.1.reputation >= to_try.1.reputation { @@ -318,35 +316,32 @@ pub enum Peer<'a> { } impl<'a> Peer<'a> { - /// If we are the `Connected` variant, returns the inner `ConnectedPeer`. Returns `None` + /// If we are the `Connected` variant, returns the inner [`ConnectedPeer`]. Returns `None` /// otherwise. pub fn into_connected(self) -> Option> { match self { - Peer::Connected(peer) => Some(peer), - Peer::NotConnected(_) => None, - Peer::Unknown(_) => None, + Self::Connected(peer) => Some(peer), + Self::NotConnected(..) | Self::Unknown(..) => None, } } - /// If we are the `Unknown` variant, returns the inner `ConnectedPeer`. Returns `None` + /// If we are the `NotConnected` variant, returns the inner [`NotConnectedPeer`]. Returns `None` /// otherwise. #[cfg(test)] // Feel free to remove this if this function is needed outside of tests pub fn into_not_connected(self) -> Option> { match self { - Peer::Connected(_) => None, - Peer::NotConnected(peer) => Some(peer), - Peer::Unknown(_) => None, + Self::NotConnected(peer) => Some(peer), + Self::Connected(..) | Self::Unknown(..) => None, } } - /// If we are the `Unknown` variant, returns the inner `ConnectedPeer`. Returns `None` + /// If we are the `Unknown` variant, returns the inner [`UnknownPeer`]. Returns `None` /// otherwise. #[cfg(test)] // Feel free to remove this if this function is needed outside of tests pub fn into_unknown(self) -> Option> { match self { - Peer::Connected(_) => None, - Peer::NotConnected(_) => None, - Peer::Unknown(peer) => Some(peer), + Self::Unknown(peer) => Some(peer), + Self::Connected(..) | Self::NotConnected(..) => None, } } } @@ -473,7 +468,7 @@ impl<'a> NotConnectedPeer<'a> { /// the slots are full, the node stays "not connected" and we return `Err`. /// /// Non-slot-occupying nodes don't count towards the number of slots. - pub fn try_outgoing(self) -> Result, NotConnectedPeer<'a>> { + pub fn try_outgoing(self) -> Result, Self> { let is_no_slot_occupy = self.state.sets[self.set].no_slot_nodes.contains(&*self.peer_id); // Note that it is possible for num_out to be strictly superior to the max, in case we were @@ -500,7 +495,7 @@ impl<'a> NotConnectedPeer<'a> { /// the slots are full, the node stays "not connected" and we return `Err`. /// /// Non-slot-occupying nodes don't count towards the number of slots. - pub fn try_accept_incoming(self) -> Result, NotConnectedPeer<'a>> { + pub fn try_accept_incoming(self) -> Result, Self> { let is_no_slot_occupy = self.state.sets[self.set].no_slot_nodes.contains(&*self.peer_id); // Note that it is possible for num_in to be strictly superior to the max, in case we were diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index 3a9ba686ee95..af4838d724cb 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/proposer-metrics/Cargo.toml b/client/proposer-metrics/Cargo.toml index ffe5045461f7..1a4b1fd4ce2c 100644 --- a/client/proposer-metrics/Cargo.toml +++ b/client/proposer-metrics/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sc-proposer-metrics" -version = "0.9.0" +version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Basic metrics for block production." readme = "README.md" @@ -14,4 +14,4 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] log = "0.4.8" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.9.0"} +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.10.0-dev"} diff --git a/client/proposer-metrics/src/lib.rs b/client/proposer-metrics/src/lib.rs index da29fb295199..c27d16ea04d7 100644 --- a/client/proposer-metrics/src/lib.rs +++ b/client/proposer-metrics/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,7 +19,8 @@ //! Prometheus basic proposer metrics. use prometheus_endpoint::{ - register, Gauge, Histogram, HistogramOpts, PrometheusError, Registry, U64, + prometheus::CounterVec, register, Gauge, Histogram, HistogramOpts, Opts, PrometheusError, + Registry, U64, }; /// Optional shareable link to basic authorship metrics. @@ -38,15 +39,26 @@ impl MetricsLink { } pub fn report(&self, do_this: impl FnOnce(&Metrics) -> O) -> Option { - Some(do_this(self.0.as_ref()?)) + self.0.as_ref().map(do_this) } } +/// The reason why proposing a block ended. +pub enum EndProposingReason { + NoMoreTransactions, + HitDeadline, + HitBlockSizeLimit, + HitBlockWeightLimit, +} + /// Authorship metrics. #[derive(Clone)] pub struct Metrics { pub block_constructed: Histogram, pub number_of_transactions: Gauge, + pub end_proposing_reason: CounterVec, + pub create_inherents_time: Histogram, + pub create_block_proposal_time: Histogram, } impl Metrics { @@ -54,18 +66,54 @@ impl Metrics { Ok(Self { block_constructed: register( Histogram::with_opts(HistogramOpts::new( - "proposer_block_constructed", + "substrate_proposer_block_constructed", "Histogram of time taken to construct new block", ))?, registry, )?, number_of_transactions: register( Gauge::new( - "proposer_number_of_transactions", + "substrate_proposer_number_of_transactions", "Number of transactions included in block", )?, registry, )?, + create_inherents_time: register( + Histogram::with_opts(HistogramOpts::new( + "substrate_proposer_create_inherents_time", + "Histogram of time taken to execute create inherents", + ))?, + registry, + )?, + create_block_proposal_time: register( + Histogram::with_opts(HistogramOpts::new( + "substrate_proposer_block_proposal_time", + "Histogram of time taken to construct a block and prepare it for proposal", + ))?, + registry, + )?, + end_proposing_reason: register( + CounterVec::new( + Opts::new( + "substrate_proposer_end_proposal_reason", + "The reason why the block proposing was ended. This doesn't include errors.", + ), + &["reason"], + )?, + registry, + )?, }) } + + /// Report the reason why the proposing ended. + pub fn report_end_proposing_reason(&self, reason: EndProposingReason) { + let reason = match reason { + EndProposingReason::HitDeadline => "hit_deadline", + EndProposingReason::NoMoreTransactions => "no_more_transactions", + EndProposingReason::HitBlockSizeLimit => "hit_block_size_limit", + EndProposingReason::HitBlockWeightLimit => "hit_block_weight_limit", + }; + + self.end_proposing_reason.with_label_values(&[reason]).inc(); + } } diff --git a/client/rpc-api/Cargo.toml b/client/rpc-api/Cargo.toml index 6342abb1a3c4..20a18ec6c61c 100644 --- a/client/rpc-api/Cargo.toml +++ b/client/rpc-api/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-rpc-api" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate RPC interfaces." readme = "README.md" @@ -13,22 +13,23 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0" } -futures = "0.3.16" +codec = { package = "parity-scale-codec", version = "3.0.0" } +futures = "0.3.21" jsonrpc-core = "18.0.0" jsonrpc-core-client = "18.0.0" jsonrpc-derive = "18.0.0" jsonrpc-pubsub = "18.0.0" log = "0.4.8" -parking_lot = "0.11.1" +parking_lot = "0.12.0" thiserror = "1.0" -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-version = { version = "4.0.0-dev", path = "../../primitives/version" } -sp-runtime = { path = "../../primitives/runtime", version = "4.0.0-dev" } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-version = { version = "5.0.0", path = "../../primitives/version" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } sc-chain-spec = { path = "../chain-spec", version = "4.0.0-dev" } -serde = { version = "1.0.126", features = ["derive"] } -serde_json = "1.0.68" +serde = { version = "1.0.136", features = ["derive"] } +serde_json = "1.0.79" sc-transaction-pool-api = { version = "4.0.0-dev", path = "../transaction-pool/api" } -sp-rpc = { version = "4.0.0-dev", path = "../../primitives/rpc" } -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } +sp-rpc = { version = "6.0.0", path = "../../primitives/rpc" } +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } diff --git a/client/rpc-api/src/author/error.rs b/client/rpc-api/src/author/error.rs index c7e3ccffabbb..fe12bd581f12 100644 --- a/client/rpc-api/src/author/error.rs +++ b/client/rpc-api/src/author/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -43,15 +43,9 @@ pub enum Error { /// Incorrect extrinsic format. #[error("Invalid extrinsic format: {}", .0)] BadFormat(#[from] codec::Error), - /// Incorrect seed phrase. - #[error("Invalid seed phrase/SURI")] - BadSeedPhrase, /// Key type ID has an unknown format. #[error("Invalid key type ID format (should be of length four)")] BadKeyType, - /// Key type ID has some unsupported crypto. - #[error("The crypto of key type ID is unknown")] - UnsupportedKeyType, /// Some random issue with the key store. Shouldn't happen. #[error("The key store is unavailable")] KeyStoreUnavailable, @@ -84,8 +78,6 @@ const POOL_TOO_LOW_PRIORITY: i64 = POOL_INVALID_TX + 4; const POOL_CYCLE_DETECTED: i64 = POOL_INVALID_TX + 5; /// The transaction was not included to the pool because of the limits. const POOL_IMMEDIATELY_DROPPED: i64 = POOL_INVALID_TX + 6; -/// The key type crypto is not known. -const UNSUPPORTED_KEY_TYPE: i64 = POOL_INVALID_TX + 7; /// The transaction was not included to the pool since it is unactionable, /// it is not propagable and the local node does not author blocks. const POOL_UNACTIONABLE: i64 = POOL_INVALID_TX + 8; @@ -103,7 +95,7 @@ impl From for rpc::Error { Error::Verification(e) => rpc::Error { code: rpc::ErrorCode::ServerError(VERIFICATION_ERROR), message: format!("Verification Error: {}", e).into(), - data: Some(format!("{:?}", e).into()), + data: Some(e.to_string().into()), }, Error::Pool(PoolError::InvalidTransaction(InvalidTransaction::Custom(e))) => rpc::Error { code: rpc::ErrorCode::ServerError(POOL_INVALID_TX), @@ -156,14 +148,6 @@ impl From for rpc::Error { the local node does not author blocks".into(), ), }, - Error::UnsupportedKeyType => rpc::Error { - code: rpc::ErrorCode::ServerError(UNSUPPORTED_KEY_TYPE), - message: "Unknown key type crypto" .into(), - data: Some( - "The crypto for the given key type is unknown, please add the public key to the \ - request to insert the key successfully.".into() - ), - }, Error::UnsafeRpcCalled(e) => e.into(), e => errors::internal(e), } diff --git a/client/rpc-api/src/author/hash.rs b/client/rpc-api/src/author/hash.rs index c4acfb819ddb..453947dd21f2 100644 --- a/client/rpc-api/src/author/hash.rs +++ b/client/rpc-api/src/author/hash.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/author/mod.rs b/client/rpc-api/src/author/mod.rs index 720598e0b32a..84167ee95d10 100644 --- a/client/rpc-api/src/author/mod.rs +++ b/client/rpc-api/src/author/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/chain/error.rs b/client/rpc-api/src/chain/error.rs index c7f14b2dfc16..a0cacb673915 100644 --- a/client/rpc-api/src/chain/error.rs +++ b/client/rpc-api/src/chain/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/chain/mod.rs b/client/rpc-api/src/chain/mod.rs index 79ae80d0c4d1..d7d598942f1e 100644 --- a/client/rpc-api/src/chain/mod.rs +++ b/client/rpc-api/src/chain/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/child_state/mod.rs b/client/rpc-api/src/child_state/mod.rs index de94790d0990..6b4cd20f2260 100644 --- a/client/rpc-api/src/child_state/mod.rs +++ b/client/rpc-api/src/child_state/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/dev/error.rs b/client/rpc-api/src/dev/error.rs new file mode 100644 index 000000000000..1a14b0d78994 --- /dev/null +++ b/client/rpc-api/src/dev/error.rs @@ -0,0 +1,71 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Error helpers for Dev RPC module. + +use crate::errors; +use jsonrpc_core as rpc; + +/// Dev RPC Result type. +pub type Result = std::result::Result; + +/// Dev RPC future Result type. +pub type FutureResult = jsonrpc_core::BoxFuture>; + +/// Dev RPC errors. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Failed to query specified block or its parent: Probably an invalid hash. + #[error("Error while querying block: {0}")] + BlockQueryError(Box), + /// The re-execution of the specified block failed. + #[error("Failed to re-execute the specified block")] + BlockExecutionFailed, + /// The witness compaction failed. + #[error("Failed to create to compact the witness")] + WitnessCompactionFailed, + /// The method is marked as unsafe but unsafe flag wasn't supplied on the CLI. + #[error(transparent)] + UnsafeRpcCalled(#[from] crate::policy::UnsafeRpcError), +} + +/// Base error code for all dev errors. +const BASE_ERROR: i64 = 6000; + +impl From for rpc::Error { + fn from(e: Error) -> Self { + match e { + Error::BlockQueryError(_) => rpc::Error { + code: rpc::ErrorCode::ServerError(BASE_ERROR + 1), + message: e.to_string(), + data: None, + }, + Error::BlockExecutionFailed => rpc::Error { + code: rpc::ErrorCode::ServerError(BASE_ERROR + 3), + message: e.to_string(), + data: None, + }, + Error::WitnessCompactionFailed => rpc::Error { + code: rpc::ErrorCode::ServerError(BASE_ERROR + 4), + message: e.to_string(), + data: None, + }, + e => errors::internal(e), + } + } +} diff --git a/client/rpc-api/src/dev/mod.rs b/client/rpc-api/src/dev/mod.rs new file mode 100644 index 000000000000..b1ae8934af8a --- /dev/null +++ b/client/rpc-api/src/dev/mod.rs @@ -0,0 +1,64 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Substrate dev API containing RPCs that are mainly meant for debugging and stats collection for +//! developers. The endpoints in this RPC module are not meant to be available to non-local users +//! and are all marked `unsafe`. + +pub mod error; + +use self::error::Result; +use codec::{Decode, Encode}; +use jsonrpc_derive::rpc; +use scale_info::TypeInfo; +use serde::{Deserialize, Serialize}; + +/// Statistics of a block returned by the `dev_getBlockStats` RPC. +#[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, Debug, TypeInfo, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BlockStats { + /// The length in bytes of the storage proof produced by executing the block. + pub witness_len: u64, + /// The length in bytes of the storage proof after compaction. + pub witness_compact_len: u64, + /// Length of the block in bytes. + /// + /// This information can also be acquired by downloading the whole block. This merely + /// saves some complexity on the client side. + pub block_len: u64, + /// Number of extrinsics in the block. + /// + /// This information can also be acquired by downloading the whole block. This merely + /// saves some complexity on the client side. + pub num_extrinsics: u64, +} + +/// Substrate dev API. +/// +/// This API contains unstable and unsafe methods only meant for development nodes. They +/// are all flagged as unsafe for this reason. +#[rpc] +pub trait DevApi { + /// Reexecute the specified `block_hash` and gather statistics while doing so. + /// + /// This function requires the specified block and its parent to be available + /// at the queried node. If either the specified block or the parent is pruned, + /// this function will return `None`. + #[rpc(name = "dev_getBlockStats")] + fn block_stats(&self, block_hash: Hash) -> Result>; +} diff --git a/client/rpc-api/src/errors.rs b/client/rpc-api/src/errors.rs index 8e4883a4cc20..e59b1b0eda5c 100644 --- a/client/rpc-api/src/errors.rs +++ b/client/rpc-api/src/errors.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,11 +18,11 @@ use log::warn; -pub fn internal(e: E) -> jsonrpc_core::Error { - warn!("Unknown error: {:?}", e); +pub fn internal(e: E) -> jsonrpc_core::Error { + warn!("Unknown error: {}", e); jsonrpc_core::Error { code: jsonrpc_core::ErrorCode::InternalError, message: "Unknown error occurred".into(), - data: Some(format!("{:?}", e).into()), + data: Some(e.to_string().into()), } } diff --git a/client/rpc-api/src/helpers.rs b/client/rpc-api/src/helpers.rs index a26adbf2e903..2fbd2f504046 100644 --- a/client/rpc-api/src/helpers.rs +++ b/client/rpc-api/src/helpers.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/lib.rs b/client/rpc-api/src/lib.rs index 92de1e7fcb34..e06f30bf9cd8 100644 --- a/client/rpc-api/src/lib.rs +++ b/client/rpc-api/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -35,6 +35,7 @@ pub use policy::{DenyUnsafe, UnsafeRpcError}; pub mod author; pub mod chain; pub mod child_state; +pub mod dev; pub mod offchain; pub mod state; pub mod system; diff --git a/client/rpc-api/src/metadata.rs b/client/rpc-api/src/metadata.rs index d493b92c11ac..3c798782062e 100644 --- a/client/rpc-api/src/metadata.rs +++ b/client/rpc-api/src/metadata.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/offchain/error.rs b/client/rpc-api/src/offchain/error.rs index 6b8e2bfe189b..41f1416bfb36 100644 --- a/client/rpc-api/src/offchain/error.rs +++ b/client/rpc-api/src/offchain/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/offchain/mod.rs b/client/rpc-api/src/offchain/mod.rs index 333892fc19c4..c76e83011072 100644 --- a/client/rpc-api/src/offchain/mod.rs +++ b/client/rpc-api/src/offchain/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/policy.rs b/client/rpc-api/src/policy.rs index 5d56c62bfece..4d1e1d7c4ce7 100644 --- a/client/rpc-api/src/policy.rs +++ b/client/rpc-api/src/policy.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/state/error.rs b/client/rpc-api/src/state/error.rs index d70086347632..4414629e2e29 100644 --- a/client/rpc-api/src/state/error.rs +++ b/client/rpc-api/src/state/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/state/helpers.rs b/client/rpc-api/src/state/helpers.rs index 718ad69ac232..25f07be1c97c 100644 --- a/client/rpc-api/src/state/helpers.rs +++ b/client/rpc-api/src/state/helpers.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/state/mod.rs b/client/rpc-api/src/state/mod.rs index 620a000c500f..453e954ce595 100644 --- a/client/rpc-api/src/state/mod.rs +++ b/client/rpc-api/src/state/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -301,7 +301,7 @@ pub trait StateApi { /// [substrate storage][1], [transparent keys in substrate][2], /// [querying substrate storage via rpc][3]. /// - /// [1]: https://substrate.dev/docs/en/knowledgebase/advanced/storage#storage-map-key + /// [1]: https://docs.substrate.io/v3/advanced/storage#storage-map-keys /// [2]: https://www.shawntabrizi.com/substrate/transparent-keys-in-substrate/ /// [3]: https://www.shawntabrizi.com/substrate/querying-substrate-storage-via-rpc/ /// diff --git a/client/rpc-api/src/system/error.rs b/client/rpc-api/src/system/error.rs index 4ba5125d82bc..050d79b6ad63 100644 --- a/client/rpc-api/src/system/error.rs +++ b/client/rpc-api/src/system/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/system/helpers.rs b/client/rpc-api/src/system/helpers.rs index 96e8aeb1ae3d..e17e0f518c3e 100644 --- a/client/rpc-api/src/system/helpers.rs +++ b/client/rpc-api/src/system/helpers.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-api/src/system/mod.rs b/client/rpc-api/src/system/mod.rs index 3ffc5f434650..b610094f5b58 100644 --- a/client/rpc-api/src/system/mod.rs +++ b/client/rpc-api/src/system/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc-servers/Cargo.toml b/client/rpc-servers/Cargo.toml index 26a05a8263dc..c5af4ba200fe 100644 --- a/client/rpc-servers/Cargo.toml +++ b/client/rpc-servers/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-rpc-server" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate RPC servers." readme = "README.md" @@ -13,13 +13,13 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -futures = "0.3.16" +futures = "0.3.21" jsonrpc-core = "18.0.0" pubsub = { package = "jsonrpc-pubsub", version = "18.0.0" } log = "0.4.8" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.9.0"} -serde_json = "1.0.68" -tokio = "1.10" +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.10.0-dev"} +serde_json = "1.0.79" +tokio = { version = "1.17.0", features = ["parking_lot"] } http = { package = "jsonrpc-http-server", version = "18.0.0" } ipc = { package = "jsonrpc-ipc-server", version = "18.0.0" } ws = { package = "jsonrpc-ws-server", version = "18.0.0" } diff --git a/client/rpc-servers/src/lib.rs b/client/rpc-servers/src/lib.rs index 65ed6a914b19..963d9aec072f 100644 --- a/client/rpc-servers/src/lib.rs +++ b/client/rpc-servers/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -33,6 +33,9 @@ const MEGABYTE: usize = 1024 * 1024; /// Maximal payload accepted by RPC servers. pub const RPC_MAX_PAYLOAD_DEFAULT: usize = 15 * MEGABYTE; +/// Maximal buffer size in WS server. +pub const WS_MAX_BUFFER_CAPACITY_DEFAULT: usize = 16 * MEGABYTE; + /// Default maximum number of connections for WS RPC servers. const WS_MAX_CONNECTIONS: usize = 100; @@ -87,7 +90,7 @@ impl ServerMetrics { Ok(Self { session_opened: register( Counter::new( - "rpc_sessions_opened", + "substrate_rpc_sessions_opened", "Number of persistent RPC sessions opened", )?, r, @@ -95,7 +98,7 @@ impl ServerMetrics { .into(), session_closed: register( Counter::new( - "rpc_sessions_closed", + "substrate_rpc_sessions_closed", "Number of persistent RPC sessions closed", )?, r, @@ -172,18 +175,32 @@ pub fn start_ws< cors: Option<&Vec>, io: RpcHandler, maybe_max_payload_mb: Option, + maybe_max_out_buffer_capacity_mb: Option, server_metrics: ServerMetrics, tokio_handle: tokio::runtime::Handle, ) -> io::Result { - let rpc_max_payload = maybe_max_payload_mb + let max_payload = maybe_max_payload_mb .map(|mb| mb.saturating_mul(MEGABYTE)) .unwrap_or(RPC_MAX_PAYLOAD_DEFAULT); + let max_out_buffer_capacity = maybe_max_out_buffer_capacity_mb + .map(|mb| mb.saturating_mul(MEGABYTE)) + .unwrap_or(WS_MAX_BUFFER_CAPACITY_DEFAULT); + + if max_payload > max_out_buffer_capacity { + log::warn!( + "maximum payload ({}) is more than maximum output buffer ({}) size in ws server, the payload will actually be limited by the buffer size", + max_payload, + max_out_buffer_capacity, + ) + } + ws::ServerBuilder::with_meta_extractor(io, |context: &ws::RequestContext| { context.sender().into() }) .event_loop_executor(tokio_handle) - .max_payload(rpc_max_payload) + .max_payload(max_payload) .max_connections(max_connections.unwrap_or(WS_MAX_CONNECTIONS)) + .max_out_buffer_capacity(max_out_buffer_capacity) .allowed_origins(map_cors(cors)) .allowed_hosts(hosts_filtering(cors.is_some())) .session_stats(server_metrics) diff --git a/client/rpc-servers/src/middleware.rs b/client/rpc-servers/src/middleware.rs index 00532b0e8d66..4adc87866009 100644 --- a/client/rpc-servers/src/middleware.rs +++ b/client/rpc-servers/src/middleware.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -48,7 +48,7 @@ impl RpcMetrics { requests_started: register( CounterVec::new( Opts::new( - "rpc_requests_started", + "substrate_rpc_requests_started", "Number of RPC requests (not calls) received by the server.", ), &["protocol"], @@ -58,7 +58,7 @@ impl RpcMetrics { requests_finished: register( CounterVec::new( Opts::new( - "rpc_requests_finished", + "substrate_rpc_requests_finished", "Number of RPC requests (not calls) processed by the server.", ), &["protocol"], @@ -68,7 +68,7 @@ impl RpcMetrics { calls_time: register( HistogramVec::new( HistogramOpts::new( - "rpc_calls_time", + "substrate_rpc_calls_time", "Total time [μs] of processed RPC calls", ), &["protocol", "method"], @@ -78,7 +78,7 @@ impl RpcMetrics { calls_started: register( CounterVec::new( Opts::new( - "rpc_calls_started", + "substrate_rpc_calls_started", "Number of received RPC calls (unique un-batched requests)", ), &["protocol", "method"], @@ -88,7 +88,7 @@ impl RpcMetrics { calls_finished: register( CounterVec::new( Opts::new( - "rpc_calls_finished", + "substrate_rpc_calls_finished", "Number of processed RPC calls (unique un-batched requests)", ), &["protocol", "method", "is_error"], diff --git a/client/rpc/Cargo.toml b/client/rpc/Cargo.toml index 427800f74ddf..4fea6501f693 100644 --- a/client/rpc/Cargo.toml +++ b/client/rpc/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-rpc" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate Client RPC" readme = "README.md" @@ -16,26 +16,26 @@ targets = ["x86_64-unknown-linux-gnu"] sc-rpc-api = { version = "0.10.0-dev", path = "../rpc-api" } sc-client-api = { version = "4.0.0-dev", path = "../api" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } -codec = { package = "parity-scale-codec", version = "2.0.0" } -futures = "0.3.16" +codec = { package = "parity-scale-codec", version = "3.0.0" } +futures = "0.3.21" jsonrpc-pubsub = "18.0.0" log = "0.4.8" -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } rpc = { package = "jsonrpc-core", version = "18.0.0" } -sp-version = { version = "4.0.0-dev", path = "../../primitives/version" } -serde_json = "1.0.68" +sp-version = { version = "5.0.0", path = "../../primitives/version" } +serde_json = "1.0.79" sp-session = { version = "4.0.0-dev", path = "../../primitives/session" } sp-offchain = { version = "4.0.0-dev", path = "../../primitives/offchain" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } sc-utils = { version = "4.0.0-dev", path = "../utils" } -sp-rpc = { version = "4.0.0-dev", path = "../../primitives/rpc" } -sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore" } +sp-rpc = { version = "6.0.0", path = "../../primitives/rpc" } +sp-keystore = { version = "0.12.0", path = "../../primitives/keystore" } sc-chain-spec = { version = "4.0.0-dev", path = "../chain-spec" } sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sc-tracing = { version = "4.0.0-dev", path = "../tracing" } hash-db = { version = "0.15.2", default-features = false } -parking_lot = "0.11.1" +parking_lot = "0.12.0" lazy_static = { version = "1.4.0", optional = true } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../transaction-pool/api" } @@ -43,7 +43,7 @@ sc-transaction-pool-api = { version = "4.0.0-dev", path = "../transaction-pool/a assert_matches = "1.3.0" lazy_static = "1.4.0" sc-network = { version = "0.10.0-dev", path = "../network" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } sc-transaction-pool = { version = "4.0.0-dev", path = "../transaction-pool" } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } diff --git a/client/rpc/src/author/mod.rs b/client/rpc/src/author/mod.rs index 40b477a662a6..5064e6134210 100644 --- a/client/rpc/src/author/mod.rs +++ b/client/rpc/src/author/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,8 +21,7 @@ #[cfg(test)] mod tests; -use log::warn; -use std::{convert::TryInto, sync::Arc}; +use std::sync::Arc; use sp_blockchain::HeaderBackend; @@ -188,7 +187,7 @@ where let dxt = match TransactionFor::

::decode(&mut &xt[..]).map_err(error::Error::from) { Ok(tx) => tx, Err(err) => { - warn!("Failed to submit extrinsic: {}", err); + log::debug!("Failed to submit extrinsic: {}", err); // reject the subscriber (ignore errors - we don't care if subscriber is no longer // there). let _ = subscriber.reject(err.into()); @@ -211,7 +210,7 @@ where let tx_stream = match submit.await { Ok(s) => s, Err(err) => { - warn!("Failed to submit extrinsic: {}", err); + log::debug!("Failed to submit extrinsic: {}", err); // reject the subscriber (ignore errors - we don't care if subscriber is no // longer there). let _ = subscriber.reject(err.into()); @@ -222,14 +221,16 @@ where subscriptions.add(subscriber, move |sink| { tx_stream .map(|v| Ok(Ok(v))) - .forward(sink.sink_map_err(|e| warn!("Error sending notifications: {:?}", e))) + .forward( + sink.sink_map_err(|e| log::debug!("Error sending notifications: {:?}", e)), + ) .map(drop) }); }; let res = self.subscriptions.executor().spawn_obj(future.boxed().into()); if res.is_err() { - warn!("Error spawning subscription RPC task."); + log::warn!("Error spawning subscription RPC task."); } } diff --git a/client/rpc/src/author/tests.rs b/client/rpc/src/author/tests.rs index 2349e08fee50..c555465645a7 100644 --- a/client/rpc/src/author/tests.rs +++ b/client/rpc/src/author/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -24,7 +24,7 @@ use futures::executor; use sc_transaction_pool::{BasicPool, FullChainApi}; use sp_core::{ blake2_256, - crypto::{CryptoTypePublicPair, Pair, Public}, + crypto::{ByteArray, CryptoTypePublicPair, Pair}, ed25519, hexdisplay::HexDisplay, sr25519, @@ -40,8 +40,12 @@ use substrate_test_runtime_client::{ }; fn uxt(sender: AccountKeyring, nonce: u64) -> Extrinsic { - let tx = - Transfer { amount: Default::default(), nonce, from: sender.into(), to: Default::default() }; + let tx = Transfer { + amount: Default::default(), + nonce, + from: sender.into(), + to: AccountKeyring::Bob.into(), + }; tx.into_signed_tx() } @@ -133,7 +137,7 @@ fn should_watch_extrinsic() { amount: 5, nonce: 0, from: AccountKeyring::Alice.into(), - to: Default::default(), + to: AccountKeyring::Bob.into(), }; tx.into_signed_tx() }; diff --git a/client/rpc/src/chain/chain_full.rs b/client/rpc/src/chain/chain_full.rs index 96d5b86f4249..288a825eb5be 100644 --- a/client/rpc/src/chain/chain_full.rs +++ b/client/rpc/src/chain/chain_full.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc/src/chain/chain_light.rs b/client/rpc/src/chain/chain_light.rs deleted file mode 100644 index 2d15c819e1da..000000000000 --- a/client/rpc/src/chain/chain_light.rs +++ /dev/null @@ -1,114 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Blockchain API backend for light nodes. - -use futures::{future::ready, FutureExt, TryFutureExt}; -use jsonrpc_pubsub::manager::SubscriptionManager; -use std::sync::Arc; - -use sc_client_api::light::{Fetcher, RemoteBlockchain, RemoteBodyRequest}; -use sp_runtime::{ - generic::{BlockId, SignedBlock}, - traits::Block as BlockT, -}; - -use super::{client_err, error::FutureResult, ChainBackend}; -use sc_client_api::BlockchainEvents; -use sp_blockchain::HeaderBackend; - -/// Blockchain API backend for light nodes. Reads all the data from local -/// database, if available, or fetches it from remote node otherwise. -pub struct LightChain { - /// Substrate client. - client: Arc, - /// Current subscriptions. - subscriptions: SubscriptionManager, - /// Remote blockchain reference - remote_blockchain: Arc>, - /// Remote fetcher reference. - fetcher: Arc, -} - -impl> LightChain { - /// Create new Chain API RPC handler. - pub fn new( - client: Arc, - subscriptions: SubscriptionManager, - remote_blockchain: Arc>, - fetcher: Arc, - ) -> Self { - Self { client, subscriptions, remote_blockchain, fetcher } - } -} - -impl ChainBackend for LightChain -where - Block: BlockT + 'static, - Block::Header: Unpin, - Client: BlockchainEvents + HeaderBackend + Send + Sync + 'static, - F: Fetcher + Send + Sync + 'static, -{ - fn client(&self) -> &Arc { - &self.client - } - - fn subscriptions(&self) -> &SubscriptionManager { - &self.subscriptions - } - - fn header(&self, hash: Option) -> FutureResult> { - let hash = self.unwrap_or_best(hash); - - let fetcher = self.fetcher.clone(); - let maybe_header = sc_client_api::light::future_header( - &*self.remote_blockchain, - &*fetcher, - BlockId::Hash(hash), - ); - - maybe_header.then(move |result| ready(result.map_err(client_err))).boxed() - } - - fn block(&self, hash: Option) -> FutureResult>> { - let fetcher = self.fetcher.clone(); - self.header(hash) - .and_then(move |header| async move { - match header { - Some(header) => { - let body = fetcher - .remote_body(RemoteBodyRequest { - header: header.clone(), - retry_count: Default::default(), - }) - .await; - - body.map(|body| { - Some(SignedBlock { - block: Block::new(header, body), - justifications: None, - }) - }) - .map_err(client_err) - }, - None => Ok(None), - } - }) - .boxed() - } -} diff --git a/client/rpc/src/chain/mod.rs b/client/rpc/src/chain/mod.rs index a06c3a094b40..c20e5a188ad3 100644 --- a/client/rpc/src/chain/mod.rs +++ b/client/rpc/src/chain/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,7 +19,6 @@ //! Substrate blockchain API. mod chain_full; -mod chain_light; #[cfg(test)] mod tests; @@ -33,10 +32,7 @@ use rpc::{ use std::sync::Arc; use jsonrpc_pubsub::{manager::SubscriptionManager, typed::Subscriber, SubscriptionId}; -use sc_client_api::{ - light::{Fetcher, RemoteBlockchain}, - BlockchainEvents, -}; +use sc_client_api::BlockchainEvents; use sp_rpc::{list::ListOrValue, number::NumberOrHex}; use sp_runtime::{ generic::{BlockId, SignedBlock}, @@ -83,8 +79,6 @@ where match number { None => Ok(Some(self.client().info().best_hash)), Some(num_or_hex) => { - use std::convert::TryInto; - // FIXME <2329>: Database seems to limit the block number to u32 for no reason let block_num: u32 = num_or_hex.try_into().map_err(|_| { Error::Other(format!( @@ -206,29 +200,6 @@ where Chain { backend: Box::new(self::chain_full::FullChain::new(client, subscriptions)) } } -/// Create new state API that works on light node. -pub fn new_light>( - client: Arc, - subscriptions: SubscriptionManager, - remote_blockchain: Arc>, - fetcher: Arc, -) -> Chain -where - Block: BlockT + 'static, - Block::Header: Unpin, - Client: BlockBackend + HeaderBackend + BlockchainEvents + 'static, - F: Send + Sync + 'static, -{ - Chain { - backend: Box::new(self::chain_light::LightChain::new( - client, - subscriptions, - remote_blockchain, - fetcher, - )), - } -} - /// Chain API with subscriptions support. pub struct Chain { backend: Box>, diff --git a/client/rpc/src/chain/tests.rs b/client/rpc/src/chain/tests.rs index caa9f33138b8..fa4473d35f30 100644 --- a/client/rpc/src/chain/tests.rs +++ b/client/rpc/src/chain/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc/src/dev/mod.rs b/client/rpc/src/dev/mod.rs new file mode 100644 index 000000000000..d782a03feae4 --- /dev/null +++ b/client/rpc/src/dev/mod.rs @@ -0,0 +1,118 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Implementation of the [`DevApi`] trait providing debug utilities for Substrate based +//! blockchains. + +#[cfg(test)] +mod tests; + +pub use sc_rpc_api::dev::{BlockStats, DevApi}; + +use sc_client_api::{BlockBackend, HeaderBackend}; +use sc_rpc_api::{ + dev::error::{Error, Result}, + DenyUnsafe, +}; +use sp_api::{ApiExt, Core, ProvideRuntimeApi}; +use sp_core::Encode; +use sp_runtime::{ + generic::{BlockId, DigestItem}, + traits::{Block as BlockT, Header}, +}; +use std::{ + marker::{PhantomData, Send, Sync}, + sync::Arc, +}; + +type HasherOf = <::Header as Header>::Hashing; + +/// The Dev API. All methods are unsafe. +pub struct Dev { + client: Arc, + deny_unsafe: DenyUnsafe, + _phantom: PhantomData, +} + +impl Dev { + /// Create a new Dev API. + pub fn new(client: Arc, deny_unsafe: DenyUnsafe) -> Self { + Self { client, deny_unsafe, _phantom: PhantomData::default() } + } +} + +impl DevApi for Dev +where + Block: BlockT + 'static, + Client: BlockBackend + + HeaderBackend + + ProvideRuntimeApi + + Send + + Sync + + 'static, + Client::Api: Core, +{ + fn block_stats(&self, hash: Block::Hash) -> Result> { + self.deny_unsafe.check_if_safe()?; + + let block = { + let block = self + .client + .block(&BlockId::Hash(hash)) + .map_err(|e| Error::BlockQueryError(Box::new(e)))?; + if let Some(block) = block { + let (mut header, body) = block.block.deconstruct(); + // Remove the `Seal` to ensure we have the number of digests as expected by the + // runtime. + header.digest_mut().logs.retain(|item| !matches!(item, DigestItem::Seal(_, _))); + Block::new(header, body) + } else { + return Ok(None) + } + }; + let parent_header = { + let parent_hash = *block.header().parent_hash(); + let parent_header = self + .client + .header(BlockId::Hash(parent_hash)) + .map_err(|e| Error::BlockQueryError(Box::new(e)))?; + if let Some(header) = parent_header { + header + } else { + return Ok(None) + } + }; + let block_len = block.encoded_size() as u64; + let num_extrinsics = block.extrinsics().len() as u64; + let pre_root = *parent_header.state_root(); + let mut runtime_api = self.client.runtime_api(); + runtime_api.record_proof(); + runtime_api + .execute_block(&BlockId::Hash(parent_header.hash()), block) + .map_err(|_| Error::BlockExecutionFailed)?; + let witness = runtime_api + .extract_proof() + .expect("We enabled proof recording. A proof must be available; qed"); + let witness_len = witness.encoded_size() as u64; + let witness_compact_len = witness + .into_compact_proof::>(pre_root) + .map_err(|_| Error::WitnessCompactionFailed)? + .encoded_size() as u64; + Ok(Some(BlockStats { witness_len, witness_compact_len, block_len, num_extrinsics })) + } +} diff --git a/client/rpc/src/dev/tests.rs b/client/rpc/src/dev/tests.rs new file mode 100644 index 000000000000..1d31abe38b64 --- /dev/null +++ b/client/rpc/src/dev/tests.rs @@ -0,0 +1,58 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use super::*; +use assert_matches::assert_matches; +use futures::executor; +use sc_block_builder::BlockBuilderProvider; +use sp_blockchain::HeaderBackend; +use sp_consensus::BlockOrigin; +use substrate_test_runtime_client::{prelude::*, runtime::Block}; + +#[test] +fn block_stats_work() { + let mut client = Arc::new(substrate_test_runtime_client::new()); + let api = >::new(client.clone(), DenyUnsafe::No); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + executor::block_on(client.import(BlockOrigin::Own, block)).unwrap(); + + // Can't gather stats for a block without a parent. + assert_eq!(api.block_stats(client.genesis_hash()).unwrap(), None); + + assert_eq!( + api.block_stats(client.info().best_hash).unwrap(), + Some(BlockStats { + witness_len: 597, + witness_compact_len: 500, + block_len: 99, + num_extrinsics: 0, + }), + ); +} + +#[test] +fn deny_unsafe_works() { + let mut client = Arc::new(substrate_test_runtime_client::new()); + let api = >::new(client.clone(), DenyUnsafe::Yes); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + executor::block_on(client.import(BlockOrigin::Own, block)).unwrap(); + + assert_matches!(api.block_stats(client.info().best_hash), Err(Error::UnsafeRpcCalled(_))); +} diff --git a/client/rpc/src/lib.rs b/client/rpc/src/lib.rs index 832585db4854..59a1d542d365 100644 --- a/client/rpc/src/lib.rs +++ b/client/rpc/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -34,6 +34,7 @@ pub use sc_rpc_api::{DenyUnsafe, Metadata}; pub mod author; pub mod chain; +pub mod dev; pub mod offchain; pub mod state; pub mod system; @@ -54,7 +55,8 @@ impl SubscriptionTaskExecutor { impl Spawn for SubscriptionTaskExecutor { fn spawn_obj(&self, future: FutureObj<'static, ()>) -> Result<(), SpawnError> { - self.0.spawn("substrate-rpc-subscription", future.map(drop).boxed()); + self.0 + .spawn("substrate-rpc-subscription", Some("rpc"), future.map(drop).boxed()); Ok(()) } diff --git a/client/rpc/src/offchain/mod.rs b/client/rpc/src/offchain/mod.rs index 9d1cc702b51e..67b97d31ab94 100644 --- a/client/rpc/src/offchain/mod.rs +++ b/client/rpc/src/offchain/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc/src/offchain/tests.rs b/client/rpc/src/offchain/tests.rs index f9629e70198a..219eeb192dfd 100644 --- a/client/rpc/src/offchain/tests.rs +++ b/client/rpc/src/offchain/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc/src/state/mod.rs b/client/rpc/src/state/mod.rs index 80eccc2c97de..071db5324c83 100644 --- a/client/rpc/src/state/mod.rs +++ b/client/rpc/src/state/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,7 +19,6 @@ //! Substrate state API. mod state_full; -mod state_light; #[cfg(test)] mod tests; @@ -29,7 +28,6 @@ use jsonrpc_pubsub::{manager::SubscriptionManager, typed::Subscriber, Subscripti use rpc::Result as RpcResult; use std::sync::Arc; -use sc_client_api::light::{Fetcher, RemoteBlockchain}; use sc_rpc_api::{state::ReadProof, DenyUnsafe}; use sp_core::{ storage::{PrefixedStorageKey, StorageChangeSet, StorageData, StorageKey}, @@ -217,45 +215,6 @@ where (State { backend, deny_unsafe }, ChildState { backend: child_backend }) } -/// Create new state API that works on light node. -pub fn new_light>( - client: Arc, - subscriptions: SubscriptionManager, - remote_blockchain: Arc>, - fetcher: Arc, - deny_unsafe: DenyUnsafe, -) -> (State, ChildState) -where - Block: BlockT + 'static, - Block::Hash: Unpin, - BE: Backend + 'static, - Client: ExecutorProvider - + StorageProvider - + HeaderMetadata - + ProvideRuntimeApi - + HeaderBackend - + BlockchainEvents - + Send - + Sync - + 'static, - F: Send + Sync + 'static, -{ - let child_backend = Box::new(self::state_light::LightState::new( - client.clone(), - subscriptions.clone(), - remote_blockchain.clone(), - fetcher.clone(), - )); - - let backend = Box::new(self::state_light::LightState::new( - client, - subscriptions, - remote_blockchain, - fetcher, - )); - (State { backend, deny_unsafe }, ChildState { backend: child_backend }) -} - /// State API with subscriptions support. pub struct State { backend: Box>, diff --git a/client/rpc/src/state/state_full.rs b/client/rpc/src/state/state_full.rs index 97f77a407796..1a35760bd67d 100644 --- a/client/rpc/src/state/state_full.rs +++ b/client/rpc/src/state/state_full.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -26,11 +26,7 @@ use futures::{ use jsonrpc_pubsub::{manager::SubscriptionManager, typed::Subscriber, SubscriptionId}; use log::warn; use rpc::Result as RpcResult; -use std::{ - collections::{BTreeMap, HashMap}, - ops::Range, - sync::Arc, -}; +use std::{collections::HashMap, sync::Arc}; use sc_rpc_api::state::ReadProof; use sp_blockchain::{ @@ -43,10 +39,7 @@ use sp_core::{ }, Bytes, }; -use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, CheckedSub, NumberFor, SaturatedConversion}, -}; +use sp_runtime::{generic::BlockId, traits::Block as BlockT}; use sp_version::RuntimeVersion; use sp_api::{CallApiAt, Metadata, ProvideRuntimeApi}; @@ -58,7 +51,7 @@ use super::{ }; use sc_client_api::{ Backend, BlockBackend, BlockchainEvents, CallExecutor, ExecutorProvider, ProofProvider, - StorageProvider, + StorageNotification, StorageProvider, }; use std::marker::PhantomData; @@ -66,14 +59,6 @@ use std::marker::PhantomData; struct QueryStorageRange { /// Hashes of all the blocks in the range. pub hashes: Vec, - /// Number of the first block in the range. - pub first_number: NumberFor, - /// Blocks subrange ([begin; end) indices within `hashes`) where we should read keys at - /// each state to get changes. - pub unfiltered_range: Range, - /// Blocks subrange ([begin; end) indices within `hashes`) where we could pre-filter - /// blocks-with-changes by using changes tries. - pub filtered_range: Option>, } /// State API backend for full nodes. @@ -107,10 +92,8 @@ where Ok(hash.unwrap_or_else(|| self.client.info().best_hash)) } - /// Splits the `query_storage` block range into 'filtered' and 'unfiltered' subranges. - /// Blocks that contain changes within filtered subrange could be filtered using changes tries. - /// Blocks that contain changes within unfiltered subrange must be filtered manually. - fn split_query_storage_range( + /// Validates block range. + fn query_storage_range( &self, from: Block::Hash, to: Option, @@ -156,23 +139,7 @@ where hashes }; - // check if we can filter blocks-with-changes from some (sub)range using changes tries - let changes_trie_range = self - .client - .max_key_changes_range(from_number, BlockId::Hash(to_meta.hash)) - .map_err(client_err)?; - let filtered_range_begin = changes_trie_range.and_then(|(begin, _)| { - // avoids a corner case where begin < from_number (happens when querying genesis) - begin.checked_sub(&from_number).map(|x| x.saturated_into::()) - }); - let (unfiltered_range, filtered_range) = split_range(hashes.len(), filtered_range_begin); - - Ok(QueryStorageRange { - hashes, - first_number: from_number, - unfiltered_range, - filtered_range, - }) + Ok(QueryStorageRange { hashes }) } /// Iterates through range.unfiltered_range and check each block for changes of keys' values. @@ -183,8 +150,8 @@ where last_values: &mut HashMap>, changes: &mut Vec>, ) -> Result<()> { - for block in range.unfiltered_range.start..range.unfiltered_range.end { - let block_hash = range.hashes[block].clone(); + for block_hash in &range.hashes { + let block_hash = block_hash.clone(); let mut block_changes = StorageChangeSet { block: block_hash.clone(), changes: Vec::new() }; let id = BlockId::hash(block_hash); @@ -207,57 +174,6 @@ where } Ok(()) } - - /// Iterates through all blocks that are changing keys within range.filtered_range and collects - /// these changes. - fn query_storage_filtered( - &self, - range: &QueryStorageRange, - keys: &[StorageKey], - last_values: &HashMap>, - changes: &mut Vec>, - ) -> Result<()> { - let (begin, end) = match range.filtered_range { - Some(ref filtered_range) => ( - range.first_number + filtered_range.start.saturated_into(), - BlockId::Hash(range.hashes[filtered_range.end - 1].clone()), - ), - None => return Ok(()), - }; - let mut changes_map: BTreeMap, StorageChangeSet> = - BTreeMap::new(); - for key in keys { - let mut last_block = None; - let mut last_value = last_values.get(key).cloned().unwrap_or_default(); - let key_changes = self.client.key_changes(begin, end, None, key).map_err(client_err)?; - for (block, _) in key_changes.into_iter().rev() { - if last_block == Some(block) { - continue - } - - let block_hash = - range.hashes[(block - range.first_number).saturated_into::()].clone(); - let id = BlockId::Hash(block_hash); - let value_at_block = self.client.storage(&id, key).map_err(client_err)?; - if last_value == value_at_block { - continue - } - - changes_map - .entry(block) - .or_insert_with(|| StorageChangeSet { block: block_hash, changes: Vec::new() }) - .changes - .push((key.clone(), value_at_block.clone())); - last_block = Some(block); - last_value = value_at_block; - } - } - if let Some(additional_capacity) = changes_map.len().checked_sub(changes.len()) { - changes.reserve(additional_capacity); - } - changes.extend(changes_map.into_iter().map(|(_, cs)| cs)); - Ok(()) - } } impl StateBackend for FullState @@ -430,11 +346,10 @@ where keys: Vec, ) -> FutureResult>> { let call_fn = move || { - let range = self.split_query_storage_range(from, to)?; + let range = self.query_storage_range(from, to)?; let mut changes = Vec::new(); let mut last_values = HashMap::new(); self.query_storage_unfiltered(&range, &keys, &mut last_values, &mut changes)?; - self.query_storage_filtered(&range, &keys, &last_values, &mut changes)?; Ok(changes) }; @@ -526,10 +441,7 @@ where keys: Option>, ) { let keys = Into::>>::into(keys); - let stream = match self - .client - .storage_changes_notification_stream(keys.as_ref().map(|x| &**x), None) - { + let stream = match self.client.storage_changes_notification_stream(keys.as_deref(), None) { Ok(stream) => stream, Err(err) => { let _ = subscriber.reject(client_err(err).into()); @@ -554,7 +466,7 @@ where ); self.subscriptions.add(subscriber, |sink| { - let stream = stream.map(|(block, changes)| { + let stream = stream.map(|StorageNotification { block, changes }| { Ok(Ok::<_, rpc::Error>(StorageChangeSet { block, changes: changes @@ -768,30 +680,6 @@ where } } -/// Splits passed range into two subranges where: -/// - first range has at least one element in it; -/// - second range (optionally) starts at given `middle` element. -pub(crate) fn split_range( - size: usize, - middle: Option, -) -> (Range, Option>) { - // check if we can filter blocks-with-changes from some (sub)range using changes tries - let range2_begin = match middle { - // some of required changes tries are pruned => use available tries - Some(middle) if middle != 0 => Some(middle), - // all required changes tries are available, but we still want values at first block - // => do 'unfiltered' read for the first block and 'filtered' for the rest - Some(_) if size > 1 => Some(1), - // range contains single element => do not use changes tries - Some(_) => None, - // changes tries are not available => do 'unfiltered' read for the whole range - None => None, - }; - let range1 = 0..range2_begin.unwrap_or(size); - let range2 = range2_begin.map(|begin| begin..size); - (range1, range2) -} - fn invalid_block_range( from: &CachedHeaderMetadata, to: &CachedHeaderMetadata, diff --git a/client/rpc/src/state/state_light.rs b/client/rpc/src/state/state_light.rs deleted file mode 100644 index 749e57c365cc..000000000000 --- a/client/rpc/src/state/state_light.rs +++ /dev/null @@ -1,873 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! State API backend for light nodes. - -use codec::Decode; -use futures::{ - channel::oneshot::{channel, Sender}, - future::{ready, Either}, - Future, FutureExt, SinkExt, Stream, StreamExt as _, TryFutureExt, TryStreamExt as _, -}; -use hash_db::Hasher; -use jsonrpc_pubsub::{manager::SubscriptionManager, typed::Subscriber, SubscriptionId}; -use log::warn; -use parking_lot::Mutex; -use rpc::Result as RpcResult; -use std::{ - collections::{hash_map::Entry, HashMap, HashSet}, - sync::Arc, -}; - -use sc_client_api::{ - light::{ - future_header, Fetcher, RemoteBlockchain, RemoteCallRequest, RemoteReadChildRequest, - RemoteReadRequest, - }, - BlockchainEvents, -}; -use sc_rpc_api::state::ReadProof; -use sp_blockchain::{Error as ClientError, HeaderBackend}; -use sp_core::{ - storage::{PrefixedStorageKey, StorageChangeSet, StorageData, StorageKey}, - Bytes, OpaqueMetadata, -}; -use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, HashFor}, -}; -use sp_version::RuntimeVersion; - -use super::{ - client_err, - error::{Error, FutureResult}, - ChildStateBackend, StateBackend, -}; - -/// Storage data map of storage keys => (optional) storage value. -type StorageMap = HashMap>; - -/// State API backend for light nodes. -#[derive(Clone)] -pub struct LightState, Client> { - client: Arc, - subscriptions: SubscriptionManager, - version_subscriptions: SimpleSubscriptions, - storage_subscriptions: Arc>>, - remote_blockchain: Arc>, - fetcher: Arc, -} - -/// Shared requests container. -trait SharedRequests: Clone + Send + Sync { - /// Tries to listen for already issued request, or issues request. - /// - /// Returns true if requests has been issued. - fn listen_request(&self, block: Hash, sender: Sender>) -> bool; - - /// Returns (and forgets) all listeners for given request. - fn on_response_received(&self, block: Hash) -> Vec>>; -} - -/// Storage subscriptions data. -struct StorageSubscriptions { - /// Active storage requests. - active_requests: HashMap>>>, - /// Map of subscription => keys that this subscription watch for. - keys_by_subscription: HashMap>, - /// Map of key => set of subscriptions that watch this key. - subscriptions_by_key: HashMap>, -} - -impl SharedRequests - for Arc>> -{ - fn listen_request(&self, block: Block::Hash, sender: Sender>) -> bool { - let mut subscriptions = self.lock(); - let active_requests_at = subscriptions.active_requests.entry(block).or_default(); - active_requests_at.push(sender); - active_requests_at.len() == 1 - } - - fn on_response_received(&self, block: Block::Hash) -> Vec>> { - self.lock().active_requests.remove(&block).unwrap_or_default() - } -} - -/// Simple, maybe shared, subscription data that shares per block requests. -type SimpleSubscriptions = Arc>>>>>; - -impl SharedRequests for SimpleSubscriptions -where - Hash: Send + Eq + std::hash::Hash, - V: Send, -{ - fn listen_request(&self, block: Hash, sender: Sender>) -> bool { - let mut subscriptions = self.lock(); - let active_requests_at = subscriptions.entry(block).or_default(); - active_requests_at.push(sender); - active_requests_at.len() == 1 - } - - fn on_response_received(&self, block: Hash) -> Vec>> { - self.lock().remove(&block).unwrap_or_default() - } -} - -impl + 'static, Client> LightState -where - Block: BlockT, - Client: HeaderBackend + Send + Sync + 'static, -{ - /// Create new state API backend for light nodes. - pub fn new( - client: Arc, - subscriptions: SubscriptionManager, - remote_blockchain: Arc>, - fetcher: Arc, - ) -> Self { - Self { - client, - subscriptions, - version_subscriptions: Arc::new(Mutex::new(HashMap::new())), - storage_subscriptions: Arc::new(Mutex::new(StorageSubscriptions { - active_requests: HashMap::new(), - keys_by_subscription: HashMap::new(), - subscriptions_by_key: HashMap::new(), - })), - remote_blockchain, - fetcher, - } - } - - /// Returns given block hash or best block hash if None is passed. - fn block_or_best(&self, hash: Option) -> Block::Hash { - hash.unwrap_or_else(|| self.client.info().best_hash) - } -} - -impl StateBackend for LightState -where - Block: BlockT, - Block::Hash: Unpin, - Client: BlockchainEvents + HeaderBackend + Send + Sync + 'static, - F: Fetcher + 'static, -{ - fn call( - &self, - block: Option, - method: String, - call_data: Bytes, - ) -> FutureResult { - call( - &*self.remote_blockchain, - self.fetcher.clone(), - self.block_or_best(block), - method, - call_data, - ) - .boxed() - } - - fn storage_keys( - &self, - _block: Option, - _prefix: StorageKey, - ) -> FutureResult> { - async move { Err(client_err(ClientError::NotAvailableOnLightClient)) }.boxed() - } - - fn storage_pairs( - &self, - _block: Option, - _prefix: StorageKey, - ) -> FutureResult> { - async move { Err(client_err(ClientError::NotAvailableOnLightClient)) }.boxed() - } - - fn storage_keys_paged( - &self, - _block: Option, - _prefix: Option, - _count: u32, - _start_key: Option, - ) -> FutureResult> { - async move { Err(client_err(ClientError::NotAvailableOnLightClient)) }.boxed() - } - - fn storage_size(&self, _: Option, _: StorageKey) -> FutureResult> { - async move { Err(client_err(ClientError::NotAvailableOnLightClient)) }.boxed() - } - - fn storage( - &self, - block: Option, - key: StorageKey, - ) -> FutureResult> { - storage( - &*self.remote_blockchain, - self.fetcher.clone(), - self.block_or_best(block), - vec![key.0.clone()], - ) - .map_ok(move |mut values| { - values - .remove(&key) - .expect("successful request has entries for all requested keys; qed") - }) - .boxed() - } - - fn storage_hash( - &self, - block: Option, - key: StorageKey, - ) -> FutureResult> { - let res = StateBackend::storage(self, block, key); - async move { res.await.map(|r| r.map(|s| HashFor::::hash(&s.0))) }.boxed() - } - - fn metadata(&self, block: Option) -> FutureResult { - self.call(block, "Metadata_metadata".into(), Bytes(Vec::new())) - .and_then(|metadata| async move { - OpaqueMetadata::decode(&mut &metadata.0[..]) - .map(Into::into) - .map_err(|decode_err| { - client_err(ClientError::CallResultDecode( - "Unable to decode metadata", - decode_err, - )) - }) - }) - .boxed() - } - - fn runtime_version(&self, block: Option) -> FutureResult { - runtime_version(&*self.remote_blockchain, self.fetcher.clone(), self.block_or_best(block)) - .boxed() - } - - fn query_storage( - &self, - _from: Block::Hash, - _to: Option, - _keys: Vec, - ) -> FutureResult>> { - async move { Err(client_err(ClientError::NotAvailableOnLightClient)) }.boxed() - } - - fn query_storage_at( - &self, - _keys: Vec, - _at: Option, - ) -> FutureResult>> { - async move { Err(client_err(ClientError::NotAvailableOnLightClient)) }.boxed() - } - - fn read_proof( - &self, - _block: Option, - _keys: Vec, - ) -> FutureResult> { - async move { Err(client_err(ClientError::NotAvailableOnLightClient)) }.boxed() - } - - fn subscribe_storage( - &self, - _meta: crate::Metadata, - subscriber: Subscriber>, - keys: Option>, - ) { - let keys = match keys { - Some(keys) if !keys.is_empty() => keys, - _ => { - warn!("Cannot subscribe to all keys on light client. Subscription rejected."); - return - }, - }; - - let keys = keys.iter().cloned().collect::>(); - let keys_to_check = keys.iter().map(|k| k.0.clone()).collect::>(); - let subscription_id = self.subscriptions.add(subscriber, move |sink| { - let fetcher = self.fetcher.clone(); - let remote_blockchain = self.remote_blockchain.clone(); - let storage_subscriptions = self.storage_subscriptions.clone(); - let initial_block = self.block_or_best(None); - let initial_keys = keys_to_check.iter().cloned().collect::>(); - - let changes_stream = subscription_stream::( - storage_subscriptions.clone(), - self.client.import_notification_stream().map(|notification| notification.hash), - display_error( - storage(&*remote_blockchain, fetcher.clone(), initial_block, initial_keys) - .map(move |r| r.map(|r| (initial_block, r))), - ), - move |block| { - // there'll be single request per block for all active subscriptions - // with all subscribed keys - let keys = storage_subscriptions - .lock() - .subscriptions_by_key - .keys() - .map(|k| k.0.clone()) - .collect(); - - storage(&*remote_blockchain, fetcher.clone(), block, keys) - }, - move |block, old_value, new_value| { - // let's only select keys which are valid for this subscription - let new_value = new_value - .iter() - .filter(|(k, _)| keys_to_check.contains(&k.0)) - .map(|(k, v)| (k.clone(), v.clone())) - .collect::>(); - let value_differs = old_value - .as_ref() - .map(|old_value| **old_value != new_value) - .unwrap_or(true); - - value_differs.then(|| StorageChangeSet { - block, - changes: new_value.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), - }) - }, - ); - - changes_stream - .map_ok(Ok) - .forward(sink.sink_map_err(|e| warn!("Error sending notifications: {:?}", e))) - // we ignore the resulting Stream (if the first stream is over we are unsubscribed) - .map(|_| ()) - }); - - // remember keys associated with this subscription - let mut storage_subscriptions = self.storage_subscriptions.lock(); - storage_subscriptions - .keys_by_subscription - .insert(subscription_id.clone(), keys.clone()); - for key in keys { - storage_subscriptions - .subscriptions_by_key - .entry(key) - .or_default() - .insert(subscription_id.clone()); - } - } - - fn unsubscribe_storage( - &self, - _meta: Option, - id: SubscriptionId, - ) -> RpcResult { - if !self.subscriptions.cancel(id.clone()) { - return Ok(false) - } - - // forget subscription keys - let mut storage_subscriptions = self.storage_subscriptions.lock(); - let keys = storage_subscriptions.keys_by_subscription.remove(&id); - for key in keys.into_iter().flat_map(|keys| keys.into_iter()) { - match storage_subscriptions.subscriptions_by_key.entry(key) { - Entry::Vacant(_) => unreachable!( - "every key from keys_by_subscription has\ - corresponding entry in subscriptions_by_key; qed" - ), - Entry::Occupied(mut entry) => { - entry.get_mut().remove(&id); - if entry.get().is_empty() { - entry.remove(); - } - }, - } - } - - Ok(true) - } - - fn subscribe_runtime_version( - &self, - _meta: crate::Metadata, - subscriber: Subscriber, - ) { - self.subscriptions.add(subscriber, move |sink| { - let fetcher = self.fetcher.clone(); - let remote_blockchain = self.remote_blockchain.clone(); - let version_subscriptions = self.version_subscriptions.clone(); - let initial_block = self.block_or_best(None); - - let versions_stream = subscription_stream::( - version_subscriptions, - self.client.import_notification_stream().map(|notification| notification.hash), - display_error( - runtime_version(&*remote_blockchain, fetcher.clone(), initial_block) - .map(move |r| r.map(|r| (initial_block, r))), - ), - move |block| runtime_version(&*remote_blockchain, fetcher.clone(), block), - |_, old_version, new_version| { - let version_differs = old_version - .as_ref() - .map(|old_version| *old_version != new_version) - .unwrap_or(true); - - version_differs.then(|| new_version.clone()) - }, - ); - - versions_stream - .map_ok(Ok) - .forward(sink.sink_map_err(|e| warn!("Error sending notifications: {:?}", e))) - // we ignore the resulting Stream (if the first stream is over we are unsubscribed) - .map(|_| ()) - }); - } - - fn unsubscribe_runtime_version( - &self, - _meta: Option, - id: SubscriptionId, - ) -> RpcResult { - Ok(self.subscriptions.cancel(id)) - } - - fn trace_block( - &self, - _block: Block::Hash, - _targets: Option, - _storage_keys: Option, - _methods: Option, - ) -> FutureResult { - async move { Err(client_err(ClientError::NotAvailableOnLightClient)) }.boxed() - } -} - -impl ChildStateBackend for LightState -where - Block: BlockT, - Client: BlockchainEvents + HeaderBackend + Send + Sync + 'static, - F: Fetcher + 'static, -{ - fn read_child_proof( - &self, - _block: Option, - _storage_key: PrefixedStorageKey, - _keys: Vec, - ) -> FutureResult> { - async move { Err(client_err(ClientError::NotAvailableOnLightClient)) }.boxed() - } - - fn storage_keys( - &self, - _block: Option, - _storage_key: PrefixedStorageKey, - _prefix: StorageKey, - ) -> FutureResult> { - async move { Err(client_err(ClientError::NotAvailableOnLightClient)) }.boxed() - } - - fn storage_keys_paged( - &self, - _block: Option, - _storage_key: PrefixedStorageKey, - _prefix: Option, - _count: u32, - _start_key: Option, - ) -> FutureResult> { - async move { Err(client_err(ClientError::NotAvailableOnLightClient)) }.boxed() - } - - fn storage( - &self, - block: Option, - storage_key: PrefixedStorageKey, - key: StorageKey, - ) -> FutureResult> { - let block = self.block_or_best(block); - let fetcher = self.fetcher.clone(); - let child_storage = - resolve_header(&*self.remote_blockchain, &*self.fetcher, block).then(move |result| { - match result { - Ok(header) => Either::Left( - fetcher - .remote_read_child(RemoteReadChildRequest { - block, - header, - storage_key, - keys: vec![key.0.clone()], - retry_count: Default::default(), - }) - .then(move |result| { - ready( - result - .map(|mut data| { - data.remove(&key.0) - .expect( - "successful result has entry for all keys; qed", - ) - .map(StorageData) - }) - .map_err(client_err), - ) - }), - ), - Err(error) => Either::Right(ready(Err(error))), - } - }); - - child_storage.boxed() - } - - fn storage_entries( - &self, - block: Option, - storage_key: PrefixedStorageKey, - keys: Vec, - ) -> FutureResult>> { - let block = self.block_or_best(block); - let fetcher = self.fetcher.clone(); - let keys = keys.iter().map(|k| k.0.clone()).collect::>(); - let child_storage = - resolve_header(&*self.remote_blockchain, &*self.fetcher, block).then(move |result| { - match result { - Ok(header) => Either::Left( - fetcher - .remote_read_child(RemoteReadChildRequest { - block, - header, - storage_key, - keys: keys.clone(), - retry_count: Default::default(), - }) - .then(move |result| { - ready( - result - .map(|data| { - data.iter() - .filter_map(|(k, d)| { - keys.contains(k).then(|| { - d.as_ref().map(|v| StorageData(v.to_vec())) - }) - }) - .collect::>() - }) - .map_err(client_err), - ) - }), - ), - Err(error) => Either::Right(ready(Err(error))), - } - }); - - child_storage.boxed() - } - - fn storage_hash( - &self, - block: Option, - storage_key: PrefixedStorageKey, - key: StorageKey, - ) -> FutureResult> { - let child_storage = ChildStateBackend::storage(self, block, storage_key, key); - - async move { child_storage.await.map(|r| r.map(|s| HashFor::::hash(&s.0))) }.boxed() - } -} - -/// Resolve header by hash. -fn resolve_header>( - remote_blockchain: &dyn RemoteBlockchain, - fetcher: &F, - block: Block::Hash, -) -> impl std::future::Future> { - let maybe_header = future_header(remote_blockchain, fetcher, BlockId::Hash(block)); - - maybe_header.then(move |result| { - ready( - result - .and_then(|maybe_header| { - maybe_header.ok_or_else(|| ClientError::UnknownBlock(format!("{}", block))) - }) - .map_err(client_err), - ) - }) -} - -/// Call runtime method at given block -fn call>( - remote_blockchain: &dyn RemoteBlockchain, - fetcher: Arc, - block: Block::Hash, - method: String, - call_data: Bytes, -) -> impl std::future::Future> { - resolve_header(remote_blockchain, &*fetcher, block).then(move |result| match result { - Ok(header) => Either::Left( - fetcher - .remote_call(RemoteCallRequest { - block, - header, - method, - call_data: call_data.0, - retry_count: Default::default(), - }) - .then(|result| ready(result.map(Bytes).map_err(client_err))), - ), - Err(error) => Either::Right(ready(Err(error))), - }) -} - -/// Get runtime version at given block. -fn runtime_version>( - remote_blockchain: &dyn RemoteBlockchain, - fetcher: Arc, - block: Block::Hash, -) -> impl std::future::Future> { - call(remote_blockchain, fetcher, block, "Core_version".into(), Bytes(Vec::new())).then( - |version| { - ready(version.and_then(|version| { - Decode::decode(&mut &version.0[..]) - .map_err(|e| client_err(ClientError::VersionInvalid(e.to_string()))) - })) - }, - ) -} - -/// Get storage value at given key at given block. -fn storage>( - remote_blockchain: &dyn RemoteBlockchain, - fetcher: Arc, - block: Block::Hash, - keys: Vec>, -) -> impl std::future::Future>, Error>> { - resolve_header(remote_blockchain, &*fetcher, block).then(move |result| match result { - Ok(header) => Either::Left( - fetcher - .remote_read(RemoteReadRequest { - block, - header, - keys, - retry_count: Default::default(), - }) - .then(|result| { - ready( - result - .map(|result| { - result - .into_iter() - .map(|(key, value)| (StorageKey(key), value.map(StorageData))) - .collect() - }) - .map_err(client_err), - ) - }), - ), - Err(error) => Either::Right(ready(Err(error))), - }) -} - -/// Returns subscription stream that issues request on every imported block and -/// if value has changed from previous block, emits (stream) item. -fn subscription_stream< - Block, - Requests, - FutureBlocksStream, - V, - N, - InitialRequestFuture, - IssueRequest, - IssueRequestFuture, - CompareValues, ->( - shared_requests: Requests, - future_blocks_stream: FutureBlocksStream, - initial_request: InitialRequestFuture, - issue_request: IssueRequest, - compare_values: CompareValues, -) -> impl Stream> -where - Block: BlockT, - Requests: 'static + SharedRequests, - FutureBlocksStream: Stream, - V: Send + 'static + Clone, - InitialRequestFuture: Future> + Send + 'static, - IssueRequest: 'static + Fn(Block::Hash) -> IssueRequestFuture, - IssueRequestFuture: Future> + Send + 'static, - CompareValues: Fn(Block::Hash, Option<&V>, &V) -> Option, -{ - // we need to send initial value first, then we'll only be sending if value has changed - let previous_value = Arc::new(Mutex::new(None)); - - // prepare 'stream' of initial values - let initial_value_stream = initial_request.into_stream(); - - // prepare stream of future values - // - // we do not want to stop stream if single request fails - // (the warning should have been already issued by the request issuer) - let future_values_stream = future_blocks_stream - .then(move |block| { - maybe_share_remote_request::( - shared_requests.clone(), - block, - &issue_request, - ) - .map(move |r| r.map(|v| (block, v))) - }) - .filter(|r| ready(r.is_ok())); - - // now let's return changed values for selected blocks - initial_value_stream - .chain(future_values_stream) - .try_filter_map(move |(block, new_value)| { - let mut previous_value = previous_value.lock(); - let res = compare_values(block, previous_value.as_ref(), &new_value).map( - |notification_value| { - *previous_value = Some(new_value); - notification_value - }, - ); - async move { Ok(res) } - }) - .map_err(|_| ()) -} - -/// Request some data from remote node, probably reusing response from already -/// (in-progress) existing request. -fn maybe_share_remote_request( - shared_requests: Requests, - block: Block::Hash, - issue_request: &IssueRequest, -) -> impl std::future::Future> -where - V: Clone, - Requests: SharedRequests, - IssueRequest: Fn(Block::Hash) -> IssueRequestFuture, - IssueRequestFuture: std::future::Future>, -{ - let (sender, receiver) = channel(); - let need_issue_request = shared_requests.listen_request(block, sender); - - // if that isn't the first request - just listen for existing request' response - if !need_issue_request { - return Either::Right(receiver.then(|r| ready(r.unwrap_or(Err(()))))) - } - - // that is the first request - issue remote request + notify all listeners on - // completion - Either::Left(display_error(issue_request(block)).then(move |remote_result| { - let listeners = shared_requests.on_response_received(block); - // skip first element, because this future is the first element - for receiver in listeners.into_iter().skip(1) { - if let Err(_) = receiver.send(remote_result.clone()) { - // we don't care if receiver has been dropped already - } - } - - ready(remote_result) - })) -} - -/// Convert successful future result into Ok(result) and error into Err(()), -/// displaying warning. -fn display_error(future: F) -> impl std::future::Future> -where - F: std::future::Future>, -{ - future.then(|result| { - ready(result.or_else(|err| { - warn!("Remote request for subscription data has failed with: {:?}", err); - Err(()) - })) - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use futures::{executor, stream}; - use sp_core::H256; - use substrate_test_runtime_client::runtime::Block; - - #[test] - fn subscription_stream_works() { - let stream = subscription_stream::( - SimpleSubscriptions::default(), - stream::iter(vec![H256::from([2; 32]), H256::from([3; 32])]), - ready(Ok((H256::from([1; 32]), 100))), - |block| match block[0] { - 2 => ready(Ok(100)), - 3 => ready(Ok(200)), - _ => unreachable!("should not issue additional requests"), - }, - |_, old_value, new_value| match old_value == Some(new_value) { - true => None, - false => Some(new_value.clone()), - }, - ); - - assert_eq!(executor::block_on(stream.collect::>()), vec![Ok(100), Ok(200)]); - } - - #[test] - fn subscription_stream_ignores_failed_requests() { - let stream = subscription_stream::( - SimpleSubscriptions::default(), - stream::iter(vec![H256::from([2; 32]), H256::from([3; 32])]), - ready(Ok((H256::from([1; 32]), 100))), - |block| match block[0] { - 2 => ready(Err(client_err(ClientError::NotAvailableOnLightClient))), - 3 => ready(Ok(200)), - _ => unreachable!("should not issue additional requests"), - }, - |_, old_value, new_value| match old_value == Some(new_value) { - true => None, - false => Some(new_value.clone()), - }, - ); - - assert_eq!(executor::block_on(stream.collect::>()), vec![Ok(100), Ok(200)]); - } - - #[test] - fn maybe_share_remote_request_shares_request() { - type UnreachableFuture = futures::future::Ready>; - - let shared_requests = SimpleSubscriptions::default(); - - // let's 'issue' requests for B1 - shared_requests.lock().insert(H256::from([1; 32]), vec![channel().0]); - - // make sure that no additional requests are issued when we're asking for B1 - let _ = maybe_share_remote_request::( - shared_requests.clone(), - H256::from([1; 32]), - &|_| unreachable!("no duplicate requests issued"), - ); - - // make sure that additional requests is issued when we're asking for B2 - let request_issued = Arc::new(Mutex::new(false)); - let _ = maybe_share_remote_request::( - shared_requests.clone(), - H256::from([2; 32]), - &|_| { - *request_issued.lock() = true; - ready(Ok(Default::default())) - }, - ); - assert!(*request_issued.lock()); - } -} diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index 712fe00c5438..287dfac8c6ba 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -17,23 +17,22 @@ // along with this program. If not, see . use self::error::Error; -use super::{state_full::split_range, *}; +use super::*; use crate::testing::TaskExecutor; use assert_matches::assert_matches; use futures::{executor, StreamExt}; use sc_block_builder::BlockBuilderProvider; use sc_rpc_api::DenyUnsafe; use sp_consensus::BlockOrigin; -use sp_core::{hash::H256, storage::ChildInfo, ChangesTrieConfiguration}; +use sp_core::{hash::H256, storage::ChildInfo}; use sp_io::hashing::blake2_256; -use sp_runtime::generic::BlockId; use std::sync::Arc; use substrate_test_runtime_client::{prelude::*, runtime}; const STORAGE_KEY: &[u8] = b"child"; fn prefixed_storage_key() -> PrefixedStorageKey { - let child_info = ChildInfo::new_default(&STORAGE_KEY[..]); + let child_info = ChildInfo::new_default(STORAGE_KEY); child_info.prefixed_storage_key() } @@ -336,7 +335,7 @@ fn should_send_initial_storage_changes_and_notifications() { #[test] fn should_query_storage() { - fn run_tests(mut client: Arc, has_changes_trie_config: bool) { + fn run_tests(mut client: Arc) { let (api, _child) = new_full( client.clone(), SubscriptionManager::new(Arc::new(TaskExecutor)), @@ -369,13 +368,6 @@ fn should_query_storage() { let block2_hash = add_block(1); let genesis_hash = client.genesis_hash(); - if has_changes_trie_config { - assert_eq!( - client.max_key_changes_range(1, BlockId::Hash(block1_hash)).unwrap(), - Some((0, BlockId::Hash(block1_hash))), - ); - } - let mut expected = vec![ StorageChangeSet { block: genesis_hash, @@ -519,24 +511,8 @@ fn should_query_storage() { ); } - run_tests(Arc::new(substrate_test_runtime_client::new()), false); - run_tests( - Arc::new( - TestClientBuilder::new() - .changes_trie_config(Some(ChangesTrieConfiguration::new(4, 2))) - .build(), - ), - true, - ); -} - -#[test] -fn should_split_ranges() { - assert_eq!(split_range(1, None), (0..1, None)); - assert_eq!(split_range(100, None), (0..100, None)); - assert_eq!(split_range(1, Some(0)), (0..1, None)); - assert_eq!(split_range(100, Some(50)), (0..50, Some(50..100))); - assert_eq!(split_range(100, Some(99)), (0..99, Some(99..100))); + run_tests(Arc::new(substrate_test_runtime_client::new())); + run_tests(Arc::new(TestClientBuilder::new().build())); } #[test] @@ -550,11 +526,11 @@ fn should_return_runtime_version() { ); let result = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\ - \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",3],\ - [\"0x37e397fc7c91f5e4\",1],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",5],\ + \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",4],\ + [\"0x37e397fc7c91f5e4\",1],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\ [\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",2],\ [\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],[\"0xbc9d89904f5b923f\",1]],\ - \"transactionVersion\":1}"; + \"transactionVersion\":1,\"stateVersion\":1}"; let runtime_version = executor::block_on(api.runtime_version(None.into())).unwrap(); let serialized = serde_json::to_string(&runtime_version).unwrap(); diff --git a/client/rpc/src/system/mod.rs b/client/rpc/src/system/mod.rs index f99994e41a1b..534e446e140a 100644 --- a/client/rpc/src/system/mod.rs +++ b/client/rpc/src/system/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs index 14997545031d..5d6945b71420 100644 --- a/client/rpc/src/system/tests.rs +++ b/client/rpc/src/system/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -308,7 +308,10 @@ fn test_add_reset_log_filter() { // Enter log generation / filter reload if std::env::var("TEST_LOG_FILTER").is_ok() { - sc_tracing::logging::LoggerBuilder::new("test_before_add=debug").init().unwrap(); + let mut builder = sc_tracing::logging::LoggerBuilder::new("test_before_add=debug"); + builder.with_log_reloading(true); + builder.init().unwrap(); + for line in std::io::stdin().lock().lines() { let line = line.expect("Failed to read bytes"); if line.contains("add_reload") { @@ -351,26 +354,26 @@ fn test_add_reset_log_filter() { }; // Initiate logs loop in child process - child_in.write(b"\n").unwrap(); + child_in.write_all(b"\n").unwrap(); assert!(read_line().contains(EXPECTED_BEFORE_ADD)); // Initiate add directive & reload in child process - child_in.write(b"add_reload\n").unwrap(); + child_in.write_all(b"add_reload\n").unwrap(); assert!(read_line().contains(EXPECTED_BEFORE_ADD)); assert!(read_line().contains(EXPECTED_AFTER_ADD)); // Check that increasing the max log level works - child_in.write(b"add_trace\n").unwrap(); + child_in.write_all(b"add_trace\n").unwrap(); assert!(read_line().contains(EXPECTED_WITH_TRACE)); assert!(read_line().contains(EXPECTED_BEFORE_ADD)); assert!(read_line().contains(EXPECTED_AFTER_ADD)); // Initiate logs filter reset in child process - child_in.write(b"reset\n").unwrap(); + child_in.write_all(b"reset\n").unwrap(); assert!(read_line().contains(EXPECTED_BEFORE_ADD)); // Return from child process - child_in.write(b"exit\n").unwrap(); + child_in.write_all(b"exit\n").unwrap(); assert!(child_process.wait().expect("Error waiting for child process").success()); // Check for EOF diff --git a/client/rpc/src/testing.rs b/client/rpc/src/testing.rs index 23071ba10e0d..bfb91adb81d3 100644 --- a/client/rpc/src/testing.rs +++ b/client/rpc/src/testing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index 735f215c82b3..47474217ebd2 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-service" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate service. Starts a thread that spins up the network, client, and extrinsic pool. Manages communication between them." readme = "README.md" @@ -22,42 +22,41 @@ wasmtime = ["sc-executor/wasmtime"] test-helpers = [] [dependencies] -thiserror = "1.0.21" -futures = "0.3.16" +thiserror = "1.0.30" +futures = "0.3.21" jsonrpc-pubsub = "18.0" jsonrpc-core = "18.0" rand = "0.7.3" -parking_lot = "0.11.1" +parking_lot = "0.12.0" log = "0.4.11" futures-timer = "3.0.1" exit-future = "0.2.0" -pin-project = "1.0.4" +pin-project = "1.0.10" hash-db = "0.15.2" -serde = "1.0.126" -serde_json = "1.0.68" +serde = "1.0.136" +serde_json = "1.0.79" sc-keystore = { version = "4.0.0-dev", path = "../keystore" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sp-trie = { version = "4.0.0-dev", path = "../../primitives/trie" } -sp-externalities = { version = "0.10.0-dev", path = "../../primitives/externalities" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } +sp-trie = { version = "6.0.0", path = "../../primitives/trie" } +sp-externalities = { version = "0.12.0", path = "../../primitives/externalities" } sc-utils = { version = "4.0.0-dev", path = "../utils" } -sp-version = { version = "4.0.0-dev", path = "../../primitives/version" } +sp-version = { version = "5.0.0", path = "../../primitives/version" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-keystore = { version = "0.12.0", path = "../../primitives/keystore" } sp-session = { version = "4.0.0-dev", path = "../../primitives/session" } -sp-state-machine = { version = "0.10.0-dev", path = "../../primitives/state-machine" } -sp-application-crypto = { version = "4.0.0-dev", path = "../../primitives/application-crypto" } +sp-state-machine = { version = "0.12.0", path = "../../primitives/state-machine" } +sp-application-crypto = { version = "6.0.0", path = "../../primitives/application-crypto" } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } sc-consensus = { version = "0.10.0-dev", path = "../../client/consensus/common" } sp-inherents = { version = "4.0.0-dev", path = "../../primitives/inherents" } -sp-storage = { version = "4.0.0-dev", path = "../../primitives/storage" } +sp-storage = { version = "6.0.0", path = "../../primitives/storage" } sc-network = { version = "0.10.0-dev", path = "../network" } sc-chain-spec = { version = "4.0.0-dev", path = "../chain-spec" } -sc-light = { version = "4.0.0-dev", path = "../light" } sc-client-api = { version = "4.0.0-dev", path = "../api" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } sc-client-db = { version = "0.10.0-dev", default-features = false, path = "../db" } -codec = { package = "parity-scale-codec", version = "2.0.0" } +codec = { package = "parity-scale-codec", version = "3.0.0" } sc-executor = { version = "0.10.0-dev", path = "../executor" } sc-transaction-pool = { version = "4.0.0-dev", path = "../transaction-pool" } sp-transaction-pool = { version = "4.0.0-dev", path = "../../primitives/transaction-pool" } @@ -70,18 +69,18 @@ sp-block-builder = { version = "4.0.0-dev", path = "../../primitives/block-build sc-informant = { version = "0.10.0-dev", path = "../informant" } sc-telemetry = { version = "4.0.0-dev", path = "../telemetry" } sc-offchain = { version = "4.0.0-dev", path = "../offchain" } -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.9.0" } +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.10.0-dev" } sc-tracing = { version = "4.0.0-dev", path = "../tracing" } -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } -tracing = "0.1.25" +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } +tracing = "0.1.29" tracing-futures = { version = "0.2.4" } -parity-util-mem = { version = "0.10.0", default-features = false, features = [ +parity-util-mem = { version = "0.11.0", default-features = false, features = [ "primitive-types", ] } async-trait = "0.1.50" -tokio = { version = "1.10", features = ["time", "rt-multi-thread"] } +tokio = { version = "1.17.0", features = ["time", "rt-multi-thread", "parking_lot"] } tempfile = "3.1.0" -directories = "3.0.2" +directories = "4.0.1" [dev-dependencies] substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index edf231dec048..735a966d710c 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,8 +18,8 @@ use crate::{ build_network_future, - client::{light, Client, ClientConfig}, - config::{Configuration, KeystoreConfig, PrometheusConfig, TransactionStorageMode}, + client::{Client, ClientConfig}, + config::{Configuration, KeystoreConfig, PrometheusConfig}, error::Error, metrics::MetricsService, start_rpc_servers, RpcHandlers, SpawnTaskHandle, TaskManager, TransactionPoolAdapter, @@ -30,9 +30,8 @@ use log::info; use prometheus_endpoint::Registry; use sc_chain_spec::get_extension; use sc_client_api::{ - execution_extensions::ExecutionExtensions, light::RemoteBlockchain, - proof_provider::ProofProvider, BadBlocks, BlockBackend, BlockchainEvents, ExecutorProvider, - ForkBlocks, StorageProvider, UsageProvider, + execution_extensions::ExecutionExtensions, proof_provider::ProofProvider, BadBlocks, + BlockBackend, BlockchainEvents, ExecutorProvider, ForkBlocks, StorageProvider, UsageProvider, }; use sc_client_db::{Backend, DatabaseSettings}; use sc_consensus::import_queue::ImportQueue; @@ -40,7 +39,7 @@ use sc_executor::RuntimeVersionOf; use sc_keystore::LocalKeystore; use sc_network::{ block_request_handler::{self, BlockRequestHandler}, - config::{OnDemand, Role, SyncMode}, + config::{Role, SyncMode}, light_client_requests::{self, handler::LightClientRequestHandler}, state_request_handler::{self, StateRequestHandler}, warp_request_handler::{self, RequestHandler as WarpSyncRequestHandler, WarpSyncProvider}, @@ -58,7 +57,7 @@ use sp_core::traits::{CodeExecutor, SpawnNamed}; use sp_keystore::{CryptoStore, SyncCryptoStore, SyncCryptoStorePtr}; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, BlockIdTo, HashFor, Zero}, + traits::{Block as BlockT, BlockIdTo, NumberFor, Zero}, BuildStorage, }; use std::{str::FromStr, sync::Arc, time::SystemTime}; @@ -137,47 +136,9 @@ pub type TFullBackend = sc_client_db::Backend; pub type TFullCallExecutor = crate::client::LocalCallExecutor, TExec>; -/// Light client type. -pub type TLightClient = - TLightClientWithBackend>; - -/// Light client backend type. -pub type TLightBackend = - sc_light::Backend, HashFor>; - -/// Light call executor type. -pub type TLightCallExecutor = sc_light::GenesisCallExecutor< - sc_light::Backend, HashFor>, - crate::client::LocalCallExecutor< - TBl, - sc_light::Backend, HashFor>, - TExec, - >, ->; - type TFullParts = (TFullClient, Arc>, KeystoreContainer, TaskManager); -type TLightParts = ( - Arc>, - Arc>, - KeystoreContainer, - TaskManager, - Arc>, -); - -/// Light client backend type with a specific hash type. -pub type TLightBackendWithHash = - sc_light::Backend, THash>; - -/// Light client type with a specific backend. -pub type TLightClientWithBackend = Client< - TBackend, - sc_light::GenesisCallExecutor>, - TBl, - TRtApi, ->; - trait AsCryptoStoreRef { fn keystore_ref(&self) -> Arc; fn sync_keystore_ref(&self) -> Arc; @@ -266,7 +227,6 @@ pub fn new_full_client( where TBl: BlockT, TExec: CodeExecutor + RuntimeVersionOf + Clone, - TBl::Hash: FromStr, { new_full_parts(config, telemetry, executor).map(|parts| parts.0) } @@ -280,7 +240,6 @@ pub fn new_full_parts( where TBl: BlockT, TExec: CodeExecutor + RuntimeVersionOf + Clone, - TBl::Hash: FromStr, { let keystore_container = KeystoreContainer::new(&config.keystore)?; @@ -307,7 +266,6 @@ where state_pruning: config.state_pruning.clone(), source: config.database.clone(), keep_blocks: config.keep_blocks.clone(), - transaction_storage: config.transaction_storage.clone(), }; let backend = new_db_backend(db_config)?; @@ -322,14 +280,16 @@ where .chain_spec .code_substitutes() .into_iter() - .map(|(h, c)| { - let hash = TBl::Hash::from_str(&h).map_err(|_| { + .map(|(n, c)| { + let number = NumberFor::::from_str(&n).map_err(|_| { Error::Application(Box::from(format!( - "Failed to parse `{}` as block hash for code substitutes.", - h + "Failed to parse `{}` as block number for code substitutes. \ + In an old version the key for code substitute was a block hash. \ + Please update the chain spec to a version that is compatible with your node.", + n ))) })?; - Ok((hash, c)) + Ok((number, c)) }) .collect::, Error>>()?; @@ -361,54 +321,6 @@ where Ok((client, backend, keystore_container, task_manager)) } -/// Create the initial parts of a light node. -pub fn new_light_parts( - config: &Configuration, - telemetry: Option, - executor: TExec, -) -> Result, Error> -where - TBl: BlockT, - TExec: CodeExecutor + RuntimeVersionOf + Clone, -{ - let keystore_container = KeystoreContainer::new(&config.keystore)?; - let ipfs_rt = tokio::runtime::Runtime::new().expect("couldn't start the IPFS runtime"); - let task_manager = { - let registry = config.prometheus_config.as_ref().map(|cfg| &cfg.registry); - TaskManager::new(config.tokio_handle.clone(), Some(ipfs_rt), registry)? - }; - - let db_storage = { - let db_settings = sc_client_db::DatabaseSettings { - state_cache_size: config.state_cache_size, - state_cache_child_ratio: config.state_cache_child_ratio.map(|v| (v, 100)), - state_pruning: config.state_pruning.clone(), - source: config.database.clone(), - keep_blocks: config.keep_blocks.clone(), - transaction_storage: config.transaction_storage.clone(), - }; - sc_client_db::light::LightStorage::new(db_settings)? - }; - let light_blockchain = sc_light::new_light_blockchain(db_storage); - let fetch_checker = Arc::new(sc_light::new_fetch_checker::<_, TBl, _>( - light_blockchain.clone(), - executor.clone(), - Box::new(task_manager.spawn_handle()), - )); - let on_demand = Arc::new(sc_network::config::OnDemand::new(fetch_checker)); - let backend = sc_light::new_light_backend(light_blockchain); - let client = Arc::new(light::new_light( - backend.clone(), - config.chain_spec.as_storage_builder(), - executor, - Box::new(task_manager.spawn_handle()), - config.prometheus_config.as_ref().map(|config| config.registry.clone()), - telemetry, - )?); - - Ok((client, backend, keystore_container, task_manager, on_demand)) -} - /// Create an instance of default DB-backend backend. pub fn new_db_backend( settings: DatabaseSettings, @@ -452,7 +364,7 @@ where spawn_handle, config.clone(), )?; - Ok(crate::client::Client::new( + crate::client::Client::new( backend, executor, genesis_storage, @@ -462,30 +374,26 @@ where prometheus_registry, telemetry, config, - )?) + ) } /// Parameters to pass into `build`. pub struct SpawnTasksParams<'a, TBl: BlockT, TCl, TExPool, TRpc, Backend> { /// The service configuration. pub config: Configuration, - /// A shared client returned by `new_full_parts`/`new_light_parts`. + /// A shared client returned by `new_full_parts`. pub client: Arc, - /// A shared backend returned by `new_full_parts`/`new_light_parts`. + /// A shared backend returned by `new_full_parts`. pub backend: Arc, - /// A task manager returned by `new_full_parts`/`new_light_parts`. + /// A task manager returned by `new_full_parts`. pub task_manager: &'a mut TaskManager, - /// A shared keystore returned by `new_full_parts`/`new_light_parts`. + /// A shared keystore returned by `new_full_parts`. pub keystore: SyncCryptoStorePtr, - /// An optional, shared data fetcher for light clients. - pub on_demand: Option>>, /// A shared transaction pool. pub transaction_pool: Arc, /// A RPC extension builder. Use `NoopRpcExtensionBuilder` if you just want to pass in the /// extensions directly. pub rpc_extensions_builder: Box + Send>, - /// An optional, shared remote blockchain instance. Used for light clients. - pub remote_blockchain: Option>>, /// A shared network instance. pub network: Arc::Hash>>, /// A Sender for RPC requests. @@ -513,6 +421,7 @@ where if let Some(offchain) = offchain_workers.clone() { spawn_handle.spawn( "offchain-notifications", + Some("offchain-worker"), sc_offchain::notification_future( config.role.is_authority(), client.clone(), @@ -563,12 +472,10 @@ where mut config, task_manager, client, - on_demand, backend, keystore, transaction_pool, rpc_extensions_builder, - remote_blockchain, network, system_rpc_tx, telemetry, @@ -594,11 +501,13 @@ where // Inform the tx pool about imported and finalized blocks. spawn_handle.spawn( "txpool-notifications", + Some("transaction-pool"), sc_transaction_pool::notification_future(client.clone(), transaction_pool.clone()), ); spawn_handle.spawn( "on-transaction-imported", + Some("transaction-pool"), transaction_notifications(transaction_pool.clone(), network.clone(), telemetry.clone()), ); @@ -609,6 +518,7 @@ where let metrics = MetricsService::with_prometheus(telemetry.clone(), ®istry, &config)?; spawn_handle.spawn( "prometheus-endpoint", + None, prometheus_endpoint::init_prometheus(port, registry).map(drop), ); @@ -620,6 +530,7 @@ where // Periodically updated metrics and telemetry updates. spawn_handle.spawn( "telemetry-periodic-send", + None, metrics_service.run(client.clone(), transaction_pool.clone(), network.clone()), ); @@ -634,8 +545,6 @@ where client.clone(), transaction_pool.clone(), keystore.clone(), - on_demand.clone(), - remote_blockchain.clone(), &*rpc_extensions_builder, backend.offchain_storage(), system_rpc_tx.clone(), @@ -658,6 +567,7 @@ where // Spawn informant task spawn_handle.spawn( "informant", + None, sc_informant::build( client.clone(), network.clone(), @@ -733,8 +643,6 @@ fn gen_handler( client: Arc, transaction_pool: Arc, keystore: SyncCryptoStorePtr, - on_demand: Option>>, - remote_blockchain: Option>>, rpc_extensions_builder: &(dyn RpcExtensionBuilder + Send), offchain_storage: Option<>::OffchainStorage>, system_rpc_tx: TracingUnboundedSender>, @@ -773,34 +681,17 @@ where let task_executor = sc_rpc::SubscriptionTaskExecutor::new(spawn_handle); let subscriptions = SubscriptionManager::new(Arc::new(task_executor.clone())); - let (chain, state, child_state) = - if let (Some(remote_blockchain), Some(on_demand)) = (remote_blockchain, on_demand) { - // Light clients - let chain = sc_rpc::chain::new_light( - client.clone(), - subscriptions.clone(), - remote_blockchain.clone(), - on_demand.clone(), - ); - let (state, child_state) = sc_rpc::state::new_light( - client.clone(), - subscriptions.clone(), - remote_blockchain.clone(), - on_demand, - deny_unsafe, - ); - (chain, state, child_state) - } else { - // Full nodes - let chain = sc_rpc::chain::new_full(client.clone(), subscriptions.clone()); - let (state, child_state) = sc_rpc::state::new_full( - client.clone(), - subscriptions.clone(), - deny_unsafe, - config.rpc_max_payload, - ); - (chain, state, child_state) - }; + let (chain, state, child_state) = { + // Full nodes + let chain = sc_rpc::chain::new_full(client.clone(), subscriptions.clone()); + let (state, child_state) = sc_rpc::state::new_full( + client.clone(), + subscriptions.clone(), + deny_unsafe, + config.rpc_max_payload, + ); + (chain, state, child_state) + }; let author = sc_rpc::author::Author::new(client, transaction_pool, subscriptions, keystore, deny_unsafe); @@ -829,7 +720,7 @@ where pub struct BuildNetworkParams<'a, TBl: BlockT, TExPool, TImpQu, TCl> { /// The service configuration. pub config: &'a Configuration, - /// A shared client returned by `new_full_parts`/`new_light_parts`. + /// A shared client returned by `new_full_parts`. pub client: Arc, /// A shared transaction pool. pub transaction_pool: Arc, @@ -837,8 +728,6 @@ pub struct BuildNetworkParams<'a, TBl: BlockT, TExPool, TImpQu, TCl> { pub spawn_handle: SpawnTaskHandle, /// An import queue. pub import_queue: TImpQu, - /// An optional, shared data fetcher for light clients. - pub on_demand: Option>>, /// A block announce validator builder. pub block_announce_validator_builder: Option) -> Box + Send> + Send>>, @@ -877,11 +766,22 @@ where transaction_pool, spawn_handle, import_queue, - on_demand, block_announce_validator_builder, warp_sync, } = params; + if warp_sync.is_none() && config.network.sync_mode.is_warp() { + return Err("Warp sync enabled, but no warp sync provider configured.".into()) + } + + if config.state_pruning.is_archive() { + match config.network.sync_mode { + SyncMode::Fast { .. } => return Err("Fast sync doesn't work for archive nodes".into()), + SyncMode::Warp => return Err("Warp sync doesn't work for archive nodes".into()), + SyncMode::Full => {}, + }; + } + let transaction_pool_adapter = Arc::new(TransactionPoolAdapter { imports_external_transactions: !matches!(config.role, Role::Light), pool: transaction_pool, @@ -908,7 +808,7 @@ where config.network.default_peers_set.in_peers as usize + config.network.default_peers_set.out_peers as usize, ); - spawn_handle.spawn("block_request_handler", handler.run()); + spawn_handle.spawn("block-request-handler", Some("networking"), handler.run()); protocol_config } }; @@ -922,10 +822,9 @@ where let (handler, protocol_config) = StateRequestHandler::new( &protocol_id, client.clone(), - config.network.default_peers_set.in_peers as usize + - config.network.default_peers_set.out_peers as usize, + config.network.default_peers_set_num_full as usize, ); - spawn_handle.spawn("state_request_handler", handler.run()); + spawn_handle.spawn("state-request-handler", Some("networking"), handler.run()); protocol_config } }; @@ -938,7 +837,7 @@ where // Allow both outgoing and incoming requests. let (handler, protocol_config) = WarpSyncRequestHandler::new(protocol_id.clone(), provider.clone()); - spawn_handle.spawn("warp_sync_request_handler", handler.run()); + spawn_handle.spawn("warp-sync-request-handler", Some("networking"), handler.run()); protocol_config }; (provider, protocol_config) @@ -952,28 +851,27 @@ where // Allow both outgoing and incoming requests. let (handler, protocol_config) = LightClientRequestHandler::new(&protocol_id, client.clone()); - spawn_handle.spawn("light_client_request_handler", handler.run()); + spawn_handle.spawn("light-client-request-handler", Some("networking"), handler.run()); protocol_config } }; - let mut network_params = sc_network::config::Params { + let network_params = sc_network::config::Params { role: config.role.clone(), executor: { let spawn_handle = Clone::clone(&spawn_handle); Some(Box::new(move |fut| { - spawn_handle.spawn("libp2p-node", fut); + spawn_handle.spawn("libp2p-node", Some("networking"), fut); })) }, transactions_handler_executor: { let spawn_handle = Clone::clone(&spawn_handle); Box::new(move |fut| { - spawn_handle.spawn("network-transactions-handler", fut); + spawn_handle.spawn("network-transactions-handler", Some("networking"), fut); }) }, network_config: config.network.clone(), chain: client.clone(), - on_demand, transaction_pool: transaction_pool_adapter as _, import_queue: Box::new(import_queue), protocol_id, @@ -985,13 +883,6 @@ where light_client_request_protocol_config, }; - // Storage chains don't keep full block history and can't be synced in full mode. - // Force fast sync when storage chain mode is enabled. - if matches!(config.transaction_storage, TransactionStorageMode::StorageChain) { - network_params.network_config.sync_mode = - SyncMode::Fast { storage_chain_mode: true, skip_proofs: false }; - } - let has_bootnodes = !network_params.network_config.boot_nodes.is_empty(); let network_mut = sc_network::NetworkWorker::new(network_params)?; let network = network_mut.service().clone(); @@ -1030,7 +921,7 @@ where // issue, and ideally we would like to fix the network future to take as little time as // possible, but we also take the extra harm-prevention measure to execute the networking // future using `spawn_blocking`. - spawn_handle.spawn_blocking("network-worker", async move { + spawn_handle.spawn_blocking("network-worker", Some("networking"), async move { if network_start_rx.await.is_err() { log::warn!( "The NetworkStart returned as part of `build_network` has been silently dropped" diff --git a/client/service/src/chain_ops/check_block.rs b/client/service/src/chain_ops/check_block.rs index 4728e014540e..41a6c73c5f47 100644 --- a/client/service/src/chain_ops/check_block.rs +++ b/client/service/src/chain_ops/check_block.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,7 +19,7 @@ use crate::error::Error; use codec::Encode; use futures::{future, prelude::*}; -use sc_client_api::{BlockBackend, UsageProvider}; +use sc_client_api::{BlockBackend, HeaderBackend}; use sc_consensus::import_queue::ImportQueue; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; @@ -33,7 +33,7 @@ pub fn check_block( block_id: BlockId, ) -> Pin> + Send>> where - C: BlockBackend + UsageProvider + Send + Sync + 'static, + C: BlockBackend + HeaderBackend + Send + Sync + 'static, B: BlockT + for<'de> serde::Deserialize<'de>, IQ: ImportQueue + 'static, { @@ -46,6 +46,6 @@ where import_blocks(client, import_queue, reader, true, true) }, Ok(None) => Box::pin(future::err("Unknown block".into())), - Err(e) => Box::pin(future::err(format!("Error reading block: {:?}", e).into())), + Err(e) => Box::pin(future::err(format!("Error reading block: {}", e).into())), } } diff --git a/client/service/src/chain_ops/export_blocks.rs b/client/service/src/chain_ops/export_blocks.rs index 888718010318..d442a11f2c39 100644 --- a/client/service/src/chain_ops/export_blocks.rs +++ b/client/service/src/chain_ops/export_blocks.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/service/src/chain_ops/export_raw_state.rs b/client/service/src/chain_ops/export_raw_state.rs index 975149c61cfa..ffe91d0d7355 100644 --- a/client/service/src/chain_ops/export_raw_state.rs +++ b/client/service/src/chain_ops/export_raw_state.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/service/src/chain_ops/import_blocks.rs b/client/service/src/chain_ops/import_blocks.rs index 1ba9e0bd6144..c0612124dd0c 100644 --- a/client/service/src/chain_ops/import_blocks.rs +++ b/client/service/src/chain_ops/import_blocks.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -22,7 +22,7 @@ use futures::{future, prelude::*}; use futures_timer::Delay; use log::{info, warn}; use sc_chain_spec::ChainSpec; -use sc_client_api::UsageProvider; +use sc_client_api::HeaderBackend; use sc_consensus::import_queue::{ BlockImportError, BlockImportStatus, ImportQueue, IncomingBlock, Link, }; @@ -35,18 +35,17 @@ use sp_runtime::{ }, }; use std::{ - convert::{TryFrom, TryInto}, - io::{Read, Seek}, + io::Read, pin::Pin, task::Poll, time::{Duration, Instant}, }; /// Number of blocks we will add to the queue before waiting for the queue to catch up. -const MAX_PENDING_BLOCKS: u64 = 1_024; +const MAX_PENDING_BLOCKS: u64 = 10_000; /// Number of milliseconds to wait until next poll. -const DELAY_TIME: u64 = 2_000; +const DELAY_TIME: u64 = 200; /// Number of milliseconds that must have passed between two updates. const TIME_BETWEEN_UPDATES: u64 = 3_000; @@ -63,7 +62,7 @@ pub fn build_spec(spec: &dyn ChainSpec, raw: bool) -> error::Result { /// SignedBlock and return it. enum BlockIter where - R: std::io::Read + std::io::Seek, + R: std::io::Read, { Binary { // Total number of blocks we are expecting to decode. @@ -83,7 +82,7 @@ where impl BlockIter where - R: Read + Seek + 'static, + R: Read + 'static, B: BlockT + MaybeSerializeDeserialize, { fn new(input: R, binary: bool) -> Result { @@ -119,7 +118,7 @@ where impl Iterator for BlockIter where - R: Read + Seek + 'static, + R: Read + 'static, B: BlockT + MaybeSerializeDeserialize, { type Item = Result, String>; @@ -267,10 +266,11 @@ impl Speedometer { /// Different State that the `import_blocks` future could be in. enum ImportState where - R: Read + Seek + 'static, + R: Read + 'static, B: BlockT + MaybeSerializeDeserialize, { - /// We are reading from the BlockIter structure, adding those blocks to the queue if possible. + /// We are reading from the [`BlockIter`] structure, adding those blocks to the queue if + /// possible. Reading { block_iter: BlockIter }, /// The queue is full (contains at least MAX_PENDING_BLOCKS blocks) and we are waiting for it /// to catch up. @@ -291,12 +291,12 @@ where pub fn import_blocks( client: Arc, mut import_queue: IQ, - input: impl Read + Seek + Send + 'static, + input: impl Read + Send + 'static, force: bool, binary: bool, ) -> Pin> + Send>> where - C: UsageProvider + Send + Sync + 'static, + C: HeaderBackend + Send + Sync + 'static, B: BlockT + for<'de> serde::Deserialize<'de>, IQ: ImportQueue + 'static, { @@ -322,7 +322,7 @@ where for result in results { if let (Err(err), hash) = result { - warn!("There was an error importing block with hash {:?}: {:?}", hash, err); + warn!("There was an error importing block with hash {:?}: {}", hash, err); self.has_error = true; break } @@ -438,7 +438,7 @@ where info!( "🎉 Imported {} blocks. Best: #{}", read_block_count, - client.usage_info().chain.best_number + client.info().best_number ); return Poll::Ready(Ok(())) } else { @@ -469,7 +469,7 @@ where queue.poll_actions(cx, &mut link); - let best_number = client.usage_info().chain.best_number; + let best_number = client.info().best_number; speedometer.notify_user(best_number); if link.has_error { diff --git a/client/service/src/chain_ops/mod.rs b/client/service/src/chain_ops/mod.rs index c213e745a5d6..98245eeb98d0 100644 --- a/client/service/src/chain_ops/mod.rs +++ b/client/service/src/chain_ops/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/service/src/chain_ops/revert_chain.rs b/client/service/src/chain_ops/revert_chain.rs index 63f1cbd15dd6..9a3ce6024ed9 100644 --- a/client/service/src/chain_ops/revert_chain.rs +++ b/client/service/src/chain_ops/revert_chain.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/service/src/client/block_rules.rs b/client/service/src/client/block_rules.rs index 4bdf33836296..519d5a13f516 100644 --- a/client/service/src/client/block_rules.rs +++ b/client/service/src/client/block_rules.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index d7a8b6f227e8..f271b35a69ce 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -26,10 +26,7 @@ use sp_core::{ NativeOrEncoded, NeverNativeValue, }; use sp_externalities::Extensions; -use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, NumberFor}, -}; +use sp_runtime::{generic::BlockId, traits::Block as BlockT}; use sp_state_machine::{ self, backend::Backend as _, ExecutionManager, ExecutionStrategy, Ext, OverlayedChanges, StateMachine, StorageProof, @@ -41,7 +38,7 @@ use std::{cell::RefCell, panic::UnwindSafe, result, sync::Arc}; pub struct LocalCallExecutor { backend: Arc, executor: E, - wasm_override: Option, + wasm_override: Arc>, wasm_substitutes: WasmSubstitutes, spawn_handle: Box, client_config: ClientConfig, @@ -74,7 +71,7 @@ where Ok(LocalCallExecutor { backend, executor, - wasm_override, + wasm_override: Arc::new(wasm_override), spawn_handle, client_config, wasm_substitutes, @@ -93,16 +90,19 @@ where Block: BlockT, B: backend::Backend, { - let spec = self.runtime_version(id)?.spec_version; + let spec = CallExecutor::runtime_version(self, id)?; let code = if let Some(d) = self .wasm_override .as_ref() - .map(|o| o.get(&spec, onchain_code.heap_pages)) + .as_ref() + .map(|o| o.get(&spec.spec_version, onchain_code.heap_pages, &spec.spec_name)) .flatten() { log::debug!(target: "wasm_overrides", "using WASM override for block {}", id); d - } else if let Some(s) = self.wasm_substitutes.get(spec, onchain_code.heap_pages, id) { + } else if let Some(s) = + self.wasm_substitutes.get(spec.spec_version, onchain_code.heap_pages, id) + { log::debug!(target: "wasm_substitutes", "Using WASM substitute for block {:?}", id); s } else { @@ -153,8 +153,6 @@ where extensions: Option, ) -> sp_blockchain::Result> { let mut changes = OverlayedChanges::default(); - let changes_trie = - backend::changes_tries_state_at_block(at, self.backend.changes_trie_storage())?; let state = self.backend.state_at(*at)?; let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); let runtime_code = @@ -168,7 +166,6 @@ where let return_data = StateMachine::new( &state, - changes_trie, &mut changes, &self.executor, method, @@ -208,8 +205,6 @@ where where ExecutionManager: Clone, { - let changes_trie_state = - backend::changes_tries_state_at_block(at, self.backend.changes_trie_storage())?; let mut storage_transaction_cache = storage_transaction_cache.map(|c| c.borrow_mut()); let state = self.backend.state_at(*at)?; @@ -243,7 +238,6 @@ where let mut state_machine = StateMachine::new( &backend, - changes_trie_state, changes, &self.executor, method, @@ -252,8 +246,8 @@ where &runtime_code, self.spawn_handle.clone(), ) + .with_storage_transaction_cache(storage_transaction_cache.as_deref_mut()) .set_parent_hash(at_hash); - // TODO: https://github.com/paritytech/substrate/issues/4455 state_machine.execute_using_consensus_failure_handler( execution_manager, native_call.map(|n| || (n)().map_err(|e| Box::new(e) as Box<_>)), @@ -262,7 +256,6 @@ where None => { let mut state_machine = StateMachine::new( &state, - changes_trie_state, changes, &self.executor, method, @@ -271,9 +264,7 @@ where &runtime_code, self.spawn_handle.clone(), ) - .with_storage_transaction_cache( - storage_transaction_cache.as_mut().map(|c| &mut **c), - ) + .with_storage_transaction_cache(storage_transaction_cache.as_deref_mut()) .set_parent_hash(at_hash); state_machine.execute_using_consensus_failure_handler( execution_manager, @@ -286,17 +277,15 @@ where fn runtime_version(&self, id: &BlockId) -> sp_blockchain::Result { let mut overlay = OverlayedChanges::default(); - let changes_trie_state = - backend::changes_tries_state_at_block(id, self.backend.changes_trie_storage())?; let state = self.backend.state_at(*id)?; let mut cache = StorageTransactionCache::::default(); - let mut ext = Ext::new(&mut overlay, &mut cache, &state, changes_trie_state, None); + let mut ext = Ext::new(&mut overlay, &mut cache, &state, None); let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); let runtime_code = state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; self.executor .runtime_version(&mut ext, &runtime_code) - .map_err(|e| sp_blockchain::Error::VersionInvalid(format!("{:?}", e)).into()) + .map_err(|e| sp_blockchain::Error::VersionInvalid(e.to_string()).into()) } fn prove_execution( @@ -317,7 +306,7 @@ where state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; let runtime_code = self.check_override(runtime_code, at)?; - sp_state_machine::prove_execution_on_trie_backend::<_, _, NumberFor, _, _>( + sp_state_machine::prove_execution_on_trie_backend( &trie_backend, &mut Default::default(), &self.executor, @@ -330,6 +319,20 @@ where } } +impl RuntimeVersionOf for LocalCallExecutor +where + E: RuntimeVersionOf, + Block: BlockT, +{ + fn runtime_version( + &self, + ext: &mut dyn sp_externalities::Externalities, + runtime_code: &sp_core::traits::RuntimeCode, + ) -> Result { + RuntimeVersionOf::runtime_version(&self.executor, ext, runtime_code) + } +} + impl sp_version::GetRuntimeVersionAt for LocalCallExecutor where B: backend::Backend, @@ -337,7 +340,7 @@ where Block: BlockT, { fn runtime_version(&self, at: &BlockId) -> Result { - CallExecutor::runtime_version(self, at).map_err(|e| format!("{:?}", e)) + CallExecutor::runtime_version(self, at).map_err(|e| e.to_string()) } } @@ -369,6 +372,7 @@ mod tests { WasmExecutionMethod::Interpreted, Some(128), 1, + 2, ); let overrides = crate::client::wasm_override::dummy_overrides(); @@ -407,7 +411,7 @@ mod tests { let call_executor = LocalCallExecutor { backend: backend.clone(), executor: executor.clone(), - wasm_override: Some(overrides), + wasm_override: Arc::new(Some(overrides)), spawn_handle: Box::new(TaskExecutor::new()), client_config, wasm_substitutes: WasmSubstitutes::new( diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index f7d93d036a3f..004d4b711e09 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,7 +23,6 @@ use super::{ genesis, }; use codec::{Decode, Encode}; -use hash_db::Prefix; use log::{info, trace, warn}; use parking_lot::{Mutex, RwLock}; use prometheus_endpoint::Registry; @@ -31,60 +30,60 @@ use rand::Rng; use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider, RecordProof}; use sc_client_api::{ backend::{ - self, apply_aux, changes_tries_state_at_block, BlockImportOperation, ClientImportOperation, - Finalizer, ImportSummary, LockImportRun, NewBlockState, PrunableStateChangesTrieStorage, - StorageProvider, + self, apply_aux, BlockImportOperation, ClientImportOperation, FinalizeSummary, Finalizer, + ImportSummary, LockImportRun, NewBlockState, StorageProvider, }, - cht, client::{ BadBlocks, BlockBackend, BlockImportNotification, BlockOf, BlockchainEvents, ClientInfo, FinalityNotification, FinalityNotifications, ForkBlocks, ImportNotifications, - ProvideUncles, + PreCommitActions, ProvideUncles, }, execution_extensions::ExecutionExtensions, notifications::{StorageEventStream, StorageNotifications}, - CallExecutor, ExecutorProvider, KeyIterator, ProofProvider, UsageProvider, + CallExecutor, ExecutorProvider, KeyIterator, OnFinalityAction, OnImportAction, ProofProvider, + UsageProvider, }; use sc_consensus::{ BlockCheckParams, BlockImportParams, ForkChoiceStrategy, ImportResult, StateAction, }; -use sc_executor::RuntimeVersion; -use sc_light::fetcher::ChangesProof; +use sc_executor::{RuntimeVersion, RuntimeVersionOf}; use sc_telemetry::{telemetry, TelemetryHandle, SUBSTRATE_INFO}; use sp_api::{ ApiExt, ApiRef, CallApiAt, CallApiAtParams, ConstructRuntimeApi, Core as CoreApi, ProvideRuntimeApi, }; use sp_blockchain::{ - self as blockchain, well_known_cache_keys::Id as CacheKeyId, Backend as ChainBackend, Cache, - CachedHeaderMetadata, Error, HeaderBackend as ChainHeaderBackend, HeaderMetadata, ProvideCache, + self as blockchain, well_known_cache_keys::Id as CacheKeyId, Backend as ChainBackend, + CachedHeaderMetadata, Error, HeaderBackend as ChainHeaderBackend, HeaderMetadata, }; use sp_consensus::{BlockOrigin, BlockStatus, Error as ConsensusError}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_core::{ - convert_hash, - storage::{well_known_keys, ChildInfo, PrefixedStorageKey, StorageData, StorageKey}, - ChangesTrieConfiguration, NativeOrEncoded, + storage::{ + well_known_keys, ChildInfo, ChildType, PrefixedStorageKey, Storage, StorageChild, + StorageData, StorageKey, + }, + NativeOrEncoded, }; #[cfg(feature = "test-helpers")] use sp_keystore::SyncCryptoStorePtr; use sp_runtime::{ - generic::{BlockId, DigestItem, SignedBlock}, + generic::{BlockId, SignedBlock}, traits::{ - Block as BlockT, DigestFor, HashFor, Header as HeaderT, NumberFor, One, - SaturatedConversion, Zero, + Block as BlockT, HashFor, Header as HeaderT, NumberFor, One, SaturatedConversion, Zero, }, - BuildStorage, Justification, Justifications, + BuildStorage, Digest, Justification, Justifications, StateVersion, }; use sp_state_machine::{ - key_changes, key_changes_proof, prove_child_read, prove_range_read_with_size, prove_read, - read_range_proof_check, Backend as StateBackend, ChangesTrieAnchorBlockId, - ChangesTrieConfigurationRange, ChangesTrieRootsStorage, ChangesTrieStorage, DBValue, + prove_child_read, prove_range_read_with_child_with_size, prove_read, + read_range_proof_check_with_child_on_proving_backend, Backend as StateBackend, + ChildStorageCollection, KeyValueStates, KeyValueStorageLevel, StorageCollection, + MAX_NESTED_TRIE_DEPTH, }; -use sp_trie::StorageProof; +use sp_trie::{CompactProof, StorageProof}; use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{hash_map::DefaultHasher, HashMap, HashSet}, marker::PhantomData, panic::UnwindSafe, path::PathBuf, @@ -96,7 +95,6 @@ use std::{ use { super::call_executor::LocalCallExecutor, sc_client_api::in_mem, - sc_executor::RuntimeVersionOf, sp_core::traits::{CodeExecutor, SpawnNamed}, }; @@ -109,10 +107,16 @@ where { backend: Arc, executor: E, - storage_notifications: Mutex>, + storage_notifications: StorageNotifications, import_notification_sinks: NotificationSinks>, finality_notification_sinks: NotificationSinks>, - // holds the block hash currently being imported. TODO: replace this with block queue + // Collects auxiliary operations to be performed atomically together with + // block import operations. + import_actions: Mutex>>, + // Collects auxiliary operations to be performed atomically together with + // block finalization operations. + finality_actions: Mutex>>, + // Holds the block hash currently being imported. TODO: replace this with block queue. importing_block: RwLock>, block_rules: BlockRules, execution_extensions: ExecutionExtensions, @@ -198,7 +202,7 @@ pub struct ClientConfig { pub no_genesis: bool, /// Map of WASM runtime substitute starting at the child of the given block until the runtime /// version doesn't match anymore. - pub wasm_runtime_substitutes: HashMap>, + pub wasm_runtime_substitutes: HashMap, Vec>, } impl Default for ClientConfig { @@ -278,16 +282,37 @@ where let mut op = ClientImportOperation { op: self.backend.begin_operation()?, notify_imported: None, - notify_finalized: Vec::new(), + notify_finalized: None, }; let r = f(&mut op)?; - let ClientImportOperation { op, notify_imported, notify_finalized } = op; + let ClientImportOperation { mut op, notify_imported, notify_finalized } = op; + + let finality_notification = notify_finalized.map(|summary| summary.into()); + let (import_notification, storage_changes) = match notify_imported { + Some(mut summary) => { + let storage_changes = summary.storage_changes.take(); + (Some(summary.into()), storage_changes) + }, + None => (None, None), + }; + + if let Some(ref notification) = finality_notification { + for action in self.finality_actions.lock().iter_mut() { + op.insert_aux(action(notification))?; + } + } + if let Some(ref notification) = import_notification { + for action in self.import_actions.lock().iter_mut() { + op.insert_aux(action(notification))?; + } + } + self.backend.commit_operation(op)?; - self.notify_finalized(notify_finalized)?; - self.notify_imported(notify_imported)?; + self.notify_finalized(finality_notification)?; + self.notify_imported(import_notification, storage_changes)?; Ok(r) }; @@ -337,8 +362,11 @@ where if info.finalized_state.is_none() { let genesis_storage = build_genesis_storage.build_storage().map_err(sp_blockchain::Error::Storage)?; + let genesis_state_version = + Self::resolve_state_version_from_wasm(&genesis_storage, &executor)?; let mut op = backend.begin_operation()?; - let state_root = op.set_genesis_state(genesis_storage, !config.no_genesis)?; + let state_root = + op.set_genesis_state(genesis_storage, !config.no_genesis, genesis_state_version)?; let genesis_block = genesis::construct_genesis_block::(state_root.into()); info!( "🔨 Initializing Genesis block/state (state: {}, header-hash: {})", @@ -365,9 +393,11 @@ where Ok(Client { backend, executor, - storage_notifications: Mutex::new(StorageNotifications::new(prometheus_registry)), + storage_notifications: StorageNotifications::new(prometheus_registry), import_notification_sinks: Default::default(), finality_notification_sinks: Default::default(), + import_actions: Default::default(), + finality_actions: Default::default(), importing_block: Default::default(), block_rules: BlockRules::new(fork_blocks, bad_blocks), execution_extensions, @@ -406,251 +436,7 @@ where /// Get the RuntimeVersion at a given block. pub fn runtime_version_at(&self, id: &BlockId) -> sp_blockchain::Result { - self.executor.runtime_version(id) - } - - /// Reads given header and generates CHT-based header proof for CHT of given size. - pub fn header_proof_with_cht_size( - &self, - id: &BlockId, - cht_size: NumberFor, - ) -> sp_blockchain::Result<(Block::Header, StorageProof)> { - let proof_error = || { - sp_blockchain::Error::Backend(format!("Failed to generate header proof for {:?}", id)) - }; - let header = self.backend.blockchain().expect_header(*id)?; - let block_num = *header.number(); - let cht_num = cht::block_to_cht_number(cht_size, block_num).ok_or_else(proof_error)?; - let cht_start = cht::start_number(cht_size, cht_num); - let mut current_num = cht_start; - let cht_range = ::std::iter::from_fn(|| { - let old_current_num = current_num; - current_num = current_num + One::one(); - Some(old_current_num) - }); - let headers = cht_range.map(|num| self.block_hash(num)); - let proof = cht::build_proof::, _, _>( - cht_size, - cht_num, - std::iter::once(block_num), - headers, - )?; - Ok((header, proof)) - } - - /// Does the same work as `key_changes_proof`, but assumes that CHTs are of passed size. - pub fn key_changes_proof_with_cht_size( - &self, - first: Block::Hash, - last: Block::Hash, - min: Block::Hash, - max: Block::Hash, - storage_key: Option<&PrefixedStorageKey>, - key: &StorageKey, - cht_size: NumberFor, - ) -> sp_blockchain::Result> { - struct AccessedRootsRecorder<'a, Block: BlockT> { - storage: &'a dyn ChangesTrieStorage, NumberFor>, - min: NumberFor, - required_roots_proofs: Mutex, Block::Hash>>, - } - - impl<'a, Block: BlockT> ChangesTrieRootsStorage, NumberFor> - for AccessedRootsRecorder<'a, Block> - { - fn build_anchor( - &self, - hash: Block::Hash, - ) -> Result>, String> { - self.storage.build_anchor(hash) - } - - fn root( - &self, - anchor: &ChangesTrieAnchorBlockId>, - block: NumberFor, - ) -> Result, String> { - let root = self.storage.root(anchor, block)?; - if block < self.min { - if let Some(ref root) = root { - self.required_roots_proofs.lock().insert(block, root.clone()); - } - } - Ok(root) - } - } - - impl<'a, Block: BlockT> ChangesTrieStorage, NumberFor> - for AccessedRootsRecorder<'a, Block> - { - fn as_roots_storage( - &self, - ) -> &dyn sp_state_machine::ChangesTrieRootsStorage, NumberFor> { - self - } - - fn with_cached_changed_keys( - &self, - root: &Block::Hash, - functor: &mut dyn FnMut(&HashMap, HashSet>>), - ) -> bool { - self.storage.with_cached_changed_keys(root, functor) - } - - fn get(&self, key: &Block::Hash, prefix: Prefix) -> Result, String> { - self.storage.get(key, prefix) - } - } - - let first_number = - self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(first))?; - let (storage, configs) = self.require_changes_trie(first_number, last, true)?; - let min_number = - self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(min))?; - - let recording_storage = AccessedRootsRecorder:: { - storage: storage.storage(), - min: min_number, - required_roots_proofs: Mutex::new(BTreeMap::new()), - }; - - let max_number = std::cmp::min( - self.backend.blockchain().info().best_number, - self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(max))?, - ); - - // fetch key changes proof - let mut proof = Vec::new(); - for (config_zero, config_end, config) in configs { - let last_number = - self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(last))?; - let config_range = ChangesTrieConfigurationRange { - config: &config, - zero: config_zero, - end: config_end.map(|(config_end_number, _)| config_end_number), - }; - let proof_range = key_changes_proof::, _>( - config_range, - &recording_storage, - first_number, - &ChangesTrieAnchorBlockId { hash: convert_hash(&last), number: last_number }, - max_number, - storage_key, - &key.0, - ) - .map_err(|err| sp_blockchain::Error::ChangesTrieAccessFailed(err))?; - proof.extend(proof_range); - } - - // now gather proofs for all changes tries roots that were touched during key_changes_proof - // execution AND are unknown (i.e. replaced with CHT) to the requester - let roots = recording_storage.required_roots_proofs.into_inner(); - let roots_proof = self.changes_trie_roots_proof(cht_size, roots.keys().cloned())?; - - Ok(ChangesProof { - max_block: max_number, - proof, - roots: roots.into_iter().map(|(n, h)| (n, convert_hash(&h))).collect(), - roots_proof, - }) - } - - /// Generate CHT-based proof for roots of changes tries at given blocks. - fn changes_trie_roots_proof>>( - &self, - cht_size: NumberFor, - blocks: I, - ) -> sp_blockchain::Result { - // most probably we have touched several changes tries that are parts of the single CHT - // => GroupBy changes tries by CHT number and then gather proof for the whole group at once - let mut proofs = Vec::new(); - - cht::for_each_cht_group::( - cht_size, - blocks, - |_, cht_num, cht_blocks| { - let cht_proof = - self.changes_trie_roots_proof_at_cht(cht_size, cht_num, cht_blocks)?; - proofs.push(cht_proof); - Ok(()) - }, - (), - )?; - - Ok(StorageProof::merge(proofs)) - } - - /// Generates CHT-based proof for roots of changes tries at given blocks - /// (that are part of single CHT). - fn changes_trie_roots_proof_at_cht( - &self, - cht_size: NumberFor, - cht_num: NumberFor, - blocks: Vec>, - ) -> sp_blockchain::Result { - let cht_start = cht::start_number(cht_size, cht_num); - let mut current_num = cht_start; - let cht_range = ::std::iter::from_fn(|| { - let old_current_num = current_num; - current_num = current_num + One::one(); - Some(old_current_num) - }); - let roots = cht_range.map(|num| { - self.header(&BlockId::Number(num)).map(|block| { - block - .and_then(|block| block.digest().log(DigestItem::as_changes_trie_root).cloned()) - }) - }); - let proof = cht::build_proof::, _, _>( - cht_size, cht_num, blocks, roots, - )?; - Ok(proof) - } - - /// Returns changes trie storage and all configurations that have been active - /// in the range [first; last]. - /// - /// Configurations are returned in descending order (and obviously never overlap). - /// If fail_if_disabled is false, returns maximal consequent configurations ranges, - /// starting from last and stopping on either first, or when CT have been disabled. - /// If fail_if_disabled is true, fails when there's a subrange where CT have been disabled - /// inside first..last blocks range. - fn require_changes_trie( - &self, - first: NumberFor, - last: Block::Hash, - fail_if_disabled: bool, - ) -> sp_blockchain::Result<( - &dyn PrunableStateChangesTrieStorage, - Vec<(NumberFor, Option<(NumberFor, Block::Hash)>, ChangesTrieConfiguration)>, - )> { - let storage = self - .backend - .changes_trie_storage() - .ok_or_else(|| sp_blockchain::Error::ChangesTriesNotSupported)?; - - let mut configs = Vec::with_capacity(1); - let mut current = last; - loop { - let config_range = storage.configuration_at(&BlockId::Hash(current))?; - match config_range.config { - Some(config) => configs.push((config_range.zero.0, config_range.end, config)), - None if !fail_if_disabled => return Ok((storage, configs)), - None => return Err(sp_blockchain::Error::ChangesTriesNotSupported), - } - - if config_range.zero.0 < first { - break - } - - current = *self - .backend - .blockchain() - .expect_header(BlockId::Hash(config_range.zero.1))? - .parent_hash(); - } - - Ok((storage, configs)) + CallExecutor::runtime_version(&self.executor, id) } /// Apply a checked and validated block to an operation. If a justification is provided @@ -684,8 +470,6 @@ where .. } = import_block; - assert!(justifications.is_some() && finalized || justifications.is_none()); - if !intermediates.is_empty() { return Err(Error::IncompletePipeline) } @@ -779,11 +563,17 @@ where } let info = self.backend.blockchain().info(); + let gap_block = info + .block_gap + .map_or(false, |(start, _)| *import_headers.post().number() == start); + + assert!(justifications.is_some() && finalized || justifications.is_none() || gap_block); // the block is lower than our last finalized block so it must revert // finality, refusing import. if status == blockchain::BlockStatus::Unknown && - *import_headers.post().number() <= info.finalized_number + *import_headers.post().number() <= info.finalized_number && + !gap_block { return Err(sp_blockchain::Error::NotInFinalizedChain) } @@ -803,7 +593,7 @@ where sc_consensus::StorageChanges::Changes(storage_changes) => { self.backend .begin_state_operation(&mut operation.op, BlockId::Hash(parent_hash))?; - let (main_sc, child_sc, offchain_sc, tx, _, changes_trie_tx, tx_index) = + let (main_sc, child_sc, offchain_sc, tx, _, tx_index) = storage_changes.into_inner(); if self.config.offchain_indexing_api { @@ -814,18 +604,46 @@ where operation.op.update_storage(main_sc.clone(), child_sc.clone())?; operation.op.update_transaction_index(tx_index)?; - if let Some(changes_trie_transaction) = changes_trie_tx { - operation.op.update_changes_trie(changes_trie_transaction)?; - } Some((main_sc, child_sc)) }, sc_consensus::StorageChanges::Import(changes) => { - let storage = sp_storage::Storage { - top: changes.state.into_iter().collect(), - children_default: Default::default(), - }; + let mut storage = sp_storage::Storage::default(); + for state in changes.state.0.into_iter() { + if state.parent_storage_keys.len() == 0 && state.state_root.len() == 0 { + for (key, value) in state.key_values.into_iter() { + storage.top.insert(key, value); + } + } else { + for parent_storage in state.parent_storage_keys { + let storage_key = PrefixedStorageKey::new_ref(&parent_storage); + let storage_key = + match ChildType::from_prefixed_key(&storage_key) { + Some((ChildType::ParentKeyId, storage_key)) => + storage_key, + None => + return Err(Error::Backend( + "Invalid child storage key.".to_string(), + )), + }; + let entry = storage + .children_default + .entry(storage_key.to_vec()) + .or_insert_with(|| StorageChild { + data: Default::default(), + child_info: ChildInfo::new_default(storage_key), + }); + for (key, value) in state.key_values.iter() { + entry.data.insert(key.clone(), value.clone()); + } + } + } + } - let state_root = operation.op.reset_storage(storage)?; + // This is use by fast sync for runtime version to be resolvable from + // changes. + let state_version = + Self::resolve_state_version_from_wasm(&storage, &self.executor)?; + let state_root = operation.op.reset_storage(storage, state_version)?; if state_root != *import_headers.post().state_root() { // State root mismatch when importing state. This should not happen in // safe fast sync mode, but may happen in unsafe mode. @@ -835,18 +653,6 @@ where None }, }; - // Ensure parent chain is finalized to maintain invariant that - // finality is called sequentially. This will also send finality - // notifications for top 250 newly finalized blocks. - if finalized && parent_exists { - self.apply_finality_with_block_hash( - operation, - parent_hash, - None, - info.best_hash, - make_notifications, - )?; - } operation.op.update_cache(new_cache); storage_changes @@ -854,12 +660,25 @@ where None => None, }; - let is_new_best = finalized || - match fork_choice { - ForkChoiceStrategy::LongestChain => - import_headers.post().number() > &info.best_number, - ForkChoiceStrategy::Custom(v) => v, - }; + // Ensure parent chain is finalized to maintain invariant that finality is called + // sequentially. + if finalized && parent_exists { + self.apply_finality_with_block_hash( + operation, + parent_hash, + None, + info.best_hash, + make_notifications, + )?; + } + + let is_new_best = !gap_block && + (finalized || + match fork_choice { + ForkChoiceStrategy::LongestChain => + import_headers.post().number() > &info.best_number, + ForkChoiceStrategy::Custom(v) => v, + }); let leaf_state = if finalized { NewBlockState::Final @@ -895,17 +714,51 @@ where operation.op.insert_aux(aux)?; - // we only notify when we are already synced to the tip of the chain + // We only notify when we are already synced to the tip of the chain // or if this import triggers a re-org if make_notifications || tree_route.is_some() { + let header = import_headers.into_post(); if finalized { - operation.notify_finalized.push(hash); + let mut summary = match operation.notify_finalized.take() { + Some(mut summary) => { + summary.header = header.clone(); + summary.finalized.push(hash); + summary + }, + None => FinalizeSummary { + header: header.clone(), + finalized: vec![hash], + stale_heads: Vec::new(), + }, + }; + + if parent_exists { + // Add to the stale list all heads that are branching from parent besides our + // current `head`. + for head in self + .backend + .blockchain() + .leaves()? + .into_iter() + .filter(|h| *h != parent_hash) + { + let route_from_parent = sp_blockchain::tree_route( + self.backend.blockchain(), + parent_hash, + head, + )?; + if route_from_parent.retracted().is_empty() { + summary.stale_heads.push(head); + } + } + } + operation.notify_finalized = Some(summary); } operation.notify_imported = Some(ImportSummary { hash, origin, - header: import_headers.into_post(), + header, is_new_best, storage_changes, tree_route, @@ -967,11 +820,8 @@ where )?; let state = self.backend.state_at(at)?; - let changes_trie_state = - changes_tries_state_at_block(&at, self.backend.changes_trie_storage())?; - let gen_storage_changes = runtime_api - .into_storage_changes(&state, changes_trie_state.as_ref(), *parent_hash) + .into_storage_changes(&state, *parent_hash) .map_err(sp_blockchain::Error::Storage)?; if import_block.header.state_root() != &gen_storage_changes.transaction_storage_root @@ -1046,98 +896,103 @@ where operation.op.mark_finalized(BlockId::Hash(block), justification)?; if notify { - // sometimes when syncing, tons of blocks can be finalized at once. - // we'll send notifications spuriously in that case. - const MAX_TO_NOTIFY: usize = 256; - let enacted = route_from_finalized.enacted(); - let start = enacted.len() - std::cmp::min(enacted.len(), MAX_TO_NOTIFY); - for finalized in &enacted[start..] { - operation.notify_finalized.push(finalized.hash); + let finalized = + route_from_finalized.enacted().iter().map(|elem| elem.hash).collect::>(); + + let last_finalized_number = self + .backend + .blockchain() + .number(last_finalized)? + .expect("Previous finalized block expected to be onchain; qed"); + let mut stale_heads = Vec::new(); + for head in self.backend.blockchain().leaves()? { + let route_from_finalized = + sp_blockchain::tree_route(self.backend.blockchain(), block, head)?; + let retracted = route_from_finalized.retracted(); + let pivot = route_from_finalized.common_block(); + // It is not guaranteed that `backend.blockchain().leaves()` doesn't return + // heads that were in a stale state before this finalization and thus already + // included in previous notifications. We want to skip such heads. + // Given the "route" from the currently finalized block to the head under + // analysis, the condition for it to be added to the new stale heads list is: + // `!retracted.is_empty() && last_finalized_number <= pivot.number` + // 1. "route" has some "retractions". + // 2. previously finalized block number is not greater than the "route" pivot: + // - if `last_finalized_number <= pivot.number` then this is a new stale head; + // - else the stale head was already included by some previous finalization. + if !retracted.is_empty() && last_finalized_number <= pivot.number { + stale_heads.push(head); + } } + let header = self + .backend + .blockchain() + .header(BlockId::Hash(block))? + .expect("Finalized block expected to be onchain; qed"); + operation.notify_finalized = Some(FinalizeSummary { header, finalized, stale_heads }); } Ok(()) } - fn notify_finalized(&self, notify_finalized: Vec) -> sp_blockchain::Result<()> { + fn notify_finalized( + &self, + notification: Option>, + ) -> sp_blockchain::Result<()> { let mut sinks = self.finality_notification_sinks.lock(); - if notify_finalized.is_empty() { - // cleanup any closed finality notification sinks - // since we won't be running the loop below which - // would also remove any closed sinks. - sinks.retain(|sink| !sink.is_closed()); - - return Ok(()) - } - - // We assume the list is sorted and only want to inform the - // telemetry once about the finalized block. - if let Some(last) = notify_finalized.last() { - let header = self.header(&BlockId::Hash(*last))?.expect( - "Header already known to exist in DB because it is indicated in the tree route; \ - qed", - ); - - telemetry!( - self.telemetry; - SUBSTRATE_INFO; - "notify.finalized"; - "height" => format!("{}", header.number()), - "best" => ?last, - ); - } - - for finalized_hash in notify_finalized { - let header = self.header(&BlockId::Hash(finalized_hash))?.expect( - "Header already known to exist in DB because it is indicated in the tree route; \ - qed", - ); + let notification = match notification { + Some(notify_finalized) => notify_finalized, + None => { + // Cleanup any closed finality notification sinks + // since we won't be running the loop below which + // would also remove any closed sinks. + sinks.retain(|sink| !sink.is_closed()); + return Ok(()) + }, + }; - let notification = FinalityNotification { header, hash: finalized_hash }; + telemetry!( + self.telemetry; + SUBSTRATE_INFO; + "notify.finalized"; + "height" => format!("{}", notification.header.number()), + "best" => ?notification.hash, + ); - sinks.retain(|sink| sink.unbounded_send(notification.clone()).is_ok()); - } + sinks.retain(|sink| sink.unbounded_send(notification.clone()).is_ok()); Ok(()) } fn notify_imported( &self, - notify_import: Option>, + notification: Option>, + storage_changes: Option<(StorageCollection, ChildStorageCollection)>, ) -> sp_blockchain::Result<()> { - let notify_import = match notify_import { + let notification = match notification { Some(notify_import) => notify_import, None => { - // cleanup any closed import notification sinks since we won't + // Cleanup any closed import notification sinks since we won't // be sending any notifications below which would remove any // closed sinks. this is necessary since during initial sync we // won't send any import notifications which could lead to a // temporary leak of closed/discarded notification sinks (e.g. // from consensus code). self.import_notification_sinks.lock().retain(|sink| !sink.is_closed()); - return Ok(()) }, }; - if let Some(storage_changes) = notify_import.storage_changes { + if let Some(storage_changes) = storage_changes { // TODO [ToDr] How to handle re-orgs? Should we re-emit all storage changes? - self.storage_notifications.lock().trigger( - ¬ify_import.hash, + self.storage_notifications.trigger( + ¬ification.hash, storage_changes.0.into_iter(), storage_changes.1.into_iter().map(|(sk, v)| (sk, v.into_iter())), ); } - let notification = BlockImportNotification:: { - hash: notify_import.hash, - origin: notify_import.origin, - header: notify_import.header, - is_new_best: notify_import.is_new_best, - tree_route: notify_import.tree_route.map(Arc::new), - }; - self.import_notification_sinks .lock() .retain(|sink| sink.unbounded_send(notification.clone()).is_ok()); @@ -1262,6 +1117,35 @@ where trace!("Collected {} uncles", uncles.len()); Ok(uncles) } + + fn resolve_state_version_from_wasm( + storage: &Storage, + executor: &E, + ) -> sp_blockchain::Result { + if let Some(wasm) = storage.top.get(well_known_keys::CODE) { + let mut ext = sp_state_machine::BasicExternalities::new_empty(); // just to read runtime version. + + let code_fetcher = sp_core::traits::WrappedRuntimeCode(wasm.as_slice().into()); + let runtime_code = sp_core::traits::RuntimeCode { + code_fetcher: &code_fetcher, + heap_pages: None, + hash: { + use std::hash::{Hash, Hasher}; + let mut state = DefaultHasher::new(); + wasm.hash(&mut state); + state.finish().to_le_bytes().to_vec() + }, + }; + let runtime_version = + RuntimeVersionOf::runtime_version(executor, &mut ext, &runtime_code) + .map_err(|e| sp_blockchain::Error::VersionInvalid(e.to_string()))?; + Ok(runtime_version.state_version()) + } else { + Err(sp_blockchain::Error::VersionInvalid( + "Runtime missing from initial storage, could not read state version.".to_string(), + )) + } + } } impl UsageProvider for Client @@ -1306,98 +1190,162 @@ where method: &str, call_data: &[u8], ) -> sp_blockchain::Result<(Vec, StorageProof)> { - // Make sure we include the `:code` and `:heap_pages` in the execution proof to be - // backwards compatible. - // - // TODO: Remove when solved: https://github.com/paritytech/substrate/issues/5047 - let code_proof = self.read_proof( - id, - &mut [well_known_keys::CODE, well_known_keys::HEAP_PAGES].iter().map(|v| *v), - )?; - - self.executor - .prove_execution(id, method, call_data) - .map(|(r, p)| (r, StorageProof::merge(vec![p, code_proof]))) - } - - fn header_proof( - &self, - id: &BlockId, - ) -> sp_blockchain::Result<(Block::Header, StorageProof)> { - self.header_proof_with_cht_size(id, cht::size()) - } - - fn key_changes_proof( - &self, - first: Block::Hash, - last: Block::Hash, - min: Block::Hash, - max: Block::Hash, - storage_key: Option<&PrefixedStorageKey>, - key: &StorageKey, - ) -> sp_blockchain::Result> { - self.key_changes_proof_with_cht_size(first, last, min, max, storage_key, key, cht::size()) + self.executor.prove_execution(id, method, call_data) } fn read_proof_collection( &self, id: &BlockId, - start_key: &[u8], + start_key: &[Vec], size_limit: usize, - ) -> sp_blockchain::Result<(StorageProof, u32)> { + ) -> sp_blockchain::Result<(CompactProof, u32)> { let state = self.state_at(id)?; - Ok(prove_range_read_with_size::<_, HashFor>( - state, - None, - None, - size_limit, - Some(start_key), - )?) + // this is a read proof, using version V0 or V1 is equivalent. + let root = state.storage_root(std::iter::empty(), StateVersion::V0).0; + + let (proof, count) = prove_range_read_with_child_with_size::<_, HashFor>( + state, size_limit, start_key, + )?; + // This is read proof only, we can use either LayoutV0 or LayoutV1. + let proof = sp_trie::encode_compact::>>(proof, root) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))?; + Ok((proof, count)) } fn storage_collection( &self, id: &BlockId, - start_key: &[u8], + start_key: &[Vec], size_limit: usize, - ) -> sp_blockchain::Result, Vec)>> { + ) -> sp_blockchain::Result> { + if start_key.len() > MAX_NESTED_TRIE_DEPTH { + return Err(Error::Backend("Invalid start key.".to_string())) + } let state = self.state_at(id)?; - let mut current_key = start_key.to_vec(); - let mut total_size = 0; - let mut entries = Vec::new(); - while let Some(next_key) = state - .next_storage_key(¤t_key) - .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? - { - let value = state - .storage(next_key.as_ref()) + let child_info = |storage_key: &Vec| -> sp_blockchain::Result { + let storage_key = PrefixedStorageKey::new_ref(&storage_key); + match ChildType::from_prefixed_key(&storage_key) { + Some((ChildType::ParentKeyId, storage_key)) => + Ok(ChildInfo::new_default(storage_key)), + None => Err(Error::Backend("Invalid child storage key.".to_string())), + } + }; + let mut current_child = if start_key.len() == 2 { + let start_key = start_key.get(0).expect("checked len"); + if let Some(child_root) = state + .storage(&start_key) .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? - .unwrap_or_default(); - let size = value.len() + next_key.len(); - if total_size + size > size_limit && !entries.is_empty() { + { + Some((child_info(start_key)?, child_root)) + } else { + return Err(Error::Backend("Invalid root start key.".to_string())) + } + } else { + None + }; + let mut current_key = start_key.last().map(Clone::clone).unwrap_or(Vec::new()); + let mut total_size = 0; + let mut result = vec![( + KeyValueStorageLevel { + state_root: Vec::new(), + key_values: Vec::new(), + parent_storage_keys: Vec::new(), + }, + false, + )]; + + let mut child_roots = HashSet::new(); + loop { + let mut entries = Vec::new(); + let mut complete = true; + let mut switch_child_key = None; + while let Some(next_key) = if let Some(child) = current_child.as_ref() { + state + .next_child_storage_key(&child.0, ¤t_key) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? + } else { + state + .next_storage_key(¤t_key) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? + } { + let value = if let Some(child) = current_child.as_ref() { + state + .child_storage(&child.0, next_key.as_ref()) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? + .unwrap_or_default() + } else { + state + .storage(next_key.as_ref()) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))? + .unwrap_or_default() + }; + let size = value.len() + next_key.len(); + if total_size + size > size_limit && !entries.is_empty() { + complete = false; + break + } + total_size += size; + + if current_child.is_none() && + sp_core::storage::well_known_keys::is_child_storage_key(next_key.as_slice()) + { + if !child_roots.contains(value.as_slice()) { + child_roots.insert(value.clone()); + switch_child_key = Some((next_key.clone(), value.clone())); + entries.push((next_key.clone(), value)); + break + } + } + entries.push((next_key.clone(), value)); + current_key = next_key; + } + if let Some((child, child_root)) = switch_child_key.take() { + result[0].0.key_values.extend(entries.into_iter()); + current_child = Some((child_info(&child)?, child_root)); + current_key = Vec::new(); + } else if let Some((child, child_root)) = current_child.take() { + current_key = child.into_prefixed_storage_key().into_inner(); + result.push(( + KeyValueStorageLevel { + state_root: child_root, + key_values: entries, + parent_storage_keys: Vec::new(), + }, + complete, + )); + if !complete { + break + } + } else { + result[0].0.key_values.extend(entries.into_iter()); + result[0].1 = complete; break } - total_size += size; - entries.push((next_key.clone(), value)); - current_key = next_key; } - Ok(entries) + Ok(result) } fn verify_range_proof( &self, root: Block::Hash, - proof: StorageProof, - start_key: &[u8], - ) -> sp_blockchain::Result<(Vec<(Vec, Vec)>, bool)> { - Ok(read_range_proof_check::>( - root, - proof, - None, - None, - None, - Some(start_key), - )?) + proof: CompactProof, + start_key: &[Vec], + ) -> sp_blockchain::Result<(KeyValueStates, usize)> { + let mut db = sp_state_machine::MemoryDB::>::new(&[]); + // Compact encoding + let _ = sp_trie::decode_compact::>, _, _>( + &mut db, + proof.iter_compact_encoded_nodes(), + Some(&root), + ) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))?; + let proving_backend = sp_state_machine::TrieBackend::new(db, root); + let state = read_range_proof_check_with_child_on_proving_backend::>( + &proving_backend, + start_key, + )?; + + Ok(state) } } @@ -1413,7 +1361,7 @@ where fn new_block_at>( &self, parent: &BlockId, - inherent_digests: DigestFor, + inherent_digests: Digest, record_proof: R, ) -> sp_blockchain::Result> { sc_block_builder::BlockBuilder::new( @@ -1428,7 +1376,7 @@ where fn new_block( &self, - inherent_digests: DigestFor, + inherent_digests: Digest, ) -> sp_blockchain::Result> { let info = self.chain_info(); sc_block_builder::BlockBuilder::new( @@ -1531,10 +1479,9 @@ where id: &BlockId, key: &StorageKey, ) -> sp_blockchain::Result> { - Ok(self - .state_at(id)? + self.state_at(id)? .storage_hash(&key.0) - .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))?) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e))) } fn child_storage_keys( @@ -1571,93 +1518,9 @@ where child_info: &ChildInfo, key: &StorageKey, ) -> sp_blockchain::Result> { - Ok(self - .state_at(id)? + self.state_at(id)? .child_storage_hash(child_info, &key.0) - .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))?) - } - - fn max_key_changes_range( - &self, - first: NumberFor, - last: BlockId, - ) -> sp_blockchain::Result, BlockId)>> { - let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?; - let last_hash = self.backend.blockchain().expect_block_hash_from_id(&last)?; - if first > last_number { - return Err(sp_blockchain::Error::ChangesTrieAccessFailed( - "Invalid changes trie range".into(), - )) - } - - let (storage, configs) = match self.require_changes_trie(first, last_hash, false).ok() { - Some((storage, configs)) => (storage, configs), - None => return Ok(None), - }; - - let first_available_changes_trie = configs.last().map(|config| config.0); - match first_available_changes_trie { - Some(first_available_changes_trie) => { - let oldest_unpruned = storage.oldest_pruned_digest_range_end(); - let first = std::cmp::max(first_available_changes_trie, oldest_unpruned); - Ok(Some((first, last))) - }, - None => Ok(None), - } - } - - fn key_changes( - &self, - first: NumberFor, - last: BlockId, - storage_key: Option<&PrefixedStorageKey>, - key: &StorageKey, - ) -> sp_blockchain::Result, u32)>> { - let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?; - let last_hash = self.backend.blockchain().expect_block_hash_from_id(&last)?; - let (storage, configs) = self.require_changes_trie(first, last_hash, true)?; - - let mut result = Vec::new(); - let best_number = self.backend.blockchain().info().best_number; - for (config_zero, config_end, config) in configs { - let range_first = ::std::cmp::max(first, config_zero + One::one()); - let range_anchor = match config_end { - Some((config_end_number, config_end_hash)) => - if last_number > config_end_number { - ChangesTrieAnchorBlockId { - hash: config_end_hash, - number: config_end_number, - } - } else { - ChangesTrieAnchorBlockId { - hash: convert_hash(&last_hash), - number: last_number, - } - }, - None => - ChangesTrieAnchorBlockId { hash: convert_hash(&last_hash), number: last_number }, - }; - - let config_range = ChangesTrieConfigurationRange { - config: &config, - zero: config_zero.clone(), - end: config_end.map(|(config_end_number, _)| config_end_number), - }; - let result_range: Vec<(NumberFor, u32)> = key_changes::, _>( - config_range, - storage.storage(), - range_first, - &range_anchor, - best_number, - storage_key, - &key.0, - ) - .and_then(|r| r.map(|r| r.map(|(block, tx)| (block, tx))).collect::>()) - .map_err(|err| sp_blockchain::Error::ChangesTrieAccessFailed(err))?; - result.extend(result_range); - } - - Ok(result) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e))) } } @@ -1786,16 +1649,6 @@ where } } -impl ProvideCache for Client -where - B: backend::Backend, - Block: BlockT, -{ - fn cache(&self) -> Option>> { - self.backend.blockchain().cache() - } -} - impl ProvideRuntimeApi for Client where B: backend::Backend, @@ -1847,7 +1700,7 @@ where } fn runtime_version_at(&self, at: &BlockId) -> Result { - self.runtime_version_at(at).map_err(Into::into) + CallExecutor::runtime_version(&self.executor, at).map_err(Into::into) } } @@ -1888,7 +1741,7 @@ where let storage_changes = match self.prepare_block_storage_changes(&mut import_block).map_err(|e| { - warn!("Block prepare storage changes error:\n{:?}", e); + warn!("Block prepare storage changes error: {}", e); ConsensusError::ClientImport(e.to_string()) })? { PrepareStorageChangesResult::Discard(res) => return Ok(res), @@ -1899,7 +1752,7 @@ where self.apply_block(operation, import_block, new_cache, storage_changes) }) .map_err(|e| { - warn!("Block import error:\n{:?}", e); + warn!("Block import error: {}", e); ConsensusError::ClientImport(e.to_string()).into() }) } @@ -1943,9 +1796,8 @@ where .block_status(&BlockId::Hash(hash)) .map_err(|e| ConsensusError::ClientImport(e.to_string()))? { - BlockStatus::InChainWithState | BlockStatus::Queued if !import_existing => + BlockStatus::InChainWithState | BlockStatus::Queued => return Ok(ImportResult::AlreadyInChain), - BlockStatus::InChainWithState | BlockStatus::Queued => {}, BlockStatus::InChainPruned if !import_existing => return Ok(ImportResult::AlreadyInChain), BlockStatus::InChainPruned => {}, @@ -2062,6 +1914,19 @@ where } } +impl PreCommitActions for Client +where + Block: BlockT, +{ + fn register_import_action(&self, action: OnImportAction) { + self.import_actions.lock().push(action); + } + + fn register_finality_action(&self, action: OnFinalityAction) { + self.finality_actions.lock().push(action); + } +} + impl BlockchainEvents for Client where E: CallExecutor, @@ -2086,7 +1951,7 @@ where filter_keys: Option<&[StorageKey]>, child_filter_keys: Option<&[(StorageKey, Option>)]>, ) -> sp_blockchain::Result> { - Ok(self.storage_notifications.lock().listen(filter_keys, child_filter_keys)) + Ok(self.storage_notifications.listen(filter_keys, child_filter_keys)) } } diff --git a/client/service/src/client/genesis.rs b/client/service/src/client/genesis.rs index e764e8e24f10..35fb11f04972 100644 --- a/client/service/src/client/genesis.rs +++ b/client/service/src/client/genesis.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -22,8 +22,10 @@ use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header as HeaderT, Zero /// Create a genesis block, given the initial storage. pub fn construct_genesis_block(state_root: Block::Hash) -> Block { - let extrinsics_root = - <<::Header as HeaderT>::Hashing as HashT>::trie_root(Vec::new()); + let extrinsics_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( + Vec::new(), + sp_runtime::StateVersion::V0, + ); Block::new( <::Header as HeaderT>::new( diff --git a/client/service/src/client/light.rs b/client/service/src/client/light.rs deleted file mode 100644 index 7c13b98843e0..000000000000 --- a/client/service/src/client/light.rs +++ /dev/null @@ -1,82 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Light client utilities. - -use std::sync::Arc; - -use prometheus_endpoint::Registry; -use sc_executor::RuntimeVersionOf; -use sc_telemetry::TelemetryHandle; -use sp_blockchain::Result as ClientResult; -use sp_core::traits::{CodeExecutor, SpawnNamed}; -use sp_runtime::{ - traits::{Block as BlockT, HashFor}, - BuildStorage, -}; - -use super::{ - call_executor::LocalCallExecutor, - client::{Client, ClientConfig}, -}; -use sc_client_api::light::Storage as BlockchainStorage; -use sc_light::{Backend, GenesisCallExecutor}; - -/// Create an instance of light client. -pub fn new_light( - backend: Arc>>, - genesis_storage: &dyn BuildStorage, - code_executor: E, - spawn_handle: Box, - prometheus_registry: Option, - telemetry: Option, -) -> ClientResult< - Client< - Backend>, - GenesisCallExecutor< - Backend>, - LocalCallExecutor>, E>, - >, - B, - RA, - >, -> -where - B: BlockT, - S: BlockchainStorage + 'static, - E: CodeExecutor + RuntimeVersionOf + Clone + 'static, -{ - let local_executor = LocalCallExecutor::new( - backend.clone(), - code_executor, - spawn_handle.clone(), - ClientConfig::default(), - )?; - let executor = GenesisCallExecutor::new(backend.clone(), local_executor); - Client::new( - backend, - executor, - genesis_storage, - Default::default(), - Default::default(), - Default::default(), - prometheus_registry, - telemetry, - ClientConfig::default(), - ) -} diff --git a/client/service/src/client/mod.rs b/client/service/src/client/mod.rs index 754309e864eb..eac744923d50 100644 --- a/client/service/src/client/mod.rs +++ b/client/service/src/client/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -37,9 +37,8 @@ //! The latter typically requires passing one of: //! //! - A [`LocalCallExecutor`] running the runtime locally. -//! - A [`RemoteCallExecutor`](sc_client_api::light::RemoteCallRequest) that will ask a -//! third-party to perform the executions. -//! - A [`RemoteOrLocalCallExecutor`](sc_client_api::light::LocalOrRemote), combination of the two. +//! - A `RemoteCallExecutor` that will ask a third-party to perform the executions. +//! - A `RemoteOrLocalCallExecutor` combination of the two. //! //! Additionally, the fourth generic parameter of the `Client` is a marker type representing //! the ways in which the runtime can interface with the outside. Any code that builds a `Client` @@ -49,7 +48,6 @@ mod block_rules; mod call_executor; mod client; pub mod genesis; -pub mod light; mod wasm_override; mod wasm_substitutes; diff --git a/client/service/src/client/wasm_override.rs b/client/service/src/client/wasm_override.rs index 3d28467a9cbd..267aea070987 100644 --- a/client/service/src/client/wasm_override.rs +++ b/client/service/src/client/wasm_override.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -35,9 +35,10 @@ //! A custom WASM blob will override on-chain WASM if the spec version matches. If it is //! required to overrides multiple runtimes, multiple WASM blobs matching each of the spec versions //! needed must be provided in the given directory. + use sc_executor::RuntimeVersionOf; use sp_blockchain::Result; -use sp_core::traits::{FetchRuntimeCode, RuntimeCode}; +use sp_core::traits::{FetchRuntimeCode, RuntimeCode, WrappedRuntimeCode}; use sp_state_machine::BasicExternalities; use sp_version::RuntimeVersion; use std::{ @@ -45,19 +46,32 @@ use std::{ fs, hash::Hasher as _, path::{Path, PathBuf}, + time::{Duration, Instant}, }; -#[derive(Clone, Debug, PartialEq)] +/// The interval in that we will print a warning when a wasm blob `spec_name` +/// doesn't match with the on-chain `spec_name`. +const WARN_INTERVAL: Duration = Duration::from_secs(30); + /// Auxiliary structure that holds a wasm blob and its hash. +#[derive(Debug)] struct WasmBlob { + /// The actual wasm blob, aka the code. code: Vec, + /// The hash of [`Self::code`]. hash: Vec, + /// The path where this blob was found. + path: PathBuf, + /// The `spec_name` found in the runtime version of this blob. + spec_name: String, + /// When was the last time we have warned about the wasm blob having + /// a wrong `spec_name`? + last_warn: parking_lot::Mutex>, } impl WasmBlob { - fn new(code: Vec) -> Self { - let hash = make_hash(&code); - Self { code, hash } + fn new(code: Vec, hash: Vec, path: PathBuf, spec_name: String) -> Self { + Self { code, hash, path, spec_name, last_warn: Default::default() } } fn runtime_code(&self, heap_pages: Option) -> RuntimeCode { @@ -103,7 +117,7 @@ impl From for sp_blockchain::Error { /// Scrapes WASM from a folder and returns WASM from that folder /// if the runtime spec version matches. -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct WasmOverride { // Map of runtime spec version -> Wasm Blob overrides: HashMap, @@ -122,8 +136,35 @@ impl WasmOverride { /// Gets an override by it's runtime spec version. /// /// Returns `None` if an override for a spec version does not exist. - pub fn get<'a, 'b: 'a>(&'b self, spec: &u32, pages: Option) -> Option> { - self.overrides.get(spec).map(|w| w.runtime_code(pages)) + pub fn get<'a, 'b: 'a>( + &'b self, + spec: &u32, + pages: Option, + spec_name: &str, + ) -> Option> { + self.overrides.get(spec).and_then(|w| { + if spec_name == w.spec_name { + Some(w.runtime_code(pages)) + } else { + let mut last_warn = w.last_warn.lock(); + let now = Instant::now(); + + if last_warn.map_or(true, |l| l + WARN_INTERVAL <= now) { + *last_warn = Some(now); + + tracing::warn!( + target = "wasm_overrides", + on_chain_spec_name = %spec_name, + override_spec_name = %w.spec_name, + spec_version = %spec, + wasm_file = %w.path.display(), + "On chain and override `spec_name` do not match! Ignoring override.", + ); + } + + None + } + }) } /// Scrapes a folder for WASM runtimes. @@ -147,22 +188,29 @@ impl WasmOverride { let path = entry.path(); match path.extension().map(|e| e.to_str()).flatten() { Some("wasm") => { - let wasm = WasmBlob::new(fs::read(&path).map_err(handle_err)?); - let version = Self::runtime_version(executor, &wasm, Some(128))?; - log::info!( + let code = fs::read(&path).map_err(handle_err)?; + let code_hash = make_hash(&code); + let version = Self::runtime_version(executor, &code, &code_hash, Some(128))?; + + tracing::info!( target: "wasm_overrides", - "Found wasm override in file: `{:?}`, version: {}", - path.to_str(), - version, + version = %version, + file = %path.display(), + "Found wasm override.", ); - if let Some(_duplicate) = overrides.insert(version.spec_version, wasm) { - log::info!( + + let wasm = + WasmBlob::new(code, code_hash, path.clone(), version.spec_name.to_string()); + + if let Some(other) = overrides.insert(version.spec_version, wasm) { + tracing::info!( target: "wasm_overrides", - "Found duplicate spec version for runtime in file: `{:?}`, version: {}", - path.to_str(), - version, + first = %other.path.display(), + second = %path.display(), + %version, + "Found duplicate spec version for runtime.", ); - duplicates.push(format!("{}", path.display())); + duplicates.push(path.display().to_string()); } }, _ => (), @@ -178,7 +226,8 @@ impl WasmOverride { fn runtime_version( executor: &E, - code: &WasmBlob, + code: &[u8], + code_hash: &[u8], heap_pages: Option, ) -> Result where @@ -186,8 +235,15 @@ impl WasmOverride { { let mut ext = BasicExternalities::default(); executor - .runtime_version(&mut ext, &code.runtime_code(heap_pages)) - .map_err(|e| WasmOverrideError::VersionInvalid(format!("{:?}", e)).into()) + .runtime_version( + &mut ext, + &RuntimeCode { + code_fetcher: &WrappedRuntimeCode(code.into()), + heap_pages, + hash: code_hash.into(), + }, + ) + .map_err(|e| WasmOverrideError::VersionInvalid(e.to_string()).into()) } } @@ -195,9 +251,18 @@ impl WasmOverride { #[cfg(test)] pub fn dummy_overrides() -> WasmOverride { let mut overrides = HashMap::new(); - overrides.insert(0, WasmBlob::new(vec![0, 0, 0, 0, 0, 0, 0, 0])); - overrides.insert(1, WasmBlob::new(vec![1, 1, 1, 1, 1, 1, 1, 1])); - overrides.insert(2, WasmBlob::new(vec![2, 2, 2, 2, 2, 2, 2, 2])); + overrides.insert( + 0, + WasmBlob::new(vec![0, 0, 0, 0, 0, 0, 0, 0], vec![0], PathBuf::new(), "test".into()), + ); + overrides.insert( + 1, + WasmBlob::new(vec![1, 1, 1, 1, 1, 1, 1, 1], vec![1], PathBuf::new(), "test".into()), + ); + overrides.insert( + 2, + WasmBlob::new(vec![2, 2, 2, 2, 2, 2, 2, 2], vec![2], PathBuf::new(), "test".into()), + ); WasmOverride { overrides } } @@ -217,6 +282,7 @@ mod tests { WasmExecutionMethod::Interpreted, Some(128), 1, + 2, ); let bytes = substrate_test_runtime::wasm_binary_unwrap(); let dir = tempfile::tempdir().expect("Create a temporary directory"); @@ -226,15 +292,20 @@ mod tests { #[test] fn should_get_runtime_version() { - let wasm = WasmBlob::new(substrate_test_runtime::wasm_binary_unwrap().to_vec()); let executor = NativeElseWasmExecutor::::new( WasmExecutionMethod::Interpreted, Some(128), 1, + 2, ); - let version = WasmOverride::runtime_version(&executor, &wasm, Some(128)) - .expect("should get the `RuntimeVersion` of the test-runtime wasm blob"); + let version = WasmOverride::runtime_version( + &executor, + substrate_test_runtime::wasm_binary_unwrap(), + &[1], + Some(128), + ) + .expect("should get the `RuntimeVersion` of the test-runtime wasm blob"); assert_eq!(version.spec_version, 2); } diff --git a/client/service/src/client/wasm_substitutes.rs b/client/service/src/client/wasm_substitutes.rs index 28975790e9b5..369067251267 100644 --- a/client/service/src/client/wasm_substitutes.rs +++ b/client/service/src/client/wasm_substitutes.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,7 +18,6 @@ //! # WASM substitutes -use parking_lot::RwLock; use sc_client_api::backend; use sc_executor::RuntimeVersionOf; use sp_blockchain::{HeaderBackend, Result}; @@ -40,21 +39,14 @@ use std::{ struct WasmSubstitute { code: Vec, hash: Vec, - /// The hash of the block from that on we should use the substitute. - block_hash: Block::Hash, - /// The block number of `block_hash`. If `None`, the block is still unknown. - block_number: RwLock>>, + /// The block number on which we should start using the substitute. + block_number: NumberFor, } impl WasmSubstitute { - fn new( - code: Vec, - block_hash: Block::Hash, - backend: &impl backend::Backend, - ) -> Result { - let block_number = RwLock::new(backend.blockchain().number(block_hash)?); + fn new(code: Vec, block_number: NumberFor) -> Self { let hash = make_hash(&code); - Ok(Self { code, hash, block_hash, block_number }) + Self { code, hash, block_number } } fn runtime_code(&self, heap_pages: Option) -> RuntimeCode { @@ -63,32 +55,10 @@ impl WasmSubstitute { /// Returns `true` when the substitute matches for the given `block_id`. fn matches(&self, block_id: &BlockId, backend: &impl backend::Backend) -> bool { - let block_number = *self.block_number.read(); - let block_number = if let Some(block_number) = block_number { - block_number - } else { - let block_number = match backend.blockchain().number(self.block_hash) { - Ok(Some(n)) => n, - // still unknown - Ok(None) => return false, - Err(e) => { - log::debug!( - target: "wasm_substitutes", - "Failed to get block number for block hash {:?}: {:?}", - self.block_hash, - e, - ); - return false - }, - }; - *self.block_number.write() = Some(block_number); - block_number - }; - let requested_block_number = backend.blockchain().block_number_from_id(&block_id).ok().flatten(); - Some(block_number) <= requested_block_number + Some(self.block_number) <= requested_block_number } } @@ -145,14 +115,14 @@ where { /// Create a new instance. pub fn new( - substitutes: HashMap>, + substitutes: HashMap, Vec>, executor: Executor, backend: Arc, ) -> Result { let substitutes = substitutes .into_iter() - .map(|(parent_block_hash, code)| { - let substitute = WasmSubstitute::new(code, parent_block_hash, &*backend)?; + .map(|(block_number, code)| { + let substitute = WasmSubstitute::new(code, block_number); let version = Self::runtime_version(&executor, &substitute)?; Ok((version.spec_version, substitute)) }) @@ -181,6 +151,6 @@ where let mut ext = BasicExternalities::default(); executor .runtime_version(&mut ext, &code.runtime_code(None)) - .map_err(|e| WasmSubstituteError::VersionInvalid(format!("{:?}", e)).into()) + .map_err(|e| WasmSubstituteError::VersionInvalid(e.to_string()).into()) } } diff --git a/client/service/src/config.rs b/client/service/src/config.rs index a98a34b473ce..7c7bb480aa5c 100644 --- a/client/service/src/config.rs +++ b/client/service/src/config.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,7 +19,7 @@ //! Service configuration. pub use sc_client_api::execution_extensions::{ExecutionStrategies, ExecutionStrategy}; -pub use sc_client_db::{Database, DatabaseSource, KeepBlocks, PruningMode, TransactionStorageMode}; +pub use sc_client_db::{Database, DatabaseSource, KeepBlocks, PruningMode}; pub use sc_executor::WasmExecutionMethod; pub use sc_network::{ config::{ @@ -36,7 +36,7 @@ pub use sc_telemetry::TelemetryEndpoints; pub use sc_transaction_pool::Options as TransactionPoolOptions; use sp_core::crypto::SecretString; use std::{ - io, + io, iter, net::SocketAddr, path::{Path, PathBuf}, }; @@ -71,8 +71,6 @@ pub struct Configuration { pub state_pruning: PruningMode, /// Number of blocks to keep in the db. pub keep_blocks: KeepBlocks, - /// Transaction storage scheme. - pub transaction_storage: TransactionStorageMode, /// Chain configuration. pub chain_spec: Box, /// Wasm execution method. @@ -97,6 +95,8 @@ pub struct Configuration { pub rpc_methods: RpcMethods, /// Maximum payload of rpc request/responses. pub rpc_max_payload: Option, + /// Maximum size of the output buffer capacity for websocket connections. + pub ws_max_out_buffer_capacity: Option, /// Prometheus endpoint configuration. `None` if disabled. pub prometheus_config: Option, /// Telemetry service URL. `None` if disabled. @@ -118,8 +118,6 @@ pub struct Configuration { pub dev_key_seed: Option, /// Tracing targets pub tracing_targets: Option, - /// Is log filter reloading disabled - pub disable_log_reloading: bool, /// Tracing receiver pub tracing_receiver: sc_tracing::TracingReceiver, /// The size of the instances cache. @@ -132,6 +130,8 @@ pub struct Configuration { pub base_path: Option, /// Configuration of the output format that the informant uses. pub informant_output_format: sc_informant::OutputFormat, + /// Maximum number of different runtime versions that can be cached. + pub runtime_cache_size: u8, } /// Type for tasks spawned by the executor. @@ -186,12 +186,11 @@ pub struct PrometheusConfig { impl PrometheusConfig { /// Create a new config using the default registry. - /// - /// The default registry prefixes metrics with `substrate`. - pub fn new_with_default_registry(port: SocketAddr) -> Self { + pub fn new_with_default_registry(port: SocketAddr, chain_id: String) -> Self { + let param = iter::once((String::from("chain"), chain_id)).collect(); Self { port, - registry: Registry::new_custom(Some("substrate".into()), None) + registry: Registry::new_custom(None, Some(param)) .expect("this can only fail if the prefix is empty"), } } @@ -295,7 +294,7 @@ impl BasePath { } } -impl std::convert::From for BasePath { +impl From for BasePath { fn from(path: PathBuf) -> Self { BasePath::new(path) } diff --git a/client/service/src/error.rs b/client/service/src/error.rs index 1acd33ead677..be2199de2643 100644 --- a/client/service/src/error.rs +++ b/client/service/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 7284747424aa..6d9d99994288 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -34,10 +34,10 @@ mod client; mod metrics; mod task_manager; -use std::{collections::HashMap, io, net::SocketAddr, pin::Pin, task::Poll}; +use std::{collections::HashMap, io, net::SocketAddr, pin::Pin}; use codec::{Decode, Encode}; -use futures::{stream, Future, FutureExt, Stream, StreamExt}; +use futures::{Future, FutureExt, StreamExt}; use log::{debug, error, warn}; use sc_network::PeerId; use sc_utils::mpsc::TracingUnboundedReceiver; @@ -49,17 +49,15 @@ use sp_runtime::{ pub use self::{ builder::{ build_network, build_offchain_workers, new_client, new_db_backend, new_full_client, - new_full_parts, new_light_parts, spawn_tasks, BuildNetworkParams, KeystoreContainer, - NetworkStarter, NoopRpcExtensionBuilder, RpcExtensionBuilder, SpawnTasksParams, - TFullBackend, TFullCallExecutor, TFullClient, TLightBackend, TLightBackendWithHash, - TLightCallExecutor, TLightClient, TLightClientWithBackend, + new_full_parts, spawn_tasks, BuildNetworkParams, KeystoreContainer, NetworkStarter, + NoopRpcExtensionBuilder, RpcExtensionBuilder, SpawnTasksParams, TFullBackend, + TFullCallExecutor, TFullClient, }, client::{ClientConfig, LocalCallExecutor}, error::Error, }; pub use config::{ BasePath, Configuration, DatabaseSource, KeepBlocks, PruningMode, Role, RpcMethods, TaskType, - TransactionStorageMode, }; pub use sc_chain_spec::{ ChainSpec, ChainType, Extension as ChainSpecExtension, GenericChainSpec, NoExtension, @@ -69,14 +67,14 @@ use sc_client_api::{blockchain::HeaderBackend, BlockchainEvents}; pub use sc_consensus::ImportQueue; pub use sc_executor::NativeExecutionDispatch; #[doc(hidden)] -pub use sc_network::config::{OnDemand, TransactionImport, TransactionImportFuture}; +pub use sc_network::config::{TransactionImport, TransactionImportFuture}; pub use sc_rpc::Metadata as RpcMetadata; pub use sc_tracing::TracingReceiver; pub use sc_transaction_pool::Options as TransactionPoolOptions; pub use sc_transaction_pool_api::{error::IntoPoolError, InPoolTransaction, TransactionPool}; #[doc(hidden)] pub use std::{ops::Deref, result::Result, sync::Arc}; -pub use task_manager::{SpawnTaskHandle, TaskManager}; +pub use task_manager::{SpawnTaskHandle, TaskManager, DEFAULT_GROUP_NAME}; const DEFAULT_PROTOCOL_ID: &str = "sup"; @@ -153,26 +151,7 @@ async fn build_network_future< let starting_block = client.info().best_number; // Stream of finalized blocks reported by the client. - let mut finality_notification_stream = { - let mut finality_notification_stream = client.finality_notification_stream().fuse(); - - // We tweak the `Stream` in order to merge together multiple items if they happen to be - // ready. This way, we only get the latest finalized block. - stream::poll_fn(move |cx| { - let mut last = None; - while let Poll::Ready(Some(item)) = - Pin::new(&mut finality_notification_stream).poll_next(cx) - { - last = Some(item); - } - if let Some(last) = last { - Poll::Ready(Some(last)) - } else { - Poll::Pending - } - }) - .fuse() - }; + let mut finality_notification_stream = client.finality_notification_stream().fuse(); loop { futures::select! { @@ -424,6 +403,7 @@ fn start_rpc_servers< ), )?, config.rpc_max_payload, + config.ws_max_out_buffer_capacity, server_metrics.clone(), config.tokio_handle.clone(), ) @@ -498,7 +478,7 @@ where fn import(&self, transaction: B::Extrinsic) -> TransactionImportFuture { if !self.imports_external_transactions { debug!("Transaction rejected"); - Box::pin(futures::future::ready(TransactionImport::None)); + return Box::pin(futures::future::ready(TransactionImport::None)) } let encoded = transaction.encode(); @@ -528,7 +508,7 @@ where TransactionImport::Bad }, Err(e) => { - debug!("Error converting pool error: {:?}", e); + debug!("Error converting pool error: {}", e); // it is not bad at least, just some internal node logic error, so peer is // innocent. TransactionImport::KnownGood @@ -576,7 +556,7 @@ mod tests { amount: 5, nonce: 0, from: AccountKeyring::Alice.into(), - to: Default::default(), + to: AccountKeyring::Bob.into(), } .into_signed_tx(); block_on(pool.submit_one(&BlockId::hash(best.hash()), source, transaction.clone())) diff --git a/client/service/src/metrics.rs b/client/service/src/metrics.rs index 4d3c6df92fee..9fbf2c3ea3fc 100644 --- a/client/service/src/metrics.rs +++ b/client/service/src/metrics.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,7 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use std::{convert::TryFrom, time::SystemTime}; +use std::time::SystemTime; use crate::config::Configuration; use futures_timer::Delay; @@ -55,7 +55,7 @@ impl PrometheusMetrics { register( Gauge::::with_opts( Opts::new( - "build_info", + "substrate_build_info", "A metric with a constant '1' value labeled by name, version", ) .const_label("name", name) @@ -65,8 +65,11 @@ impl PrometheusMetrics { )? .set(1); - register(Gauge::::new("node_roles", "The roles the node is running as")?, ®istry)? - .set(roles); + register( + Gauge::::new("substrate_node_roles", "The roles the node is running as")?, + ®istry, + )? + .set(roles); register_globals(registry)?; @@ -74,7 +77,7 @@ impl PrometheusMetrics { SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default(); register( Gauge::::new( - "process_start_time_seconds", + "substrate_process_start_time_seconds", "Number of seconds between the UNIX epoch and the moment the process started", )?, registry, @@ -85,20 +88,20 @@ impl PrometheusMetrics { // generic internals block_height: register( GaugeVec::new( - Opts::new("block_height", "Block height info of the chain"), + Opts::new("substrate_block_height", "Block height info of the chain"), &["status"], )?, registry, )?, number_leaves: register( - Gauge::new("number_leaves", "Number of known chain leaves (aka forks)")?, + Gauge::new("substrate_number_leaves", "Number of known chain leaves (aka forks)")?, registry, )?, ready_transactions_number: register( Gauge::new( - "ready_transactions_number", + "substrate_ready_transactions_number", "Number of transactions in the ready queue", )?, registry, @@ -106,16 +109,16 @@ impl PrometheusMetrics { // I/ O database_cache: register( - Gauge::new("database_cache_bytes", "RocksDB cache size in bytes")?, + Gauge::new("substrate_database_cache_bytes", "RocksDB cache size in bytes")?, registry, )?, state_cache: register( - Gauge::new("state_cache_bytes", "State cache size in bytes")?, + Gauge::new("substrate_state_cache_bytes", "State cache size in bytes")?, registry, )?, state_db: register( GaugeVec::new( - Opts::new("state_db_cache_bytes", "State DB cache in bytes"), + Opts::new("substrate_state_db_cache_bytes", "State DB cache in bytes"), &["subtype"], )?, registry, diff --git a/client/service/src/task_manager/mod.rs b/client/service/src/task_manager/mod.rs index 527a744553f3..236bf2a301e1 100644 --- a/client/service/src/task_manager/mod.rs +++ b/client/service/src/task_manager/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -21,83 +21,119 @@ use crate::{config::TaskType, Error}; use exit_future::Signal; use futures::{ - future::{join_all, pending, select, try_join_all, BoxFuture, Either}, + future::{pending, select, try_join_all, BoxFuture, Either}, Future, FutureExt, StreamExt, }; -use log::debug; use prometheus_endpoint::{ exponential_buckets, register, CounterVec, HistogramOpts, HistogramVec, Opts, PrometheusError, Registry, U64, }; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use std::{panic, pin::Pin, result::Result}; -use tokio::{runtime::Handle, task::JoinHandle}; +use tokio::runtime::Handle; use tracing_futures::Instrument; mod prometheus_future; #[cfg(test)] mod tests; +/// Default task group name. +pub const DEFAULT_GROUP_NAME: &'static str = "default"; + +/// The name of a group a task belongs to. +/// +/// This name is passed belong-side the task name to the prometheus metrics and can be used +/// to group tasks. +pub enum GroupName { + /// Sets the group name to `default`. + Default, + /// Use the specifically given name as group name. + Specific(&'static str), +} + +impl From> for GroupName { + fn from(name: Option<&'static str>) -> Self { + match name { + Some(name) => Self::Specific(name), + None => Self::Default, + } + } +} + +impl From<&'static str> for GroupName { + fn from(name: &'static str) -> Self { + Self::Specific(name) + } +} + /// An handle for spawning tasks in the service. #[derive(Clone)] pub struct SpawnTaskHandle { on_exit: exit_future::Exit, tokio_handle: Handle, metrics: Option, - task_notifier: TracingUnboundedSender>, } impl SpawnTaskHandle { - /// Spawns the given task with the given name. + /// Spawns the given task with the given name and a group name. + /// If group is not specified `DEFAULT_GROUP_NAME` will be used. /// - /// Note that the `name` is a `&'static str`. The reason for this choice is that statistics - /// about this task are getting reported to the Prometheus endpoint (if enabled), and that - /// therefore the set of possible task names must be bounded. + /// Note that the `name` is a `&'static str`. The reason for this choice is that + /// statistics about this task are getting reported to the Prometheus endpoint (if enabled), and + /// that therefore the set of possible task names must be bounded. /// /// In other words, it would be a bad idea for someone to do for example /// `spawn(format!("{:?}", some_public_key))`. - pub fn spawn(&self, name: &'static str, task: impl Future + Send + 'static) { - self.spawn_inner(name, task, TaskType::Async) + pub fn spawn( + &self, + name: &'static str, + group: impl Into, + task: impl Future + Send + 'static, + ) { + self.spawn_inner(name, group, task, TaskType::Async) } /// Spawns the blocking task with the given name. See also `spawn`. pub fn spawn_blocking( &self, name: &'static str, + group: impl Into, task: impl Future + Send + 'static, ) { - self.spawn_inner(name, task, TaskType::Blocking) + self.spawn_inner(name, group, task, TaskType::Blocking) } /// Helper function that implements the spawning logic. See `spawn` and `spawn_blocking`. fn spawn_inner( &self, name: &'static str, + group: impl Into, task: impl Future + Send + 'static, task_type: TaskType, ) { - if self.task_notifier.is_closed() { - debug!("Attempt to spawn a new task has been prevented: {}", name); - return - } - let on_exit = self.on_exit.clone(); let metrics = self.metrics.clone(); + let group = match group.into() { + GroupName::Specific(var) => var, + // If no group is specified use default. + GroupName::Default => DEFAULT_GROUP_NAME, + }; + // Note that we increase the started counter here and not within the future. This way, // we could properly visualize on Prometheus situations where the spawning doesn't work. if let Some(metrics) = &self.metrics { - metrics.tasks_spawned.with_label_values(&[name]).inc(); + metrics.tasks_spawned.with_label_values(&[name, group]).inc(); // We do a dummy increase in order for the task to show up in metrics. - metrics.tasks_ended.with_label_values(&[name, "finished"]).inc_by(0); + metrics.tasks_ended.with_label_values(&[name, "finished", group]).inc_by(0); } let future = async move { if let Some(metrics) = metrics { // Add some wrappers around `task`. let task = { - let poll_duration = metrics.poll_duration.with_label_values(&[name]); - let poll_start = metrics.poll_start.with_label_values(&[name]); + let poll_duration = metrics.poll_duration.with_label_values(&[name, group]); + let poll_start = metrics.poll_start.with_label_values(&[name, group]); let inner = prometheus_future::with_poll_durations(poll_duration, poll_start, task); // The logic of `AssertUnwindSafe` here is ok considering that we throw @@ -108,15 +144,15 @@ impl SpawnTaskHandle { match select(on_exit, task).await { Either::Right((Err(payload), _)) => { - metrics.tasks_ended.with_label_values(&[name, "panic"]).inc(); + metrics.tasks_ended.with_label_values(&[name, "panic", group]).inc(); panic::resume_unwind(payload) }, Either::Right((Ok(()), _)) => { - metrics.tasks_ended.with_label_values(&[name, "finished"]).inc(); + metrics.tasks_ended.with_label_values(&[name, "finished", group]).inc(); }, Either::Left(((), _)) => { // The `on_exit` has triggered. - metrics.tasks_ended.with_label_values(&[name, "interrupted"]).inc(); + metrics.tasks_ended.with_label_values(&[name, "interrupted", group]).inc(); }, } } else { @@ -126,27 +162,37 @@ impl SpawnTaskHandle { } .in_current_span(); - let join_handle = match task_type { - TaskType::Async => self.tokio_handle.spawn(future), + match task_type { + TaskType::Async => { + self.tokio_handle.spawn(future); + }, TaskType::Blocking => { let handle = self.tokio_handle.clone(); self.tokio_handle.spawn_blocking(move || { handle.block_on(future); - }) + }); }, - }; - - let _ = self.task_notifier.unbounded_send(join_handle); + } } } impl sp_core::traits::SpawnNamed for SpawnTaskHandle { - fn spawn_blocking(&self, name: &'static str, future: BoxFuture<'static, ()>) { - self.spawn_blocking(name, future); + fn spawn_blocking( + &self, + name: &'static str, + group: Option<&'static str>, + future: BoxFuture<'static, ()>, + ) { + self.spawn_inner(name, group, future, TaskType::Blocking) } - fn spawn(&self, name: &'static str, future: BoxFuture<'static, ()>) { - self.spawn(name, future); + fn spawn( + &self, + name: &'static str, + group: Option<&'static str>, + future: BoxFuture<'static, ()>, + ) { + self.spawn_inner(name, group, future, TaskType::Async) } } @@ -172,8 +218,13 @@ impl SpawnEssentialTaskHandle { /// Spawns the given task with the given name. /// /// See also [`SpawnTaskHandle::spawn`]. - pub fn spawn(&self, name: &'static str, task: impl Future + Send + 'static) { - self.spawn_inner(name, task, TaskType::Async) + pub fn spawn( + &self, + name: &'static str, + group: impl Into, + task: impl Future + Send + 'static, + ) { + self.spawn_inner(name, group, task, TaskType::Async) } /// Spawns the blocking task with the given name. @@ -182,14 +233,16 @@ impl SpawnEssentialTaskHandle { pub fn spawn_blocking( &self, name: &'static str, + group: impl Into, task: impl Future + Send + 'static, ) { - self.spawn_inner(name, task, TaskType::Blocking) + self.spawn_inner(name, group, task, TaskType::Blocking) } fn spawn_inner( &self, name: &'static str, + group: impl Into, task: impl Future + Send + 'static, task_type: TaskType, ) { @@ -199,17 +252,27 @@ impl SpawnEssentialTaskHandle { let _ = essential_failed.close_channel(); }); - let _ = self.inner.spawn_inner(name, essential_task, task_type); + let _ = self.inner.spawn_inner(name, group, essential_task, task_type); } } impl sp_core::traits::SpawnEssentialNamed for SpawnEssentialTaskHandle { - fn spawn_essential_blocking(&self, name: &'static str, future: BoxFuture<'static, ()>) { - self.spawn_blocking(name, future); + fn spawn_essential_blocking( + &self, + name: &'static str, + group: Option<&'static str>, + future: BoxFuture<'static, ()>, + ) { + self.spawn_blocking(name, group, future); } - fn spawn_essential(&self, name: &'static str, future: BoxFuture<'static, ()>) { - self.spawn(name, future); + fn spawn_essential( + &self, + name: &'static str, + group: Option<&'static str>, + future: BoxFuture<'static, ()>, + ) { + self.spawn(name, group, future); } } @@ -218,8 +281,8 @@ pub struct TaskManager { /// A future that resolves when the service has exited, this is useful to /// make sure any internally spawned futures stop when the service does. on_exit: exit_future::Exit, - /// A signal that makes the exit future above resolve, fired on service drop. - signal: Option, + /// A signal that makes the exit future above resolve, fired on drop. + _signal: Signal, /// Tokio runtime handle that is used to spawn futures. tokio_handle: Handle, /// Prometheus metric where to report the polling times. @@ -231,10 +294,6 @@ pub struct TaskManager { essential_failed_rx: TracingUnboundedReceiver<()>, /// Things to keep alive until the task manager is dropped. keep_alive: Box, - /// A sender to a stream of background tasks. This is used for the completion future. - task_notifier: TracingUnboundedSender>, - /// This future will complete when all the tasks are joined and the stream is closed. - completion_future: JoinHandle<()>, /// A list of other `TaskManager`'s to terminate and gracefully shutdown when the parent /// terminates and gracefully shutdown. Also ends the parent `future()` if a child's essential /// task fails. @@ -259,16 +318,6 @@ impl TaskManager { let metrics = prometheus_registry.map(Metrics::register).transpose()?; - let (task_notifier, background_tasks) = tracing_unbounded("mpsc_background_tasks"); - // NOTE: for_each_concurrent will await on all the JoinHandle futures at the same time. It - // is possible to limit this but it's actually better for the memory foot print to await - // them all to not accumulate anything on that stream. - let completion_future = - tokio_handle.spawn(background_tasks.for_each_concurrent(None, |x| async move { - let _ = x.await; - })); - - let ipfs_rt = if ipfs_rt.is_none() { None } else { @@ -278,14 +327,12 @@ impl TaskManager { Ok(Self { on_exit, - signal: Some(signal), + _signal: signal, tokio_handle, metrics, essential_failed_tx, essential_failed_rx, keep_alive: Box::new(()), - task_notifier, - completion_future, children: Vec::new(), ipfs_rt, }) @@ -297,7 +344,6 @@ impl TaskManager { on_exit: self.on_exit.clone(), tokio_handle: self.tokio_handle.clone(), metrics: self.metrics.clone(), - task_notifier: self.task_notifier.clone(), } } @@ -306,36 +352,12 @@ impl TaskManager { SpawnEssentialTaskHandle::new(self.essential_failed_tx.clone(), self.spawn_handle()) } - /// Send the signal for termination, prevent new tasks to be created, await for all the existing - /// tasks to be finished and drop the object. You can consider this as an async drop. - /// - /// It's always better to call and await this function before exiting the process as background - /// tasks may be running in the background. If the process exit and the background tasks are not - /// cancelled, this will lead to objects not getting dropped properly. - /// - /// This is an issue in some cases as some of our dependencies do require that we drop all the - /// objects properly otherwise it triggers a SIGABRT on exit. - pub fn clean_shutdown(mut self) -> Pin + Send>> { - self.terminate(); - let children_shutdowns = self.children.into_iter().map(|x| x.clean_shutdown()); - let keep_alive = self.keep_alive; - let completion_future = self.completion_future; - - Box::pin(async move { - join_all(children_shutdowns).await; - let _ = completion_future.await; - - let _ = keep_alive; - }) - } - /// Return a future that will end with success if the signal to terminate was sent /// (`self.terminate()`) or with an error if an essential task fails. /// /// # Warning /// - /// This function will not wait until the end of the remaining task. You must call and await - /// `clean_shutdown()` after this. + /// This function will not wait until the end of the remaining task. pub fn future<'a>( &'a mut self, ) -> Pin> + Send + 'a>> { @@ -360,18 +382,6 @@ impl TaskManager { }) } - /// Signal to terminate all the running tasks. - pub fn terminate(&mut self) { - if let Some(signal) = self.signal.take() { - let _ = signal.fire(); - // NOTE: this will prevent new tasks to be spawned - self.task_notifier.close_channel(); - for child in self.children.iter_mut() { - child.terminate(); - } - } - } - /// Set what the task manager should keep alive, can be called multiple times. pub fn keep_alive(&mut self, to_keep_alive: T) { // allows this fn to safely called multiple times. @@ -403,34 +413,34 @@ impl Metrics { poll_duration: register(HistogramVec::new( HistogramOpts { common_opts: Opts::new( - "tasks_polling_duration", + "substrate_tasks_polling_duration", "Duration in seconds of each invocation of Future::poll" ), buckets: exponential_buckets(0.001, 4.0, 9) .expect("function parameters are constant and always valid; qed"), }, - &["task_name"] + &["task_name", "task_group"] )?, registry)?, poll_start: register(CounterVec::new( Opts::new( - "tasks_polling_started_total", + "substrate_tasks_polling_started_total", "Total number of times we started invoking Future::poll" ), - &["task_name"] + &["task_name", "task_group"] )?, registry)?, tasks_spawned: register(CounterVec::new( Opts::new( - "tasks_spawned_total", + "substrate_tasks_spawned_total", "Total number of tasks that have been spawned on the Service" ), - &["task_name"] + &["task_name", "task_group"] )?, registry)?, tasks_ended: register(CounterVec::new( Opts::new( - "tasks_ended_total", + "substrate_tasks_ended_total", "Total number of tasks for which Future::poll has returned Ready(()) or panicked" ), - &["task_name", "reason"] + &["task_name", "reason", "task_group"] )?, registry)?, }) } diff --git a/client/service/src/task_manager/prometheus_future.rs b/client/service/src/task_manager/prometheus_future.rs index 43a76a0f596c..88296a67336f 100644 --- a/client/service/src/task_manager/prometheus_future.rs +++ b/client/service/src/task_manager/prometheus_future.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/service/src/task_manager/tests.rs b/client/service/src/task_manager/tests.rs index ced281ecf339..0af63f2f787e 100644 --- a/client/service/src/task_manager/tests.rs +++ b/client/service/src/task_manager/tests.rs @@ -1,7 +1,7 @@ // // This file is part of Substrate. -// // Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. -// // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // // This program is free software: you can redistribute it and/or modify // // it under the terms of the GNU General Public License as published by @@ -90,162 +90,97 @@ fn new_task_manager(tokio_handle: tokio::runtime::Handle, ipfs_rt: Option panic!("task should not have stopped: {:?}", res), - _ = t2 => {}, - } - }); - assert_eq!(drop_tester, 4); - runtime.block_on(task_manager.clean_shutdown()); + { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let handle = runtime.handle().clone(); + let ipfs_rt = tokio::runtime::Runtime::new().unwrap(); + + let mut task_manager = new_task_manager(handle.clone(), Some(ipfs_rt)); + let child_1 = new_task_manager(handle.clone()); + let spawn_handle_child_1 = child_1.spawn_handle(); + let child_2 = new_task_manager(handle.clone()); + let spawn_handle_child_2 = child_2.spawn_handle(); + task_manager.add_child(child_1); + task_manager.add_child(child_2); + let spawn_handle = task_manager.spawn_handle(); + spawn_handle.spawn("task1", None, run_background_task(drop_tester.new_ref())); + spawn_handle.spawn("task2", None, run_background_task(drop_tester.new_ref())); + spawn_handle_child_1.spawn("task3", None, run_background_task(drop_tester.new_ref())); + spawn_handle_child_2.spawn("task4", None, run_background_task(drop_tester.new_ref())); + assert_eq!(drop_tester, 4); + // allow the tasks to even start + runtime.block_on(async { tokio::time::sleep(Duration::from_secs(1)).await }); + assert_eq!(drop_tester, 4); + spawn_handle_child_1.spawn("task5", None, async { panic!("task failed") }); + runtime.block_on(async { + let t1 = task_manager.future().fuse(); + let t2 = tokio::time::sleep(Duration::from_secs(3)).fuse(); + + pin_mut!(t1, t2); + + select! { + res = t1 => panic!("task should not have stopped: {:?}", res), + _ = t2 => {}, + } + }); + assert_eq!(drop_tester, 4); + } drop_tester.wait_on_drop(); } diff --git a/client/service/test/Cargo.toml b/client/service/test/Cargo.toml index 85a6dcc9e8b2..ab92de4e84f8 100644 --- a/client/service/test/Cargo.toml +++ b/client/service/test/Cargo.toml @@ -2,43 +2,43 @@ name = "sc-service-test" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" publish = false -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -hex-literal = "0.3.1" +hex = "0.4" +hex-literal = "0.3.4" tempfile = "3.1.0" -tokio = { version = "1.10.0", features = ["time"] } +tokio = { version = "1.17.0", features = ["time"] } log = "0.4.8" fdlimit = "0.2.1" -parking_lot = "0.11.1" -sc-light = { version = "4.0.0-dev", path = "../../light" } +parking_lot = "0.12.0" sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } -sp-state-machine = { version = "0.10.0-dev", path = "../../../primitives/state-machine" } -sp-externalities = { version = "0.10.0-dev", path = "../../../primitives/externalities" } -sp-trie = { version = "4.0.0-dev", path = "../../../primitives/trie" } -sp-storage = { version = "4.0.0-dev", path = "../../../primitives/storage" } +sp-state-machine = { version = "0.12.0", path = "../../../primitives/state-machine" } +sp-externalities = { version = "0.12.0", path = "../../../primitives/externalities" } +sp-trie = { version = "6.0.0", path = "../../../primitives/trie" } +sp-storage = { version = "6.0.0", path = "../../../primitives/storage" } sc-client-db = { version = "0.10.0-dev", default-features = false, path = "../../db" } -futures = "0.3.16" +futures = "0.3.21" sc-service = { version = "0.10.0-dev", features = ["test-helpers"], path = "../../service" } sc-network = { version = "0.10.0-dev", path = "../../network" } sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } sc-consensus = { version = "0.10.0-dev", path = "../../../client/consensus/common" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../../client/transaction-pool/api" } substrate-test-runtime = { version = "2.0.0", path = "../../../test-utils/runtime" } substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } sc-client-api = { version = "4.0.0-dev", path = "../../api" } sc-block-builder = { version = "0.10.0-dev", path = "../../block-builder" } sc-executor = { version = "0.10.0-dev", path = "../../executor" } -sp-panic-handler = { version = "3.0.0", path = "../../../primitives/panic-handler" } -parity-scale-codec = "2.0.0" -sp-tracing = { version = "4.0.0-dev", path = "../../../primitives/tracing" } +sp-panic-handler = { version = "4.0.0", path = "../../../primitives/panic-handler" } +parity-scale-codec = "3.0.0" +sp-tracing = { version = "5.0.0", path = "../../../primitives/tracing" } diff --git a/client/service/test/src/client/db.rs b/client/service/test/src/client/db.rs index 5278c9a13a4d..5c1315fc870c 100644 --- a/client/service/test/src/client/db.rs +++ b/client/service/test/src/client/db.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs deleted file mode 100644 index fb9566d208f7..000000000000 --- a/client/service/test/src/client/light.rs +++ /dev/null @@ -1,981 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use super::prepare_client_with_key_changes; -use parity_scale_codec::{Decode, Encode}; -use parking_lot::Mutex; -use sc_block_builder::BlockBuilderProvider; -use sc_client_api::{ - backend::NewBlockState, - blockchain::Info, - cht, - in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, - AuxStore, Backend as ClientBackend, BlockBackend, BlockImportOperation, CallExecutor, - ChangesProof, ExecutionStrategy, FetchChecker, ProofProvider, ProvideChtRoots, - RemoteBodyRequest, RemoteCallRequest, RemoteChangesRequest, RemoteHeaderRequest, - RemoteReadChildRequest, RemoteReadRequest, Storage, StorageProof, StorageProvider, -}; -use sc_executor::{NativeElseWasmExecutor, RuntimeVersion, WasmExecutionMethod}; -use sc_light::{ - backend::{Backend, GenesisOrUnavailableState}, - blockchain::{Blockchain, BlockchainCache}, - call_executor::{check_execution_proof, GenesisCallExecutor}, - fetcher::LightDataChecker, -}; -use sp_api::{ProofRecorder, StorageTransactionCache}; -use sp_blockchain::{ - well_known_cache_keys, BlockStatus, CachedHeaderMetadata, Error as ClientError, HeaderBackend, - Result as ClientResult, -}; -use sp_consensus::BlockOrigin; -use sp_core::{testing::TaskExecutor, NativeOrEncoded, H256}; -use sp_externalities::Extensions; -use sp_runtime::{ - generic::BlockId, - traits::{BlakeTwo256, Block as _, Header as HeaderT, NumberFor}, - Digest, Justifications, -}; -use sp_state_machine::{ExecutionManager, OverlayedChanges}; -use std::{cell::RefCell, collections::HashMap, panic::UnwindSafe, sync::Arc}; -use substrate_test_runtime_client::{ - runtime::{self, Block, Extrinsic, Hash, Header}, - AccountKeyring, ClientBlockImportExt, TestClient, -}; - -use sp_core::{ - blake2_256, - storage::{well_known_keys, ChildInfo, StorageKey}, - ChangesTrieConfiguration, -}; -use sp_state_machine::Backend as _; - -pub type DummyBlockchain = Blockchain; - -pub struct DummyStorage { - pub changes_tries_cht_roots: HashMap, - pub aux_store: Mutex, Vec>>, -} - -impl DummyStorage { - pub fn new() -> Self { - DummyStorage { - changes_tries_cht_roots: HashMap::new(), - aux_store: Mutex::new(HashMap::new()), - } - } -} - -impl sp_blockchain::HeaderBackend for DummyStorage { - fn header(&self, _id: BlockId) -> ClientResult> { - Err(ClientError::Backend("Test error".into())) - } - - fn info(&self) -> Info { - panic!("Test error") - } - - fn status(&self, _id: BlockId) -> ClientResult { - Err(ClientError::Backend("Test error".into())) - } - - fn number(&self, hash: Hash) -> ClientResult>> { - if hash == Default::default() { - Ok(Some(Default::default())) - } else { - Err(ClientError::Backend("Test error".into())) - } - } - - fn hash(&self, number: u64) -> ClientResult> { - if number == 0 { - Ok(Some(Default::default())) - } else { - Err(ClientError::Backend("Test error".into())) - } - } -} - -impl sp_blockchain::HeaderMetadata for DummyStorage { - type Error = ClientError; - - fn header_metadata(&self, hash: Hash) -> Result, Self::Error> { - self.header(BlockId::hash(hash))? - .map(|header| CachedHeaderMetadata::from(&header)) - .ok_or(ClientError::UnknownBlock("header not found".to_owned())) - } - fn insert_header_metadata(&self, _hash: Hash, _metadata: CachedHeaderMetadata) {} - fn remove_header_metadata(&self, _hash: Hash) {} -} - -impl AuxStore for DummyStorage { - fn insert_aux< - 'a, - 'b: 'a, - 'c: 'a, - I: IntoIterator, - D: IntoIterator, - >( - &self, - insert: I, - _delete: D, - ) -> ClientResult<()> { - for (k, v) in insert.into_iter() { - self.aux_store.lock().insert(k.to_vec(), v.to_vec()); - } - Ok(()) - } - - fn get_aux(&self, key: &[u8]) -> ClientResult>> { - Ok(self.aux_store.lock().get(key).cloned()) - } -} - -impl Storage for DummyStorage { - fn import_header( - &self, - _header: Header, - _cache: HashMap>, - _state: NewBlockState, - _aux_ops: Vec<(Vec, Option>)>, - ) -> ClientResult<()> { - Ok(()) - } - - fn set_head(&self, _block: BlockId) -> ClientResult<()> { - Err(ClientError::Backend("Test error".into())) - } - - fn finalize_header(&self, _block: BlockId) -> ClientResult<()> { - Err(ClientError::Backend("Test error".into())) - } - - fn last_finalized(&self) -> ClientResult { - Err(ClientError::Backend("Test error".into())) - } - - fn cache(&self) -> Option>> { - None - } - - fn usage_info(&self) -> Option { - None - } -} - -impl ProvideChtRoots for DummyStorage { - fn header_cht_root(&self, _cht_size: u64, _block: u64) -> ClientResult> { - Err(ClientError::Backend("Test error".into())) - } - - fn changes_trie_cht_root(&self, cht_size: u64, block: u64) -> ClientResult> { - cht::block_to_cht_number(cht_size, block) - .and_then(|cht_num| self.changes_tries_cht_roots.get(&cht_num)) - .cloned() - .ok_or_else(|| { - ClientError::Backend(format!("Test error: CHT for block #{} not found", block)) - .into() - }) - .map(Some) - } -} - -struct DummyCallExecutor; - -impl CallExecutor for DummyCallExecutor { - type Error = ClientError; - - type Backend = substrate_test_runtime_client::Backend; - - fn call( - &self, - _id: &BlockId, - _method: &str, - _call_data: &[u8], - _strategy: ExecutionStrategy, - _extensions: Option, - ) -> Result, ClientError> { - Ok(vec![42]) - } - - fn contextual_call< - EM: Fn( - Result, Self::Error>, - Result, Self::Error>, - ) -> Result, Self::Error>, - R: Encode + Decode + PartialEq, - NC: FnOnce() -> Result + UnwindSafe, - >( - &self, - _at: &BlockId, - _method: &str, - _call_data: &[u8], - _changes: &RefCell, - _storage_transaction_cache: Option< - &RefCell< - StorageTransactionCache< - Block, - >::State, - >, - >, - >, - _execution_manager: ExecutionManager, - _native_call: Option, - _proof_recorder: &Option>, - _extensions: Option, - ) -> ClientResult> - where - ExecutionManager: Clone, - { - unreachable!() - } - - fn runtime_version(&self, _id: &BlockId) -> Result { - unreachable!() - } - - fn prove_execution( - &self, - _: &BlockId, - _: &str, - _: &[u8], - ) -> Result<(Vec, StorageProof), ClientError> { - unreachable!() - } -} - -fn local_executor() -> NativeElseWasmExecutor -{ - NativeElseWasmExecutor::new(WasmExecutionMethod::Interpreted, None, 8) -} - -#[test] -fn local_state_is_created_when_genesis_state_is_available() { - let def = Default::default(); - let header0 = - substrate_test_runtime_client::runtime::Header::new(0, def, def, def, Default::default()); - - let backend: Backend<_, BlakeTwo256> = - Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new()))); - let mut op = backend.begin_operation().unwrap(); - op.set_block_data(header0, None, None, None, NewBlockState::Final).unwrap(); - op.set_genesis_state(Default::default(), true).unwrap(); - backend.commit_operation(op).unwrap(); - - match backend.state_at(BlockId::Number(0)).unwrap() { - GenesisOrUnavailableState::Genesis(_) => (), - _ => panic!("unexpected state"), - } -} - -#[test] -fn unavailable_state_is_created_when_genesis_state_is_unavailable() { - let backend: Backend<_, BlakeTwo256> = - Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new()))); - - match backend.state_at(BlockId::Number(0)).unwrap() { - GenesisOrUnavailableState::Unavailable => (), - _ => panic!("unexpected state"), - } -} - -#[test] -fn light_aux_store_is_updated_via_non_importing_op() { - let backend = Backend::new(Arc::new(DummyBlockchain::new(DummyStorage::new()))); - let mut op = ClientBackend::::begin_operation(&backend).unwrap(); - BlockImportOperation::::insert_aux(&mut op, vec![(vec![1], Some(vec![2]))]).unwrap(); - ClientBackend::::commit_operation(&backend, op).unwrap(); - - assert_eq!(AuxStore::get_aux(&backend, &[1]).unwrap(), Some(vec![2])); -} - -#[test] -fn execution_proof_is_generated_and_checked() { - fn execute(remote_client: &TestClient, at: u64, method: &'static str) -> (Vec, Vec) { - let remote_block_id = BlockId::Number(at); - let remote_header = remote_client.header(&remote_block_id).unwrap().unwrap(); - - // 'fetch' execution proof from remote node - let (remote_result, remote_execution_proof) = - remote_client.execution_proof(&remote_block_id, method, &[]).unwrap(); - - // check remote execution proof locally - let local_result = check_execution_proof::<_, _, BlakeTwo256>( - &local_executor(), - Box::new(TaskExecutor::new()), - &RemoteCallRequest { - block: substrate_test_runtime_client::runtime::Hash::default(), - header: remote_header, - method: method.into(), - call_data: vec![], - retry_count: None, - }, - remote_execution_proof, - ) - .unwrap(); - - (remote_result, local_result) - } - - fn execute_with_proof_failure(remote_client: &TestClient, at: u64) { - let remote_block_id = BlockId::Number(at); - let remote_header = remote_client.header(&remote_block_id).unwrap().unwrap(); - - // 'fetch' execution proof from remote node - let (_, remote_execution_proof) = remote_client - .execution_proof( - &remote_block_id, - "Core_initialize_block", - &Header::new( - at, - Default::default(), - Default::default(), - Default::default(), - Default::default(), - ) - .encode(), - ) - .unwrap(); - - // check remote execution proof locally - let execution_result = check_execution_proof::<_, _, BlakeTwo256>( - &local_executor(), - Box::new(TaskExecutor::new()), - &RemoteCallRequest { - block: substrate_test_runtime_client::runtime::Hash::default(), - header: remote_header.clone(), - method: "Core_initialize_block".into(), - call_data: Header::new( - at + 1, - Default::default(), - Default::default(), - remote_header.hash(), - remote_header.digest().clone(), // this makes next header wrong - ) - .encode(), - retry_count: None, - }, - remote_execution_proof, - ); - match execution_result { - Err(sp_blockchain::Error::Execution(_)) => (), - _ => panic!("Unexpected execution result: {:?}", execution_result), - } - } - - // prepare remote client - let mut remote_client = substrate_test_runtime_client::new(); - for i in 1u32..3u32 { - let mut digest = Digest::default(); - digest.push(sp_runtime::generic::DigestItem::Other::(i.to_le_bytes().to_vec())); - futures::executor::block_on(remote_client.import_justified( - BlockOrigin::Own, - remote_client.new_block(digest).unwrap().build().unwrap().block, - Justifications::from((*b"TEST", Default::default())), - )) - .unwrap(); - } - - // check method that doesn't requires environment - let (remote, local) = execute(&remote_client, 0, "Core_version"); - assert_eq!(remote, local); - - let (remote, local) = execute(&remote_client, 2, "Core_version"); - assert_eq!(remote, local); - - // check that proof check doesn't panic even if proof is incorrect AND no panic handler is set - execute_with_proof_failure(&remote_client, 2); - - // check that proof check doesn't panic even if proof is incorrect AND panic handler is set - sp_panic_handler::set("TEST", "1.2.3"); - execute_with_proof_failure(&remote_client, 2); -} - -#[test] -fn code_is_executed_at_genesis_only() { - let backend = Arc::new(InMemBackend::::new()); - let def = H256::default(); - let header0 = - substrate_test_runtime_client::runtime::Header::new(0, def, def, def, Default::default()); - let hash0 = header0.hash(); - let header1 = - substrate_test_runtime_client::runtime::Header::new(1, def, def, hash0, Default::default()); - let hash1 = header1.hash(); - backend - .blockchain() - .insert(hash0, header0, None, None, NewBlockState::Final) - .unwrap(); - backend - .blockchain() - .insert(hash1, header1, None, None, NewBlockState::Final) - .unwrap(); - - let genesis_executor = GenesisCallExecutor::new(backend, DummyCallExecutor); - assert_eq!( - genesis_executor - .call(&BlockId::Number(0), "test_method", &[], ExecutionStrategy::NativeElseWasm, None,) - .unwrap(), - vec![42], - ); - - let call_on_unavailable = genesis_executor.call( - &BlockId::Number(1), - "test_method", - &[], - ExecutionStrategy::NativeElseWasm, - None, - ); - - match call_on_unavailable { - Err(ClientError::NotAvailableOnLightClient) => (), - _ => unreachable!("unexpected result: {:?}", call_on_unavailable), - } -} - -type TestChecker = LightDataChecker< - NativeElseWasmExecutor, - Block, - DummyStorage, ->; - -fn prepare_for_read_proof_check() -> (TestChecker, Header, StorageProof, u32) { - // prepare remote client - let remote_client = substrate_test_runtime_client::new(); - let remote_block_id = BlockId::Number(0); - let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); - let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); - remote_block_header.state_root = remote_client - .state_at(&remote_block_id) - .unwrap() - .storage_root(::std::iter::empty()) - .0 - .into(); - - // 'fetch' read proof from remote node - let heap_pages = remote_client - .storage(&remote_block_id, &StorageKey(well_known_keys::HEAP_PAGES.to_vec())) - .unwrap() - .and_then(|v| Decode::decode(&mut &v.0[..]).ok()) - .unwrap(); - let remote_read_proof = remote_client - .read_proof(&remote_block_id, &mut std::iter::once(well_known_keys::HEAP_PAGES)) - .unwrap(); - - // check remote read proof locally - let local_storage = InMemoryBlockchain::::new(); - local_storage - .insert(remote_block_hash, remote_block_header.clone(), None, None, NewBlockState::Final) - .unwrap(); - let local_checker = LightDataChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - Box::new(TaskExecutor::new()), - ); - (local_checker, remote_block_header, remote_read_proof, heap_pages) -} - -fn prepare_for_read_child_proof_check() -> (TestChecker, Header, StorageProof, Vec) { - use substrate_test_runtime_client::{DefaultTestClientBuilderExt, TestClientBuilderExt}; - let child_info = ChildInfo::new_default(b"child1"); - let child_info = &child_info; - // prepare remote client - let remote_client = substrate_test_runtime_client::TestClientBuilder::new() - .add_extra_child_storage(child_info, b"key1".to_vec(), b"value1".to_vec()) - .build(); - let remote_block_id = BlockId::Number(0); - let remote_block_hash = remote_client.block_hash(0).unwrap().unwrap(); - let mut remote_block_header = remote_client.header(&remote_block_id).unwrap().unwrap(); - remote_block_header.state_root = remote_client - .state_at(&remote_block_id) - .unwrap() - .storage_root(::std::iter::empty()) - .0 - .into(); - - // 'fetch' child read proof from remote node - let child_value = remote_client - .child_storage(&remote_block_id, child_info, &StorageKey(b"key1".to_vec())) - .unwrap() - .unwrap() - .0; - assert_eq!(b"value1"[..], child_value[..]); - let remote_read_proof = remote_client - .read_child_proof(&remote_block_id, child_info, &mut std::iter::once("key1".as_bytes())) - .unwrap(); - - // check locally - let local_storage = InMemoryBlockchain::::new(); - local_storage - .insert(remote_block_hash, remote_block_header.clone(), None, None, NewBlockState::Final) - .unwrap(); - let local_checker = LightDataChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - Box::new(TaskExecutor::new()), - ); - (local_checker, remote_block_header, remote_read_proof, child_value) -} - -fn prepare_for_header_proof_check(insert_cht: bool) -> (TestChecker, Hash, Header, StorageProof) { - // prepare remote client - let mut remote_client = substrate_test_runtime_client::new(); - let mut local_headers_hashes = Vec::new(); - for i in 0..4 { - let block = remote_client.new_block(Default::default()).unwrap().build().unwrap().block; - futures::executor::block_on(remote_client.import(BlockOrigin::Own, block)).unwrap(); - local_headers_hashes.push( - remote_client - .block_hash(i + 1) - .map_err(|_| ClientError::Backend("TestError".into())), - ); - } - - // 'fetch' header proof from remote node - let remote_block_id = BlockId::Number(1); - let (remote_block_header, remote_header_proof) = - remote_client.header_proof_with_cht_size(&remote_block_id, 4).unwrap(); - - // check remote read proof locally - let local_storage = InMemoryBlockchain::::new(); - let local_cht_root = - cht::compute_root::(4, 0, local_headers_hashes).unwrap(); - if insert_cht { - local_storage.insert_cht_root(1, local_cht_root); - } - let local_checker = LightDataChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - Box::new(TaskExecutor::new()), - ); - (local_checker, local_cht_root, remote_block_header, remote_header_proof) -} - -fn header_with_computed_extrinsics_root(extrinsics: Vec) -> Header { - use sp_trie::{trie_types::Layout, TrieConfiguration}; - let iter = extrinsics.iter().map(Encode::encode); - let extrinsics_root = Layout::::ordered_trie_root(iter); - - // only care about `extrinsics_root` - Header::new(0, extrinsics_root, H256::zero(), H256::zero(), Default::default()) -} - -#[test] -fn storage_read_proof_is_generated_and_checked() { - let (local_checker, remote_block_header, remote_read_proof, heap_pages) = - prepare_for_read_proof_check(); - assert_eq!( - (&local_checker as &dyn FetchChecker) - .check_read_proof( - &RemoteReadRequest::

{ - block: remote_block_header.hash(), - header: remote_block_header, - keys: vec![well_known_keys::HEAP_PAGES.to_vec()], - retry_count: None, - }, - remote_read_proof - ) - .unwrap() - .remove(well_known_keys::HEAP_PAGES) - .unwrap() - .unwrap()[0], - heap_pages as u8 - ); -} - -#[test] -fn storage_child_read_proof_is_generated_and_checked() { - let child_info = ChildInfo::new_default(&b"child1"[..]); - let (local_checker, remote_block_header, remote_read_proof, result) = - prepare_for_read_child_proof_check(); - assert_eq!( - (&local_checker as &dyn FetchChecker) - .check_read_child_proof( - &RemoteReadChildRequest::
{ - block: remote_block_header.hash(), - header: remote_block_header, - storage_key: child_info.prefixed_storage_key(), - keys: vec![b"key1".to_vec()], - retry_count: None, - }, - remote_read_proof - ) - .unwrap() - .remove(b"key1".as_ref()) - .unwrap() - .unwrap(), - result - ); -} - -#[test] -fn header_proof_is_generated_and_checked() { - let (local_checker, local_cht_root, remote_block_header, remote_header_proof) = - prepare_for_header_proof_check(true); - assert_eq!( - (&local_checker as &dyn FetchChecker) - .check_header_proof( - &RemoteHeaderRequest::
{ - cht_root: local_cht_root, - block: 1, - retry_count: None, - }, - Some(remote_block_header.clone()), - remote_header_proof - ) - .unwrap(), - remote_block_header - ); -} - -#[test] -fn check_header_proof_fails_if_cht_root_is_invalid() { - let (local_checker, _, mut remote_block_header, remote_header_proof) = - prepare_for_header_proof_check(true); - remote_block_header.number = 100; - assert!((&local_checker as &dyn FetchChecker) - .check_header_proof( - &RemoteHeaderRequest::
{ - cht_root: Default::default(), - block: 1, - retry_count: None, - }, - Some(remote_block_header.clone()), - remote_header_proof - ) - .is_err()); -} - -#[test] -fn check_header_proof_fails_if_invalid_header_provided() { - let (local_checker, local_cht_root, mut remote_block_header, remote_header_proof) = - prepare_for_header_proof_check(true); - remote_block_header.number = 100; - assert!((&local_checker as &dyn FetchChecker) - .check_header_proof( - &RemoteHeaderRequest::
{ - cht_root: local_cht_root, - block: 1, - retry_count: None, - }, - Some(remote_block_header.clone()), - remote_header_proof - ) - .is_err()); -} - -#[test] -fn changes_proof_is_generated_and_checked_when_headers_are_not_pruned() { - let (remote_client, local_roots, test_cases) = prepare_client_with_key_changes(); - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - Box::new(TaskExecutor::new()), - ); - let local_checker = &local_checker as &dyn FetchChecker; - let max = remote_client.chain_info().best_number; - let max_hash = remote_client.chain_info().best_hash; - - for (index, (begin, end, key, expected_result)) in test_cases.into_iter().enumerate() { - let begin_hash = remote_client.block_hash(begin).unwrap().unwrap(); - let end_hash = remote_client.block_hash(end).unwrap().unwrap(); - - // 'fetch' changes proof from remote node - let key = StorageKey(key); - let remote_proof = remote_client - .key_changes_proof(begin_hash, end_hash, begin_hash, max_hash, None, &key) - .unwrap(); - - // check proof on local client - let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec(); - let config = ChangesTrieConfiguration::new(4, 2); - let request = RemoteChangesRequest::
{ - changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { - zero: (0, Default::default()), - end: None, - config: Some(config), - }], - first_block: (begin, begin_hash), - last_block: (end, end_hash), - max_block: (max, max_hash), - tries_roots: (begin, begin_hash, local_roots_range), - key: key.0, - storage_key: None, - retry_count: None, - }; - let local_result = local_checker - .check_changes_proof( - &request, - ChangesProof { - max_block: remote_proof.max_block, - proof: remote_proof.proof, - roots: remote_proof.roots, - roots_proof: remote_proof.roots_proof, - }, - ) - .unwrap(); - - // ..and ensure that result is the same as on remote node - if local_result != expected_result { - panic!( - "Failed test {}: local = {:?}, expected = {:?}", - index, local_result, expected_result, - ); - } - } -} - -#[test] -fn changes_proof_is_generated_and_checked_when_headers_are_pruned() { - // we're testing this test case here: - // (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]), - let (remote_client, remote_roots, _) = prepare_client_with_key_changes(); - let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec(); - let dave = StorageKey(dave); - - // 'fetch' changes proof from remote node: - // we're fetching changes for range b1..b4 - // we do not know changes trie roots before b3 (i.e. we only know b3+b4) - // but we have changes trie CHT root for b1...b4 - let b1 = remote_client.block_hash_from_id(&BlockId::Number(1)).unwrap().unwrap(); - let b3 = remote_client.block_hash_from_id(&BlockId::Number(3)).unwrap().unwrap(); - let b4 = remote_client.block_hash_from_id(&BlockId::Number(4)).unwrap().unwrap(); - let remote_proof = remote_client - .key_changes_proof_with_cht_size(b1, b4, b3, b4, None, &dave, 4) - .unwrap(); - - // prepare local checker, having a root of changes trie CHT#0 - let local_cht_root = cht::compute_root::( - 4, - 0, - remote_roots.iter().cloned().map(|ct| Ok(Some(ct))), - ) - .unwrap(); - let mut local_storage = DummyStorage::new(); - local_storage.changes_tries_cht_roots.insert(0, local_cht_root); - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(local_storage)), - local_executor(), - Box::new(TaskExecutor::new()), - ); - - // check proof on local client - let config = ChangesTrieConfiguration::new(4, 2); - let request = RemoteChangesRequest::
{ - changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { - zero: (0, Default::default()), - end: None, - config: Some(config), - }], - first_block: (1, b1), - last_block: (4, b4), - max_block: (4, b4), - tries_roots: (3, b3, vec![remote_roots[2].clone(), remote_roots[3].clone()]), - storage_key: None, - key: dave.0, - retry_count: None, - }; - let local_result = local_checker - .check_changes_proof_with_cht_size( - &request, - ChangesProof { - max_block: remote_proof.max_block, - proof: remote_proof.proof, - roots: remote_proof.roots, - roots_proof: remote_proof.roots_proof, - }, - 4, - ) - .unwrap(); - - assert_eq!(local_result, vec![(4, 0), (1, 1), (1, 0)]); -} - -#[test] -fn check_changes_proof_fails_if_proof_is_wrong() { - let (remote_client, local_roots, test_cases) = prepare_client_with_key_changes(); - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - Box::new(TaskExecutor::new()), - ); - let local_checker = &local_checker as &dyn FetchChecker; - let max = remote_client.chain_info().best_number; - let max_hash = remote_client.chain_info().best_hash; - - let (begin, end, key, _) = test_cases[0].clone(); - let begin_hash = remote_client.block_hash(begin).unwrap().unwrap(); - let end_hash = remote_client.block_hash(end).unwrap().unwrap(); - - // 'fetch' changes proof from remote node - let key = StorageKey(key); - let remote_proof = remote_client - .key_changes_proof(begin_hash, end_hash, begin_hash, max_hash, None, &key) - .unwrap(); - - let local_roots_range = local_roots.clone()[(begin - 1) as usize..].to_vec(); - let config = ChangesTrieConfiguration::new(4, 2); - let request = RemoteChangesRequest::
{ - changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { - zero: (0, Default::default()), - end: None, - config: Some(config), - }], - first_block: (begin, begin_hash), - last_block: (end, end_hash), - max_block: (max, max_hash), - tries_roots: (begin, begin_hash, local_roots_range.clone()), - storage_key: None, - key: key.0, - retry_count: None, - }; - - // check proof on local client using max from the future - assert!(local_checker - .check_changes_proof( - &request, - ChangesProof { - max_block: remote_proof.max_block + 1, - proof: remote_proof.proof.clone(), - roots: remote_proof.roots.clone(), - roots_proof: remote_proof.roots_proof.clone(), - } - ) - .is_err()); - - // check proof on local client using broken proof - assert!(local_checker - .check_changes_proof( - &request, - ChangesProof { - max_block: remote_proof.max_block, - proof: local_roots_range.clone().into_iter().map(|v| v.as_ref().to_vec()).collect(), - roots: remote_proof.roots, - roots_proof: remote_proof.roots_proof, - } - ) - .is_err()); - - // extra roots proofs are provided - assert!(local_checker - .check_changes_proof( - &request, - ChangesProof { - max_block: remote_proof.max_block, - proof: remote_proof.proof.clone(), - roots: vec![(begin - 1, Default::default())].into_iter().collect(), - roots_proof: StorageProof::empty(), - } - ) - .is_err()); - assert!(local_checker - .check_changes_proof( - &request, - ChangesProof { - max_block: remote_proof.max_block, - proof: remote_proof.proof.clone(), - roots: vec![(end + 1, Default::default())].into_iter().collect(), - roots_proof: StorageProof::empty(), - } - ) - .is_err()); -} - -#[test] -fn check_changes_tries_proof_fails_if_proof_is_wrong() { - // we're testing this test case here: - // (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]), - let (remote_client, remote_roots, _) = prepare_client_with_key_changes(); - let local_cht_root = cht::compute_root::( - 4, - 0, - remote_roots.iter().cloned().map(|ct| Ok(Some(ct))), - ) - .unwrap(); - let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec(); - let dave = StorageKey(dave); - - // 'fetch' changes proof from remote node: - // we're fetching changes for range b1..b4 - // we do not know changes trie roots before b3 (i.e. we only know b3+b4) - // but we have changes trie CHT root for b1...b4 - let b1 = remote_client.block_hash_from_id(&BlockId::Number(1)).unwrap().unwrap(); - let b3 = remote_client.block_hash_from_id(&BlockId::Number(3)).unwrap().unwrap(); - let b4 = remote_client.block_hash_from_id(&BlockId::Number(4)).unwrap().unwrap(); - let remote_proof = remote_client - .key_changes_proof_with_cht_size(b1, b4, b3, b4, None, &dave, 4) - .unwrap(); - - // fails when changes trie CHT is missing from the local db - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - Box::new(TaskExecutor::new()), - ); - assert!(local_checker - .check_changes_tries_proof(4, &remote_proof.roots, remote_proof.roots_proof.clone()) - .is_err()); - - // fails when proof is broken - let mut local_storage = DummyStorage::new(); - local_storage.changes_tries_cht_roots.insert(0, local_cht_root); - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(local_storage)), - local_executor(), - Box::new(TaskExecutor::new()), - ); - let result = - local_checker.check_changes_tries_proof(4, &remote_proof.roots, StorageProof::empty()); - assert!(result.is_err()); -} - -#[test] -fn check_body_proof_faulty() { - let header = - header_with_computed_extrinsics_root(vec![Extrinsic::IncludeData(vec![1, 2, 3, 4])]); - let block = Block::new(header.clone(), Vec::new()); - - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - Box::new(TaskExecutor::new()), - ); - - let body_request = RemoteBodyRequest { header: header.clone(), retry_count: None }; - - assert!( - local_checker.check_body_proof(&body_request, block.extrinsics).is_err(), - "vec![1, 2, 3, 4] != vec![]" - ); -} - -#[test] -fn check_body_proof_of_same_data_should_succeed() { - let extrinsics = vec![Extrinsic::IncludeData(vec![1, 2, 3, 4, 5, 6, 7, 8, 255])]; - - let header = header_with_computed_extrinsics_root(extrinsics.clone()); - let block = Block::new(header.clone(), extrinsics); - - let local_checker = TestChecker::new( - Arc::new(DummyBlockchain::new(DummyStorage::new())), - local_executor(), - Box::new(TaskExecutor::new()), - ); - - let body_request = RemoteBodyRequest { header: header.clone(), retry_count: None }; - - assert!(local_checker.check_body_proof(&body_request, block.extrinsics).is_ok()); -} diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index d82a839936d7..c86b23347d41 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,36 +20,32 @@ use futures::executor::block_on; use hex_literal::hex; use parity_scale_codec::{Decode, Encode, Joiner}; use sc_block_builder::BlockBuilderProvider; -use sc_client_api::{in_mem, BlockBackend, BlockchainEvents, StorageProvider}; -use sc_client_db::{ - Backend, DatabaseSettings, DatabaseSource, KeepBlocks, PruningMode, TransactionStorageMode, +use sc_client_api::{ + in_mem, BlockBackend, BlockchainEvents, FinalityNotifications, StorageProvider, }; +use sc_client_db::{Backend, DatabaseSettings, DatabaseSource, KeepBlocks, PruningMode}; use sc_consensus::{ BlockCheckParams, BlockImport, BlockImportParams, ForkChoiceStrategy, ImportResult, }; -use sc_service::client::{self, new_in_mem, Client, LocalCallExecutor}; +use sc_service::client::{new_in_mem, Client, LocalCallExecutor}; use sp_api::ProvideRuntimeApi; use sp_consensus::{BlockOrigin, BlockStatus, Error as ConsensusError, SelectChain}; -use sp_core::{blake2_256, testing::TaskExecutor, ChangesTrieConfiguration, H256}; +use sp_core::{testing::TaskExecutor, H256}; use sp_runtime::{ generic::BlockId, traits::{BlakeTwo256, Block as BlockT, Header as HeaderT}, - ConsensusEngineId, DigestItem, Justifications, + ConsensusEngineId, Justifications, StateVersion, }; use sp_state_machine::{ backend::Backend as _, ExecutionStrategy, InMemoryBackend, OverlayedChanges, StateMachine, }; use sp_storage::{ChildInfo, StorageKey}; -use sp_trie::{trie_types::Layout, TrieConfiguration}; -use std::{ - collections::{HashMap, HashSet}, - sync::Arc, -}; +use sp_trie::{LayoutV0, TrieConfiguration}; +use std::{collections::HashSet, sync::Arc}; use substrate_test_runtime::TestAPI; use substrate_test_runtime_client::{ prelude::*, runtime::{ - self, genesismap::{insert_genesis_block, GenesisConfig}, Block, BlockNumber, Digest, Hash, Header, RuntimeApi, Transfer, }, @@ -58,7 +54,6 @@ use substrate_test_runtime_client::{ }; mod db; -mod light; const TEST_ENGINE_ID: ConsensusEngineId = *b"TEST"; @@ -77,87 +72,12 @@ impl sc_executor::NativeExecutionDispatch for ExecutorDispatch { } fn executor() -> sc_executor::NativeElseWasmExecutor { - sc_executor::NativeElseWasmExecutor::new(sc_executor::WasmExecutionMethod::Interpreted, None, 8) -} - -pub fn prepare_client_with_key_changes() -> ( - client::Client< - substrate_test_runtime_client::Backend, - substrate_test_runtime_client::ExecutorDispatch, - Block, - RuntimeApi, - >, - Vec, - Vec<(u64, u64, Vec, Vec<(u64, u32)>)>, -) { - // prepare block structure - let blocks_transfers = vec![ - vec![ - (AccountKeyring::Alice, AccountKeyring::Dave), - (AccountKeyring::Bob, AccountKeyring::Dave), - ], - vec![(AccountKeyring::Charlie, AccountKeyring::Eve)], - vec![], - vec![(AccountKeyring::Alice, AccountKeyring::Dave)], - ]; - - // prepare client ang import blocks - let mut local_roots = Vec::new(); - let config = Some(ChangesTrieConfiguration::new(4, 2)); - let mut remote_client = TestClientBuilder::new().changes_trie_config(config).build(); - let mut nonces: HashMap<_, u64> = Default::default(); - for (i, block_transfers) in blocks_transfers.into_iter().enumerate() { - let mut builder = remote_client.new_block(Default::default()).unwrap(); - for (from, to) in block_transfers { - builder - .push_transfer(Transfer { - from: from.into(), - to: to.into(), - amount: 1, - nonce: *nonces.entry(from).and_modify(|n| *n = *n + 1).or_default(), - }) - .unwrap(); - } - let block = builder.build().unwrap().block; - block_on(remote_client.import(BlockOrigin::Own, block)).unwrap(); - - let header = remote_client.header(&BlockId::Number(i as u64 + 1)).unwrap().unwrap(); - let trie_root = header - .digest() - .log(DigestItem::as_changes_trie_root) - .map(|root| H256::from_slice(root.as_ref())) - .unwrap(); - local_roots.push(trie_root); - } - - // prepare test cases - let alice = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Alice.into())).to_vec(); - let bob = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Bob.into())).to_vec(); - let charlie = - blake2_256(&runtime::system::balance_of_key(AccountKeyring::Charlie.into())).to_vec(); - let dave = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Dave.into())).to_vec(); - let eve = blake2_256(&runtime::system::balance_of_key(AccountKeyring::Eve.into())).to_vec(); - let ferdie = - blake2_256(&runtime::system::balance_of_key(AccountKeyring::Ferdie.into())).to_vec(); - let test_cases = vec![ - (1, 4, alice.clone(), vec![(4, 0), (1, 0)]), - (1, 3, alice.clone(), vec![(1, 0)]), - (2, 4, alice.clone(), vec![(4, 0)]), - (2, 3, alice.clone(), vec![]), - (1, 4, bob.clone(), vec![(1, 1)]), - (1, 1, bob.clone(), vec![(1, 1)]), - (2, 4, bob.clone(), vec![]), - (1, 4, charlie.clone(), vec![(2, 0)]), - (1, 4, dave.clone(), vec![(4, 0), (1, 1), (1, 0)]), - (1, 1, dave.clone(), vec![(1, 1), (1, 0)]), - (3, 4, dave.clone(), vec![(4, 0)]), - (1, 4, eve.clone(), vec![(2, 0)]), - (1, 1, eve.clone(), vec![]), - (3, 4, eve.clone(), vec![]), - (1, 4, ferdie.clone(), vec![]), - ]; - - (remote_client, local_roots, test_cases) + sc_executor::NativeElseWasmExecutor::new( + sc_executor::WasmExecutionMethod::Interpreted, + None, + 8, + 2, + ) } fn construct_block( @@ -170,7 +90,7 @@ fn construct_block( let transactions = txs.into_iter().map(|tx| tx.into_signed_tx()).collect::>(); let iter = transactions.iter().map(Encode::encode); - let extrinsics_root = Layout::::ordered_trie_root(iter).into(); + let extrinsics_root = LayoutV0::::ordered_trie_root(iter).into(); let mut header = Header { parent_hash, @@ -187,7 +107,6 @@ fn construct_block( StateMachine::new( backend, - sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, &executor(), "Core_initialize_block", @@ -202,7 +121,6 @@ fn construct_block( for tx in transactions.iter() { StateMachine::new( backend, - sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, &executor(), "BlockBuilder_apply_extrinsic", @@ -217,7 +135,6 @@ fn construct_block( let ret_data = StateMachine::new( backend, - sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, &executor(), "BlockBuilder_finalize_block", @@ -248,10 +165,27 @@ fn block1(genesis_hash: Hash, backend: &InMemoryBackend) -> (Vec, + finalized: &[Hash], + stale_heads: &[Hash], +) { + match notifications.try_next() { + Ok(Some(notif)) => { + let stale_heads_expected: HashSet<_> = stale_heads.iter().collect(); + let stale_heads: HashSet<_> = notif.stale_heads.iter().collect(); + assert_eq!(notif.tree_route.as_ref(), &finalized[..finalized.len() - 1]); + assert_eq!(notif.hash, *finalized.last().unwrap()); + assert_eq!(stale_heads, stale_heads_expected); + }, + Ok(None) => panic!("unexpected notification result, client send channel was closed"), + Err(_) => assert!(finalized.is_empty()), + } +} + #[test] fn construct_genesis_should_work_with_native() { let mut storage = GenesisConfig::new( - None, vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()], vec![AccountKeyring::One.into(), AccountKeyring::Two.into()], 1000, @@ -261,7 +195,7 @@ fn construct_genesis_should_work_with_native() { .genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); - let backend = InMemoryBackend::from(storage); + let backend = InMemoryBackend::from((storage, StateVersion::default())); let (b1data, _b1hash) = block1(genesis_hash, &backend); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); @@ -270,7 +204,6 @@ fn construct_genesis_should_work_with_native() { let _ = StateMachine::new( &backend, - sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, &executor(), "Core_execute_block", @@ -286,7 +219,6 @@ fn construct_genesis_should_work_with_native() { #[test] fn construct_genesis_should_work_with_wasm() { let mut storage = GenesisConfig::new( - None, vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()], vec![AccountKeyring::One.into(), AccountKeyring::Two.into()], 1000, @@ -296,7 +228,7 @@ fn construct_genesis_should_work_with_wasm() { .genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); - let backend = InMemoryBackend::from(storage); + let backend = InMemoryBackend::from((storage, StateVersion::default())); let (b1data, _b1hash) = block1(genesis_hash, &backend); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); @@ -305,7 +237,6 @@ fn construct_genesis_should_work_with_wasm() { let _ = StateMachine::new( &backend, - sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, &executor(), "Core_execute_block", @@ -321,7 +252,6 @@ fn construct_genesis_should_work_with_wasm() { #[test] fn construct_genesis_with_bad_transaction_should_panic() { let mut storage = GenesisConfig::new( - None, vec![Sr25519Keyring::One.public().into(), Sr25519Keyring::Two.public().into()], vec![AccountKeyring::One.into(), AccountKeyring::Two.into()], 68, @@ -331,7 +261,7 @@ fn construct_genesis_with_bad_transaction_should_panic() { .genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); - let backend = InMemoryBackend::from(storage); + let backend = InMemoryBackend::from((storage, StateVersion::default())); let (b1data, _b1hash) = block1(genesis_hash, &backend); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); @@ -340,7 +270,6 @@ fn construct_genesis_with_bad_transaction_should_panic() { let r = StateMachine::new( &backend, - sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, &executor(), "Core_execute_block", @@ -507,8 +436,8 @@ fn uncles_with_multiple_forks() { // block tree: // G -> A1 -> A2 -> A3 -> A4 -> A5 // A1 -> B2 -> B3 -> B4 - // B2 -> C3 - // A1 -> D2 + // B2 -> C3 + // A1 -> D2 let mut client = substrate_test_runtime_client::new(); // G -> A1 @@ -909,27 +838,14 @@ fn best_containing_on_longest_chain_with_max_depth_higher_than_best() { ); } -#[test] -fn key_changes_works() { - let (client, _, test_cases) = prepare_client_with_key_changes(); - - for (index, (begin, end, key, expected_result)) in test_cases.into_iter().enumerate() { - let end = client.block_hash(end).unwrap().unwrap(); - let actual_result = - client.key_changes(begin, BlockId::Hash(end), None, &StorageKey(key)).unwrap(); - if actual_result != expected_result { - panic!( - "Failed test {}: actual = {:?}, expected = {:?}", - index, actual_result, expected_result, - ); - } - } -} - #[test] fn import_with_justification() { + // block tree: + // G -> A1 -> A2 -> A3 let mut client = substrate_test_runtime_client::new(); + let mut finality_notifications = client.finality_notification_stream(); + // G -> A1 let a1 = client.new_block(Default::default()).unwrap().build().unwrap().block; block_on(client.import(BlockOrigin::Own, a1.clone())).unwrap(); @@ -961,6 +877,10 @@ fn import_with_justification() { assert_eq!(client.justifications(&BlockId::Hash(a1.hash())).unwrap(), None); assert_eq!(client.justifications(&BlockId::Hash(a2.hash())).unwrap(), None); + + finality_notification_check(&mut finality_notifications, &[a1.hash(), a2.hash()], &[]); + finality_notification_check(&mut finality_notifications, &[a3.hash()], &[]); + assert!(finality_notifications.try_next().is_err()); } #[test] @@ -970,6 +890,9 @@ fn importing_diverged_finalized_block_should_trigger_reorg() { // G -> A1 -> A2 // \ // -> B1 + + let mut finality_notifications = client.finality_notification_stream(); + let a1 = client .new_block_at(&BlockId::Number(0), Default::default(), false) .unwrap() @@ -1008,6 +931,9 @@ fn importing_diverged_finalized_block_should_trigger_reorg() { assert_eq!(client.chain_info().best_hash, b1.hash()); assert_eq!(client.chain_info().finalized_hash, b1.hash()); + + finality_notification_check(&mut finality_notifications, &[b1.hash()], &[a2.hash()]); + assert!(finality_notifications.try_next().is_err()); } #[test] @@ -1017,6 +943,9 @@ fn finalizing_diverged_block_should_trigger_reorg() { // G -> A1 -> A2 // \ // -> B1 -> B2 + + let mut finality_notifications = client.finality_notification_stream(); + let a1 = client .new_block_at(&BlockId::Number(0), Default::default(), false) .unwrap() @@ -1081,6 +1010,113 @@ fn finalizing_diverged_block_should_trigger_reorg() { block_on(client.import(BlockOrigin::Own, b3.clone())).unwrap(); assert_eq!(client.chain_info().best_hash, b3.hash()); + + finality_notification_check(&mut finality_notifications, &[b1.hash()], &[a2.hash()]); + assert!(finality_notifications.try_next().is_err()); +} + +#[test] +fn finality_notifications_content() { + let (mut client, _select_chain) = TestClientBuilder::new().build_with_longest_chain(); + + // -> D3 -> D4 + // G -> A1 -> A2 -> A3 + // -> B1 -> B2 + // -> C1 + + let mut finality_notifications = client.finality_notification_stream(); + + let a1 = client + .new_block_at(&BlockId::Number(0), Default::default(), false) + .unwrap() + .build() + .unwrap() + .block; + block_on(client.import(BlockOrigin::Own, a1.clone())).unwrap(); + + let a2 = client + .new_block_at(&BlockId::Hash(a1.hash()), Default::default(), false) + .unwrap() + .build() + .unwrap() + .block; + block_on(client.import(BlockOrigin::Own, a2.clone())).unwrap(); + + let a3 = client + .new_block_at(&BlockId::Hash(a2.hash()), Default::default(), false) + .unwrap() + .build() + .unwrap() + .block; + block_on(client.import(BlockOrigin::Own, a3.clone())).unwrap(); + + let mut b1 = client.new_block_at(&BlockId::Number(0), Default::default(), false).unwrap(); + // needed to make sure B1 gets a different hash from A1 + b1.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 1, + nonce: 0, + }) + .unwrap(); + let b1 = b1.build().unwrap().block; + block_on(client.import(BlockOrigin::Own, b1.clone())).unwrap(); + + let b2 = client + .new_block_at(&BlockId::Hash(b1.hash()), Default::default(), false) + .unwrap() + .build() + .unwrap() + .block; + block_on(client.import(BlockOrigin::Own, b2.clone())).unwrap(); + + let mut c1 = client.new_block_at(&BlockId::Number(0), Default::default(), false).unwrap(); + // needed to make sure B1 gets a different hash from A1 + c1.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 2, + nonce: 0, + }) + .unwrap(); + let c1 = c1.build().unwrap().block; + block_on(client.import(BlockOrigin::Own, c1.clone())).unwrap(); + + let mut d3 = client + .new_block_at(&BlockId::Hash(a2.hash()), Default::default(), false) + .unwrap(); + // needed to make sure D3 gets a different hash from A3 + d3.push_transfer(Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 2, + nonce: 0, + }) + .unwrap(); + let d3 = d3.build().unwrap().block; + block_on(client.import(BlockOrigin::Own, d3.clone())).unwrap(); + + let d4 = client + .new_block_at(&BlockId::Hash(d3.hash()), Default::default(), false) + .unwrap() + .build() + .unwrap() + .block; + + // Postpone import to test behavior of import of finalized block. + + ClientExt::finalize_block(&client, BlockId::Hash(a2.hash()), None).unwrap(); + + // Import and finalize D4 + block_on(client.import_as_final(BlockOrigin::Own, d4.clone())).unwrap(); + + finality_notification_check( + &mut finality_notifications, + &[a1.hash(), a2.hash()], + &[c1.hash(), b2.hash()], + ); + finality_notification_check(&mut finality_notifications, &[d3.hash(), d4.hash()], &[a3.hash()]); + assert!(finality_notifications.try_next().is_err()); } #[test] @@ -1165,7 +1201,6 @@ fn doesnt_import_blocks_that_revert_finality() { state_cache_child_ratio: None, state_pruning: PruningMode::ArchiveAll, keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorageMode::BlockBody, source: DatabaseSource::RocksDb { path: tmp.path().into(), cache_size: 1024 }, }, u64::MAX, @@ -1175,6 +1210,8 @@ fn doesnt_import_blocks_that_revert_finality() { let mut client = TestClientBuilder::with_backend(backend).build(); + let mut finality_notifications = client.finality_notification_stream(); + // -> C1 // / // G -> A1 -> A2 @@ -1232,12 +1269,8 @@ fn doesnt_import_blocks_that_revert_finality() { ClientExt::finalize_block(&client, BlockId::Hash(a2.hash()), None).unwrap(); let import_err = block_on(client.import(BlockOrigin::Own, b3)).err().unwrap(); - let expected_err = ConsensusError::ClientImport( - sp_blockchain::Error::RuntimeApiError(sp_api::ApiError::Application(Box::new( - sp_blockchain::Error::NotInFinalizedChain, - ))) - .to_string(), - ); + let expected_err = + ConsensusError::ClientImport(sp_blockchain::Error::NotInFinalizedChain.to_string()); assert_eq!(import_err.to_string(), expected_err.to_string()); @@ -1260,6 +1293,9 @@ fn doesnt_import_blocks_that_revert_finality() { ConsensusError::ClientImport(sp_blockchain::Error::NotInFinalizedChain.to_string()); assert_eq!(import_err.to_string(), expected_err.to_string()); + + finality_notification_check(&mut finality_notifications, &[a1.hash(), a2.hash()], &[b2.hash()]); + assert!(finality_notifications.try_next().is_err()); } #[test] @@ -1380,7 +1416,6 @@ fn returns_status_for_pruned_blocks() { state_cache_child_ratio: None, state_pruning: PruningMode::keep_blocks(1), keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorageMode::BlockBody, source: DatabaseSource::RocksDb { path: tmp.path().into(), cache_size: 1024 }, }, u64::MAX, @@ -1539,152 +1574,6 @@ fn returns_status_for_pruned_blocks() { ); } -#[test] -fn imports_blocks_with_changes_tries_config_change() { - // create client with initial 4^2 configuration - let mut client = TestClientBuilder::with_default_backend() - .changes_trie_config(Some(ChangesTrieConfiguration { - digest_interval: 4, - digest_levels: 2, - })) - .build(); - - // =================================================================== - // blocks 1,2,3,4,5,6,7,8,9,10 are empty - // block 11 changes the key - // block 12 is the L1 digest that covers this change - // blocks 13,14,15,16,17,18,19,20,21,22 are empty - // block 23 changes the configuration to 5^1 AND is skewed digest - // =================================================================== - // blocks 24,25 are changing the key - // block 26 is empty - // block 27 changes the key - // block 28 is the L1 digest (NOT SKEWED!!!) that covers changes AND changes configuration to - // `3^1` - // =================================================================== - // block 29 is empty - // block 30 changes the key - // block 31 is L1 digest that covers this change - // =================================================================== - (1..11).for_each(|number| { - let block = client - .new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap() - .build() - .unwrap() - .block; - block_on(client.import(BlockOrigin::Own, block)).unwrap(); - }); - (11..12).for_each(|number| { - let mut block = client - .new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap(); - block - .push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())) - .unwrap(); - let block = block.build().unwrap().block; - block_on(client.import(BlockOrigin::Own, block)).unwrap(); - }); - (12..23).for_each(|number| { - let block = client - .new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap() - .build() - .unwrap() - .block; - block_on(client.import(BlockOrigin::Own, block)).unwrap(); - }); - (23..24).for_each(|number| { - let mut block = client - .new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap(); - block - .push_changes_trie_configuration_update(Some(ChangesTrieConfiguration { - digest_interval: 5, - digest_levels: 1, - })) - .unwrap(); - let block = block.build().unwrap().block; - block_on(client.import(BlockOrigin::Own, block)).unwrap(); - }); - (24..26).for_each(|number| { - let mut block = client - .new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap(); - block - .push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())) - .unwrap(); - let block = block.build().unwrap().block; - block_on(client.import(BlockOrigin::Own, block)).unwrap(); - }); - (26..27).for_each(|number| { - let block = client - .new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap() - .build() - .unwrap() - .block; - block_on(client.import(BlockOrigin::Own, block)).unwrap(); - }); - (27..28).for_each(|number| { - let mut block = client - .new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap(); - block - .push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())) - .unwrap(); - let block = block.build().unwrap().block; - block_on(client.import(BlockOrigin::Own, block)).unwrap(); - }); - (28..29).for_each(|number| { - let mut block = client - .new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap(); - block - .push_changes_trie_configuration_update(Some(ChangesTrieConfiguration { - digest_interval: 3, - digest_levels: 1, - })) - .unwrap(); - let block = block.build().unwrap().block; - block_on(client.import(BlockOrigin::Own, block)).unwrap(); - }); - (29..30).for_each(|number| { - let block = client - .new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap() - .build() - .unwrap() - .block; - block_on(client.import(BlockOrigin::Own, block)).unwrap(); - }); - (30..31).for_each(|number| { - let mut block = client - .new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap(); - block - .push_storage_change(vec![42], Some(number.to_le_bytes().to_vec())) - .unwrap(); - let block = block.build().unwrap().block; - block_on(client.import(BlockOrigin::Own, block)).unwrap(); - }); - (31..32).for_each(|number| { - let block = client - .new_block_at(&BlockId::Number(number - 1), Default::default(), false) - .unwrap() - .build() - .unwrap() - .block; - block_on(client.import(BlockOrigin::Own, block)).unwrap(); - }); - - // now check that configuration cache works - assert_eq!( - client.key_changes(1, BlockId::Number(31), None, &StorageKey(vec![42])).unwrap(), - vec![(30, 0), (27, 0), (25, 0), (24, 0), (11, 0)] - ); -} - #[test] fn storage_keys_iter_prefix_and_start_key_works() { let child_info = ChildInfo::new_default(b"child"); @@ -1759,12 +1648,21 @@ fn storage_keys_iter_works() { let res: Vec<_> = client .storage_keys_iter(&BlockId::Number(0), Some(&prefix), None) .unwrap() - .take(2) - .map(|x| x.0) + .take(8) + .map(|x| hex::encode(&x.0)) .collect(); assert_eq!( res, - [hex!("0befda6e1ca4ef40219d588a727f1271").to_vec(), hex!("3a636f6465").to_vec()] + [ + "00c232cf4e70a5e343317016dc805bf80a6a8cd8ad39958d56f99891b07851e0", + "085b2407916e53a86efeb8b72dbe338c4b341dab135252f96b6ed8022209b6cb", + "0befda6e1ca4ef40219d588a727f1271", + "1a560ecfd2a62c2b8521ef149d0804eb621050e3988ed97dca55f0d7c3e6aa34", + "1d66850d32002979d67dd29dc583af5b2ae2a1f71c1f35ad90fff122be7a3824", + "237498b98d8803334286e9f0483ef513098dd3c1c22ca21c4dc155b4ef6cc204", + "29b9db10ec5bf7907d8f74b5e60aa8140c4fbdd8127a1ee5600cb98e5ec01729", + "3a636f6465", + ] ); let res: Vec<_> = client @@ -1774,15 +1672,19 @@ fn storage_keys_iter_works() { Some(&StorageKey(hex!("3a636f6465").to_vec())), ) .unwrap() - .take(3) - .map(|x| x.0) + .take(7) + .map(|x| hex::encode(&x.0)) .collect(); assert_eq!( res, [ - hex!("3a686561707061676573").to_vec(), - hex!("6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081").to_vec(), - hex!("79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d").to_vec(), + "3a686561707061676573", + "52008686cc27f6e5ed83a216929942f8bcd32a396f09664a5698f81371934b56", + "5348d72ac6cc66e5d8cbecc27b0e0677503b845fe2382d819f83001781788fd5", + "5c2d5fda66373dabf970e4fb13d277ce91c5233473321129d32b5a8085fa8133", + "6644b9b8bc315888ac8e41a7968dc2b4141a5403c58acdf70b7e8f7e07bf5081", + "66484000ed3f75c95fc7b03f39c20ca1e1011e5999278247d3b2f5e3c3273808", + "79c07e2b1d2e2abfd4855b936617eeff5e0621c4869aa60c02be9adcc98a0d1d", ] ); @@ -1795,12 +1697,18 @@ fn storage_keys_iter_works() { )), ) .unwrap() - .take(1) - .map(|x| x.0) + .take(5) + .map(|x| hex::encode(x.0)) .collect(); assert_eq!( res, - [hex!("cf722c0832b5231d35e29f319ff27389f5032bfc7bfc3ba5ed7839f2042fb99f").to_vec()] + [ + "7d5007603a7f5dd729d51d93cf695d6465789443bb967c0d1fe270e388c96eaa", + "811ecfaadcf5f2ee1d67393247e2f71a1662d433e8ce7ff89fb0d4aa9561820b", + "a93d74caa7ec34ea1b04ce1e5c090245f867d333f0f88278a451e45299654dc5", + "a9ee1403384afbfc13f13be91ff70bfac057436212e53b9733914382ac942892", + "cf722c0832b5231d35e29f319ff27389f5032bfc7bfc3ba5ed7839f2042fb99f", + ] ); } diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index 8000c536cdf9..c492ed30f7d9 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -30,11 +30,12 @@ use sc_service::{ client::Client, config::{BasePath, DatabaseSource, KeystoreConfig}, ChainSpecExtension, Configuration, Error, GenericChainSpec, KeepBlocks, Role, RuntimeGenesis, - SpawnTaskHandle, TaskManager, TransactionStorageMode, + SpawnTaskHandle, TaskManager, }; use sc_transaction_pool_api::TransactionPool; +use sp_api::BlockId; use sp_blockchain::HeaderBackend; -use sp_runtime::{generic::BlockId, traits::Block as BlockT}; +use sp_runtime::traits::Block as BlockT; use std::{iter, net::Ipv4Addr, pin::Pin, sync::Arc, task::Context, time::Duration}; use tempfile::TempDir; use tokio::{runtime::Runtime, time}; @@ -45,22 +46,20 @@ mod client; /// Maximum duration of single wait call. const MAX_WAIT_TIME: Duration = Duration::from_secs(60 * 3); -struct TestNet { +struct TestNet { runtime: Runtime, authority_nodes: Vec<(usize, F, U, Multiaddr)>, full_nodes: Vec<(usize, F, U, Multiaddr)>, - light_nodes: Vec<(usize, L, Multiaddr)>, chain_spec: GenericChainSpec, base_port: u16, nodes: usize, } -impl Drop for TestNet { +impl Drop for TestNet { fn drop(&mut self) { // Drop the nodes before dropping the runtime, as the runtime otherwise waits for all // futures to be ended and we run into a dead lock. self.full_nodes.drain(..); - self.light_nodes.drain(..); self.authority_nodes.drain(..); } } @@ -156,39 +155,26 @@ where } } -impl TestNet +impl TestNet where F: Clone + Send + 'static, - L: Clone + Send + 'static, U: Clone + Send + 'static, { - pub fn run_until_all_full(&mut self, full_predicate: FP, light_predicate: LP) + pub fn run_until_all_full(&mut self, full_predicate: FP) where FP: Send + Fn(usize, &F) -> bool + 'static, - LP: Send + Fn(usize, &L) -> bool + 'static, { let full_nodes = self.full_nodes.clone(); - let light_nodes = self.light_nodes.clone(); let future = async move { let mut interval = time::interval(Duration::from_millis(100)); - loop { interval.tick().await; - let full_ready = full_nodes + if full_nodes .iter() - .all(|&(ref id, ref service, _, _)| full_predicate(*id, service)); - - if !full_ready { - continue - } - - let light_ready = light_nodes - .iter() - .all(|&(ref id, ref service, _)| light_predicate(*id, service)); - - if light_ready { - return + .all(|&(ref id, ref service, _, _)| full_predicate(*id, service)) + { + break } } }; @@ -249,7 +235,6 @@ fn node_config< state_cache_child_ratio: None, state_pruning: Default::default(), keep_blocks: KeepBlocks::All, - transaction_storage: TransactionStorageMode::BlockBody, chain_spec: Box::new((*spec).clone()), wasm_method: sc_service::config::WasmExecutionMethod::Interpreted, wasm_runtime_overrides: Default::default(), @@ -261,6 +246,7 @@ fn node_config< rpc_cors: None, rpc_methods: Default::default(), rpc_max_payload: None, + ws_max_out_buffer_capacity: None, prometheus_config: None, telemetry_endpoints: None, default_heap_pages: None, @@ -274,14 +260,13 @@ fn node_config< announce_block: true, base_path: Some(BasePath::new(root)), informant_output_format: Default::default(), - disable_log_reloading: false, + runtime_cache_size: 2, } } -impl TestNet +impl TestNet where F: TestNetNode, - L: TestNetNode, E: ChainSpecExtension + Clone + 'static + Send + Sync, G: RuntimeGenesis + 'static, { @@ -289,10 +274,9 @@ where temp: &TempDir, spec: GenericChainSpec, full: impl Iterator Result<(F, U), Error>>, - light: impl Iterator Result>, authorities: impl Iterator Result<(F, U), Error>)>, base_port: u16, - ) -> TestNet { + ) -> TestNet { sp_tracing::try_init_simple(); fdlimit::raise_fd_limit(); let runtime = Runtime::new().expect("Error creating tokio runtime"); @@ -300,12 +284,11 @@ where runtime, authority_nodes: Default::default(), full_nodes: Default::default(), - light_nodes: Default::default(), chain_spec: spec, base_port, nodes: 0, }; - net.insert_nodes(temp, full, light, authorities); + net.insert_nodes(temp, full, authorities); net } @@ -313,7 +296,6 @@ where &mut self, temp: &TempDir, full: impl Iterator Result<(F, U), Error>>, - light: impl Iterator Result>, authorities: impl Iterator Result<(F, U), Error>)>, ) { let handle = self.runtime.handle().clone(); @@ -358,26 +340,6 @@ where self.full_nodes.push((self.nodes, service, user_data, addr)); self.nodes += 1; } - - for light in light { - let node_config = node_config( - self.nodes, - &self.chain_spec, - Role::Light, - handle.clone(), - None, - self.base_port, - &temp, - ); - let addr = node_config.network.listen_addresses.iter().next().unwrap().clone(); - let service = light(node_config).expect("Error creating test node service"); - - handle.spawn(service.clone().map_err(|_| ())); - let addr = addr - .with(multiaddr::Protocol::P2p(service.network().local_peer_id().clone().into())); - self.light_nodes.push((self.nodes, service, addr)); - self.nodes += 1; - } } } @@ -388,23 +350,16 @@ fn tempdir_with_prefix(prefix: &str) -> TempDir { .expect("Error creating test dir") } -pub fn connectivity( - spec: GenericChainSpec, - full_builder: Fb, - light_builder: Lb, -) where +pub fn connectivity(spec: GenericChainSpec, full_builder: Fb) +where E: ChainSpecExtension + Clone + 'static + Send + Sync, G: RuntimeGenesis + 'static, Fb: Fn(Configuration) -> Result, F: TestNetNode, - Lb: Fn(Configuration) -> Result, - L: TestNetNode, { const NUM_FULL_NODES: usize = 5; - const NUM_LIGHT_NODES: usize = 5; - let expected_full_connections = NUM_FULL_NODES - 1 + NUM_LIGHT_NODES; - let expected_light_connections = NUM_FULL_NODES; + let expected_full_connections = NUM_FULL_NODES - 1; { let temp = tempdir_with_prefix("substrate-connectivity-test"); @@ -413,7 +368,6 @@ pub fn connectivity( &temp, spec.clone(), (0..NUM_FULL_NODES).map(|_| |cfg| full_builder(cfg).map(|s| (s, ()))), - (0..NUM_LIGHT_NODES).map(|_| |cfg| light_builder(cfg)), // Note: this iterator is empty but we can't just use `iter::empty()`, otherwise // the type of the closure cannot be inferred. (0..0).map(|_| (String::new(), { |cfg| full_builder(cfg).map(|s| (s, ())) })), @@ -427,25 +381,12 @@ pub fn connectivity( .add_reserved_peer(first_address.to_string()) .expect("Error adding reserved peer"); } - for (_, service, _) in network.light_nodes.iter() { - service - .network() - .add_reserved_peer(first_address.to_string()) - .expect("Error adding reserved peer"); - } - network.run_until_all_full( - move |_index, service| { - let connected = service.network().num_connected(); - debug!("Got {}/{} full connections...", connected, expected_full_connections); - connected == expected_full_connections - }, - move |_index, service| { - let connected = service.network().num_connected(); - debug!("Got {}/{} light connections...", connected, expected_light_connections); - connected == expected_light_connections - }, - ); + network.run_until_all_full(move |_index, service| { + let connected = service.network().num_connected(); + debug!("Got {}/{} full connections...", connected, expected_full_connections); + connected == expected_full_connections + }); }; temp.close().expect("Error removing temp dir"); @@ -457,7 +398,6 @@ pub fn connectivity( &temp, spec, (0..NUM_FULL_NODES).map(|_| |cfg| full_builder(cfg).map(|s| (s, ()))), - (0..NUM_LIGHT_NODES).map(|_| |cfg| light_builder(cfg)), // Note: this iterator is empty but we can't just use `iter::empty()`, otherwise // the type of the closure cannot be inferred. (0..0).map(|_| (String::new(), { |cfg| full_builder(cfg).map(|s| (s, ())) })), @@ -465,8 +405,7 @@ pub fn connectivity( ); info!("Checking linked topology"); let mut address = network.full_nodes[0].3.clone(); - let max_nodes = std::cmp::max(NUM_FULL_NODES, NUM_LIGHT_NODES); - for i in 0..max_nodes { + for i in 0..NUM_FULL_NODES { if i != 0 { if let Some((_, service, _, node_id)) = network.full_nodes.get(i) { service @@ -476,44 +415,26 @@ pub fn connectivity( address = node_id.clone(); } } - - if let Some((_, service, node_id)) = network.light_nodes.get(i) { - service - .network() - .add_reserved_peer(address.to_string()) - .expect("Error adding reserved peer"); - address = node_id.clone(); - } } - network.run_until_all_full( - move |_index, service| { - let connected = service.network().num_connected(); - debug!("Got {}/{} full connections...", connected, expected_full_connections); - connected == expected_full_connections - }, - move |_index, service| { - let connected = service.network().num_connected(); - debug!("Got {}/{} light connections...", connected, expected_light_connections); - connected == expected_light_connections - }, - ); + network.run_until_all_full(move |_index, service| { + let connected = service.network().num_connected(); + debug!("Got {}/{} full connections...", connected, expected_full_connections); + connected == expected_full_connections + }); } temp.close().expect("Error removing temp dir"); } } -pub fn sync( +pub fn sync( spec: GenericChainSpec, full_builder: Fb, - light_builder: Lb, mut make_block_and_import: B, mut extrinsic_factory: ExF, ) where Fb: Fn(Configuration) -> Result<(F, U), Error>, F: TestNetNode, - Lb: Fn(Configuration) -> Result, - L: TestNetNode, B: FnMut(&F, &mut U), ExF: FnMut(&F, &U) -> ::Extrinsic, U: Clone + Send + 'static, @@ -521,15 +442,12 @@ pub fn sync( G: RuntimeGenesis + 'static, { const NUM_FULL_NODES: usize = 10; - // FIXME: BABE light client support is currently not working. - const NUM_LIGHT_NODES: usize = 10; const NUM_BLOCKS: usize = 512; let temp = tempdir_with_prefix("substrate-sync-test"); let mut network = TestNet::new( &temp, spec, (0..NUM_FULL_NODES).map(|_| |cfg| full_builder(cfg)), - (0..NUM_LIGHT_NODES).map(|_| |cfg| light_builder(cfg)), // Note: this iterator is empty but we can't just use `iter::empty()`, otherwise // the type of the closure cannot be inferred. (0..0).map(|_| (String::new(), { |cfg| full_builder(cfg) })), @@ -560,16 +478,10 @@ pub fn sync( .add_reserved_peer(first_address.to_string()) .expect("Error adding reserved peer"); } - for (_, service, _) in network.light_nodes.iter() { - service - .network() - .add_reserved_peer(first_address.to_string()) - .expect("Error adding reserved peer"); - } - network.run_until_all_full( - |_index, service| service.client().info().best_number == (NUM_BLOCKS as u32).into(), - |_index, service| service.client().info().best_number == (NUM_BLOCKS as u32).into(), - ); + + network.run_until_all_full(|_index, service| { + service.client().info().best_number == (NUM_BLOCKS as u32).into() + }); info!("Checking extrinsic propagation"); let first_service = network.full_nodes[0].1.clone(); @@ -585,34 +497,26 @@ pub fn sync( )) .expect("failed to submit extrinsic"); - network.run_until_all_full( - |_index, service| service.transaction_pool().ready().count() == 1, - |_index, _service| true, - ); + network.run_until_all_full(|_index, service| service.transaction_pool().ready().count() == 1); } -pub fn consensus( +pub fn consensus( spec: GenericChainSpec, full_builder: Fb, - light_builder: Lb, authorities: impl IntoIterator, ) where Fb: Fn(Configuration) -> Result, F: TestNetNode, - Lb: Fn(Configuration) -> Result, - L: TestNetNode, E: ChainSpecExtension + Clone + 'static + Send + Sync, G: RuntimeGenesis + 'static, { const NUM_FULL_NODES: usize = 10; - const NUM_LIGHT_NODES: usize = 10; const NUM_BLOCKS: usize = 10; // 10 * 2 sec block production time = ~20 seconds let temp = tempdir_with_prefix("substrate-consensus-test"); let mut network = TestNet::new( &temp, spec, (0..NUM_FULL_NODES / 2).map(|_| |cfg| full_builder(cfg).map(|s| (s, ()))), - (0..NUM_LIGHT_NODES / 2).map(|_| |cfg| light_builder(cfg)), authorities .into_iter() .map(|key| (key, { |cfg| full_builder(cfg).map(|s| (s, ())) })), @@ -627,30 +531,20 @@ pub fn consensus( .add_reserved_peer(first_address.to_string()) .expect("Error adding reserved peer"); } - for (_, service, _) in network.light_nodes.iter() { - service - .network() - .add_reserved_peer(first_address.to_string()) - .expect("Error adding reserved peer"); - } for (_, service, _, _) in network.authority_nodes.iter().skip(1) { service .network() .add_reserved_peer(first_address.to_string()) .expect("Error adding reserved peer"); } - network.run_until_all_full( - |_index, service| { - service.client().info().finalized_number >= (NUM_BLOCKS as u32 / 2).into() - }, - |_index, service| service.client().info().best_number >= (NUM_BLOCKS as u32 / 2).into(), - ); + network.run_until_all_full(|_index, service| { + service.client().info().finalized_number >= (NUM_BLOCKS as u32 / 2).into() + }); info!("Adding more peers"); network.insert_nodes( &temp, (0..NUM_FULL_NODES / 2).map(|_| |cfg| full_builder(cfg).map(|s| (s, ()))), - (0..NUM_LIGHT_NODES / 2).map(|_| |cfg| light_builder(cfg)), // Note: this iterator is empty but we can't just use `iter::empty()`, otherwise // the type of the closure cannot be inferred. (0..0).map(|_| (String::new(), { |cfg| full_builder(cfg).map(|s| (s, ())) })), @@ -661,14 +555,8 @@ pub fn consensus( .add_reserved_peer(first_address.to_string()) .expect("Error adding reserved peer"); } - for (_, service, _) in network.light_nodes.iter() { - service - .network() - .add_reserved_peer(first_address.to_string()) - .expect("Error adding reserved peer"); - } - network.run_until_all_full( - |_index, service| service.client().info().finalized_number >= (NUM_BLOCKS as u32).into(), - |_index, service| service.client().info().best_number >= (NUM_BLOCKS as u32).into(), - ); + + network.run_until_all_full(|_index, service| { + service.client().info().finalized_number >= (NUM_BLOCKS as u32).into() + }); } diff --git a/client/state-db/Cargo.toml b/client/state-db/Cargo.toml index 93d5e1464b39..42ef4338b406 100644 --- a/client/state-db/Cargo.toml +++ b/client/state-db/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-state-db" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "State database maintenance. Handles canonicalization and pruning in the database." readme = "README.md" @@ -13,10 +13,10 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -parking_lot = "0.11.1" +parking_lot = "0.12.0" log = "0.4.11" sc-client-api = { version = "4.0.0-dev", path = "../api" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -codec = { package = "parity-scale-codec", version = "2.0.0", features = ["derive"] } -parity-util-mem = { version = "0.10.0", default-features = false, features = ["primitive-types"] } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +codec = { package = "parity-scale-codec", version = "3.0.0", features = ["derive"] } +parity-util-mem = { version = "0.11.0", default-features = false, features = ["primitive-types"] } parity-util-mem-derive = "0.1.0" diff --git a/client/state-db/src/lib.rs b/client/state-db/src/lib.rs index 44629975d781..74f218e88f86 100644 --- a/client/state-db/src/lib.rs +++ b/client/state-db/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -130,6 +130,8 @@ pub enum Error { InvalidPruningMode(String), /// Too many unfinalized sibling blocks inserted. TooManySiblingBlocks, + /// Trying to insert existing block. + BlockAlreadyExists, } /// Pinning error type. @@ -154,6 +156,7 @@ impl fmt::Debug for Error { Error::InvalidParent => write!(f, "Trying to insert block with unknown parent"), Error::InvalidPruningMode(e) => write!(f, "Expected pruning mode: {}", e), Error::TooManySiblingBlocks => write!(f, "Too many sibling blocks inserted"), + Error::BlockAlreadyExists => write!(f, "Block already exists"), } } } diff --git a/client/state-db/src/noncanonical.rs b/client/state-db/src/noncanonical.rs index c726ceae4b05..7d6fcbced786 100644 --- a/client/state-db/src/noncanonical.rs +++ b/client/state-db/src/noncanonical.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -134,12 +134,12 @@ fn discard_descendants( hash: &BlockHash, ) -> u32 { let (first, mut remainder) = if let Some((first, rest)) = levels.0.split_first_mut() { - (Some(first), (rest, &mut levels.1[..])) + (Some(first), (rest, &mut *levels.1)) } else { if let Some((first, rest)) = levels.1.split_first_mut() { - (Some(first), (&mut levels.0[..], rest)) + (Some(first), (&mut *levels.0, rest)) } else { - (None, (&mut levels.0[..], &mut levels.1[..])) + (None, (&mut *levels.0, &mut *levels.1)) } }; let mut pinned_children = 0; @@ -210,9 +210,10 @@ impl NonCanonicalOverlay { insert_values(&mut values, record.inserted); trace!( target: "state-db", - "Uncanonicalized journal entry {}.{} ({} inserted, {} deleted)", + "Uncanonicalized journal entry {}.{} ({:?}) ({} inserted, {} deleted)", block, index, + record.hash, overlay.inserted.len(), overlay.deleted.len() ); @@ -261,8 +262,7 @@ impl NonCanonicalOverlay { .push((to_meta_key(LAST_CANONICAL, &()), last_canonicalized.encode())); self.last_canonicalized = Some(last_canonicalized); } else if self.last_canonicalized.is_some() { - if number < front_block_number || - number >= front_block_number + self.levels.len() as u64 + 1 + if number < front_block_number || number > front_block_number + self.levels.len() as u64 { trace!(target: "state-db", "Failed to insert block {}, current is {} .. {})", number, @@ -297,6 +297,9 @@ impl NonCanonicalOverlay { if level.blocks.len() >= MAX_BLOCKS_PER_LEVEL as usize { return Err(Error::TooManySiblingBlocks) } + if level.blocks.iter().any(|b| b.hash == *hash) { + return Err(Error::BlockAlreadyExists) + } let index = level.available_index(); let journal_key = to_journal_key(number, index); @@ -642,7 +645,7 @@ mod tests { use super::{to_journal_key, NonCanonicalOverlay}; use crate::{ test::{make_changeset, make_db}, - ChangeSet, CommitSet, MetaDb, + ChangeSet, CommitSet, Error, MetaDb, }; use sp_core::H256; use std::io; @@ -711,6 +714,20 @@ mod tests { .unwrap(); } + #[test] + fn insert_existing_fails() { + let db = make_db(&[]); + let h1 = H256::random(); + let mut overlay = NonCanonicalOverlay::::new(&db).unwrap(); + overlay + .insert::(&h1, 2, &H256::default(), ChangeSet::default()) + .unwrap(); + assert!(matches!( + overlay.insert::(&h1, 2, &H256::default(), ChangeSet::default()), + Err(Error::BlockAlreadyExists) + )); + } + #[test] #[should_panic] fn canonicalize_unknown_panics() { diff --git a/client/state-db/src/pruning.rs b/client/state-db/src/pruning.rs index 465c1ecda6cc..2631405cdffa 100644 --- a/client/state-db/src/pruning.rs +++ b/client/state-db/src/pruning.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/state-db/src/test.rs b/client/state-db/src/test.rs index ad5ce8e874cc..9fb97036b2f2 100644 --- a/client/state-db/src/test.rs +++ b/client/state-db/src/test.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/sync-state-rpc/Cargo.toml b/client/sync-state-rpc/Cargo.toml index b81fd1fd5c61..e13631f210bc 100644 --- a/client/sync-state-rpc/Cargo.toml +++ b/client/sync-state-rpc/Cargo.toml @@ -3,9 +3,9 @@ name = "sc-sync-state-rpc" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "A RPC handler to create sync states for light clients." -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,7 +13,7 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -thiserror = "1.0.21" +thiserror = "1.0.30" jsonrpc-core = "18.0.0" jsonrpc-core-client = "18.0.0" jsonrpc-derive = "18.0.0" @@ -22,9 +22,8 @@ sc-client-api = { version = "4.0.0-dev", path = "../api" } sc-consensus-babe = { version = "0.10.0-dev", path = "../consensus/babe" } sc-consensus-epochs = { version = "0.10.0-dev", path = "../consensus/epochs" } sc-finality-grandpa = { version = "0.10.0-dev", path = "../finality-grandpa" } -sc-rpc-api = { version = "0.10.0-dev", path = "../rpc-api" } -serde_json = "1.0.68" -serde = { version = "1.0.126", features = ["derive"] } +serde_json = "1.0.79" +serde = { version = "1.0.136", features = ["derive"] } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -codec = { package = "parity-scale-codec", version = "2.0.0" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "3.0.0" } diff --git a/client/sync-state-rpc/src/lib.rs b/client/sync-state-rpc/src/lib.rs index a1621e3986d7..6fc0d17800fe 100644 --- a/client/sync-state-rpc/src/lib.rs +++ b/client/sync-state-rpc/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -134,7 +134,6 @@ pub struct SyncStateRpcHandler { client: Arc, shared_authority_set: SharedAuthoritySet, shared_epoch_changes: SharedEpochChanges, - deny_unsafe: sc_rpc_api::DenyUnsafe, } impl SyncStateRpcHandler @@ -148,12 +147,11 @@ where client: Arc, shared_authority_set: SharedAuthoritySet, shared_epoch_changes: SharedEpochChanges, - deny_unsafe: sc_rpc_api::DenyUnsafe, ) -> Result> { if sc_chain_spec::get_extension::(chain_spec.extensions()) .is_some() { - Ok(Self { chain_spec, client, shared_authority_set, shared_epoch_changes, deny_unsafe }) + Ok(Self { chain_spec, client, shared_authority_set, shared_epoch_changes }) } else { Err(Error::::LightSyncStateExtensionNotFound) } @@ -185,10 +183,6 @@ where Backend: HeaderBackend + sc_client_api::AuxStore + 'static, { fn system_gen_sync_spec(&self, raw: bool) -> jsonrpc_core::Result { - if let Err(err) = self.deny_unsafe.check_if_safe() { - return Err(err.into()) - } - let mut chain_spec = self.chain_spec.cloned_box(); let sync_state = self.build_sync_state().map_err(map_error::>)?; diff --git a/client/telemetry/Cargo.toml b/client/telemetry/Cargo.toml index f115017f0970..67583e325f34 100644 --- a/client/telemetry/Cargo.toml +++ b/client/telemetry/Cargo.toml @@ -3,9 +3,9 @@ name = "sc-telemetry" version = "4.0.0-dev" authors = ["Parity Technologies "] description = "Telemetry utils" -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sc-telemetry" readme = "README.md" @@ -15,14 +15,14 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -parking_lot = "0.11.1" -futures = "0.3.9" +parking_lot = "0.12.0" +futures = "0.3.21" wasm-timer = "0.2.5" -libp2p = { version = "0.39.1", default-features = false, features = ["dns-async-std", "tcp-async-io", "wasm-ext", "websocket"] } +libp2p = { version = "0.40.0", default-features = false, features = ["dns-async-std", "tcp-async-io", "wasm-ext", "websocket"] } log = "0.4.8" -pin-project = "1.0.4" +pin-project = "1.0.10" rand = "0.7.2" -serde = { version = "1.0.126", features = ["derive"] } -serde_json = "1.0.68" +serde = { version = "1.0.136", features = ["derive"] } +serde_json = "1.0.79" chrono = "0.4.19" -thiserror = "1.0.21" +thiserror = "1.0.30" diff --git a/client/telemetry/src/endpoints.rs b/client/telemetry/src/endpoints.rs index 62e618031198..fba3822a9067 100644 --- a/client/telemetry/src/endpoints.rs +++ b/client/telemetry/src/endpoints.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/telemetry/src/error.rs b/client/telemetry/src/error.rs index 90a8018f4e1d..4d9cdc05c51b 100644 --- a/client/telemetry/src/error.rs +++ b/client/telemetry/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/telemetry/src/lib.rs b/client/telemetry/src/lib.rs index 9fb86f57d839..68128e020708 100644 --- a/client/telemetry/src/lib.rs +++ b/client/telemetry/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -440,7 +440,7 @@ enum Register { /// Report a telemetry. /// -/// Translates to [`tracing::info`], but contains an additional verbosity parameter which the log +/// Translates to `tracing::info`, but contains an additional verbosity parameter which the log /// record is tagged with. Additionally the verbosity parameter is added to the record as a /// key-value pair. /// diff --git a/client/telemetry/src/node.rs b/client/telemetry/src/node.rs index 4d845c328fe8..ec857ede70fd 100644 --- a/client/telemetry/src/node.rs +++ b/client/telemetry/src/node.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/telemetry/src/transport.rs b/client/telemetry/src/transport.rs index 04ec79ebf564..23725b44a64d 100644 --- a/client/telemetry/src/transport.rs +++ b/client/telemetry/src/transport.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/tracing/Cargo.toml b/client/tracing/Cargo.toml index 3e314a82aa58..599e310a8515 100644 --- a/client/tracing/Cargo.toml +++ b/client/tracing/Cargo.toml @@ -3,8 +3,8 @@ name = "sc-tracing" version = "4.0.0-dev" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" authors = ["Parity Technologies "] -edition = "2018" -homepage = "https://substrate.dev" +edition = "2021" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Instrumentation implementation for substrate." readme = "README.md" @@ -15,23 +15,32 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] ansi_term = "0.12.1" atty = "0.2.13" +chrono = "0.4.19" lazy_static = "1.4.0" +libc = "0.2.121" log = { version = "0.4.8" } -once_cell = "1.4.1" -parking_lot = "0.11.1" -regex = "1.4.2" +once_cell = "1.8.0" +parking_lot = "0.12.0" +regex = "1.5.5" rustc-hash = "1.1.0" -serde = "1.0.126" -thiserror = "1.0.21" -tracing = "0.1.25" +serde = "1.0.136" +thiserror = "1.0.30" +tracing = "0.1.29" tracing-log = "0.1.2" -tracing-subscriber = "0.2.19" -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } -sp-rpc = { version = "4.0.0-dev", path = "../../primitives/rpc" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } +tracing-subscriber = { version = "0.2.25", features = ["parking_lot"] } +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } +sp-rpc = { version = "6.0.0", path = "../../primitives/rpc" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } sc-client-api = { version = "4.0.0-dev", path = "../api" } sc-tracing-proc-macro = { version = "4.0.0-dev", path = "./proc-macro" } sc-rpc-server = { version = "4.0.0-dev", path = "../rpc-servers" } + +[dev-dependencies] +criterion = "0.3" + +[[bench]] +name = "bench" +harness = false diff --git a/client/tracing/benches/bench.rs b/client/tracing/benches/bench.rs new file mode 100644 index 000000000000..a939a2679739 --- /dev/null +++ b/client/tracing/benches/bench.rs @@ -0,0 +1,49 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use criterion::{criterion_group, criterion_main, Criterion}; +use tracing_subscriber::fmt::time::{ChronoLocal, FormatTime}; + +fn bench_fast_local_time(c: &mut Criterion) { + c.bench_function("fast_local_time", |b| { + let mut buffer = String::new(); + let t = sc_tracing::logging::FastLocalTime { with_fractional: true }; + b.iter(|| { + buffer.clear(); + t.format_time(&mut buffer).unwrap(); + }) + }); +} + +// This is here just as a point of comparison. +fn bench_chrono_local(c: &mut Criterion) { + c.bench_function("chrono_local", |b| { + let mut buffer = String::new(); + let t = ChronoLocal::with_format("%Y-%m-%d %H:%M:%S%.3f".to_string()); + b.iter(|| { + buffer.clear(); + t.format_time(&mut buffer).unwrap(); + }) + }); +} + +criterion_group! { + name = benches; + config = Criterion::default(); + targets = bench_fast_local_time, bench_chrono_local +} +criterion_main!(benches); diff --git a/client/tracing/proc-macro/Cargo.toml b/client/tracing/proc-macro/Cargo.toml index 002370b515f2..645a6ed93a16 100644 --- a/client/tracing/proc-macro/Cargo.toml +++ b/client/tracing/proc-macro/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-tracing-proc-macro" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Helper macros for Substrate's client CLI" @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -proc-macro-crate = "1.0.0" -proc-macro2 = "1.0.29" -quote = { version = "1.0.3", features = ["proc-macro"] } -syn = { version = "1.0.58", features = ["proc-macro", "full", "extra-traits", "parsing"] } +proc-macro-crate = "1.1.3" +proc-macro2 = "1.0.36" +quote = { version = "1.0.10", features = ["proc-macro"] } +syn = { version = "1.0.82", features = ["proc-macro", "full", "extra-traits", "parsing"] } diff --git a/client/tracing/proc-macro/src/lib.rs b/client/tracing/proc-macro/src/lib.rs index e9a4f58705b4..ba757619fb5a 100644 --- a/client/tracing/proc-macro/src/lib.rs +++ b/client/tracing/proc-macro/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -56,7 +56,7 @@ use syn::{Error, Expr, Ident, ItemFn}; /// 2020-10-16 08:03:14 ✌️ version 2.0.0-47f7d3f2e-x86_64-linux-gnu /// 2020-10-16 08:03:14 ❤️ by Anonymous, 2017-2020 /// 2020-10-16 08:03:14 📋 Chain specification: Local Testnet -/// 2020-10-16 08:03:14 🏷 Node name: nice-glove-1401 +/// 2020-10-16 08:03:14 🏷 Node name: nice-glove-1401 /// 2020-10-16 08:03:14 👤 Role: LIGHT /// 2020-10-16 08:03:14 💾 Database: RocksDb at /tmp/substrate95w2Dk/chains/local_testnet/db /// 2020-10-16 08:03:14 ⛓ Native runtime: node-template-1 (node-template-1.tx1.au1) @@ -64,7 +64,7 @@ use syn::{Error, Expr, Ident, ItemFn}; /// 2020-10-16 08:03:14 [light] Loading GRANDPA authorities from genesis on what appears to be first startup. /// 2020-10-16 08:03:15 [light] ⏱ Loaded block-time = 6000 milliseconds from genesis on first-launch /// 2020-10-16 08:03:15 [light] Using default protocol ID "sup" because none is configured in the chain specs -/// 2020-10-16 08:03:15 [light] 🏷 Local node identity is: 12D3KooWHX4rkWT6a6N55Km7ZnvenGdShSKPkzJ3yj9DU5nqDtWR +/// 2020-10-16 08:03:15 [light] 🏷 Local node identity is: 12D3KooWHX4rkWT6a6N55Km7ZnvenGdShSKPkzJ3yj9DU5nqDtWR /// 2020-10-16 08:03:15 [light] 📦 Highest known block at #0 /// 2020-10-16 08:03:15 [light] 〽️ Prometheus server started at 127.0.0.1:9615 /// 2020-10-16 08:03:15 [light] Listening for new connections on 127.0.0.1:9944. @@ -90,7 +90,7 @@ use syn::{Error, Expr, Ident, ItemFn}; /// 2020-10-16 08:12:57 ✌️ version 2.0.0-efb9b822a-x86_64-linux-gnu /// 2020-10-16 08:12:57 ❤️ by Anonymous, 2017-2020 /// 2020-10-16 08:12:57 📋 Chain specification: Local Testnet -/// 2020-10-16 08:12:57 🏷 Node name: open-harbor-1619 +/// 2020-10-16 08:12:57 🏷 Node name: open-harbor-1619 /// 2020-10-16 08:12:57 👤 Role: LIGHT /// 2020-10-16 08:12:57 💾 Database: RocksDb at /tmp/substrate9T9Mtb/chains/local_testnet/db /// 2020-10-16 08:12:57 ⛓ Native runtime: node-template-1 (node-template-1.tx1.au1) @@ -98,7 +98,7 @@ use syn::{Error, Expr, Ident, ItemFn}; /// 2020-10-16 08:12:58 [open-harbor-1619] Loading GRANDPA authorities from genesis on what appears to be first startup. /// 2020-10-16 08:12:58 [open-harbor-1619] ⏱ Loaded block-time = 6000 milliseconds from genesis on first-launch /// 2020-10-16 08:12:58 [open-harbor-1619] Using default protocol ID "sup" because none is configured in the chain specs -/// 2020-10-16 08:12:58 [open-harbor-1619] 🏷 Local node identity is: 12D3KooWRzmYC8QTK1Pm8Cfvid3skTS4Hn54jc4AUtje8Rqbfgtp +/// 2020-10-16 08:12:58 [open-harbor-1619] 🏷 Local node identity is: 12D3KooWRzmYC8QTK1Pm8Cfvid3skTS4Hn54jc4AUtje8Rqbfgtp /// 2020-10-16 08:12:58 [open-harbor-1619] 📦 Highest known block at #0 /// 2020-10-16 08:12:58 [open-harbor-1619] 〽️ Prometheus server started at 127.0.0.1:9615 /// 2020-10-16 08:12:58 [open-harbor-1619] Listening for new connections on 127.0.0.1:9944. @@ -119,7 +119,7 @@ pub fn prefix_logs_with(arg: TokenStream, item: TokenStream) -> TokenStream { let name = syn::parse_macro_input!(arg as Expr); let crate_name = match crate_name("sc-tracing") { - Ok(FoundCrate::Itself) => Ident::from(Ident::new("sc_tracing", Span::call_site())), + Ok(FoundCrate::Itself) => Ident::new("sc_tracing", Span::call_site()), Ok(FoundCrate::Name(crate_name)) => Ident::new(&crate_name, Span::call_site()), Err(e) => return Error::new(Span::call_site(), e).to_compile_error().into(), }; diff --git a/client/tracing/src/block/mod.rs b/client/tracing/src/block/mod.rs index 8280d4613a18..259827e4b47d 100644 --- a/client/tracing/src/block/mod.rs +++ b/client/tracing/src/block/mod.rs @@ -253,7 +253,7 @@ where self.client.runtime_api().execute_block(&parent_id, block) }) { return Err(Error::Dispatch( - format!("Failed to collect traces and execute block: {:?}", e).to_string(), + format!("Failed to collect traces and execute block: {}", e).to_string(), )) } } @@ -267,7 +267,7 @@ where .lock() .drain() // Patch wasm identifiers - .filter_map(|(_, s)| patch_and_filter(SpanDatum::from(s), targets)) + .filter_map(|(_, s)| patch_and_filter(s, targets)) .collect(); let events: Vec<_> = block_subscriber .events @@ -315,7 +315,7 @@ fn event_values_filter(event: &TraceEvent, filter_kind: &str, values: &str) -> b .values .string_values .get(filter_kind) - .and_then(|value| Some(check_target(values, value, &event.level))) + .map(|value| check_target(values, value, &event.level)) .unwrap_or(false) } diff --git a/client/tracing/src/lib.rs b/client/tracing/src/lib.rs index bf6e3d780c6e..ff3723e0d1a9 100644 --- a/client/tracing/src/lib.rs +++ b/client/tracing/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -58,7 +58,7 @@ const ZERO_DURATION: Duration = Duration::from_nanos(0); /// Responsible for assigning ids to new spans, which are not re-used. pub struct ProfilingLayer { targets: Vec<(String, Level)>, - trace_handler: Box, + trace_handlers: Vec>, } /// Used to configure how to receive the metrics @@ -76,14 +76,14 @@ impl Default for TracingReceiver { /// A handler for tracing `SpanDatum` pub trait TraceHandler: Send + Sync { - /// Process a `SpanDatum` - fn handle_span(&self, span: SpanDatum); - /// Process a `TraceEvent` - fn handle_event(&self, event: TraceEvent); + /// Process a `SpanDatum`. + fn handle_span(&self, span: &SpanDatum); + /// Process a `TraceEvent`. + fn handle_event(&self, event: &TraceEvent); } /// Represents a tracing event, complete with values -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct TraceEvent { /// Name of the event. pub name: String, @@ -98,7 +98,7 @@ pub struct TraceEvent { } /// Represents a single instance of a tracing span -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct SpanDatum { /// id for this span pub id: Id, @@ -213,6 +213,15 @@ impl fmt::Display for Values { } } +/// Trace handler event types. +#[derive(Debug)] +pub enum TraceHandlerEvents { + /// An event. + Event(TraceEvent), + /// A span. + Span(SpanDatum), +} + impl ProfilingLayer { /// Takes a `TracingReceiver` and a comma separated list of targets, /// either with a level: "pallet=trace,frame=debug" @@ -231,7 +240,12 @@ impl ProfilingLayer { /// wasm_tracing indicates whether to enable wasm traces pub fn new_with_handler(trace_handler: Box, targets: &str) -> Self { let targets: Vec<_> = targets.split(',').map(|s| parse_target(s)).collect(); - Self { targets, trace_handler } + Self { targets, trace_handlers: vec![trace_handler] } + } + + /// Attach additional handlers to allow handling of custom events/spans. + pub fn add_handler(&mut self, trace_handler: Box) { + self.trace_handlers.push(trace_handler); } fn check_target(&self, target: &str, level: &Level) -> bool { @@ -242,6 +256,18 @@ impl ProfilingLayer { } false } + + /// Sequentially dispatch a trace event to all handlers. + fn dispatch_event(&self, event: TraceHandlerEvents) { + match &event { + TraceHandlerEvents::Span(span_datum) => { + self.trace_handlers.iter().for_each(|handler| handler.handle_span(span_datum)); + }, + TraceHandlerEvents::Event(event) => { + self.trace_handlers.iter().for_each(|handler| handler.handle_event(event)); + }, + } + } } // Default to TRACE if no level given or unable to parse Level @@ -320,7 +346,7 @@ where values, parent_id, }; - self.trace_handler.handle_event(trace_event); + self.dispatch_event(TraceHandlerEvents::Event(trace_event)); } fn on_enter(&self, span: &Id, ctx: Context) { @@ -348,10 +374,10 @@ where span_datum.target = t; } if self.check_target(&span_datum.target, &span_datum.level) { - self.trace_handler.handle_span(span_datum); + self.dispatch_event(TraceHandlerEvents::Span(span_datum)); } } else { - self.trace_handler.handle_span(span_datum); + self.dispatch_event(TraceHandlerEvents::Span(span_datum)); } } } @@ -374,7 +400,7 @@ fn log_level(level: Level) -> log::Level { } impl TraceHandler for LogTraceHandler { - fn handle_span(&self, span_datum: SpanDatum) { + fn handle_span(&self, span_datum: &SpanDatum) { if span_datum.values.is_empty() { log::log!( log_level(span_datum.level), @@ -383,7 +409,7 @@ impl TraceHandler for LogTraceHandler { span_datum.name, span_datum.overall_time.as_nanos(), span_datum.id.into_u64(), - span_datum.parent_id.map(|s| s.into_u64()), + span_datum.parent_id.as_ref().map(|s| s.into_u64()), ); } else { log::log!( @@ -393,18 +419,18 @@ impl TraceHandler for LogTraceHandler { span_datum.name, span_datum.overall_time.as_nanos(), span_datum.id.into_u64(), - span_datum.parent_id.map(|s| s.into_u64()), + span_datum.parent_id.as_ref().map(|s| s.into_u64()), span_datum.values, ); } } - fn handle_event(&self, event: TraceEvent) { + fn handle_event(&self, event: &TraceEvent) { log::log!( log_level(event.level), "{}, parent_id: {:?}, {}", event.target, - event.parent_id.map(|s| s.into_u64()), + event.parent_id.as_ref().map(|s| s.into_u64()), event.values, ); } @@ -438,7 +464,10 @@ impl From for sp_rpc::tracing::Span { mod tests { use super::*; use parking_lot::Mutex; - use std::sync::Arc; + use std::sync::{ + mpsc::{Receiver, Sender}, + Arc, + }; use tracing_subscriber::layer::SubscriberExt; struct TestTraceHandler { @@ -447,12 +476,12 @@ mod tests { } impl TraceHandler for TestTraceHandler { - fn handle_span(&self, sd: SpanDatum) { - self.spans.lock().push(sd); + fn handle_span(&self, sd: &SpanDatum) { + self.spans.lock().push(sd.clone()); } - fn handle_event(&self, event: TraceEvent) { - self.events.lock().push(event); + fn handle_event(&self, event: &TraceEvent) { + self.events.lock().push(event.clone()); } } @@ -591,14 +620,14 @@ mod tests { let span1 = tracing::info_span!(target: "test_target", "test_span1"); let _guard1 = span1.enter(); - let (tx, rx) = mpsc::channel(); + let (tx, rx): (Sender, Receiver) = mpsc::channel(); let handle = thread::spawn(move || { let span2 = tracing::info_span!(target: "test_target", "test_span2"); let _guard2 = span2.enter(); // emit event tracing::event!(target: "test_target", tracing::Level::INFO, "test_event1"); for msg in rx.recv() { - if msg == false { + if !msg { break } } diff --git a/client/tracing/src/logging/directives.rs b/client/tracing/src/logging/directives.rs index 5aaeb4d17e7d..fe7d6a780dbf 100644 --- a/client/tracing/src/logging/directives.rs +++ b/client/tracing/src/logging/directives.rs @@ -17,8 +17,7 @@ use once_cell::sync::OnceCell; use parking_lot::Mutex; use tracing_subscriber::{ - filter::Directive, fmt as tracing_fmt, fmt::time::ChronoLocal, layer, reload::Handle, - EnvFilter, Registry, + filter::Directive, fmt as tracing_fmt, layer, reload::Handle, EnvFilter, Registry, }; // Handle to reload the tracing log filter @@ -109,6 +108,6 @@ pub(crate) fn set_reload_handle(handle: Handle) { // Used in the reload `Handle`. type SCSubscriber< N = tracing_fmt::format::DefaultFields, - E = crate::logging::EventFormat, - W = fn() -> std::io::Stderr, + E = crate::logging::EventFormat, + W = crate::logging::DefaultLogger, > = layer::Layered, Registry>; diff --git a/client/tracing/src/logging/event_format.rs b/client/tracing/src/logging/event_format.rs index 61d7fe77aec6..aec6b76843da 100644 --- a/client/tracing/src/logging/event_format.rs +++ b/client/tracing/src/logging/event_format.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -16,6 +16,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +use crate::logging::fast_local_time::FastLocalTime; use ansi_term::Colour; use regex::Regex; use std::fmt::{self, Write}; @@ -23,16 +24,13 @@ use tracing::{Event, Level, Subscriber}; use tracing_log::NormalizeEvent; use tracing_subscriber::{ field::RecordFields, - fmt::{ - time::{FormatTime, SystemTime}, - FmtContext, FormatEvent, FormatFields, - }, + fmt::{time::FormatTime, FmtContext, FormatEvent, FormatFields}, layer::Context, registry::{LookupSpan, SpanRef}, }; /// A pre-configured event formatter. -pub struct EventFormat { +pub struct EventFormat { /// Use the given timer for log message timestamps. pub timer: T, /// Sets whether or not an event's target is displayed. @@ -64,7 +62,7 @@ where S: Subscriber + for<'a> LookupSpan<'a>, N: for<'a> FormatFields<'a> + 'static, { - let writer = &mut MaybeColorWriter::new(self.enable_color, writer); + let writer = &mut ControlCodeSanitizer::new(!self.enable_color, writer); let normalized_meta = event.normalized_metadata(); let meta = normalized_meta.as_ref().unwrap_or_else(|| event.metadata()); time::write(&self.timer, writer, self.enable_color)?; @@ -102,10 +100,18 @@ where } } + // The writer only sanitizes its output once it's flushed, so if we don't actually need + // to sanitize everything we need to flush out what was already buffered as-is and only + // force-sanitize what follows. + if !writer.sanitize { + writer.flush()?; + writer.sanitize = true; + } + ctx.format_fields(writer, event)?; writeln!(writer)?; - writer.write() + writer.flush() } } @@ -296,43 +302,60 @@ where } } -/// A writer that may write to `inner_writer` with colors. +/// A writer which (optionally) strips out terminal control codes from the logs. /// -/// This is used by [`EventFormat`] to kill colors when `enable_color` is `false`. +/// This is used by [`EventFormat`] to sanitize the log messages. /// -/// It is required to call [`MaybeColorWriter::write`] after all writes are done, +/// It is required to call [`ControlCodeSanitizer::flush`] after all writes are done, /// because the content of these writes is buffered and will only be written to the /// `inner_writer` at that point. -struct MaybeColorWriter<'a> { - enable_color: bool, +struct ControlCodeSanitizer<'a> { + sanitize: bool, buffer: String, inner_writer: &'a mut dyn fmt::Write, } -impl<'a> fmt::Write for MaybeColorWriter<'a> { +impl<'a> fmt::Write for ControlCodeSanitizer<'a> { fn write_str(&mut self, buf: &str) -> fmt::Result { self.buffer.push_str(buf); Ok(()) } } -impl<'a> MaybeColorWriter<'a> { +// NOTE: When making any changes here make sure to also change this function in `sp-panic-handler`. +fn strip_control_codes(input: &str) -> std::borrow::Cow { + lazy_static::lazy_static! { + static ref RE: Regex = Regex::new(r#"(?x) + \x1b\[[^m]+m| # VT100 escape codes + [ + \x00-\x09\x0B-\x1F # ASCII control codes / Unicode C0 control codes, except \n + \x7F # ASCII delete + \u{80}-\u{9F} # Unicode C1 control codes + \u{202A}-\u{202E} # Unicode left-to-right / right-to-left control characters + \u{2066}-\u{2069} # Same as above + ] + "#).expect("regex parsing doesn't fail; qed"); + } + + RE.replace_all(input, "") +} + +impl<'a> ControlCodeSanitizer<'a> { /// Creates a new instance. - fn new(enable_color: bool, inner_writer: &'a mut dyn fmt::Write) -> Self { - Self { enable_color, inner_writer, buffer: String::new() } + fn new(sanitize: bool, inner_writer: &'a mut dyn fmt::Write) -> Self { + Self { sanitize, inner_writer, buffer: String::new() } } /// Write the buffered content to the `inner_writer`. - fn write(&mut self) -> fmt::Result { - lazy_static::lazy_static! { - static ref RE: Regex = Regex::new("\x1b\\[[^m]+m").expect("Error initializing color regex"); - } - - if !self.enable_color { - let replaced = RE.replace_all(&self.buffer, ""); - self.inner_writer.write_str(&replaced) + fn flush(&mut self) -> fmt::Result { + if self.sanitize { + let replaced = strip_control_codes(&self.buffer); + self.inner_writer.write_str(&replaced)? } else { - self.inner_writer.write_str(&self.buffer) + self.inner_writer.write_str(&self.buffer)? } + + self.buffer.clear(); + Ok(()) } } diff --git a/client/tracing/src/logging/fast_local_time.rs b/client/tracing/src/logging/fast_local_time.rs new file mode 100644 index 000000000000..47fc23340482 --- /dev/null +++ b/client/tracing/src/logging/fast_local_time.rs @@ -0,0 +1,160 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use chrono::{Datelike, Timelike}; +use std::{cell::RefCell, fmt::Write, time::SystemTime}; +use tracing_subscriber::fmt::time::FormatTime; + +/// A structure which, when `Display`d, will print out the current local time. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Default)] +pub struct FastLocalTime { + /// Decides whenever the fractional timestamp with be included in the output. + /// + /// If `false` the output will match the following `chrono` format string: + /// `%Y-%m-%d %H:%M:%S` + /// + /// If `true` the output will match the following `chrono` format string: + /// `%Y-%m-%d %H:%M:%S%.3f` + pub with_fractional: bool, +} + +// This is deliberately slightly larger than we actually need, just in case. +const TIMESTAMP_MAXIMUM_LENGTH: usize = 32; + +#[derive(Default)] +struct InlineString { + buffer: [u8; TIMESTAMP_MAXIMUM_LENGTH], + length: usize, +} + +impl Write for InlineString { + fn write_str(&mut self, s: &str) -> std::fmt::Result { + let new_length = self.length + s.len(); + assert!( + new_length <= TIMESTAMP_MAXIMUM_LENGTH, + "buffer overflow when formatting the current timestamp" + ); + + self.buffer[self.length..new_length].copy_from_slice(s.as_bytes()); + self.length = new_length; + Ok(()) + } +} + +impl InlineString { + fn as_str(&self) -> &str { + // SAFETY: this is safe since the only place we append to the buffer + // is in `write_str` from an `&str` + unsafe { std::str::from_utf8_unchecked(&self.buffer[..self.length]) } + } +} + +#[derive(Default)] +struct CachedTimestamp { + buffer: InlineString, + last_regenerated_at: u64, + last_fractional: u32, +} + +thread_local! { + static TIMESTAMP: RefCell = Default::default(); +} + +impl FormatTime for FastLocalTime { + fn format_time(&self, w: &mut dyn Write) -> std::fmt::Result { + const TIMESTAMP_PARTIAL_LENGTH: usize = "0000-00-00 00:00:00".len(); + + let elapsed = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("system time is never before UNIX epoch; qed"); + let unix_time = elapsed.as_secs(); + + TIMESTAMP.with(|cache| { + let mut cache = cache.borrow_mut(); + + // Regenerate the timestamp only at most once each second. + if cache.last_regenerated_at != unix_time { + let ts = chrono::Local::now(); + let fractional = (ts.nanosecond() % 1_000_000_000) / 1_000_000; + cache.last_regenerated_at = unix_time; + cache.last_fractional = fractional; + cache.buffer.length = 0; + + write!( + &mut cache.buffer, + "{:04}-{:02}-{:02} {:02}:{:02}:{:02}.{:03}", + ts.year(), + ts.month(), + ts.day(), + ts.hour(), + ts.minute(), + ts.second(), + fractional + )?; + } else if self.with_fractional { + let fractional = elapsed.subsec_millis(); + + // Regenerate the fractional part at most once each millisecond. + if cache.last_fractional != fractional { + cache.last_fractional = fractional; + cache.buffer.length = TIMESTAMP_PARTIAL_LENGTH + 1; + write!(&mut cache.buffer, "{:03}", fractional)?; + } + } + + let mut slice = cache.buffer.as_str(); + if !self.with_fractional { + slice = &slice[..TIMESTAMP_PARTIAL_LENGTH]; + } + + w.write_str(slice) + }) + } +} + +impl std::fmt::Display for FastLocalTime { + fn fmt(&self, w: &mut std::fmt::Formatter) -> std::fmt::Result { + self.format_time(w) + } +} + +#[test] +fn test_format_fast_local_time() { + assert_eq!( + chrono::Local::now().format("%Y-%m-%d %H:%M:%S").to_string().len(), + FastLocalTime { with_fractional: false }.to_string().len() + ); + assert_eq!( + chrono::Local::now().format("%Y-%m-%d %H:%M:%S%.3f").to_string().len(), + FastLocalTime { with_fractional: true }.to_string().len() + ); + + // A simple trick to make sure this test won't randomly fail if we so happen + // to land on the exact moment when we tick over to the next second. + let now_1 = FastLocalTime { with_fractional: false }.to_string(); + let expected = chrono::Local::now().format("%Y-%m-%d %H:%M:%S").to_string(); + let now_2 = FastLocalTime { with_fractional: false }.to_string(); + + assert!( + now_1 == expected || now_2 == expected, + "'{}' or '{}' should have been equal to '{}'", + now_1, + now_2, + expected + ); +} diff --git a/client/tracing/src/logging/layers/mod.rs b/client/tracing/src/logging/layers/mod.rs index 7dd0c4d120ad..382b79bc85d7 100644 --- a/client/tracing/src/logging/layers/mod.rs +++ b/client/tracing/src/logging/layers/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/tracing/src/logging/layers/prefix_layer.rs b/client/tracing/src/logging/layers/prefix_layer.rs index 2ad786a09223..836ffd2adda8 100644 --- a/client/tracing/src/logging/layers/prefix_layer.rs +++ b/client/tracing/src/logging/layers/prefix_layer.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/tracing/src/logging/mod.rs b/client/tracing/src/logging/mod.rs index dd4830fe8975..c325a3f73c41 100644 --- a/client/tracing/src/logging/mod.rs +++ b/client/tracing/src/logging/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -24,7 +24,11 @@ mod directives; mod event_format; +mod fast_local_time; mod layers; +mod stderr_writer; + +pub(crate) type DefaultLogger = stderr_writer::MakeStderrWriter; pub use directives::*; pub use sc_tracing_proc_macro::*; @@ -34,8 +38,8 @@ use tracing::Subscriber; use tracing_subscriber::{ filter::LevelFilter, fmt::{ - format, time::ChronoLocal, FormatEvent, FormatFields, Formatter, Layer as FmtLayer, - MakeWriter, SubscriberBuilder, + format, FormatEvent, FormatFields, Formatter, Layer as FmtLayer, MakeWriter, + SubscriberBuilder, }, layer::{self, SubscriberExt}, registry::LookupSpan, @@ -43,8 +47,11 @@ use tracing_subscriber::{ }; pub use event_format::*; +pub use fast_local_time::FastLocalTime; pub use layers::*; +use stderr_writer::MakeStderrWriter; + /// Logging Result typedef. pub type Result = std::result::Result; @@ -88,13 +95,9 @@ fn prepare_subscriber( directives: &str, profiling_targets: Option<&str>, force_colors: Option, + detailed_output: bool, builder_hook: impl Fn( - SubscriberBuilder< - format::DefaultFields, - EventFormat, - EnvFilter, - fn() -> std::io::Stderr, - >, + SubscriberBuilder, ) -> SubscriberBuilder, ) -> Result LookupSpan<'a>> where @@ -155,23 +158,19 @@ where tracing_log::LogTracer::builder().with_max_level(max_level).init()?; // If we're only logging `INFO` entries then we'll use a simplified logging format. - let simple = match max_level_hint { - Some(level) if level <= tracing_subscriber::filter::LevelFilter::INFO => true, - _ => false, - }; + let detailed_output = match max_level_hint { + Some(level) if level <= tracing_subscriber::filter::LevelFilter::INFO => false, + _ => true, + } || detailed_output; let enable_color = force_colors.unwrap_or_else(|| atty::is(atty::Stream::Stderr)); - let timer = ChronoLocal::with_format(if simple { - "%Y-%m-%d %H:%M:%S".to_string() - } else { - "%Y-%m-%d %H:%M:%S%.3f".to_string() - }); + let timer = fast_local_time::FastLocalTime { with_fractional: detailed_output }; let event_format = EventFormat { timer, - display_target: !simple, - display_level: !simple, - display_thread_name: !simple, + display_target: detailed_output, + display_level: detailed_output, + display_thread_name: detailed_output, enable_color, dup_to_stdout: !atty::is(atty::Stream::Stderr) && atty::is(atty::Stream::Stdout), }; @@ -179,7 +178,7 @@ where let builder = builder.with_span_events(format::FmtSpan::NONE); - let builder = builder.with_writer(std::io::stderr as _); + let builder = builder.with_writer(MakeStderrWriter::default()); let builder = builder.event_format(event_format); @@ -194,8 +193,10 @@ where pub struct LoggerBuilder { directives: String, profiling: Option<(crate::TracingReceiver, String)>, + custom_profiler: Option>, log_reloading: bool, force_colors: Option, + detailed_output: bool, } impl LoggerBuilder { @@ -204,8 +205,10 @@ impl LoggerBuilder { Self { directives: directives.into(), profiling: None, - log_reloading: true, + custom_profiler: None, + log_reloading: false, force_colors: None, + detailed_output: false, } } @@ -219,12 +222,32 @@ impl LoggerBuilder { self } + /// Add a custom profiler. + pub fn with_custom_profiling( + &mut self, + custom_profiler: Box, + ) -> &mut Self { + self.custom_profiler = Some(custom_profiler); + self + } + /// Wether or not to disable log reloading. pub fn with_log_reloading(&mut self, enabled: bool) -> &mut Self { self.log_reloading = enabled; self } + /// Whether detailed log output should be enabled. + /// + /// This includes showing the log target, log level and thread name. + /// + /// This will be automatically enabled when there is a log level enabled that is higher than + /// `info`. + pub fn with_detailed_output(&mut self, detailed: bool) -> &mut Self { + self.detailed_output = detailed; + self + } + /// Force enable/disable colors. pub fn with_colors(&mut self, enable: bool) -> &mut Self { self.force_colors = Some(enable); @@ -241,9 +264,15 @@ impl LoggerBuilder { &self.directives, Some(&profiling_targets), self.force_colors, + self.detailed_output, |builder| enable_log_reloading!(builder), )?; - let profiling = crate::ProfilingLayer::new(tracing_receiver, &profiling_targets); + let mut profiling = + crate::ProfilingLayer::new(tracing_receiver, &profiling_targets); + + self.custom_profiler + .into_iter() + .for_each(|profiler| profiling.add_handler(profiler)); tracing::subscriber::set_global_default(subscriber.with(profiling))?; @@ -253,9 +282,15 @@ impl LoggerBuilder { &self.directives, Some(&profiling_targets), self.force_colors, + self.detailed_output, |builder| builder, )?; - let profiling = crate::ProfilingLayer::new(tracing_receiver, &profiling_targets); + let mut profiling = + crate::ProfilingLayer::new(tracing_receiver, &profiling_targets); + + self.custom_profiler + .into_iter() + .for_each(|profiler| profiling.add_handler(profiler)); tracing::subscriber::set_global_default(subscriber.with(profiling))?; @@ -263,19 +298,25 @@ impl LoggerBuilder { } } else { if self.log_reloading { - let subscriber = - prepare_subscriber(&self.directives, None, self.force_colors, |builder| { - enable_log_reloading!(builder) - })?; + let subscriber = prepare_subscriber( + &self.directives, + None, + self.force_colors, + self.detailed_output, + |builder| enable_log_reloading!(builder), + )?; tracing::subscriber::set_global_default(subscriber)?; Ok(()) } else { - let subscriber = - prepare_subscriber(&self.directives, None, self.force_colors, |builder| { - builder - })?; + let subscriber = prepare_subscriber( + &self.directives, + None, + self.force_colors, + self.detailed_output, + |builder| builder, + )?; tracing::subscriber::set_global_default(subscriber)?; @@ -289,7 +330,16 @@ impl LoggerBuilder { mod tests { use super::*; use crate as sc_tracing; - use std::{env, process::Command}; + use log::info; + use std::{ + collections::BTreeMap, + env, + process::Command, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, + }, + }; use tracing::{metadata::Kind, subscriber::Interest, Callsite, Level, Metadata}; const EXPECTED_LOG_MESSAGE: &'static str = "yeah logging works as expected"; @@ -299,9 +349,28 @@ mod tests { let _ = LoggerBuilder::new(directives).init().unwrap(); } + fn run_test_in_another_process( + test_name: &str, + test_body: impl FnOnce(), + ) -> Option { + if env::var("RUN_FORKED_TEST").is_ok() { + test_body(); + None + } else { + let output = Command::new(env::current_exe().unwrap()) + .arg(test_name) + .env("RUN_FORKED_TEST", "1") + .output() + .unwrap(); + + assert!(output.status.success()); + Some(output) + } + } + #[test] fn test_logger_filters() { - if env::var("RUN_TEST_LOGGER_FILTERS").is_ok() { + run_test_in_another_process("test_logger_filters", || { let test_directives = "afg=debug,sync=trace,client=warn,telemetry,something-with-dash=error"; init_logger(&test_directives); @@ -338,15 +407,7 @@ mod tests { assert!(test_filter("telemetry", Level::TRACE)); assert!(test_filter("something-with-dash", Level::ERROR)); }); - } else { - let status = Command::new(env::current_exe().unwrap()) - .arg("test_logger_filters") - .env("RUN_TEST_LOGGER_FILTERS", "1") - .output() - .unwrap() - .status; - assert!(status.success()); - } + }); } /// This test ensures that using dash (`-`) in the target name in logs and directives actually @@ -481,4 +542,132 @@ mod tests { assert_eq!("MAX_LOG_LEVEL=Trace", run_test(None, Some("test=info".into()))); } } + + // This creates a bunch of threads and makes sure they start executing + // a given callback almost exactly at the same time. + fn run_on_many_threads(thread_count: usize, callback: impl Fn(usize) + 'static + Send + Clone) { + let started_count = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(AtomicBool::new(false)); + let threads: Vec<_> = (0..thread_count) + .map(|nth_thread| { + let started_count = started_count.clone(); + let barrier = barrier.clone(); + let callback = callback.clone(); + + std::thread::spawn(move || { + started_count.fetch_add(1, Ordering::SeqCst); + while !barrier.load(Ordering::SeqCst) { + std::thread::yield_now(); + } + + callback(nth_thread); + }) + }) + .collect(); + + while started_count.load(Ordering::SeqCst) != thread_count { + std::thread::yield_now(); + } + barrier.store(true, Ordering::SeqCst); + + for thread in threads { + if let Err(error) = thread.join() { + println!("error: failed to join thread: {:?}", error); + unsafe { libc::abort() } + } + } + } + + #[test] + fn parallel_logs_from_multiple_threads_are_properly_gathered() { + const THREAD_COUNT: usize = 128; + const LOGS_PER_THREAD: usize = 1024; + + let output = run_test_in_another_process( + "parallel_logs_from_multiple_threads_are_properly_gathered", + || { + let builder = LoggerBuilder::new(""); + builder.init().unwrap(); + + run_on_many_threads(THREAD_COUNT, |nth_thread| { + for _ in 0..LOGS_PER_THREAD { + info!("Thread <<{}>>", nth_thread); + } + }); + }, + ); + + if let Some(output) = output { + let stderr = String::from_utf8(output.stderr).unwrap(); + let mut count_per_thread = BTreeMap::new(); + for line in stderr.split("\n") { + if let Some(index_s) = line.find("Thread <<") { + let index_s = index_s + "Thread <<".len(); + let index_e = line.find(">>").unwrap(); + let nth_thread: usize = line[index_s..index_e].parse().unwrap(); + *count_per_thread.entry(nth_thread).or_insert(0) += 1; + } + } + + assert_eq!(count_per_thread.len(), THREAD_COUNT); + for (_, count) in count_per_thread { + assert_eq!(count, LOGS_PER_THREAD); + } + } + } + + #[test] + fn huge_single_line_log_is_properly_printed_out() { + let mut line = String::new(); + line.push_str("$$START$$"); + for n in 0..16 * 1024 * 1024 { + let ch = b'a' + (n as u8 % (b'z' - b'a')); + line.push(char::from(ch)); + } + line.push_str("$$END$$"); + + let output = + run_test_in_another_process("huge_single_line_log_is_properly_printed_out", || { + let builder = LoggerBuilder::new(""); + builder.init().unwrap(); + info!("{}", line); + }); + + if let Some(output) = output { + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains(&line)); + } + } + + #[test] + fn control_characters_are_always_stripped_out_from_the_log_messages() { + const RAW_LINE: &str = "$$START$$\x1B[1;32mIn\u{202a}\u{202e}\u{2066}\u{2069}ner\n\r\x7ftext!\u{80}\u{9f}\x1B[0m$$END$$"; + const SANITIZED_LINE: &str = "$$START$$Inner\ntext!$$END$$"; + + let output = run_test_in_another_process( + "control_characters_are_always_stripped_out_from_the_log_messages", + || { + std::env::set_var("RUST_LOG", "trace"); + let mut builder = LoggerBuilder::new(""); + builder.with_colors(true); + builder.init().unwrap(); + log::error!("{}", RAW_LINE); + }, + ); + + if let Some(output) = output { + let stderr = String::from_utf8(output.stderr).unwrap(); + // The log messages should always be sanitized. + assert!(!stderr.contains(RAW_LINE)); + assert!(stderr.contains(SANITIZED_LINE)); + + // The part where the timestamp, the logging level, etc. is printed out doesn't + // always have to be sanitized unless it's necessary, and here it shouldn't be. + assert!(stderr.contains("\x1B[31mERROR\x1B[0m")); + + // Make sure the logs aren't being duplicated. + assert_eq!(stderr.find("ERROR"), stderr.rfind("ERROR")); + assert_eq!(stderr.find(SANITIZED_LINE), stderr.rfind(SANITIZED_LINE)); + } + } } diff --git a/client/tracing/src/logging/stderr_writer.rs b/client/tracing/src/logging/stderr_writer.rs new file mode 100644 index 000000000000..e62c5e82c1ac --- /dev/null +++ b/client/tracing/src/logging/stderr_writer.rs @@ -0,0 +1,228 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! This module contains a buffered semi-asynchronous stderr writer. +//! +//! Depending on how we were started writing to stderr can take a surprisingly long time. +//! +//! If the other side takes their sweet sweet time reading whatever we send them then writing +//! to stderr might block for a long time, since it is effectively a synchronous operation. +//! And every time we write to stderr we need to grab a global lock, which affects every thread +//! which also tries to log something at the same time. +//! +//! Of course we *will* be ultimately limited by how fast the recipient can ingest our logs, +//! but it's not like logging is the only thing we're doing. And we still can't entirely +//! avoid the problem of multiple threads contending for the same lock. (Well, technically +//! we could employ something like a lock-free circular buffer, but that might be like +//! killing a fly with a sledgehammer considering the complexity involved; this is only +//! a logger after all.) +//! +//! But we can try to make things a little better. We can offload actually writing to stderr +//! to another thread and flush the logs in bulk instead of doing it per-line, which should +//! reduce the amount of CPU time we waste on making syscalls and on spinning waiting for locks. +//! +//! How much this helps depends on a multitude of factors, including the hardware we're running on, +//! how much we're logging, from how many threads, which exact set of threads are logging, to what +//! stderr is actually connected to (is it a terminal emulator? a file? an UDP socket?), etc. +//! +//! In general this can reduce the real time execution time as much as 75% in certain cases, or it +//! can make absolutely no difference in others. + +use parking_lot::{Condvar, Mutex, Once}; +use std::{ + io::Write, + sync::atomic::{AtomicBool, Ordering}, + time::Duration, +}; +use tracing::{Level, Metadata}; + +/// How many bytes of buffered logs will trigger an async flush on another thread? +const ASYNC_FLUSH_THRESHOLD: usize = 16 * 1024; + +/// How many bytes of buffered logs will trigger a sync flush on the current thread? +const SYNC_FLUSH_THRESHOLD: usize = 768 * 1024; + +/// How many bytes can be buffered at maximum? +const EMERGENCY_FLUSH_THRESHOLD: usize = 2 * 1024 * 1024; + +/// If there isn't enough printed out this is how often the logs will be automatically flushed. +const AUTOFLUSH_EVERY: Duration = Duration::from_millis(50); + +/// The least serious level at which a synchronous flush will be triggered. +const SYNC_FLUSH_LEVEL_THRESHOLD: Level = Level::ERROR; + +/// The amount of time we'll block until the buffer is fully flushed on exit. +/// +/// This should be completely unnecessary in normal circumstances. +const ON_EXIT_FLUSH_TIMEOUT: Duration = Duration::from_secs(5); + +/// A global buffer to which we'll append all of our logs before flushing them out to stderr. +static BUFFER: Mutex> = parking_lot::const_mutex(Vec::new()); + +/// A spare buffer which we'll swap with the main buffer on each flush to minimize lock contention. +static SPARE_BUFFER: Mutex> = parking_lot::const_mutex(Vec::new()); + +/// A conditional variable used to forcefully trigger asynchronous flushes. +static ASYNC_FLUSH_CONDVAR: Condvar = Condvar::new(); + +static ENABLE_ASYNC_LOGGING: AtomicBool = AtomicBool::new(true); + +fn flush_logs(mut buffer: parking_lot::lock_api::MutexGuard>) { + let mut spare_buffer = SPARE_BUFFER.lock(); + std::mem::swap(&mut *spare_buffer, &mut *buffer); + std::mem::drop(buffer); + + let stderr = std::io::stderr(); + let mut stderr_lock = stderr.lock(); + let _ = stderr_lock.write_all(&*spare_buffer); + std::mem::drop(stderr_lock); + + spare_buffer.clear(); +} + +fn log_autoflush_thread() { + let mut buffer = BUFFER.lock(); + loop { + ASYNC_FLUSH_CONDVAR.wait_for(&mut buffer, AUTOFLUSH_EVERY); + loop { + flush_logs(buffer); + + buffer = BUFFER.lock(); + if buffer.len() >= ASYNC_FLUSH_THRESHOLD { + // While we were busy flushing we picked up enough logs to do another flush. + continue + } else { + break + } + } + } +} + +#[cold] +fn initialize() { + std::thread::Builder::new() + .name("log-autoflush".to_owned()) + .spawn(log_autoflush_thread) + .expect("thread spawning doesn't normally fail; qed"); + + // SAFETY: This is safe since we pass a valid pointer to `atexit`. + let errcode = unsafe { libc::atexit(on_exit) }; + assert_eq!(errcode, 0, "atexit failed while setting up the logger: {}", errcode); +} + +extern "C" fn on_exit() { + ENABLE_ASYNC_LOGGING.store(false, Ordering::SeqCst); + + if let Some(buffer) = BUFFER.try_lock_for(ON_EXIT_FLUSH_TIMEOUT) { + flush_logs(buffer); + } +} + +/// A drop-in replacement for [`std::io::stderr`] for use anywhere +/// a [`tracing_subscriber::fmt::MakeWriter`] is accepted. +pub struct MakeStderrWriter { + // A dummy field so that the structure is not publicly constructible. + _dummy: (), +} + +impl Default for MakeStderrWriter { + fn default() -> Self { + static ONCE: Once = Once::new(); + ONCE.call_once(initialize); + MakeStderrWriter { _dummy: () } + } +} + +impl tracing_subscriber::fmt::MakeWriter for MakeStderrWriter { + type Writer = StderrWriter; + + fn make_writer(&self) -> Self::Writer { + StderrWriter::new(false) + } + + // The `tracing-subscriber` crate calls this for every line logged. + fn make_writer_for(&self, meta: &Metadata<'_>) -> Self::Writer { + StderrWriter::new(*meta.level() <= SYNC_FLUSH_LEVEL_THRESHOLD) + } +} + +pub struct StderrWriter { + buffer: Option>>, + sync_flush_on_drop: bool, + original_len: usize, +} + +impl StderrWriter { + fn new(mut sync_flush_on_drop: bool) -> Self { + if !ENABLE_ASYNC_LOGGING.load(Ordering::Relaxed) { + sync_flush_on_drop = true; + } + + // This lock isn't as expensive as it might look, since this is only called once the full + // line to be logged is already serialized into a thread-local buffer inside of the + // `tracing-subscriber` crate, and basically the only thing we'll do when holding this lock + // is to copy that over to our global shared buffer in one go in `Write::write_all` and be + // immediately dropped. + let buffer = BUFFER.lock(); + StderrWriter { original_len: buffer.len(), buffer: Some(buffer), sync_flush_on_drop } + } +} + +#[cold] +fn emergency_flush(buffer: &mut Vec, input: &[u8]) { + let stderr = std::io::stderr(); + let mut stderr_lock = stderr.lock(); + let _ = stderr_lock.write_all(buffer); + buffer.clear(); + + let _ = stderr_lock.write_all(input); +} + +impl Write for StderrWriter { + fn write(&mut self, input: &[u8]) -> Result { + let buffer = self.buffer.as_mut().expect("buffer is only None after `drop`; qed"); + if buffer.len() + input.len() >= EMERGENCY_FLUSH_THRESHOLD { + // Make sure we don't blow our memory budget. Normally this should never happen, + // but there are cases where we directly print out untrusted user input which + // can potentially be megabytes in size. + emergency_flush(buffer, input); + } else { + buffer.extend_from_slice(input); + } + Ok(input.len()) + } + + fn write_all(&mut self, input: &[u8]) -> Result<(), std::io::Error> { + self.write(input).map(|_| ()) + } + + fn flush(&mut self) -> Result<(), std::io::Error> { + Ok(()) + } +} + +impl Drop for StderrWriter { + fn drop(&mut self) { + let buf = self.buffer.take().expect("buffer is only None after `drop`; qed"); + if self.sync_flush_on_drop || buf.len() >= SYNC_FLUSH_THRESHOLD { + flush_logs(buf); + } else if self.original_len < ASYNC_FLUSH_THRESHOLD && buf.len() >= ASYNC_FLUSH_THRESHOLD { + ASYNC_FLUSH_CONDVAR.notify_one(); + } + } +} diff --git a/client/transaction-pool/Cargo.toml b/client/transaction-pool/Cargo.toml index 2184af819adf..df309760b847 100644 --- a/client/transaction-pool/Cargo.toml +++ b/client/transaction-pool/Cargo.toml @@ -2,9 +2,9 @@ name = "sc-transaction-pool" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate transaction pool implementation." readme = "README.md" @@ -13,26 +13,26 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0" } -thiserror = "1.0.21" -futures = "0.3.16" -intervalier = "0.4.0" +codec = { package = "parity-scale-codec", version = "3.0.0" } +thiserror = "1.0.30" +futures = "0.3.21" +futures-timer = "3.0.2" log = "0.4.8" -parity-util-mem = { version = "0.10.0", default-features = false, features = ["primitive-types"] } -parking_lot = "0.11.1" -prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.9.0"} +parity-util-mem = { version = "0.11.0", default-features = false, features = ["primitive-types"] } +parking_lot = "0.12.0" +prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.10.0-dev"} sc-client-api = { version = "4.0.0-dev", path = "../api" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } sp-transaction-pool = { version = "4.0.0-dev", path = "../../primitives/transaction-pool" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "./api" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sc-utils = { version = "4.0.0-dev", path = "../utils" } -serde = { version = "1.0.126", features = ["derive"] } +serde = { version = "1.0.136", features = ["derive"] } linked-hash-map = "0.5.4" -retain_mut = "0.1.3" +retain_mut = "0.1.4" [dev-dependencies] assert_matches = "1.3.0" @@ -41,13 +41,9 @@ sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/comm substrate-test-runtime-transaction-pool = { version = "2.0.0", path = "../../test-utils/runtime/transaction-pool" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } sc-block-builder = { version = "0.10.0-dev", path = "../block-builder" } -codec = { package = "parity-scale-codec", version = "2.0.0" } substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" } criterion = "0.3" [[bench]] name = "basics" harness = false - -[features] -test-helpers = [] diff --git a/client/transaction-pool/README.md b/client/transaction-pool/README.md index e4f8ccb3d810..4a2bbb8838f9 100644 --- a/client/transaction-pool/README.md +++ b/client/transaction-pool/README.md @@ -39,7 +39,7 @@ runtime (queried at current best imported block). Since the blockchain is not always linear, forks need to be correctly handled by the transaction pool as well. In case of a fork, some blocks are *retracted* from the canonical chain, and some other blocks get *enacted* on top of some -common ancestor. The transactions from retrated blocks could simply be discarded, +common ancestor. The transactions from retracted blocks could simply be discarded, but it's desirable to make sure they are still considered for inclusion in case they are deemed valid by the runtime state at best, recently enacted block (fork the chain re-organized to). @@ -49,7 +49,7 @@ pool, it's broadcasting status, block inclusion, finality, etc. ## Transaction Validity details -Information retrieved from the the runtime are encapsulated in `TransactionValidity` +Information retrieved from the the runtime are encapsulated in the `TransactionValidity` type. ```rust @@ -147,7 +147,7 @@ choosing the ones with highest priority to include to the next block first. - `priority` of transaction may change over time - on-chain conditions may affect `priority` -- Given two transactions with overlapping `provides` tags, the one with higher +- given two transactions with overlapping `provides` tags, the one with higher `priority` should be preferred. However we can also look at the total priority of a subtree rooted at that transaction and compare that instead (i.e. even though the transaction itself has lower `priority` it "unlocks" other high priority transactions). @@ -163,7 +163,7 @@ the transaction is valid all that time though. - `longevity` of transaction may change over time - on-chain conditions may affect `longevity` -- After `longevity` lapses the transaction may still be valid +- after `longevity` lapses, the transaction may still be valid ### `propagate` @@ -231,15 +231,16 @@ to instead of gossiping everyting have other peers request transactions they are interested in. Since the pool is expected to store more transactions than what can fit -to a single block. Validating the entire pool on every block might not be -feasible, so the actual implementation might need to take some shortcuts. +in a single block, validating the entire pool on every block might not be +feasible. This means that the actual implementation might need to take some +shortcuts. ## Suggestions & caveats -1. The validity of transaction should not change significantly from block to +1. The validity of a transaction should not change significantly from block to block. I.e. changes in validity should happen predictably, e.g. `longevity` decrements by 1, `priority` stays the same, `requires` changes if transaction - that provided a tag was included in block. `provides` does not change, etc. + that provided a tag was included in block, `provides` does not change, etc. 1. That means we don't have to revalidate every transaction after every block import, but we need to take care of removing potentially stale transactions. @@ -253,9 +254,9 @@ feasible, so the actual implementation might need to take some shortcuts. 1. In the past there were many issues found when running small networks with a lot of re-orgs. Make sure that transactions are never lost. -1. UTXO model is quite challenging. The transaction becomes valid right after - it's included in block, however it is waiting for exactly the same inputs to - be spent, so it will never really be included again. +1. The UTXO model is quite challenging. A transaction becomes valid right after + it's included in a block, however it is waiting for exactly the same inputs + to be spent, so it will never really be included again. 1. Note that in a non-ideal implementation the state of the pool will most likely always be a bit off, i.e. some transactions might be still in the pool, @@ -277,8 +278,8 @@ feasible, so the actual implementation might need to take some shortcuts. 1. We periodically validate all transactions in the pool in batches. -1. To minimize runtime calls, we introduce batch-verify call. Note it should reset - the state (overlay) after every verification. +1. To minimize runtime calls, we introduce the batch-verify call. Note it should + reset the state (overlay) after every verification. 1. Consider leveraging finality. Maybe we could verify against latest finalised block instead. With this the pool in different nodes can be more similar @@ -286,16 +287,16 @@ feasible, so the actual implementation might need to take some shortcuts. is not a strict requirement for a Substrate chain to have though. 1. Perhaps we could avoid maintaining ready/future queues as currently, but - rather if transaction doesn't have all requirements satisfied by existing + rather if a transaction doesn't have all requirements satisfied by existing transactions we attempt to re-import it in the future. 1. Instead of maintaining a full pool with total ordering we attempt to maintain a set of next (couple of) blocks. We could introduce batch-validate runtime - api method that pretty much attempts to simulate actual block inclusion of + api method that pretty much attempts to simulate actual block inclusion of a set of such transactions (without necessarily fully running/dispatching them). Importing a transaction would consist of figuring out which next block - this transaction have a chance to be included in and then attempting to - either push it back or replace some of existing transactions. + this transaction has a chance to be included in and then attempting to + either push it back or replace some existing transactions. 1. Perhaps we could use some immutable graph structure to easily add/remove transactions. We need some traversal method that takes priority and @@ -320,7 +321,7 @@ The pool consists of basically two independent parts: The pool is split into `ready` pool and `future` pool. The latter contains transactions that don't have their requirements satisfied, and the former holds transactions that can be used to build a graph of dependencies. Note that the -graph is build ad-hoc during the traversal process (getting the `ready` +graph is built ad-hoc during the traversal process (using the `ready` iterator). This makes the importing process cheaper (we don't need to find the exact position in the queue or graph), but traversal process slower (logarithmic). However most of the time we will only need the beginning of the @@ -342,26 +343,26 @@ to limit number of runtime verification calls. Each time a transaction is imported, we first verify it's validity and later find if the tags it `requires` can be satisfied by transactions already in `ready` pool. In case the transaction is imported to the `ready` pool we -additionally *promote* transactions from `future` pool if the transaction +additionally *promote* transactions from the `future` pool if the transaction happened to fulfill their requirements. -Note we need to cater for cases where transaction might replace a already +Note we need to cater for cases where a transaction might replace an already existing transaction in the pool. In such case we check the entire sub-tree of transactions that we are about to replace, compare their cumulative priority to determine which subtree to keep. -After a block is imported we kick-off pruning procedure. We first attempt to -figure out what tags were satisfied by transaction in that block. For each block -transaction we either call into runtime to get it's `ValidTransaction` object, +After a block is imported we kick-off the pruning procedure. We first attempt to +figure out what tags were satisfied by a transaction in that block. For each block +transaction we either call into the runtime to get it's `ValidTransaction` object, or we check the pool if that transaction is already known to spare the runtime -call. From this we gather full set of `provides` tags and perform pruning of -`ready` pool based on that. Also we promote all transactions from `future` that -have their tags satisfied. +call. From this we gather the full set of `provides` tags and perform pruning of +the `ready` pool based on that. Also, we promote all transactions from `future` +that have their tags satisfied. In case we remove transactions that we are unsure if they were already included -in current block or some block in the past, it is being added to revalidation -queue and attempted to be re-imported by the background task in the future. +in the current block or some block in the past, it gets added to the revalidation +queue and attempts to be re-imported by the background task in the future. Runtime calls to verify transactions are performed from a separate (limited) -thread pool to avoid interferring too much with other subsystems of the node. We +thread pool to avoid interfering too much with other subsystems of the node. We definitely don't want to have all cores validating network transactions, because all of these transactions need to be considered untrusted (potentially DoS). diff --git a/client/transaction-pool/api/Cargo.toml b/client/transaction-pool/api/Cargo.toml index efef36071f08..8dd5e7253f96 100644 --- a/client/transaction-pool/api/Cargo.toml +++ b/client/transaction-pool/api/Cargo.toml @@ -2,18 +2,17 @@ name = "sc-transaction-pool-api" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Transaction pool client facing API." [dependencies] -futures = { version = "0.3.1" } -log = { version = "0.4.8" } -serde = { version = "1.0.126", features = ["derive"] } -thiserror = { version = "1.0.21" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +futures = "0.3.21" +log = "0.4.8" +serde = { version = "1.0.136", features = ["derive"] } +thiserror = "1.0.30" -derive_more = { version = "0.99.11" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } diff --git a/client/transaction-pool/api/src/error.rs b/client/transaction-pool/api/src/error.rs index feee3b0a949c..b093657f739b 100644 --- a/client/transaction-pool/api/src/error.rs +++ b/client/transaction-pool/api/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -26,7 +26,7 @@ use sp_runtime::transaction_validity::{ pub type Result = std::result::Result; /// Transaction pool error type. -#[derive(Debug, thiserror::Error, derive_more::From)] +#[derive(Debug, thiserror::Error)] #[allow(missing_docs)] pub enum Error { #[error("Unknown transaction validity: {0:?}")] @@ -64,7 +64,6 @@ pub enum Error { #[error("Transaction cannot be propagated and the local node does not author blocks")] Unactionable, - #[from(ignore)] #[error("{0}")] InvalidBlockId(String), diff --git a/client/transaction-pool/api/src/lib.rs b/client/transaction-pool/api/src/lib.rs index a6252f1373c5..d7b3ddc99636 100644 --- a/client/transaction-pool/api/src/lib.rs +++ b/client/transaction-pool/api/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -223,13 +223,14 @@ pub trait TransactionPool: Send + Sync { at: NumberFor, ) -> Pin< Box< - dyn Future> + Send>> - + Send, + dyn Future< + Output = Box> + Send>, + > + Send, >, >; /// Get an iterator for ready transactions ordered by priority. - fn ready(&self) -> Box> + Send>; + fn ready(&self) -> Box> + Send>; // *** Block production /// Remove transactions identified by given hashes (and dependent transactions) from the pool. @@ -254,9 +255,30 @@ pub trait TransactionPool: Send + Sync { fn ready_transaction(&self, hash: &TxHash) -> Option>; } +/// An iterator of ready transactions. +/// +/// The trait extends regular [`std::iter::Iterator`] trait and allows reporting +/// last-returned element as invalid. +/// +/// The implementation is then allowed, for performance reasons, to change the elements +/// returned next, by e.g. skipping elements that are known to depend on the reported +/// transaction, which yields them invalid as well. +pub trait ReadyTransactions: Iterator { + /// Report given transaction as invalid. + /// + /// This might affect subsequent elements returned by the iterator, so dependent transactions + /// are skipped for performance reasons. + fn report_invalid(&mut self, _tx: &Self::Item); +} + +/// A no-op implementation for an empty iterator. +impl ReadyTransactions for std::iter::Empty { + fn report_invalid(&mut self, _tx: &T) {} +} + /// Events that the transaction pool listens for. pub enum ChainEvent { - /// New best block have been added to the chain + /// New best block have been added to the chain. NewBestBlock { /// Hash of the block. hash: B::Hash, @@ -267,8 +289,10 @@ pub enum ChainEvent { }, /// An existing block has been finalized. Finalized { - /// Hash of just finalized block + /// Hash of just finalized block. hash: B::Hash, + /// Path from old finalized to new finalized parent. + tree_route: Arc<[B::Hash]>, }, } @@ -331,7 +355,7 @@ impl OffchainSubmitTransaction for TP result.map(|_| ()).map_err(|e| { log::warn!( target: "txpool", - "(offchain call) Error submitting a transaction to the pool: {:?}", + "(offchain call) Error submitting a transaction to the pool: {}", e ) }) diff --git a/client/transaction-pool/benches/basics.rs b/client/transaction-pool/benches/basics.rs index cf30a0200ad7..c3577a45faf0 100644 --- a/client/transaction-pool/benches/basics.rs +++ b/client/transaction-pool/benches/basics.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,16 +18,16 @@ use criterion::{criterion_group, criterion_main, Criterion}; -use codec::Encode; +use codec::{Decode, Encode}; use futures::{ executor::block_on, future::{ready, Ready}, }; -use sc_transaction_pool::{test_helpers::*, *}; +use sc_transaction_pool::*; use sp_core::blake2_256; use sp_runtime::{ generic::BlockId, - traits::Block as BlockT, + traits::{Block as BlockT, NumberFor}, transaction_validity::{ InvalidTransaction, TransactionSource, TransactionTag as Tag, TransactionValidity, ValidTransaction, @@ -63,7 +63,7 @@ impl ChainApi for TestApi { &self, at: &BlockId, _source: TransactionSource, - uxt: test_helpers::ExtrinsicFor, + uxt: ::Extrinsic, ) -> Self::ValidationFuture { let nonce = uxt.transfer().nonce; let from = uxt.transfer().from.clone(); @@ -89,7 +89,7 @@ impl ChainApi for TestApi { fn block_id_to_number( &self, at: &BlockId, - ) -> Result>, Self::Error> { + ) -> Result>, Self::Error> { Ok(match at { BlockId::Number(num) => Some(*num), BlockId::Hash(_) => None, @@ -99,14 +99,14 @@ impl ChainApi for TestApi { fn block_id_to_hash( &self, at: &BlockId, - ) -> Result>, Self::Error> { + ) -> Result::Hash>, Self::Error> { Ok(match at { BlockId::Number(num) => Some(H256::from_low_u64_be(*num)).into(), BlockId::Hash(_) => None, }) } - fn hash_and_length(&self, uxt: &test_helpers::ExtrinsicFor) -> (H256, usize) { + fn hash_and_length(&self, uxt: &::Extrinsic) -> (H256, usize) { let encoded = uxt.encode(); (blake2_256(&encoded).into(), encoded.len()) } @@ -126,7 +126,8 @@ impl ChainApi for TestApi { fn uxt(transfer: Transfer) -> Extrinsic { Extrinsic::Transfer { transfer, - signature: Default::default(), + signature: Decode::decode(&mut sp_runtime::traits::TrailingZeroInput::zeroes()) + .expect("infinite input; no dead input space; qed"), exhaust_resources_when_not_first: false, } } diff --git a/client/transaction-pool/graph/Cargo.toml b/client/transaction-pool/graph/Cargo.toml deleted file mode 100644 index b49cadc51c33..000000000000 --- a/client/transaction-pool/graph/Cargo.toml +++ /dev/null @@ -1,39 +0,0 @@ -[package] -name = "sc-transaction-graph" -version = "4.0.0-dev" -authors = ["Parity Technologies "] -edition = "2018" -license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -homepage = "https://substrate.dev" -repository = "https://github.com/paritytech/substrate/" -description = "Generic Transaction Pool" -readme = "README.md" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[dependencies] -derive_more = "0.99.2" -thiserror = "1.0.21" -futures = "0.3.9" -log = "0.4.8" -parking_lot = "0.11.1" -serde = { version = "1.0.101", features = ["derive"] } -sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sc-utils = { version = "4.0.0-dev", path = "../../utils" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -sp-transaction-pool = { version = "4.0.0-dev", path = "../../../primitives/transaction-pool" } -parity-util-mem = { version = "0.10.0", default-features = false, features = ["primitive-types"] } -linked-hash-map = "0.5.4" -retain_mut = "0.1.3" - -[dev-dependencies] -assert_matches = "1.3.0" -codec = { package = "parity-scale-codec", version = "2.0.0" } -substrate-test-runtime = { version = "2.0.0", path = "../../../test-utils/runtime" } -criterion = "0.3" - -[[bench]] -name = "basics" -harness = false diff --git a/client/transaction-pool/src/api.rs b/client/transaction-pool/src/api.rs index a735c67d846c..12909f313d10 100644 --- a/client/transaction-pool/src/api.rs +++ b/client/transaction-pool/src/api.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -18,7 +18,7 @@ //! Chain api required for the transaction pool. -use codec::{Decode, Encode}; +use codec::Encode; use futures::{ channel::{mpsc, oneshot}, future::{ready, Future, FutureExt, Ready}, @@ -28,16 +28,12 @@ use futures::{ use std::{marker::PhantomData, pin::Pin, sync::Arc}; use prometheus_endpoint::Registry as PrometheusRegistry; -use sc_client_api::{ - blockchain::HeaderBackend, - light::{Fetcher, RemoteBodyRequest, RemoteCallRequest}, - BlockBackend, -}; +use sc_client_api::{blockchain::HeaderBackend, BlockBackend}; use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_core::traits::SpawnEssentialNamed; use sp_runtime::{ generic::BlockId, - traits::{self, Block as BlockT, BlockIdTo, Hash as HashT, Header as HeaderT}, + traits::{self, Block as BlockT, BlockIdTo}, transaction_validity::{TransactionSource, TransactionValidity}, }; use sp_transaction_pool::runtime_api::TaggedTransactionQueue; @@ -64,6 +60,7 @@ fn spawn_validation_pool_task( ) { spawner.spawn_essential_blocking( name, + Some("transaction-pool"), async move { loop { let task = receiver.lock().await.next().await; @@ -170,18 +167,14 @@ where &self, at: &BlockId, ) -> error::Result>> { - self.client - .to_number(at) - .map_err(|e| Error::BlockIdConversion(format!("{:?}", e))) + self.client.to_number(at).map_err(|e| Error::BlockIdConversion(e.to_string())) } fn block_id_to_hash( &self, at: &BlockId, ) -> error::Result>> { - self.client - .to_hash(at) - .map_err(|e| Error::BlockIdConversion(format!("{:?}", e))) + self.client.to_hash(at).map_err(|e| Error::BlockIdConversion(e.to_string())) } fn hash_and_length( @@ -227,7 +220,7 @@ where }?; let block_hash = client.to_hash(at) - .map_err(|e| Error::RuntimeApi(format!("{:?}", e)))? + .map_err(|e| Error::RuntimeApi(e.to_string()))? .ok_or_else(|| Error::RuntimeApi(format!("Could not get hash for block `{:?}`.", at)))?; use sp_api::Core; @@ -240,7 +233,7 @@ where .map_err(|e| Error::RuntimeApi(e.to_string())) } else { let block_number = client.to_number(at) - .map_err(|e| Error::RuntimeApi(format!("{:?}", e)))? + .map_err(|e| Error::RuntimeApi(e.to_string()))? .ok_or_else(|| Error::RuntimeApi(format!("Could not get number for block `{:?}`.", at)) )?; @@ -289,127 +282,3 @@ where validate_transaction_blocking(&*self.client, at, source, uxt) } } - -/// The transaction pool logic for light client. -pub struct LightChainApi { - client: Arc, - fetcher: Arc, - _phantom: PhantomData, -} - -impl LightChainApi { - /// Create new transaction pool logic. - pub fn new(client: Arc, fetcher: Arc) -> Self { - LightChainApi { client, fetcher, _phantom: Default::default() } - } -} - -impl graph::ChainApi for LightChainApi -where - Block: BlockT, - Client: HeaderBackend + 'static, - F: Fetcher + 'static, -{ - type Block = Block; - type Error = error::Error; - type ValidationFuture = - Box> + Send + Unpin>; - type BodyFuture = Pin< - Box< - dyn Future::Extrinsic>>>> - + Send, - >, - >; - - fn validate_transaction( - &self, - at: &BlockId, - source: TransactionSource, - uxt: graph::ExtrinsicFor, - ) -> Self::ValidationFuture { - let header_hash = self.client.expect_block_hash_from_id(at); - let header_and_hash = header_hash.and_then(|header_hash| { - self.client - .expect_header(BlockId::Hash(header_hash)) - .map(|header| (header_hash, header)) - }); - let (block, header) = match header_and_hash { - Ok((header_hash, header)) => (header_hash, header), - Err(err) => return Box::new(ready(Err(err.into()))), - }; - let remote_validation_request = self.fetcher.remote_call(RemoteCallRequest { - block, - header, - method: "TaggedTransactionQueue_validate_transaction".into(), - call_data: (source, uxt, block).encode(), - retry_count: None, - }); - let remote_validation_request = remote_validation_request.then(move |result| { - let result: error::Result = - result.map_err(Into::into).and_then(|result| { - Decode::decode(&mut &result[..]).map_err(|e| { - Error::RuntimeApi(format!("Error decoding tx validation result: {:?}", e)) - }) - }); - ready(result) - }); - - Box::new(remote_validation_request) - } - - fn block_id_to_number( - &self, - at: &BlockId, - ) -> error::Result>> { - Ok(self.client.block_number_from_id(at)?) - } - - fn block_id_to_hash( - &self, - at: &BlockId, - ) -> error::Result>> { - Ok(self.client.block_hash_from_id(at)?) - } - - fn hash_and_length( - &self, - ex: &graph::ExtrinsicFor, - ) -> (graph::ExtrinsicHash, usize) { - ex.using_encoded(|x| (<::Hashing as HashT>::hash(x), x.len())) - } - - fn block_body(&self, id: &BlockId) -> Self::BodyFuture { - let header = self - .client - .header(*id) - .and_then(|h| h.ok_or_else(|| sp_blockchain::Error::UnknownBlock(format!("{}", id)))); - let header = match header { - Ok(header) => header, - Err(err) => { - log::warn!(target: "txpool", "Failed to query header: {:?}", err); - return Box::pin(ready(Ok(None))) - }, - }; - - let fetcher = self.fetcher.clone(); - async move { - let transactions = fetcher - .remote_body(RemoteBodyRequest { header, retry_count: None }) - .await - .unwrap_or_else(|e| { - log::warn!(target: "txpool", "Failed to fetch block body: {:?}", e); - Vec::new() - }); - - Ok(Some(transactions)) - } - .boxed() - } - - fn block_header( - &self, - at: &BlockId, - ) -> Result::Header>, Self::Error> { - self.client.header(*at).map_err(Into::into) - } -} diff --git a/client/transaction-pool/src/error.rs b/client/transaction-pool/src/error.rs index b14e0569f083..adc19d52c7e4 100644 --- a/client/transaction-pool/src/error.rs +++ b/client/transaction-pool/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/transaction-pool/src/graph/base_pool.rs b/client/transaction-pool/src/graph/base_pool.rs index 890a87e82929..8e0422739cc6 100644 --- a/client/transaction-pool/src/graph/base_pool.rs +++ b/client/transaction-pool/src/graph/base_pool.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -20,7 +20,7 @@ //! //! For a more full-featured pool, have a look at the `pool` module. -use std::{collections::HashSet, fmt, hash, sync::Arc}; +use std::{cmp::Ordering, collections::HashSet, fmt, hash, sync::Arc}; use log::{debug, trace, warn}; use sc_transaction_pool_api::{error, InPoolTransaction, PoolStatus}; @@ -36,7 +36,7 @@ use sp_runtime::{ use super::{ future::{FutureTransactions, WaitingTransaction}, - ready::ReadyTransactions, + ready::{BestIterator, ReadyTransactions, TransactionRef}, }; /// Successful import result. @@ -355,7 +355,7 @@ impl BasePool impl Iterator>> { + pub fn ready(&self) -> BestIterator { self.ready.get() } @@ -384,8 +384,8 @@ impl BasePool BasePool, _>(|worst, current| { let transaction = ¤t.transaction; - match minimal { - None => Some(transaction.clone()), - Some(ref tx) if tx.insertion_id > transaction.insertion_id => - Some(transaction.clone()), - other => other, - } + worst + .map(|worst| { + // Here we don't use `TransactionRef`'s ordering implementation because + // while it prefers priority like need here, it also prefers older + // transactions for inclusion purposes and limit enforcement needs to prefer + // newer transactions instead and drop the older ones. + match worst.transaction.priority.cmp(&transaction.transaction.priority) { + Ordering::Less => worst, + Ordering::Equal => + if worst.insertion_id > transaction.insertion_id { + transaction.clone() + } else { + worst + }, + Ordering::Greater => transaction.clone(), + } + }) + .or_else(|| Some(transaction.clone())) }); - if let Some(minimal) = minimal { - removed.append(&mut self.remove_subtree(&[minimal.transaction.hash.clone()])) + if let Some(worst) = worst { + removed.append(&mut self.remove_subtree(&[worst.transaction.hash.clone()])) } else { break } @@ -414,14 +426,14 @@ impl BasePool Some(current.clone()), Some(ref tx) if tx.imported_at > current.imported_at => Some(current.clone()), other => other, }); - if let Some(minimal) = minimal { - removed.append(&mut self.remove_subtree(&[minimal.transaction.hash.clone()])) + if let Some(worst) = worst { + removed.append(&mut self.remove_subtree(&[worst.transaction.hash.clone()])) } else { break } diff --git a/client/transaction-pool/src/graph/future.rs b/client/transaction-pool/src/graph/future.rs index 6ed1f1014304..ae49e3f2d3ae 100644 --- a/client/transaction-pool/src/graph/future.rs +++ b/client/transaction-pool/src/graph/future.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/transaction-pool/src/graph/listener.rs b/client/transaction-pool/src/graph/listener.rs index b8149018f783..d4f42b32fdbb 100644 --- a/client/transaction-pool/src/graph/listener.rs +++ b/client/transaction-pool/src/graph/listener.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/transaction-pool/src/graph/mod.rs b/client/transaction-pool/src/graph/mod.rs index 3ecfb8fe68c6..42812897a954 100644 --- a/client/transaction-pool/src/graph/mod.rs +++ b/client/transaction-pool/src/graph/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/transaction-pool/src/graph/pool.rs b/client/transaction-pool/src/graph/pool.rs index 2af5a8a19a5a..39be43f82c8b 100644 --- a/client/transaction-pool/src/graph/pool.rs +++ b/client/transaction-pool/src/graph/pool.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -224,12 +224,8 @@ impl Pool { hashes: &[ExtrinsicHash], ) -> Result<(), B::Error> { // Get details of all extrinsics that are already in the pool - let in_pool_tags = self - .validated_pool - .extrinsics_tags(hashes) - .into_iter() - .filter_map(|x| x) - .flatten(); + let in_pool_tags = + self.validated_pool.extrinsics_tags(hashes).into_iter().flatten().flatten(); // Prune all transactions that provide given tags let prune_status = self.validated_pool.prune_tags(in_pool_tags)?; @@ -444,159 +440,16 @@ impl Clone for Pool { #[cfg(test)] mod tests { use super::{super::base_pool::Limit, *}; + use crate::tests::{pool, uxt, TestApi, INVALID_NONCE}; use assert_matches::assert_matches; - use codec::Encode; use futures::executor::block_on; use parking_lot::Mutex; use sc_transaction_pool_api::TransactionStatus; - use sp_runtime::{ - traits::Hash, - transaction_validity::{InvalidTransaction, TransactionSource, ValidTransaction}, - }; - use std::{ - collections::{HashMap, HashSet}, - time::Instant, - }; - use substrate_test_runtime::{AccountId, Block, Extrinsic, Hashing, Transfer, H256}; - - const INVALID_NONCE: u64 = 254; - const SOURCE: TransactionSource = TransactionSource::External; - - #[derive(Clone, Debug, Default)] - struct TestApi { - delay: Arc>>>, - invalidate: Arc>>, - clear_requirements: Arc>>, - add_requirements: Arc>>, - } - - impl ChainApi for TestApi { - type Block = Block; - type Error = error::Error; - type ValidationFuture = futures::future::Ready>; - type BodyFuture = futures::future::Ready>>>; - - /// Verify extrinsic at given block. - fn validate_transaction( - &self, - at: &BlockId, - _source: TransactionSource, - uxt: ExtrinsicFor, - ) -> Self::ValidationFuture { - let hash = self.hash_and_length(&uxt).0; - let block_number = self.block_id_to_number(at).unwrap().unwrap(); - - let res = match uxt { - Extrinsic::Transfer { transfer, .. } => { - let nonce = transfer.nonce; - - // This is used to control the test flow. - if nonce > 0 { - let opt = self.delay.lock().take(); - if let Some(delay) = opt { - if delay.recv().is_err() { - println!("Error waiting for delay!"); - } - } - } + use sp_runtime::transaction_validity::TransactionSource; + use std::{collections::HashMap, time::Instant}; + use substrate_test_runtime::{AccountId, Extrinsic, Transfer, H256}; - if self.invalidate.lock().contains(&hash) { - InvalidTransaction::Custom(0).into() - } else if nonce < block_number { - InvalidTransaction::Stale.into() - } else { - let mut transaction = ValidTransaction { - priority: 4, - requires: if nonce > block_number { - vec![vec![nonce as u8 - 1]] - } else { - vec![] - }, - provides: if nonce == INVALID_NONCE { - vec![] - } else { - vec![vec![nonce as u8]] - }, - longevity: 3, - propagate: true, - }; - - if self.clear_requirements.lock().contains(&hash) { - transaction.requires.clear(); - } - - if self.add_requirements.lock().contains(&hash) { - transaction.requires.push(vec![128]); - } - - Ok(transaction) - } - }, - Extrinsic::IncludeData(_) => Ok(ValidTransaction { - priority: 9001, - requires: vec![], - provides: vec![vec![42]], - longevity: 9001, - propagate: false, - }), - _ => unimplemented!(), - }; - - futures::future::ready(Ok(res)) - } - - /// Returns a block number given the block id. - fn block_id_to_number( - &self, - at: &BlockId, - ) -> Result>, Self::Error> { - Ok(match at { - BlockId::Number(num) => Some(*num), - BlockId::Hash(_) => None, - }) - } - - /// Returns a block hash given the block id. - fn block_id_to_hash( - &self, - at: &BlockId, - ) -> Result::Hash>, Self::Error> { - Ok(match at { - BlockId::Number(num) => Some(H256::from_low_u64_be(*num)).into(), - BlockId::Hash(_) => None, - }) - } - - /// Hash the extrinsic. - fn hash_and_length(&self, uxt: &ExtrinsicFor) -> (BlockHash, usize) { - let encoded = uxt.encode(); - let len = encoded.len(); - (Hashing::hash(&encoded), len) - } - - fn block_body(&self, _id: &BlockId) -> Self::BodyFuture { - futures::future::ready(Ok(None)) - } - - fn block_header( - &self, - _: &BlockId, - ) -> Result::Header>, Self::Error> { - Ok(None) - } - } - - fn uxt(transfer: Transfer) -> Extrinsic { - Extrinsic::Transfer { - transfer, - signature: Default::default(), - exhaust_resources_when_not_first: false, - } - } - - fn pool() -> Pool { - Pool::new(Default::default(), true.into(), TestApi::default().into()) - } + const SOURCE: TransactionSource = TransactionSource::External; #[test] fn should_validate_and_import_transaction() { @@ -632,7 +485,7 @@ mod tests { }); // when - pool.validated_pool.rotator().ban(&Instant::now(), vec![pool.hash_of(&uxt)]); + pool.validated_pool.ban(&Instant::now(), vec![pool.hash_of(&uxt)]); let res = block_on(pool.submit_one(&BlockId::Number(0), SOURCE, uxt)); assert_eq!(pool.validated_pool().status().ready, 0); assert_eq!(pool.validated_pool().status().future, 0); @@ -763,9 +616,9 @@ mod tests { assert_eq!(pool.validated_pool().status().future, 0); assert_eq!(pool.validated_pool().status().ready, 0); // make sure they are temporarily banned as well - assert!(pool.validated_pool.rotator().is_banned(&hash1)); - assert!(pool.validated_pool.rotator().is_banned(&hash2)); - assert!(pool.validated_pool.rotator().is_banned(&hash3)); + assert!(pool.validated_pool.is_banned(&hash1)); + assert!(pool.validated_pool.is_banned(&hash2)); + assert!(pool.validated_pool.is_banned(&hash3)); } #[test] @@ -788,7 +641,7 @@ mod tests { block_on(pool.prune_tags(&BlockId::Number(1), vec![vec![0]], vec![hash1.clone()])).unwrap(); // then - assert!(pool.validated_pool.rotator().is_banned(&hash1)); + assert!(pool.validated_pool.is_banned(&hash1)); } #[test] @@ -828,8 +681,8 @@ mod tests { // then assert_eq!(pool.validated_pool().status().future, 1); - assert!(pool.validated_pool.rotator().is_banned(&hash1)); - assert!(!pool.validated_pool.rotator().is_banned(&hash2)); + assert!(pool.validated_pool.is_banned(&hash1)); + assert!(!pool.validated_pool.is_banned(&hash2)); } #[test] @@ -1044,7 +897,7 @@ mod tests { } #[test] - fn should_trigger_dropped() { + fn should_trigger_dropped_older() { // given let limit = Limit { count: 1, total_bytes: 1000 }; let options = @@ -1077,6 +930,67 @@ mod tests { assert_eq!(stream.next(), Some(TransactionStatus::Dropped)); } + #[test] + fn should_trigger_dropped_lower_priority() { + { + // given + let limit = Limit { count: 1, total_bytes: 1000 }; + let options = + Options { ready: limit.clone(), future: limit.clone(), ..Default::default() }; + + let pool = Pool::new(options, true.into(), TestApi::default().into()); + + let xt = Extrinsic::IncludeData(Vec::new()); + block_on(pool.submit_one(&BlockId::Number(0), SOURCE, xt)).unwrap(); + assert_eq!(pool.validated_pool().status().ready, 1); + + // then + let xt = uxt(Transfer { + from: AccountId::from_h256(H256::from_low_u64_be(2)), + to: AccountId::from_h256(H256::from_low_u64_be(1)), + amount: 4, + nonce: 1, + }); + let result = block_on(pool.submit_one(&BlockId::Number(1), SOURCE, xt)); + assert!(matches!( + result, + Err(sc_transaction_pool_api::error::Error::ImmediatelyDropped) + )); + } + { + // given + let limit = Limit { count: 2, total_bytes: 1000 }; + let options = + Options { ready: limit.clone(), future: limit.clone(), ..Default::default() }; + + let pool = Pool::new(options, true.into(), TestApi::default().into()); + + let xt = Extrinsic::IncludeData(Vec::new()); + block_on(pool.submit_and_watch(&BlockId::Number(0), SOURCE, xt)).unwrap(); + assert_eq!(pool.validated_pool().status().ready, 1); + + let xt = uxt(Transfer { + from: AccountId::from_h256(H256::from_low_u64_be(1)), + to: AccountId::from_h256(H256::from_low_u64_be(2)), + amount: 5, + nonce: 0, + }); + let watcher = + block_on(pool.submit_and_watch(&BlockId::Number(0), SOURCE, xt)).unwrap(); + assert_eq!(pool.validated_pool().status().ready, 2); + + // when + let xt = Extrinsic::Store(Vec::new()); + block_on(pool.submit_one(&BlockId::Number(1), SOURCE, xt)).unwrap(); + assert_eq!(pool.validated_pool().status().ready, 2); + + // then + let mut stream = futures::executor::block_on_stream(watcher.into_stream()); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::Dropped)); + } + } + #[test] fn should_handle_pruning_in_the_middle_of_import() { // given diff --git a/client/transaction-pool/src/graph/ready.rs b/client/transaction-pool/src/graph/ready.rs index 03689aeb32e6..ebaa73f14924 100644 --- a/client/transaction-pool/src/graph/ready.rs +++ b/client/transaction-pool/src/graph/ready.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,7 +23,7 @@ use std::{ sync::Arc, }; -use log::trace; +use log::{debug, trace}; use sc_transaction_pool_api::error; use serde::Serialize; use sp_runtime::{traits::Member, transaction_validity::TransactionTag as Tag}; @@ -31,7 +31,7 @@ use sp_runtime::{traits::Member, transaction_validity::TransactionTag as Tag}; use super::{ base_pool::Transaction, future::WaitingTransaction, - tracked_map::{self, ReadOnlyTrackedMap, TrackedMap}, + tracked_map::{self, TrackedMap}, }; /// An in-pool transaction reference. @@ -156,11 +156,16 @@ impl ReadyTransactions { /// - transactions that are valid for a shorter time go first /// 4. Lastly we sort by the time in the queue /// - transactions that are longer in the queue go first - pub fn get(&self) -> impl Iterator>> { + /// + /// The iterator is providing a way to report transactions that the receiver considers invalid. + /// In such case the entire subgraph of transactions that depend on the reported one will be + /// skipped. + pub fn get(&self) -> BestIterator { BestIterator { - all: self.ready.clone(), + all: self.ready.clone_map(), best: self.best.clone(), awaiting: Default::default(), + invalid: Default::default(), } } @@ -479,9 +484,10 @@ impl ReadyTransactions { /// Iterator of ready transactions ordered by priority. pub struct BestIterator { - all: ReadOnlyTrackedMap>, + all: HashMap>, awaiting: HashMap)>, best: BTreeSet>, + invalid: HashSet, } impl BestIterator { @@ -498,6 +504,34 @@ impl BestIterator { } } +impl sc_transaction_pool_api::ReadyTransactions + for BestIterator +{ + fn report_invalid(&mut self, tx: &Self::Item) { + BestIterator::report_invalid(self, tx) + } +} + +impl BestIterator { + /// Report given transaction as invalid. + /// + /// As a consequence, all values that depend on the invalid one will be skipped. + /// When given transaction is not in the pool it has no effect. + /// When invoked on a fully drained iterator it has no effect either. + pub fn report_invalid(&mut self, tx: &Arc>) { + if let Some(to_report) = self.all.get(&tx.hash) { + debug!( + target: "txpool", + "[{:?}] Reported as invalid. Will skip sub-chains while iterating.", + to_report.transaction.transaction.hash + ); + for hash in &to_report.unlocks { + self.invalid.insert(hash.clone()); + } + } + } +} + impl Iterator for BestIterator { type Item = Arc>; @@ -505,9 +539,19 @@ impl Iterator for BestIterator { loop { let best = self.best.iter().next_back()?.clone(); let best = self.best.take(&best)?; + let hash = &best.transaction.hash; + + // Check if the transaction was marked invalid. + if self.invalid.contains(hash) { + debug!( + target: "txpool", + "[{:?}] Skipping invalid child transaction while iterating.", + hash + ); + continue + } - let next = self.all.read().get(&best.transaction.hash).cloned(); - let ready = match next { + let ready = match self.all.get(&hash).cloned() { Some(ready) => ready, // The transaction is not in all, maybe it was removed in the meantime? None => continue, @@ -522,7 +566,6 @@ impl Iterator for BestIterator { // then get from the pool } else { self.all - .read() .get(hash) .map(|next| (next.requires_offset + 1, next.transaction.clone())) }; @@ -635,10 +678,13 @@ mod tests { assert_eq!(ready.get().count(), 3); } - #[test] - fn should_return_best_transactions_in_correct_order() { - // given - let mut ready = ReadyTransactions::default(); + /// Populate the pool, with a graph that looks like so: + /// + /// tx1 -> tx2 \ + /// -> -> tx3 + /// -> tx4 -> tx5 -> tx6 + /// -> tx7 + fn populate_pool(ready: &mut ReadyTransactions>) { let mut tx1 = tx(1); tx1.requires.clear(); let mut tx2 = tx(2); @@ -649,11 +695,17 @@ mod tests { tx3.provides = vec![]; let mut tx4 = tx(4); tx4.requires = vec![tx1.provides[0].clone()]; - tx4.provides = vec![]; - let tx5 = Transaction { - data: vec![5], + tx4.provides = vec![vec![107]]; + let mut tx5 = tx(5); + tx5.requires = vec![tx4.provides[0].clone()]; + tx5.provides = vec![vec![108]]; + let mut tx6 = tx(6); + tx6.requires = vec![tx5.provides[0].clone()]; + tx6.provides = vec![]; + let tx7 = Transaction { + data: vec![7], bytes: 1, - hash: 5, + hash: 7, priority: 1, valid_till: u64::MAX, // use the max here for testing. requires: vec![tx1.provides[0].clone()], @@ -663,20 +715,30 @@ mod tests { }; // when - for tx in vec![tx1, tx2, tx3, tx4, tx5] { - import(&mut ready, tx).unwrap(); + for tx in vec![tx1, tx2, tx3, tx7, tx4, tx5, tx6] { + import(ready, tx).unwrap(); } - // then assert_eq!(ready.best.len(), 1); + } + + #[test] + fn should_return_best_transactions_in_correct_order() { + // given + let mut ready = ReadyTransactions::default(); + populate_pool(&mut ready); + // when let mut it = ready.get().map(|tx| tx.data[0]); + // then assert_eq!(it.next(), Some(1)); assert_eq!(it.next(), Some(2)); assert_eq!(it.next(), Some(3)); assert_eq!(it.next(), Some(4)); assert_eq!(it.next(), Some(5)); + assert_eq!(it.next(), Some(6)); + assert_eq!(it.next(), Some(7)); assert_eq!(it.next(), None); } @@ -725,4 +787,26 @@ mod tests { TransactionRef { transaction: Arc::new(with_priority(3, 3)), insertion_id: 2 } ); } + + #[test] + fn should_skip_invalid_transactions_while_iterating() { + // given + let mut ready = ReadyTransactions::default(); + populate_pool(&mut ready); + + // when + let mut it = ready.get(); + let data = |tx: &Arc>>| tx.data[0]; + + // then + assert_eq!(it.next().as_ref().map(data), Some(1)); + assert_eq!(it.next().as_ref().map(data), Some(2)); + assert_eq!(it.next().as_ref().map(data), Some(3)); + let tx4 = it.next(); + assert_eq!(tx4.as_ref().map(data), Some(4)); + // report 4 as invalid, which should skip 5 & 6. + it.report_invalid(&tx4.unwrap()); + assert_eq!(it.next().as_ref().map(data), Some(7)); + assert_eq!(it.next().as_ref().map(data), None); + } } diff --git a/client/transaction-pool/src/graph/rotator.rs b/client/transaction-pool/src/graph/rotator.rs index 910f86b5ed5b..b897fe788503 100644 --- a/client/transaction-pool/src/graph/rotator.rs +++ b/client/transaction-pool/src/graph/rotator.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/transaction-pool/src/graph/tracked_map.rs b/client/transaction-pool/src/graph/tracked_map.rs index c1fdda227c6a..32d04b006887 100644 --- a/client/transaction-pool/src/graph/tracked_map.rs +++ b/client/transaction-pool/src/graph/tracked_map.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -57,11 +57,6 @@ impl TrackedMap { std::cmp::max(self.bytes.load(AtomicOrdering::Relaxed), 0) as usize } - /// Read-only clone of the interior. - pub fn clone(&self) -> ReadOnlyTrackedMap { - ReadOnlyTrackedMap(self.index.clone()) - } - /// Lock map for read. pub fn read(&self) -> TrackedMapReadAccess { TrackedMapReadAccess { inner_guard: self.index.read() } @@ -77,18 +72,10 @@ impl TrackedMap { } } -/// Read-only access to map. -/// -/// The only thing can be done is .read(). -pub struct ReadOnlyTrackedMap(Arc>>); - -impl ReadOnlyTrackedMap -where - K: Eq + std::hash::Hash, -{ - /// Lock map for read. - pub fn read(&self) -> TrackedMapReadAccess { - TrackedMapReadAccess { inner_guard: self.0.read() } +impl TrackedMap { + /// Clone the inner map. + pub fn clone_map(&self) -> HashMap { + self.index.read().clone() } } diff --git a/client/transaction-pool/src/graph/validated_pool.rs b/client/transaction-pool/src/graph/validated_pool.rs index e4aad7f342b5..4ddaf8de5c2b 100644 --- a/client/transaction-pool/src/graph/validated_pool.rs +++ b/client/transaction-pool/src/graph/validated_pool.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -25,7 +25,7 @@ use std::{ use futures::channel::mpsc::{channel, Sender}; use parking_lot::{Mutex, RwLock}; use retain_mut::RetainMut; -use sc_transaction_pool_api::{error, PoolStatus}; +use sc_transaction_pool_api::{error, PoolStatus, ReadyTransactions}; use serde::Serialize; use sp_runtime::{ generic::BlockId, @@ -203,7 +203,7 @@ impl ValidatedPool { let imported = self.pool.write().import(tx)?; if let base::Imported::Ready { ref hash, .. } = imported { - self.import_notification_sinks.lock().retain_mut(|sink| { + RetainMut::retain_mut(&mut *self.import_notification_sinks.lock(), |sink| { match sink.try_send(*hash) { Ok(()) => true, Err(e) => @@ -569,12 +569,6 @@ impl ValidatedPool { Ok(()) } - /// Get rotator reference. - #[cfg(feature = "test-helpers")] - pub fn rotator(&self) -> &PoolRotator> { - &self.rotator - } - /// Get api reference. pub fn api(&self) -> &B { &self.api @@ -630,7 +624,7 @@ impl ValidatedPool { } /// Get an iterator for ready transactions ordered by priority - pub fn ready(&self) -> impl Iterator> + Send { + pub fn ready(&self) -> impl ReadyTransactions> + Send { self.pool.read().ready() } diff --git a/client/transaction-pool/src/graph/watcher.rs b/client/transaction-pool/src/graph/watcher.rs index 975ee6608886..8cd78cfc7824 100644 --- a/client/transaction-pool/src/graph/watcher.rs +++ b/client/transaction-pool/src/graph/watcher.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/transaction-pool/src/lib.rs b/client/transaction-pool/src/lib.rs index 6eb5bd2f332e..9fd2ef1deef4 100644 --- a/client/transaction-pool/src/lib.rs +++ b/client/transaction-pool/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -23,40 +23,32 @@ #![warn(unused_extern_crates)] mod api; +pub mod error; mod graph; mod metrics; mod revalidation; +#[cfg(test)] +mod tests; -pub mod error; - -/// Common types for testing the transaction pool -#[cfg(feature = "test-helpers")] -pub mod test_helpers { - pub use super::{ - graph::{BlockHash, ChainApi, ExtrinsicFor, NumberFor, Pool}, - revalidation::RevalidationQueue, - }; -} - -pub use crate::api::{FullChainApi, LightChainApi}; +pub use crate::api::FullChainApi; use futures::{ channel::oneshot, future::{self, ready}, prelude::*, }; -pub use graph::{ChainApi, Options, Pool, Transaction}; +pub use graph::{base_pool::Limit as PoolLimit, ChainApi, Options, Pool, Transaction}; use parking_lot::Mutex; use std::{ collections::{HashMap, HashSet}, - convert::TryInto, pin::Pin, sync::Arc, }; use graph::{ExtrinsicHash, IsValidator}; use sc_transaction_pool_api::{ - ChainEvent, ImportNotificationStream, MaintainedTransactionPool, PoolFuture, PoolStatus, - TransactionFor, TransactionPool, TransactionSource, TransactionStatusStreamFor, TxHash, + error::Error as TxPoolError, ChainEvent, ImportNotificationStream, MaintainedTransactionPool, + PoolFuture, PoolStatus, ReadyTransactions, TransactionFor, TransactionPool, TransactionSource, + TransactionStatusStreamFor, TxHash, }; use sp_core::traits::SpawnEssentialNamed; use sp_runtime::{ @@ -69,7 +61,7 @@ use crate::metrics::MetricsLink as PrometheusMetrics; use prometheus_endpoint::Registry as PrometheusRegistry; type BoxedReadyIterator = - Box>> + Send>; + Box>> + Send>; type ReadyIteratorFor = BoxedReadyIterator, graph::ExtrinsicFor>; @@ -78,9 +70,6 @@ type PolledIterator = Pin = BasicPool, Block>; -/// A transaction pool for a light node. -pub type LightPool = - BasicPool, Block>; /// Basic implementation of transaction pool that can be customized by providing PoolApi. pub struct BasicPool @@ -172,13 +161,10 @@ where PoolApi: graph::ChainApi + 'static, { /// Create new basic transaction pool with provided api, for tests. - #[cfg(feature = "test-helpers")] - pub fn new_test( - pool_api: Arc, - ) -> (Self, Pin + Send>>, intervalier::BackSignalControl) { + pub fn new_test(pool_api: Arc) -> (Self, Pin + Send>>) { let pool = Arc::new(graph::Pool::new(Default::default(), true.into(), pool_api.clone())); - let (revalidation_queue, background_task, notifier) = - revalidation::RevalidationQueue::new_test(pool_api.clone(), pool.clone()); + let (revalidation_queue, background_task) = + revalidation::RevalidationQueue::new_background(pool_api.clone(), pool.clone()); ( Self { api: pool_api, @@ -189,7 +175,6 @@ where metrics: Default::default(), }, background_task, - notifier, ) } @@ -216,7 +201,7 @@ where }; if let Some(background_task) = background_task { - spawner.spawn_essential("txpool-background", background_task); + spawner.spawn_essential("txpool-background", Some("transaction-pool"), background_task); } Self { @@ -239,7 +224,6 @@ where } /// Get access to the underlying api - #[cfg(feature = "test-helpers")] pub fn api(&self) -> &PoolApi { &self.api } @@ -363,33 +347,6 @@ where } } -impl LightPool -where - Block: BlockT, - Client: sp_blockchain::HeaderBackend + sc_client_api::UsageProvider + 'static, - Fetcher: sc_client_api::Fetcher + 'static, -{ - /// Create new basic transaction pool for a light node with the provided api. - pub fn new_light( - options: graph::Options, - prometheus: Option<&PrometheusRegistry>, - spawner: impl SpawnEssentialNamed, - client: Arc, - fetcher: Arc, - ) -> Self { - let pool_api = Arc::new(LightChainApi::new(client.clone(), fetcher)); - Self::with_revalidation_type( - options, - false.into(), - pool_api, - prometheus, - RevalidationType::Light, - spawner, - client.usage_info().chain.best_number, - ) - } -} - impl FullPool where Block: BlockT, @@ -460,8 +417,8 @@ where .validate_transaction_blocking(at, TransactionSource::Local, xt.clone())? .map_err(|e| { Self::Error::Pool(match e { - TransactionValidityError::Invalid(i) => i.into(), - TransactionValidityError::Unknown(u) => u.into(), + TransactionValidityError::Invalid(i) => TxPoolError::InvalidTransaction(i), + TransactionValidityError::Unknown(u) => TxPoolError::UnknownTransaction(u), }) })?; @@ -576,7 +533,7 @@ async fn prune_known_txs_for_block { - log::debug!(target: "txpool", "Error retrieving header for {:?}: {:?}", block_id, e); + log::debug!(target: "txpool", "Error retrieving header for {:?}: {}", block_id, e); return hashes }, }; if let Err(e) = pool.prune(&block_id, &BlockId::hash(*header.parent_hash()), &extrinsics).await { - log::error!("Cannot prune known in the pool {:?}!", e); + log::error!("Cannot prune known in the pool: {}", e); } hashes @@ -681,7 +638,7 @@ where .block_body(&BlockId::hash(hash)) .await .unwrap_or_else(|e| { - log::warn!("Failed to fetch block body {:?}!", e); + log::warn!("Failed to fetch block body: {}", e); None }) .unwrap_or_default() @@ -727,7 +684,7 @@ where { log::debug!( target: "txpool", - "[{:?}] Error re-submitting transactions: {:?}", + "[{:?}] Error re-submitting transactions: {}", id, e, ) @@ -751,15 +708,17 @@ where } .boxed() }, - ChainEvent::Finalized { hash } => { + ChainEvent::Finalized { hash, tree_route } => { let pool = self.pool.clone(); async move { - if let Err(e) = pool.validated_pool().on_block_finalized(hash).await { - log::warn!( - target: "txpool", - "Error [{}] occurred while attempting to notify watchers of finalization {}", - e, hash - ) + for hash in tree_route.iter().chain(&[hash]) { + if let Err(e) = pool.validated_pool().on_block_finalized(*hash).await { + log::warn!( + target: "txpool", + "Error [{}] occurred while attempting to notify watchers of finalization {}", + e, hash + ) + } } } .boxed() diff --git a/client/transaction-pool/src/metrics.rs b/client/transaction-pool/src/metrics.rs index d62d64f13a0a..8bcefe246bff 100644 --- a/client/transaction-pool/src/metrics.rs +++ b/client/transaction-pool/src/metrics.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -56,28 +56,28 @@ impl Metrics { Ok(Self { submitted_transactions: register( Counter::new( - "sub_txpool_submitted_transactions", + "substrate_sub_txpool_submitted_transactions", "Total number of transactions submitted", )?, registry, )?, validations_invalid: register( Counter::new( - "sub_txpool_validations_invalid", + "substrate_sub_txpool_validations_invalid", "Total number of transactions that were removed from the pool as invalid", )?, registry, )?, block_transactions_pruned: register( Counter::new( - "sub_txpool_block_transactions_pruned", + "substrate_sub_txpool_block_transactions_pruned", "Total number of transactions that was requested to be pruned by block events", )?, registry, )?, block_transactions_resubmitted: register( Counter::new( - "sub_txpool_block_transactions_resubmitted", + "substrate_sub_txpool_block_transactions_resubmitted", "Total number of transactions that was requested to be resubmitted by block events", )?, registry, @@ -98,14 +98,14 @@ impl ApiMetrics { Ok(Self { validations_scheduled: register( Counter::new( - "sub_txpool_validations_scheduled", + "substrate_sub_txpool_validations_scheduled", "Total number of transactions scheduled for validation", )?, registry, )?, validations_finished: register( Counter::new( - "sub_txpool_validations_finished", + "substrate_sub_txpool_validations_finished", "Total number of transactions that finished validation", )?, registry, diff --git a/client/transaction-pool/src/revalidation.rs b/client/transaction-pool/src/revalidation.rs index a8b2c1d32036..e3641008a706 100644 --- a/client/transaction-pool/src/revalidation.rs +++ b/client/transaction-pool/src/revalidation.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -35,10 +35,7 @@ use sp_runtime::{ use futures::prelude::*; use std::time::Duration; -#[cfg(not(feature = "test-helpers"))] const BACKGROUND_REVALIDATION_INTERVAL: Duration = Duration::from_millis(200); -#[cfg(feature = "test-helpers")] -pub const BACKGROUND_REVALIDATION_INTERVAL: Duration = Duration::from_millis(1); const MIN_BACKGROUND_REVALIDATION_BATCH_SIZE: usize = 20; @@ -109,7 +106,7 @@ async fn batch_revalidate( Err(validation_err) => { log::debug!( target: "txpool", - "[{:?}]: Error during revalidation: {:?}. Removing.", + "[{:?}]: Removing due to error during revalidation: {}", ext_hash, validation_err ); @@ -213,36 +210,25 @@ impl RevalidationWorker { /// It does two things: periodically tries to process some transactions /// from the queue and also accepts messages to enqueue some more /// transactions from the pool. - pub async fn run( + pub async fn run( mut self, from_queue: TracingUnboundedReceiver>, - interval: R, - ) where - R: Send, - R::Guard: Send, - { - let interval = interval.into_stream().fuse(); + interval: Duration, + ) { + let interval_fut = futures_timer::Delay::new(interval); let from_queue = from_queue.fuse(); - futures::pin_mut!(interval, from_queue); + futures::pin_mut!(interval_fut, from_queue); let this = &mut self; loop { futures::select! { - _guard = interval.next() => { + // Using `fuse()` in here is okay, because we reset the interval when it has fired. + _ = (&mut interval_fut).fuse() => { let next_batch = this.prepare_batch(); let batch_len = next_batch.len(); batch_revalidate(this.pool.clone(), this.api.clone(), this.best_block, next_batch).await; - #[cfg(feature = "test-helpers")] - { - use intervalier::Guard; - // only trigger test events if something was processed - if batch_len == 0 { - _guard.expect("Always some() in tests").skip(); - } - } - if batch_len > 0 || this.len() > 0 { log::debug!( target: "txpool", @@ -251,6 +237,8 @@ impl RevalidationWorker { this.len(), ); } + + interval_fut.reset(interval); }, workload = from_queue.next() => { match workload { @@ -298,15 +286,11 @@ where } /// New revalidation queue with background worker. - pub fn new_with_interval( + pub fn new_with_interval( api: Arc, pool: Arc>, - interval: R, - ) -> (Self, Pin + Send>>) - where - R: Send + 'static, - R::Guard: Send, - { + interval: Duration, + ) -> (Self, Pin + Send>>) { let (to_worker, from_queue) = tracing_unbounded("mpsc_revalidation_queue"); let worker = RevalidationWorker::new(api.clone(), pool.clone()); @@ -321,24 +305,7 @@ where api: Arc, pool: Arc>, ) -> (Self, Pin + Send>>) { - Self::new_with_interval( - api, - pool, - intervalier::Interval::new(BACKGROUND_REVALIDATION_INTERVAL), - ) - } - - /// New revalidation queue with background worker and test signal. - #[cfg(feature = "test-helpers")] - pub fn new_test( - api: Arc, - pool: Arc>, - ) -> (Self, Pin + Send>>, intervalier::BackSignalControl) { - let (interval, notifier) = - intervalier::BackSignalInterval::new(BACKGROUND_REVALIDATION_INTERVAL); - let (queue, background) = Self::new_with_interval(api, pool, interval); - - (queue, background, notifier) + Self::new_with_interval(api, pool, BACKGROUND_REVALIDATION_INTERVAL) } /// Queue some transaction for later revalidation. @@ -371,4 +338,41 @@ where } #[cfg(test)] -mod tests {} +mod tests { + use super::*; + use crate::{ + graph::Pool, + tests::{uxt, TestApi}, + }; + use futures::executor::block_on; + use sc_transaction_pool_api::TransactionSource; + use sp_runtime::generic::BlockId; + use substrate_test_runtime::{AccountId, Transfer, H256}; + + #[test] + fn revalidation_queue_works() { + let api = Arc::new(TestApi::default()); + let pool = Arc::new(Pool::new(Default::default(), true.into(), api.clone())); + let queue = Arc::new(RevalidationQueue::new(api.clone(), pool.clone())); + + let uxt = uxt(Transfer { + from: AccountId::from_h256(H256::from_low_u64_be(1)), + to: AccountId::from_h256(H256::from_low_u64_be(2)), + amount: 5, + nonce: 0, + }); + let uxt_hash = block_on(pool.submit_one( + &BlockId::number(0), + TransactionSource::External, + uxt.clone(), + )) + .expect("Should be valid"); + + block_on(queue.revalidate_later(0, vec![uxt_hash])); + + // revalidated in sync offload 2nd time + assert_eq!(api.validation_requests().len(), 2); + // number of ready + assert_eq!(pool.validated_pool().status().ready, 1); + } +} diff --git a/client/transaction-pool/src/tests.rs b/client/transaction-pool/src/tests.rs new file mode 100644 index 000000000000..79142e16a1b3 --- /dev/null +++ b/client/transaction-pool/src/tests.rs @@ -0,0 +1,185 @@ +// This file is part of Substrate. + +// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Testing related primitives for internal usage in this crate. + +use crate::graph::{BlockHash, ChainApi, ExtrinsicFor, NumberFor, Pool}; +use codec::Encode; +use parking_lot::Mutex; +use sc_transaction_pool_api::error; +use sp_runtime::{ + generic::BlockId, + traits::{Block as BlockT, Hash}, + transaction_validity::{ + InvalidTransaction, TransactionSource, TransactionValidity, ValidTransaction, + }, +}; +use std::{collections::HashSet, sync::Arc}; +use substrate_test_runtime::{Block, Extrinsic, Hashing, Transfer, H256}; + +pub(crate) const INVALID_NONCE: u64 = 254; + +/// Test api that implements [`ChainApi`]. +#[derive(Clone, Debug, Default)] +pub(crate) struct TestApi { + pub delay: Arc>>>, + pub invalidate: Arc>>, + pub clear_requirements: Arc>>, + pub add_requirements: Arc>>, + pub validation_requests: Arc>>, +} + +impl TestApi { + /// Query validation requests received. + pub fn validation_requests(&self) -> Vec { + self.validation_requests.lock().clone() + } +} + +impl ChainApi for TestApi { + type Block = Block; + type Error = error::Error; + type ValidationFuture = futures::future::Ready>; + type BodyFuture = futures::future::Ready>>>; + + /// Verify extrinsic at given block. + fn validate_transaction( + &self, + at: &BlockId, + _source: TransactionSource, + uxt: ExtrinsicFor, + ) -> Self::ValidationFuture { + self.validation_requests.lock().push(uxt.clone()); + let hash = self.hash_and_length(&uxt).0; + let block_number = self.block_id_to_number(at).unwrap().unwrap(); + + let res = match uxt { + Extrinsic::Transfer { transfer, .. } => { + let nonce = transfer.nonce; + + // This is used to control the test flow. + if nonce > 0 { + let opt = self.delay.lock().take(); + if let Some(delay) = opt { + if delay.recv().is_err() { + println!("Error waiting for delay!"); + } + } + } + + if self.invalidate.lock().contains(&hash) { + InvalidTransaction::Custom(0).into() + } else if nonce < block_number { + InvalidTransaction::Stale.into() + } else { + let mut transaction = ValidTransaction { + priority: 4, + requires: if nonce > block_number { + vec![vec![nonce as u8 - 1]] + } else { + vec![] + }, + provides: if nonce == INVALID_NONCE { + vec![] + } else { + vec![vec![nonce as u8]] + }, + longevity: 3, + propagate: true, + }; + + if self.clear_requirements.lock().contains(&hash) { + transaction.requires.clear(); + } + + if self.add_requirements.lock().contains(&hash) { + transaction.requires.push(vec![128]); + } + + Ok(transaction) + } + }, + Extrinsic::IncludeData(_) => Ok(ValidTransaction { + priority: 9001, + requires: vec![], + provides: vec![vec![42]], + longevity: 9001, + propagate: false, + }), + Extrinsic::Store(_) => Ok(ValidTransaction { + priority: 9001, + requires: vec![], + provides: vec![vec![43]], + longevity: 9001, + propagate: false, + }), + _ => unimplemented!(), + }; + + futures::future::ready(Ok(res)) + } + + /// Returns a block number given the block id. + fn block_id_to_number( + &self, + at: &BlockId, + ) -> Result>, Self::Error> { + Ok(match at { + BlockId::Number(num) => Some(*num), + BlockId::Hash(_) => None, + }) + } + + /// Returns a block hash given the block id. + fn block_id_to_hash( + &self, + at: &BlockId, + ) -> Result::Hash>, Self::Error> { + Ok(match at { + BlockId::Number(num) => Some(H256::from_low_u64_be(*num)).into(), + BlockId::Hash(_) => None, + }) + } + + /// Hash the extrinsic. + fn hash_and_length(&self, uxt: &ExtrinsicFor) -> (BlockHash, usize) { + let encoded = uxt.encode(); + let len = encoded.len(); + (Hashing::hash(&encoded), len) + } + + fn block_body(&self, _id: &BlockId) -> Self::BodyFuture { + futures::future::ready(Ok(None)) + } + + fn block_header( + &self, + _: &BlockId, + ) -> Result::Header>, Self::Error> { + Ok(None) + } +} + +pub(crate) fn uxt(transfer: Transfer) -> Extrinsic { + let signature = TryFrom::try_from(&[0; 64][..]).unwrap(); + Extrinsic::Transfer { transfer, signature, exhaust_resources_when_not_first: false } +} + +pub(crate) fn pool() -> Pool { + Pool::new(Default::default(), true.into(), TestApi::default().into()) +} diff --git a/client/transaction-pool/tests/pool.rs b/client/transaction-pool/tests/pool.rs index 6c34d05cd5dc..2243f140cb22 100644 --- a/client/transaction-pool/tests/pool.rs +++ b/client/transaction-pool/tests/pool.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -17,6 +17,7 @@ // along with this program. If not, see . //! Tests for top-level transaction pool api + use codec::Encode; use futures::{ executor::{block_on, block_on_stream}, @@ -25,7 +26,7 @@ use futures::{ }; use sc_block_builder::BlockBuilderProvider; use sc_client_api::client::BlockchainEvents; -use sc_transaction_pool::{test_helpers::*, *}; +use sc_transaction_pool::*; use sc_transaction_pool_api::{ ChainEvent, MaintainedTransactionPool, TransactionPool, TransactionStatus, }; @@ -35,7 +36,7 @@ use sp_runtime::{ traits::Block as _, transaction_validity::{InvalidTransaction, TransactionSource, ValidTransaction}, }; -use std::{collections::BTreeSet, convert::TryInto, sync::Arc}; +use std::{collections::BTreeSet, sync::Arc}; use substrate_test_runtime_client::{ runtime::{Block, Extrinsic, Hash, Header, Index, Transfer}, AccountKeyring::*, @@ -47,14 +48,13 @@ fn pool() -> Pool { Pool::new(Default::default(), true.into(), TestApi::with_alice_nonce(209).into()) } -fn maintained_pool( -) -> (BasicPool, futures::executor::ThreadPool, intervalier::BackSignalControl) { - let (pool, background_task, notifier) = - BasicPool::new_test(Arc::new(TestApi::with_alice_nonce(209))); +fn maintained_pool() -> (BasicPool, Arc, futures::executor::ThreadPool) { + let api = Arc::new(TestApi::with_alice_nonce(209)); + let (pool, background_task) = BasicPool::new_test(api.clone()); let thread_pool = futures::executor::ThreadPool::new().unwrap(); thread_pool.spawn_ok(background_task); - (pool, thread_pool, notifier) + (pool, api, thread_pool) } const SOURCE: TransactionSource = TransactionSource::External; @@ -135,7 +135,7 @@ fn should_ban_invalid_transactions() { #[test] fn only_prune_on_new_best() { - let pool = maintained_pool().0; + let (pool, api, _) = maintained_pool(); let uxt = uxt(Alice, 209); let _ = block_on(pool.submit_and_watch(&BlockId::number(0), SOURCE, uxt.clone())) @@ -143,7 +143,7 @@ fn only_prune_on_new_best() { pool.api().push_block(1, vec![uxt.clone()], true); assert_eq!(pool.status().ready, 1); - let header = pool.api().push_block(2, vec![uxt], true); + let header = api.push_block(2, vec![uxt], true); let event = ChainEvent::NewBestBlock { hash: header.hash(), tree_route: None }; block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 0); @@ -205,12 +205,12 @@ fn block_event_with_retracted( fn should_prune_old_during_maintenance() { let xt = uxt(Alice, 209); - let (pool, _guard, _notifier) = maintained_pool(); + let (pool, api, _guard) = maintained_pool(); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api().push_block(1, vec![xt.clone()], true); + let header = api.push_block(1, vec![xt.clone()], true); block_on(pool.maintain(block_event(header))); assert_eq!(pool.status().ready, 0); @@ -221,33 +221,38 @@ fn should_revalidate_during_maintenance() { let xt1 = uxt(Alice, 209); let xt2 = uxt(Alice, 210); - let (pool, _guard, mut notifier) = maintained_pool(); + let (pool, api, _guard) = maintained_pool(); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt1.clone())).expect("1. Imported"); - block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt2.clone())).expect("2. Imported"); + let watcher = block_on(pool.submit_and_watch(&BlockId::number(0), SOURCE, xt2.clone())) + .expect("2. Imported"); assert_eq!(pool.status().ready, 2); - assert_eq!(pool.api().validation_requests().len(), 2); + assert_eq!(api.validation_requests().len(), 2); + + let header = api.push_block(1, vec![xt1.clone()], true); - let header = pool.api().push_block(1, vec![xt1.clone()], true); + api.add_invalid(&xt2); block_on(pool.maintain(block_event(header))); assert_eq!(pool.status().ready, 1); - block_on(notifier.next()); // test that pool revalidated transaction that left ready and not included in the block - assert_eq!(pool.api().validation_requests().len(), 3); + assert_eq!( + futures::executor::block_on_stream(watcher).collect::>(), + vec![TransactionStatus::Ready, TransactionStatus::Invalid], + ); } #[test] fn should_resubmit_from_retracted_during_maintenance() { let xt = uxt(Alice, 209); - let (pool, _guard, _notifier) = maintained_pool(); + let (pool, api, _guard) = maintained_pool(); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api().push_block(1, vec![], true); - let fork_header = pool.api().push_block(1, vec![], false); + let header = api.push_block(1, vec![], true); + let fork_header = api.push_block(1, vec![], false); let event = block_event_with_retracted(header, fork_header.hash(), &*pool.api()); @@ -259,13 +264,13 @@ fn should_resubmit_from_retracted_during_maintenance() { fn should_not_resubmit_from_retracted_during_maintenance_if_tx_is_also_in_enacted() { let xt = uxt(Alice, 209); - let (pool, _guard, _notifier) = maintained_pool(); + let (pool, api, _guard) = maintained_pool(); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api().push_block(1, vec![xt.clone()], true); - let fork_header = pool.api().push_block(1, vec![xt], false); + let header = api.push_block(1, vec![xt.clone()], true); + let fork_header = api.push_block(1, vec![xt], false); let event = block_event_with_retracted(header, fork_header.hash(), &*pool.api()); @@ -277,19 +282,23 @@ fn should_not_resubmit_from_retracted_during_maintenance_if_tx_is_also_in_enacte fn should_not_retain_invalid_hashes_from_retracted() { let xt = uxt(Alice, 209); - let (pool, _guard, mut notifier) = maintained_pool(); + let (pool, api, _guard) = maintained_pool(); - block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); + let watcher = block_on(pool.submit_and_watch(&BlockId::number(0), SOURCE, xt.clone())) + .expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api().push_block(1, vec![], true); - let fork_header = pool.api().push_block(1, vec![xt.clone()], false); - pool.api().add_invalid(&xt); + let header = api.push_block(1, vec![], true); + let fork_header = api.push_block(1, vec![xt.clone()], false); + api.add_invalid(&xt); let event = block_event_with_retracted(header, fork_header.hash(), &*pool.api()); - block_on(pool.maintain(event)); - block_on(notifier.next()); + + assert_eq!( + futures::executor::block_on_stream(watcher).collect::>(), + vec![TransactionStatus::Ready, TransactionStatus::Invalid], + ); assert_eq!(pool.status().ready, 0); } @@ -300,26 +309,30 @@ fn should_revalidate_across_many_blocks() { let xt2 = uxt(Alice, 210); let xt3 = uxt(Alice, 211); - let (pool, _guard, mut notifier) = maintained_pool(); + let (pool, api, _guard) = maintained_pool(); - block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt1.clone())).expect("1. Imported"); + let watcher1 = block_on(pool.submit_and_watch(&BlockId::number(0), SOURCE, xt1.clone())) + .expect("1. Imported"); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt2.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 2); - let header = pool.api().push_block(1, vec![], true); + let header = api.push_block(1, vec![], true); block_on(pool.maintain(block_event(header))); - block_on(notifier.next()); block_on(pool.submit_one(&BlockId::number(1), SOURCE, xt3.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 3); - let header = pool.api().push_block(2, vec![xt1.clone()], true); - block_on(pool.maintain(block_event(header))); - block_on(notifier.next()); + let header = api.push_block(2, vec![xt1.clone()], true); + let block_hash = header.hash(); + block_on(pool.maintain(block_event(header.clone()))); + + block_on( + watcher1 + .take_while(|s| future::ready(*s != TransactionStatus::InBlock(block_hash))) + .collect::>(), + ); assert_eq!(pool.status().ready, 2); - // xt1 and xt2 validated twice, then xt3 once, then xt2 and xt3 again - assert_eq!(pool.api().validation_requests().len(), 7); } #[test] @@ -329,7 +342,7 @@ fn should_push_watchers_during_maintenance() { } // given - let (pool, _guard, mut notifier) = maintained_pool(); + let (pool, api, _guard) = maintained_pool(); let tx0 = alice_uxt(0); let watcher0 = @@ -349,18 +362,16 @@ fn should_push_watchers_during_maintenance() { assert_eq!(pool.status().ready, 5); // when - pool.api().add_invalid(&tx3); - pool.api().add_invalid(&tx4); + api.add_invalid(&tx3); + api.add_invalid(&tx4); // clear timer events if any - let header = pool.api().push_block(1, vec![], true); + let header = api.push_block(1, vec![], true); block_on(pool.maintain(block_event(header))); - block_on(notifier.next()); // then // hash3 is now invalid // hash4 is now invalid - assert_eq!(pool.status().ready, 3); assert_eq!( futures::executor::block_on_stream(watcher3).collect::>(), vec![TransactionStatus::Ready, TransactionStatus::Invalid], @@ -369,13 +380,14 @@ fn should_push_watchers_during_maintenance() { futures::executor::block_on_stream(watcher4).collect::>(), vec![TransactionStatus::Ready, TransactionStatus::Invalid], ); + assert_eq!(pool.status().ready, 3); // when - let header = pool.api().push_block(2, vec![tx0, tx1, tx2], true); + let header = api.push_block(2, vec![tx0, tx1, tx2], true); let header_hash = header.hash(); block_on(pool.maintain(block_event(header))); - let event = ChainEvent::Finalized { hash: header_hash.clone() }; + let event = ChainEvent::Finalized { hash: header_hash.clone(), tree_route: Arc::from(vec![]) }; block_on(pool.maintain(event)); // then @@ -410,7 +422,7 @@ fn should_push_watchers_during_maintenance() { #[test] fn can_track_heap_size() { - let (pool, _guard, _notifier) = maintained_pool(); + let (pool, _api, _guard) = maintained_pool(); block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 209))).expect("1. Imported"); block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 210))).expect("1. Imported"); block_on(pool.submit_one(&BlockId::number(0), SOURCE, uxt(Alice, 211))).expect("1. Imported"); @@ -424,7 +436,7 @@ fn finalization() { let xt = uxt(Alice, 209); let api = TestApi::with_alice_nonce(209); api.push_block(1, vec![], true); - let (pool, _background, _) = BasicPool::new_test(api.into()); + let (pool, _background) = BasicPool::new_test(api.into()); let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), SOURCE, xt.clone())) .expect("1. Imported"); pool.api().push_block(2, vec![xt.clone()], true); @@ -433,7 +445,7 @@ fn finalization() { let event = ChainEvent::NewBestBlock { hash: header.hash(), tree_route: None }; block_on(pool.maintain(event)); - let event = ChainEvent::Finalized { hash: header.hash() }; + let event = ChainEvent::Finalized { hash: header.hash(), tree_route: Arc::from(vec![]) }; block_on(pool.maintain(event)); let mut stream = futures::executor::block_on_stream(watcher); @@ -449,7 +461,7 @@ fn fork_aware_finalization() { // starting block A1 (last finalized.) api.push_block(1, vec![], true); - let (pool, _background, _) = BasicPool::new_test(api.into()); + let (pool, _background) = BasicPool::new_test(api.into()); let mut canon_watchers = vec![]; let from_alice = uxt(Alice, 1); @@ -481,7 +493,7 @@ fn fork_aware_finalization() { b1 = header.hash(); block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 0); - let event = ChainEvent::Finalized { hash: b1 }; + let event = ChainEvent::Finalized { hash: b1, tree_route: Arc::from(vec![]) }; block_on(pool.maintain(event)); } @@ -525,7 +537,7 @@ fn fork_aware_finalization() { block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 2); - let event = ChainEvent::Finalized { hash: header.hash() }; + let event = ChainEvent::Finalized { hash: header.hash(), tree_route: Arc::from(vec![]) }; block_on(pool.maintain(event)); } @@ -542,7 +554,7 @@ fn fork_aware_finalization() { d1 = header.hash(); block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 2); - let event = ChainEvent::Finalized { hash: d1 }; + let event = ChainEvent::Finalized { hash: d1, tree_route: Arc::from(vec![]) }; block_on(pool.maintain(event)); } @@ -555,7 +567,7 @@ fn fork_aware_finalization() { let event = ChainEvent::NewBestBlock { hash: header.hash(), tree_route: None }; block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 0); - block_on(pool.maintain(ChainEvent::Finalized { hash: e1 })); + block_on(pool.maintain(ChainEvent::Finalized { hash: e1, tree_route: Arc::from(vec![]) })); } for (canon_watcher, h) in canon_watchers { @@ -597,7 +609,7 @@ fn prune_and_retract_tx_at_same_time() { // starting block A1 (last finalized.) api.push_block(1, vec![], true); - let (pool, _background, _) = BasicPool::new_test(api.into()); + let (pool, _background) = BasicPool::new_test(api.into()); let from_alice = uxt(Alice, 1); pool.api().increment_nonce(Alice.into()); @@ -625,7 +637,7 @@ fn prune_and_retract_tx_at_same_time() { block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 0); - let event = ChainEvent::Finalized { hash: header.hash() }; + let event = ChainEvent::Finalized { hash: header.hash(), tree_route: Arc::from(vec![]) }; block_on(pool.maintain(event)); header.hash() @@ -663,7 +675,7 @@ fn resubmit_tx_of_fork_that_is_not_part_of_retracted() { // starting block A1 (last finalized.) api.push_block(1, vec![], true); - let (pool, _background, _) = BasicPool::new_test(api.into()); + let (pool, _background) = BasicPool::new_test(api.into()); let tx0 = uxt(Alice, 1); let tx1 = uxt(Dave, 2); @@ -708,7 +720,7 @@ fn resubmit_from_retracted_fork() { // starting block A1 (last finalized.) api.push_block(1, vec![], true); - let (pool, _background, _) = BasicPool::new_test(api.into()); + let (pool, _background) = BasicPool::new_test(api.into()); let tx0 = uxt(Alice, 1); let tx1 = uxt(Dave, 2); @@ -800,7 +812,7 @@ fn resubmit_from_retracted_fork() { #[test] fn ready_set_should_not_resolve_before_block_update() { - let (pool, _guard, _notifier) = maintained_pool(); + let (pool, _api, _guard) = maintained_pool(); let xt1 = uxt(Alice, 209); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt1.clone())).expect("1. Imported"); @@ -809,8 +821,8 @@ fn ready_set_should_not_resolve_before_block_update() { #[test] fn ready_set_should_resolve_after_block_update() { - let (pool, _guard, _notifier) = maintained_pool(); - let header = pool.api().push_block(1, vec![], true); + let (pool, api, _guard) = maintained_pool(); + let header = api.push_block(1, vec![], true); let xt1 = uxt(Alice, 209); @@ -822,8 +834,8 @@ fn ready_set_should_resolve_after_block_update() { #[test] fn ready_set_should_eventually_resolve_when_block_update_arrives() { - let (pool, _guard, _notifier) = maintained_pool(); - let header = pool.api().push_block(1, vec![], true); + let (pool, api, _guard) = maintained_pool(); + let header = api.push_block(1, vec![], true); let xt1 = uxt(Alice, 209); @@ -833,7 +845,7 @@ fn ready_set_should_eventually_resolve_when_block_update_arrives() { let mut context = futures::task::Context::from_waker(&noop_waker); let mut ready_set_future = pool.ready_at(1); - if let Poll::Ready(_) = ready_set_future.poll_unpin(&mut context) { + if ready_set_future.poll_unpin(&mut context).is_ready() { panic!("Ready set should not be ready before block update!"); } @@ -852,8 +864,6 @@ fn ready_set_should_eventually_resolve_when_block_update_arrives() { #[test] fn should_not_accept_old_signatures() { - use std::convert::TryFrom; - let client = Arc::new(substrate_test_runtime_client::new()); let pool = Arc::new( @@ -929,13 +939,13 @@ fn import_notification_to_pool_maintain_works() { // When we prune transactions, we need to make sure that we remove #[test] fn pruning_a_transaction_should_remove_it_from_best_transaction() { - let (pool, _guard, _notifier) = maintained_pool(); + let (pool, api, _guard) = maintained_pool(); let xt1 = Extrinsic::IncludeData(Vec::new()); block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt1.clone())).expect("1. Imported"); assert_eq!(pool.status().ready, 1); - let header = pool.api().push_block(1, vec![xt1.clone()], true); + let header = api.push_block(1, vec![xt1.clone()], true); // This will prune `xt1`. block_on(pool.maintain(block_event(header))); @@ -943,26 +953,6 @@ fn pruning_a_transaction_should_remove_it_from_best_transaction() { assert_eq!(pool.status().ready, 0); } -#[test] -fn only_revalidate_on_best_block() { - let xt = uxt(Alice, 209); - - let (pool, _guard, mut notifier) = maintained_pool(); - - block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported"); - assert_eq!(pool.status().ready, 1); - - let header = pool.api().push_block(1, vec![], true); - - pool.api().push_block(2, vec![], false); - pool.api().push_block(2, vec![], false); - - block_on(pool.maintain(block_event(header))); - block_on(notifier.next()); - - assert_eq!(pool.status().ready, 1); -} - #[test] fn stale_transactions_are_pruned() { sp_tracing::try_init_simple(); @@ -974,7 +964,7 @@ fn stale_transactions_are_pruned() { Transfer { from: Alice.into(), to: Bob.into(), nonce: 3, amount: 1 }, ]; - let (pool, _guard, _notifier) = maintained_pool(); + let (pool, api, _guard) = maintained_pool(); xts.into_iter().for_each(|xt| { block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.into_signed_tx())) @@ -992,7 +982,7 @@ fn stale_transactions_are_pruned() { ]; // Import block - let header = pool.api().push_block(1, xts, true); + let header = api.push_block(1, xts, true); block_on(pool.maintain(block_event(header))); // The imported transactions have a different hash and should not evict our initial // transactions. @@ -1000,7 +990,7 @@ fn stale_transactions_are_pruned() { // Import enough blocks to make our transactions stale for n in 1..66 { - let header = pool.api().push_block(n, vec![], true); + let header = api.push_block(n, vec![], true); block_on(pool.maintain(block_event(header))); } diff --git a/client/transaction-pool/tests/revalidation.rs b/client/transaction-pool/tests/revalidation.rs deleted file mode 100644 index b2c8225b78f5..000000000000 --- a/client/transaction-pool/tests/revalidation.rs +++ /dev/null @@ -1,32 +0,0 @@ -use futures::executor::block_on; -use sc_transaction_pool::test_helpers::{Pool, RevalidationQueue}; -use sc_transaction_pool_api::TransactionSource; -use sp_runtime::generic::BlockId; -use std::sync::Arc; -use substrate_test_runtime_client::AccountKeyring::*; -use substrate_test_runtime_transaction_pool::{uxt, TestApi}; - -fn setup() -> (Arc, Pool) { - let test_api = Arc::new(TestApi::empty()); - let pool = Pool::new(Default::default(), true.into(), test_api.clone()); - (test_api, pool) -} - -#[test] -fn smoky() { - let (api, pool) = setup(); - let pool = Arc::new(pool); - let queue = Arc::new(RevalidationQueue::new(api.clone(), pool.clone())); - - let uxt = uxt(Alice, 0); - let uxt_hash = - block_on(pool.submit_one(&BlockId::number(0), TransactionSource::External, uxt.clone())) - .expect("Should be valid"); - - block_on(queue.revalidate_later(0, vec![uxt_hash])); - - // revalidated in sync offload 2nd time - assert_eq!(api.validation_requests().len(), 2); - // number of ready - assert_eq!(pool.validated_pool().status().ready, 1); -} diff --git a/client/utils/Cargo.toml b/client/utils/Cargo.toml index 99765dd501dd..2f1338577e37 100644 --- a/client/utils/Cargo.toml +++ b/client/utils/Cargo.toml @@ -2,19 +2,24 @@ name = "sc-utils" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "I/O for Substrate runtimes" readme = "README.md" [dependencies] -futures = "0.3.9" +futures = "0.3.21" lazy_static = "1.4.0" -prometheus = { version = "0.11.0", default-features = false } +parking_lot = "0.12.0" +prometheus = { version = "0.13.0", default-features = false } futures-timer = "3.0.2" +log = "0.4" [features] default = ["metered"] metered = [] + +[dev-dependencies] +tokio-test = "0.4.2" diff --git a/client/utils/src/id_sequence.rs b/client/utils/src/id_sequence.rs new file mode 100644 index 000000000000..2ccb4fc3f32e --- /dev/null +++ b/client/utils/src/id_sequence.rs @@ -0,0 +1,54 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Produce opaque sequential IDs. + +/// A Sequence of IDs. +#[derive(Debug, Default)] +// The `Clone` trait is intentionally not defined on this type. +pub struct IDSequence { + next_id: u64, +} + +/// A Sequential ID. +/// +/// Its integer value is intentionally not public: it is supposed to be instantiated from within +/// this module only. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct SeqID(u64); + +impl std::fmt::Display for SeqID { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl IDSequence { + /// Create a new ID-sequence. + pub fn new() -> Self { + Default::default() + } + + /// Obtain another ID from this sequence. + pub fn next_id(&mut self) -> SeqID { + let id = SeqID(self.next_id); + self.next_id += 1; + + id + } +} diff --git a/client/utils/src/lib.rs b/client/utils/src/lib.rs index b49cd60d67b1..0e47330335c2 100644 --- a/client/utils/src/lib.rs +++ b/client/utils/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -36,6 +36,9 @@ //! | entity | Name of channel passed to `tracing_unbounded` | //! | action | One of `send`/`received`/`dropped` | +pub mod id_sequence; pub mod metrics; pub mod mpsc; +pub mod notification; +pub mod pubsub; pub mod status_sinks; diff --git a/client/utils/src/metrics.rs b/client/utils/src/metrics.rs index 85ccce626bc2..457ef23e41ab 100644 --- a/client/utils/src/metrics.rs +++ b/client/utils/src/metrics.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -29,17 +29,17 @@ use prometheus::{core::GenericCounterVec, Opts}; lazy_static! { pub static ref TOKIO_THREADS_TOTAL: GenericCounter = - GenericCounter::new("tokio_threads_total", "Total number of threads created") + GenericCounter::new("substrate_tokio_threads_total", "Total number of threads created") .expect("Creating of statics doesn't fail. qed"); pub static ref TOKIO_THREADS_ALIVE: GenericGauge = - GenericGauge::new("tokio_threads_alive", "Number of threads alive right now") + GenericGauge::new("substrate_tokio_threads_alive", "Number of threads alive right now") .expect("Creating of statics doesn't fail. qed"); } #[cfg(feature = "metered")] lazy_static! { pub static ref UNBOUNDED_CHANNELS_COUNTER : GenericCounterVec = GenericCounterVec::new( - Opts::new("unbounded_channel_len", "Items in each mpsc::unbounded instance"), + Opts::new("substrate_unbounded_channel_len", "Items in each mpsc::unbounded instance"), &["entity", "action"] // 'name of channel, send|received|dropped ).expect("Creating of statics doesn't fail. qed"); diff --git a/client/utils/src/mpsc.rs b/client/utils/src/mpsc.rs index 1739af5e9015..ee3fba4a5ee6 100644 --- a/client/utils/src/mpsc.rs +++ b/client/utils/src/mpsc.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/client/utils/src/notification.rs b/client/utils/src/notification.rs new file mode 100644 index 000000000000..ff527c343f9f --- /dev/null +++ b/client/utils/src/notification.rs @@ -0,0 +1,122 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Provides mpsc notification channel that can be instantiated +//! _after_ it's been shared to the consumer and producers entities. +//! +//! Useful when building RPC extensions where, at service definition time, we +//! don't know whether the specific interface where the RPC extension will be +//! exposed is safe or not and we want to lazily build the RPC extension +//! whenever we bind the service to an interface. +//! +//! See [`sc-service::builder::RpcExtensionBuilder`] for more details. + +use futures::stream::{FusedStream, Stream}; +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use crate::pubsub::{Hub, Receiver}; + +mod registry; +use registry::Registry; + +#[cfg(test)] +mod tests; + +/// Trait used to define the "tracing key" string used to tag +/// and identify the mpsc channels. +pub trait TracingKeyStr { + /// Const `str` representing the "tracing key" used to tag and identify + /// the mpsc channels owned by the object implemeting this trait. + const TRACING_KEY: &'static str; +} + +/// The receiving half of the notifications channel. +/// +/// The [`NotificationStream`] entity stores the [`Hub`] so it can be +/// used to add more subscriptions. +#[derive(Clone)] +pub struct NotificationStream { + hub: Hub, + _pd: std::marker::PhantomData, +} + +/// The receiving half of the notifications channel(s). +#[derive(Debug)] +pub struct NotificationReceiver { + receiver: Receiver, +} + +/// The sending half of the notifications channel(s). +pub struct NotificationSender { + hub: Hub, +} + +impl NotificationStream { + /// Creates a new pair of receiver and sender of `Payload` notifications. + pub fn channel() -> (NotificationSender, Self) { + let hub = Hub::new(TK::TRACING_KEY); + let sender = NotificationSender { hub: hub.clone() }; + let receiver = NotificationStream { hub, _pd: Default::default() }; + (sender, receiver) + } + + /// Subscribe to a channel through which the generic payload can be received. + pub fn subscribe(&self) -> NotificationReceiver { + let receiver = self.hub.subscribe(()); + NotificationReceiver { receiver } + } +} + +impl NotificationSender { + /// Send out a notification to all subscribers that a new payload is available for a + /// block. + pub fn notify( + &self, + payload: impl FnOnce() -> Result, + ) -> Result<(), Error> + where + Payload: Clone, + { + self.hub.send(payload) + } +} + +impl Clone for NotificationSender { + fn clone(&self) -> Self { + Self { hub: self.hub.clone() } + } +} + +impl Unpin for NotificationReceiver {} + +impl Stream for NotificationReceiver { + type Item = Payload; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.get_mut().receiver).poll_next(cx) + } +} + +impl FusedStream for NotificationReceiver { + fn is_terminated(&self) -> bool { + self.receiver.is_terminated() + } +} diff --git a/client/utils/src/notification/registry.rs b/client/utils/src/notification/registry.rs new file mode 100644 index 000000000000..ebe41452a567 --- /dev/null +++ b/client/utils/src/notification/registry.rs @@ -0,0 +1,63 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use std::collections::HashSet; + +use crate::{ + id_sequence::SeqID, + pubsub::{Dispatch, Subscribe, Unsubscribe}, +}; + +/// The shared structure to keep track on subscribers. +#[derive(Debug, Default)] +pub(super) struct Registry { + pub(super) subscribers: HashSet, +} + +impl Subscribe<()> for Registry { + fn subscribe(&mut self, _subs_key: (), subs_id: SeqID) { + self.subscribers.insert(subs_id); + } +} +impl Unsubscribe for Registry { + fn unsubscribe(&mut self, subs_id: SeqID) { + self.subscribers.remove(&subs_id); + } +} + +impl Dispatch for Registry +where + MakePayload: FnOnce() -> Result, + Payload: Clone, +{ + type Item = Payload; + type Ret = Result<(), Error>; + + fn dispatch(&mut self, make_payload: MakePayload, mut dispatch: F) -> Self::Ret + where + F: FnMut(&SeqID, Self::Item), + { + if !self.subscribers.is_empty() { + let payload = make_payload()?; + for subs_id in &self.subscribers { + dispatch(subs_id, payload.clone()); + } + } + Ok(()) + } +} diff --git a/client/utils/src/notification/tests.rs b/client/utils/src/notification/tests.rs new file mode 100644 index 000000000000..a001fa7e89e9 --- /dev/null +++ b/client/utils/src/notification/tests.rs @@ -0,0 +1,52 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use super::*; +use futures::StreamExt; + +#[derive(Clone)] +pub struct DummyTracingKey; +impl TracingKeyStr for DummyTracingKey { + const TRACING_KEY: &'static str = "test_notification_stream"; +} + +type StringStream = NotificationStream; + +#[test] +fn notification_channel_simple() { + let (sender, stream) = StringStream::channel(); + + let test_payload = String::from("test payload"); + let closure_payload = test_payload.clone(); + + // Create a future to receive a single notification + // from the stream and verify its payload. + let future = stream.subscribe().take(1).for_each(move |payload| { + let test_payload = closure_payload.clone(); + async move { + assert_eq!(payload, test_payload); + } + }); + + // Send notification. + let r: std::result::Result<(), ()> = sender.notify(|| Ok(test_payload)); + r.unwrap(); + + // Run receiver future. + tokio_test::block_on(future); +} diff --git a/client/utils/src/pubsub.rs b/client/utils/src/pubsub.rs new file mode 100644 index 000000000000..c8e51e3494b9 --- /dev/null +++ b/client/utils/src/pubsub.rs @@ -0,0 +1,263 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Provides means to implement a typical Pub/Sub mechanism. +//! +//! This module provides a type [`Hub`] which can be used both to subscribe, +//! and to send the broadcast messages. +//! +//! The [`Hub`] type is parametrized by two other types: +//! - `Message` — the type of a message that shall be delivered to the subscribers; +//! - `Registry` — implementation of the subscription/dispatch logic. +//! +//! A Registry is implemented by defining the following traits: +//! - [`Subscribe`]; +//! - [`Dispatch`]; +//! - [`Unsubscribe`]. +//! +//! As a result of subscription `Hub::subscribe` method returns an instance of +//! [`Receiver`]. That can be used as a [`Stream`] to receive the messages. +//! Upon drop the [`Receiver`] shall unregister itself from the `Hub`. + +use std::{ + collections::HashMap, + pin::Pin, + sync::{Arc, Weak}, + task::{Context, Poll}, +}; + +use futures::stream::{FusedStream, Stream}; +// use parking_lot::Mutex; +use parking_lot::ReentrantMutex; +use std::cell::RefCell; + +use crate::{ + id_sequence::SeqID, + mpsc::{TracingUnboundedReceiver, TracingUnboundedSender}, +}; + +#[cfg(test)] +mod tests; + +/// Unsubscribe: unregisters a previously created subscription. +pub trait Unsubscribe { + /// Remove all registrations of the subscriber with ID `subs_id`. + fn unsubscribe(&mut self, subs_id: SeqID); +} + +/// Subscribe using a key of type `K` +pub trait Subscribe { + /// Register subscriber with the ID `subs_id` as having interest to the key `K`. + fn subscribe(&mut self, subs_key: K, subs_id: SeqID); +} + +/// Dispatch a message of type `M`. +pub trait Dispatch { + /// The type of the that shall be sent through the channel as a result of such dispatch. + type Item; + /// The type returned by the `dispatch`-method. + type Ret; + + /// Dispatch the message of type `M`. + /// + /// The implementation is given an instance of `M` and is supposed to invoke `dispatch` for + /// each matching subscriber, with an argument of type `Self::Item` matching that subscriber. + /// + /// Note that this does not have to be of the same type with the item that will be sent through + /// to the subscribers. The subscribers will receive a message of type `Self::Item`. + fn dispatch(&mut self, message: M, dispatch: F) -> Self::Ret + where + F: FnMut(&SeqID, Self::Item); +} + +/// A subscription hub. +/// +/// Does the subscription and dispatch. +/// The exact subscription and routing behaviour is to be implemented by the Registry (of type `R`). +/// The Hub under the hood uses the channel defined in `crate::mpsc` module. +#[derive(Debug)] +pub struct Hub { + tracing_key: &'static str, + shared: Arc>>>, +} + +/// The receiving side of the subscription. +/// +/// The messages are delivered as items of a [`Stream`]. +/// Upon drop this receiver unsubscribes itself from the [`Hub`]. +#[derive(Debug)] +pub struct Receiver +where + R: Unsubscribe, +{ + rx: TracingUnboundedReceiver, + + shared: Weak>>>, + subs_id: SeqID, +} + +#[derive(Debug)] +struct Shared { + id_sequence: crate::id_sequence::IDSequence, + registry: R, + sinks: HashMap>, +} + +impl Hub +where + R: Unsubscribe, +{ + /// Provide access to the registry (for test purposes). + pub fn map_registry_for_tests(&self, map: MapF) -> Ret + where + MapF: FnOnce(&R) -> Ret, + { + let shared_locked = self.shared.lock(); + let shared_borrowed = shared_locked.borrow(); + map(&shared_borrowed.registry) + } +} + +impl Drop for Receiver +where + R: Unsubscribe, +{ + fn drop(&mut self) { + if let Some(shared) = self.shared.upgrade() { + shared.lock().borrow_mut().unsubscribe(self.subs_id); + } + } +} + +impl Hub { + /// Create a new instance of Hub (with default value for the Registry). + pub fn new(tracing_key: &'static str) -> Self + where + R: Default, + { + Self::new_with_registry(tracing_key, Default::default()) + } + + /// Create a new instance of Hub over the initialized Registry. + pub fn new_with_registry(tracing_key: &'static str, registry: R) -> Self { + let shared = + Shared { registry, sinks: Default::default(), id_sequence: Default::default() }; + let shared = Arc::new(ReentrantMutex::new(RefCell::new(shared))); + Self { tracing_key, shared } + } + + /// Subscribe to this Hub using the `subs_key: K`. + /// + /// A subscription with a key `K` is possible if the Registry implements `Subscribe`. + pub fn subscribe(&self, subs_key: K) -> Receiver + where + R: Subscribe + Unsubscribe, + { + let shared_locked = self.shared.lock(); + let mut shared_borrowed = shared_locked.borrow_mut(); + + let subs_id = shared_borrowed.id_sequence.next_id(); + + // The order (registry.subscribe then sinks.insert) is important here: + // assuming that `Subscribe::subscribe` can panic, it is better to at least + // have the sink disposed. + shared_borrowed.registry.subscribe(subs_key, subs_id); + + let (tx, rx) = crate::mpsc::tracing_unbounded(self.tracing_key); + assert!(shared_borrowed.sinks.insert(subs_id, tx).is_none(), "Used IDSequence to create another ID. Should be unique until u64 is overflowed. Should be unique."); + + Receiver { shared: Arc::downgrade(&self.shared), subs_id, rx } + } + + /// Send the message produced with `Trigger`. + /// + /// This is possible if the registry implements `Dispatch`. + pub fn send(&self, trigger: Trigger) -> >::Ret + where + R: Dispatch, + { + let shared_locked = self.shared.lock(); + let mut shared_borrowed = shared_locked.borrow_mut(); + let (registry, sinks) = shared_borrowed.get_mut(); + + let dispatch_result = registry.dispatch(trigger, |subs_id, item| { + if let Some(tx) = sinks.get_mut(&subs_id) { + if let Err(send_err) = tx.unbounded_send(item) { + log::warn!("Sink with SubsID = {} failed to perform unbounded_send: {} ({} as Dispatch<{}, Item = {}>::dispatch(...))", subs_id, send_err, std::any::type_name::(), + std::any::type_name::(), + std::any::type_name::()); + } + } else { + log::warn!( + "No Sink for SubsID = {} ({} as Dispatch<{}, Item = {}>::dispatch(...))", + subs_id, + std::any::type_name::(), + std::any::type_name::(), + std::any::type_name::(), + ); + } + }); + + dispatch_result + } +} + +impl Shared { + fn get_mut(&mut self) -> (&mut R, &mut HashMap>) { + (&mut self.registry, &mut self.sinks) + } + + fn unsubscribe(&mut self, subs_id: SeqID) + where + R: Unsubscribe, + { + // The order (sinks.remove then registry.unsubscribe) is important here: + // assuming that `Unsubscribe::unsubscribe` can panic, it is better to at least + // have the sink disposed. + self.sinks.remove(&subs_id); + self.registry.unsubscribe(subs_id); + } +} + +impl Clone for Hub { + fn clone(&self) -> Self { + Self { tracing_key: self.tracing_key, shared: self.shared.clone() } + } +} + +impl Unpin for Receiver where R: Unsubscribe {} + +impl Stream for Receiver +where + R: Unsubscribe, +{ + type Item = M; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.get_mut().rx).poll_next(cx) + } +} + +impl FusedStream for Receiver +where + R: Unsubscribe, +{ + fn is_terminated(&self) -> bool { + self.rx.is_terminated() + } +} diff --git a/client/utils/src/pubsub/tests.rs b/client/utils/src/pubsub/tests.rs new file mode 100644 index 000000000000..12945c0d886b --- /dev/null +++ b/client/utils/src/pubsub/tests.rs @@ -0,0 +1,123 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use futures::StreamExt; +use tokio_test::block_on; + +use super::*; + +mod normal_operation; +mod panicking_registry; + +const TK: &str = "a_tracing_key"; + +type Message = u64; +type TestHub = Hub; +type TestReceiver = Receiver; + +#[derive(Default)] +struct Registry { + subscribers: HashMap, +} + +struct SubsKey { + _receiver: Option, + panic: SubsKeyPanic, +} + +impl SubsKey { + fn new() -> Self { + Self { _receiver: None, panic: SubsKeyPanic::None } + } + fn with_receiver(self, receiver: TestReceiver) -> Self { + Self { _receiver: Some(receiver), ..self } + } + fn with_panic(self, panic: SubsKeyPanic) -> Self { + Self { panic, ..self } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SubsKeyPanic { + None, + + OnSubscribePanicBefore, + OnSubscribePanicAfter, + + OnUnsubscribePanicBefore, + OnUnsubscribePanicAfter, + + OnDispatchPanicBefore, + OnDispatchPanicAfter, +} + +impl Hub { + fn subs_count(&self) -> usize { + self.map_registry_for_tests(|r| r.subscribers.len()) + } + fn sink_count(&self) -> usize { + self.shared.lock().borrow().sinks.len() + } +} + +impl Subscribe for Registry { + fn subscribe(&mut self, subs_key: SubsKey, subs_id: SeqID) { + let sk_panic = subs_key.panic; + + if sk_panic == SubsKeyPanic::OnSubscribePanicBefore { + panic!("on-subscribe-panic-before") + } + self.subscribers.insert(subs_id, subs_key); + if sk_panic == SubsKeyPanic::OnSubscribePanicAfter { + panic!("on-subscribe-panic-after") + } + } +} +impl Unsubscribe for Registry { + fn unsubscribe(&mut self, subs_id: SeqID) { + let sk_panic = + self.subscribers.get(&subs_id).map(|sk| sk.panic).unwrap_or(SubsKeyPanic::None); + + if sk_panic == SubsKeyPanic::OnUnsubscribePanicBefore { + panic!("on-unsubscribe-panic-before") + } + self.subscribers.remove(&subs_id); + if sk_panic == SubsKeyPanic::OnUnsubscribePanicAfter { + panic!("on-unsubscribe-panic-after") + } + } +} +impl Dispatch for Registry { + type Item = Message; + type Ret = (); + + fn dispatch(&mut self, message: Message, mut dispatch: F) -> Self::Ret + where + F: FnMut(&SeqID, Self::Item), + { + self.subscribers.iter().for_each(|(id, subs_key)| { + if subs_key.panic == SubsKeyPanic::OnDispatchPanicBefore { + panic!("on-dispatch-panic-before") + } + dispatch(id, message); + if subs_key.panic == SubsKeyPanic::OnDispatchPanicAfter { + panic!("on-dispatch-panic-after") + } + }); + } +} diff --git a/client/utils/src/pubsub/tests/normal_operation.rs b/client/utils/src/pubsub/tests/normal_operation.rs new file mode 100644 index 000000000000..a13c718d74a8 --- /dev/null +++ b/client/utils/src/pubsub/tests/normal_operation.rs @@ -0,0 +1,88 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use super::*; + +#[test] +fn positive_rx_receives_relevant_messages_and_terminates_upon_hub_drop() { + block_on(async { + let hub = TestHub::new(TK); + assert_eq!(hub.subs_count(), 0); + + // No subscribers yet. That message is not supposed to get to anyone. + hub.send(0); + + let mut rx_01 = hub.subscribe(SubsKey::new()); + assert_eq!(hub.subs_count(), 1); + + // That message is sent after subscription. Should be delivered into rx_01. + hub.send(1); + assert_eq!(Some(1), rx_01.next().await); + + // Hub is disposed. The rx_01 should be over after that. + std::mem::drop(hub); + + assert!(!rx_01.is_terminated()); + assert_eq!(None, rx_01.next().await); + assert!(rx_01.is_terminated()); + }); +} + +#[test] +fn positive_subs_count_is_correct_upon_drop_of_rxs() { + block_on(async { + let hub = TestHub::new(TK); + assert_eq!(hub.subs_count(), 0); + + let rx_01 = hub.subscribe(SubsKey::new()); + assert_eq!(hub.subs_count(), 1); + let rx_02 = hub.subscribe(SubsKey::new()); + assert_eq!(hub.subs_count(), 2); + + std::mem::drop(rx_01); + assert_eq!(hub.subs_count(), 1); + std::mem::drop(rx_02); + assert_eq!(hub.subs_count(), 0); + }); +} + +#[test] +fn positive_subs_count_is_correct_upon_drop_of_rxs_on_cloned_hubs() { + block_on(async { + let hub_01 = TestHub::new(TK); + let hub_02 = hub_01.clone(); + assert_eq!(hub_01.subs_count(), 0); + assert_eq!(hub_02.subs_count(), 0); + + let rx_01 = hub_02.subscribe(SubsKey::new()); + assert_eq!(hub_01.subs_count(), 1); + assert_eq!(hub_02.subs_count(), 1); + + let rx_02 = hub_02.subscribe(SubsKey::new()); + assert_eq!(hub_01.subs_count(), 2); + assert_eq!(hub_02.subs_count(), 2); + + std::mem::drop(rx_01); + assert_eq!(hub_01.subs_count(), 1); + assert_eq!(hub_02.subs_count(), 1); + + std::mem::drop(rx_02); + assert_eq!(hub_01.subs_count(), 0); + assert_eq!(hub_02.subs_count(), 0); + }); +} diff --git a/client/utils/src/pubsub/tests/panicking_registry.rs b/client/utils/src/pubsub/tests/panicking_registry.rs new file mode 100644 index 000000000000..26ce63bd51b0 --- /dev/null +++ b/client/utils/src/pubsub/tests/panicking_registry.rs @@ -0,0 +1,248 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use super::*; + +use std::panic::{catch_unwind, AssertUnwindSafe}; + +fn assert_hub_props(hub: &TestHub, sinks_count: usize, subs_count: usize) { + assert_eq!(hub.sink_count(), sinks_count); + assert_eq!(hub.subs_count(), subs_count); +} + +#[test] +fn t01() { + let hub = TestHub::new(TK); + assert_hub_props(&hub, 0, 0); + + let rx_01 = hub.subscribe(SubsKey::new()); + assert_hub_props(&hub, 1, 1); + + std::mem::drop(rx_01); + assert_hub_props(&hub, 0, 0); +} + +#[test] +fn t02() { + block_on(async { + // Create a Hub + let hub = TestHub::new(TK); + assert_hub_props(&hub, 0, 0); + + // Subscribe rx-01 + let rx_01 = hub.subscribe(SubsKey::new()); + assert_hub_props(&hub, 1, 1); + + // Subscribe rx-02 so that its unsubscription will lead to an attempt to drop rx-01 in the + // middle of unsubscription of rx-02 + let rx_02 = hub.subscribe(SubsKey::new().with_receiver(rx_01)); + assert_hub_props(&hub, 2, 2); + + // Subscribe rx-03 in order to see that it will receive messages after the unclean + // unsubscription + let mut rx_03 = hub.subscribe(SubsKey::new()); + assert_hub_props(&hub, 3, 3); + + // drop rx-02 leads to an attempt to unsubscribe rx-01 + assert!(catch_unwind(AssertUnwindSafe(move || { + std::mem::drop(rx_02); + })) + .is_err()); + + // One of the rxes could not unsubscribe + assert_hub_props(&hub, 2, 2); + + // Subscribe rx-04 in order to see that it will receive messages after the unclean + // unsubscription + let mut rx_04 = hub.subscribe(SubsKey::new()); + assert_hub_props(&hub, 3, 3); + + hub.send(2); + + // The messages are still received + assert_eq!(rx_03.next().await, Some(2)); + assert_eq!(rx_04.next().await, Some(2)); + + // Perform a clean unsubscription + std::mem::drop(rx_04); + + hub.send(3); + + // The messages are still received + assert_eq!(rx_03.next().await, Some(3)); + + std::mem::drop(rx_03); + + hub.send(4); + + // The stuck subscription is still there + assert_hub_props(&hub, 1, 1); + }); +} + +async fn add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(hub: &TestHub) { + let rx_01 = hub.subscribe(SubsKey::new()); + let rx_02 = hub.subscribe(SubsKey::new()); + + hub.send(1); + hub.send(2); + hub.send(3); + + assert_eq!(rx_01.take(3).collect::>().await, vec![1, 2, 3]); + + hub.send(4); + hub.send(5); + hub.send(6); + + assert_eq!(rx_02.take(6).collect::>().await, vec![1, 2, 3, 4, 5, 6]); +} + +#[test] +fn t03() { + block_on(async { + // Create a Hub + let hub = TestHub::new(TK); + assert_hub_props(&hub, 0, 0); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 0, 0); + + assert!(catch_unwind(AssertUnwindSafe( + || hub.subscribe(SubsKey::new().with_panic(SubsKeyPanic::OnSubscribePanicBefore)) + )) + .is_err()); + + assert_hub_props(&hub, 0, 0); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 0, 0); + }); +} + +#[test] +fn t04() { + block_on(async { + let hub = TestHub::new(TK); + + assert_hub_props(&hub, 0, 0); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 0, 0); + + assert!(catch_unwind(AssertUnwindSafe( + || hub.subscribe(SubsKey::new().with_panic(SubsKeyPanic::OnSubscribePanicAfter)) + )) + .is_err()); + + // the registry has panicked after it has added a subs-id into its internal storage — the + // sinks do not leak, although the subscriptions storage contains some garbage + assert_hub_props(&hub, 0, 1); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 0, 1); + }) +} + +#[test] +fn t05() { + block_on(async { + let hub = TestHub::new(TK); + + assert_hub_props(&hub, 0, 0); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 0, 0); + + let rx_01 = + hub.subscribe(SubsKey::new().with_panic(SubsKeyPanic::OnUnsubscribePanicBefore)); + + assert_hub_props(&hub, 1, 1); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 1, 1); + + assert!(catch_unwind(AssertUnwindSafe(move || std::mem::drop(rx_01))).is_err()); + + // the registry has panicked on-unsubscribe before it removed the subs-id from its internal + // storage — the sinks do not leak, although the subscriptions storage contains some garbage + assert_hub_props(&hub, 0, 1); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 0, 1); + }) +} + +#[test] +fn t06() { + block_on(async { + let hub = TestHub::new(TK); + + assert_hub_props(&hub, 0, 0); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 0, 0); + + let rx_01 = hub.subscribe(SubsKey::new().with_panic(SubsKeyPanic::OnUnsubscribePanicAfter)); + + assert_hub_props(&hub, 1, 1); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 1, 1); + + assert!(catch_unwind(AssertUnwindSafe(move || std::mem::drop(rx_01))).is_err()); + + // the registry has panicked on-unsubscribe after it removed the subs-id from its internal + // storage — the sinks do not leak, the subscriptions storage does not contain any garbage + assert_hub_props(&hub, 0, 0); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 0, 0); + }) +} + +#[test] +fn t07() { + block_on(async { + let hub = TestHub::new(TK); + + assert_hub_props(&hub, 0, 0); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 0, 0); + + let rx_01 = hub.subscribe(SubsKey::new().with_panic(SubsKeyPanic::OnDispatchPanicBefore)); + assert_hub_props(&hub, 1, 1); + assert!(catch_unwind(AssertUnwindSafe(|| hub.send(1))).is_err()); + assert_hub_props(&hub, 1, 1); + + std::mem::drop(rx_01); + assert_hub_props(&hub, 0, 0); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 0, 0); + }) +} + +#[test] +fn t08() { + block_on(async { + let hub = TestHub::new(TK); + + assert_hub_props(&hub, 0, 0); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 0, 0); + + let rx_01 = hub.subscribe(SubsKey::new().with_panic(SubsKeyPanic::OnDispatchPanicAfter)); + assert_hub_props(&hub, 1, 1); + assert!(catch_unwind(AssertUnwindSafe(|| hub.send(1))).is_err()); + assert_hub_props(&hub, 1, 1); + + std::mem::drop(rx_01); + assert_hub_props(&hub, 0, 0); + add_some_subscribers_see_that_messages_are_delivered_and_unsubscribe(&hub).await; + assert_hub_props(&hub, 0, 0); + }) +} diff --git a/client/utils/src/status_sinks.rs b/client/utils/src/status_sinks.rs index a87f0e0ad6e8..a1d965d08085 100644 --- a/client/utils/src/status_sinks.rs +++ b/client/utils/src/status_sinks.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000000..ca3c1bde4e32 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,22 @@ +# Substrate Builder Docker Image + +The Docker image in this folder is a `builder` image. It is self contained and allow users to build the binaries themselves. +There is no requirement on having Rust or any other toolchain installed but a working Docker environment. + +Unlike the `parity/polkadot` image which contains a single binary (`polkadot`!) used by default, the image in this folder builds and contains several binaries and you need to provide the name of the binary to be called. + +You should refer to the .Dockerfile for the actual list. At the time of editing, the list of included binaries is: + +- substrate +- subkey +- node-template +- chain-spec-builder + +The image can be used by passing the selected binary followed by the appropriate tags for this binary. + +Your best guess to get started is to pass the `--help flag`. Here are a few examples: + +- `docker run --rm -it parity/substrate substrate --version` +- `docker run --rm -it parity/substrate subkey --help` +- `docker run --rm -it parity/substrate node-template --version` +- `docker run --rm -it parity/substrate chain-spec-builder --help` diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 000000000000..f0a4560ff8fe --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -e + +pushd . + +# The following line ensure we run from the project root +PROJECT_ROOT=`git rev-parse --show-toplevel` +cd $PROJECT_ROOT + +# Find the current version from Cargo.toml +VERSION=`grep "^version" ./bin/node/cli/Cargo.toml | egrep -o "([0-9\.]+)"` +GITUSER=parity +GITREPO=substrate + +# Build the image +echo "Building ${GITUSER}/${GITREPO}:latest docker image, hang on!" +time docker build -f ./docker/substrate_builder.Dockerfile -t ${GITUSER}/${GITREPO}:latest . +docker tag ${GITUSER}/${GITREPO}:latest ${GITUSER}/${GITREPO}:v${VERSION} + +# Show the list of available images for this repo +echo "Image is ready" +docker images | grep ${GITREPO} + +popd diff --git a/docker/substrate_builder.Dockerfile b/docker/substrate_builder.Dockerfile new file mode 100644 index 000000000000..d0812c1a80c4 --- /dev/null +++ b/docker/substrate_builder.Dockerfile @@ -0,0 +1,35 @@ +# This is the build stage for Substrate. Here we create the binary. +FROM docker.io/paritytech/ci-linux:production as builder + +WORKDIR /substrate +COPY . /substrate +RUN cargo build --locked --release + +# This is the 2nd stage: a very small image where we copy the Substrate binary." +FROM docker.io/library/ubuntu:20.04 +LABEL description="Multistage Docker image for Substrate: a platform for web3" \ + io.parity.image.type="builder" \ + io.parity.image.authors="chevdor@gmail.com, devops-team@parity.io" \ + io.parity.image.vendor="Parity Technologies" \ + io.parity.image.description="Substrate is a next-generation framework for blockchain innovation 🚀" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/docker/substrate_builder.Dockerfile" \ + io.parity.image.documentation="https://github.com/paritytech/polkadot/" + +COPY --from=builder /substrate/target/release/substrate /usr/local/bin +COPY --from=builder /substrate/target/release/subkey /usr/local/bin +COPY --from=builder /substrate/target/release/node-template /usr/local/bin +COPY --from=builder /substrate/target/release/chain-spec-builder /usr/local/bin + +RUN useradd -m -u 1000 -U -s /bin/sh -d /substrate substrate && \ + mkdir -p /data /substrate/.local/share/substrate && \ + chown -R substrate:substrate /data && \ + ln -s /data /substrate/.local/share/substrate && \ +# unclutter and minimize the attack surface + rm -rf /usr/bin /usr/sbin && \ +# Sanity checks + ldd /usr/local/bin/substrate && \ + /usr/local/bin/substrate --version + +USER substrate +EXPOSE 30333 9933 9944 9615 +VOLUME ["/data"] diff --git a/docs/CODEOWNERS b/docs/CODEOWNERS index ee6382b72f1b..0b9e6e778305 100644 --- a/docs/CODEOWNERS +++ b/docs/CODEOWNERS @@ -19,7 +19,7 @@ # - The latest matching rule, if multiple, takes precedence. # CI -/.maintain/ @paritytech/ci +/scripts/ci/ @paritytech/ci /.github/ @paritytech/ci /.gitlab-ci.yml @paritytech/ci @@ -27,10 +27,6 @@ /primitives/sr-sandbox/ @pepyakin /primitives/core/src/sandbox.rs @pepyakin -# Offchain -/client/offchain/ @tomusdrw -/primitives/offchain/ @tomusdrw - # GRANDPA, BABE, consensus stuff /frame/babe/ @andresilva /frame/grandpa/ @andresilva @@ -41,10 +37,10 @@ /primitives/consensus/pow/ @sorpaas # BEEFY -/client/beefy/ @adoerr -/frame/beefy/ @adoerr -/frame/beefy-mmr/ @adoerr -/primitives/beefy/ @adoerr +/client/beefy/ @acatangiu +/frame/beefy/ @acatangiu +/frame/beefy-mmr/ @acatangiu +/primitives/beefy/ @acatangiu # Contracts /frame/contracts/ @athei diff --git a/docs/CONTRIBUTING.adoc b/docs/CONTRIBUTING.adoc index 0a9a7ebacff5..5b1920e775bb 100644 --- a/docs/CONTRIBUTING.adoc +++ b/docs/CONTRIBUTING.adoc @@ -79,16 +79,17 @@ To create a Polkadot companion PR: . Override substrate deps to point to your local path or branch using https://github.com/bkchr/diener. (E.g. from the polkadot clone dir run `diener patch --crates-to-patch ../substrate --substrate` assuming substrate clone is in a sibling dir. If you do use diener, ensure that you _do not_ commit the changes diener makes to the Cargo.tomls.) . Make the changes required and build polkadot locally. . Submit all this as a PR against the Polkadot Repo. -. Link to your Polkadot PR in the _description_ of your _Substrate_ PR as "polkadot companion: [URL]" +. In the _description_ of your _Substrate_ PR add "polkadot companion: [Polkadot_PR_URL]" . Now you should see that the `check_polkadot` CI job will build your Substrate PR agains the mentioned Polkadot branch in your PR description. . Someone will need to approve the Polkadot PR before the Substrate CI will go green. (The Polkadot CI failing can be ignored as long as the polkadot job in the _substrate_ PR is green). . Wait for reviews on both the Substrate and the Polkadot PRs. . Once the Substrate PR runs green, a member of the `parity` github group can comment on the Substrate PR with `bot merge` which will: - Merge the Substrate PR. - - The bot will push a commit to the Polkadot PR updating its Substrate reference. + - The bot will push a commit to the Polkadot PR updating its Substrate reference. (effecively doing `cargo update -p sp-io`) - If the polkadot PR origins from a fork then a project member may need to press `approve run` on the polkadot PR. - The bot will merge the Polkadot PR once all its CI `{"build_allow_failure":false}` checks are green. Note: The merge-bot currently doesn't work with forks on org accounts, only individual accounts. + (Hint: it's recommended to use `bot merge` to merge all substrate PRs, not just ones with a polkadot companion.) If your PR is reviewed well, but a Polkadot PR is missing, signal it with https://github.com/paritytech/substrate/labels/A7-needspolkadotpr[`A7-needspolkadotpr`] to prevent it from getting automatically merged. diff --git a/docs/README.adoc b/docs/README.adoc index 71052420b1aa..0b82f0ed82a1 100644 --- a/docs/README.adoc +++ b/docs/README.adoc @@ -190,7 +190,7 @@ You will also need to install the following packages: - Linux: [source, shell] -sudo apt install cmake pkg-config libssl-dev git clang libclang-dev +sudo apt install cmake pkg-config libssl-dev git clang libclang-dev llvm - Linux on ARM: `rust-lld` is required for linking wasm, but is missing on non Tier 1 platforms. @@ -250,6 +250,20 @@ If you are trying to set up Substrate on Windows, you should do the following: 7. Finally, you need to install `cmake`: https://cmake.org/download/ +==== Docker + +You can use https://github.com/paritytech/scripts/tree/master/dockerfiles/ci-linux[Parity CI docker image] with all necessary dependencies to build Substrate: + +[source, shell] +---- +#run it in the folder with the Substrate source code +docker run --rm -it -w /shellhere/substrate \ + -v $(pwd):/shellhere/substrate \ + paritytech/ci-linux:production +---- + +You can find necessary cargo commands in <> + ==== Shared Steps Then, grab the Substrate source code: diff --git a/docs/STYLE_GUIDE.md b/docs/STYLE_GUIDE.md index ea070cdbc59f..8854f885a4b2 100644 --- a/docs/STYLE_GUIDE.md +++ b/docs/STYLE_GUIDE.md @@ -140,8 +140,8 @@ let mut target_path = ``` - Unsafe code requires explicit proofs just as panickers do. When introducing unsafe code, - consider tradeoffs between efficiency on one hand and reliability, maintenance costs, and - security on the other. Here is a list of questions that may help evaluating the tradeoff while + consider trade-offs between efficiency on one hand and reliability, maintenance costs, and + security on the other. Here is a list of questions that may help evaluating the trade-off while preparing or reviewing a PR: - how much more performant or compact the resulting code will be using unsafe code, - how likely is it that invariants could be violated, diff --git a/docs/Upgrading-2.0-to-3.0.md b/docs/Upgrading-2.0-to-3.0.md index 45da3811220f..017467ede2d7 100644 --- a/docs/Upgrading-2.0-to-3.0.md +++ b/docs/Upgrading-2.0-to-3.0.md @@ -199,6 +199,7 @@ As mentioned above, Bounties, Tips and Lottery have been extracted out of treasu type OnSlash = (); type ProposalBond = ProposalBond; type ProposalBondMinimum = ProposalBondMinimum; + type ProposalBondMaximum = (); type SpendPeriod = SpendPeriod; type Burn = Burn; + type BurnDestination = (); diff --git a/docs/node-template-release.md b/docs/node-template-release.md index 25834ae99f43..4f4977a9df03 100644 --- a/docs/node-template-release.md +++ b/docs/node-template-release.md @@ -7,7 +7,7 @@ the existence of your current git commit ID in the remote repository. Assume you are in root directory of Substrate. Run: ```bash - cd .maintain/ + cd scripts/ci/ ./node-template-release.sh ``` @@ -50,7 +50,7 @@ commit in Substrate remote repository, such as: ``` P.S: This step can be automated if we update `node-template-release` package in - `.maintain/node-template-release`. + `scripts/ci/node-template-release`. 4. Once the three `Cargo.toml`s are updated, compile and confirm that the Node Template builds. Then commit the changes to a new branch in [Substrate Node Template](https://github.com/substrate-developer-hub/substrate-node-template), and make a PR. diff --git a/docs/rustdocs-release.md b/docs/rustdocs-release.md new file mode 100644 index 000000000000..5c7e2db40a2c --- /dev/null +++ b/docs/rustdocs-release.md @@ -0,0 +1,20 @@ +# Rustdocs Release Process + +There is [a script in place](../.maintain/rustdocs-release.sh) to manage the deployment of Substrate rustdocs at +https://paritytech.github.io/substrate, which is pushing the rustdocs file in `gh-pages` branch of +https://github.com/paritytech/substrate. + +The documentation at the top of the `rustdocs-release.sh` explains most of the mechanics of the script. + +Manage the rustdocs deployment with one of the following commands. + +```bash +# Deploy rustdocs of `monthly-2021-10` tag +.maintain/rustdocs-release.sh deploy monthly-2021-10 + +# In addition to the above, the `latest` symlink will point to this version of rustdocs +.maintain/rustdocs-release.sh deploy -l monthly-2021-10 + +# Remove the rustdocs of `monthly-2021-10` from `gh-pages`. +.maintain/rustdocs-release.sh remove monthly-2021-10 +``` diff --git a/frame/assets/Cargo.toml b/frame/assets/Cargo.toml index 05e7912dd07c..d3685e658232 100644 --- a/frame/assets/Cargo.toml +++ b/frame/assets/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-assets" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME asset management pallet" readme = "README.md" @@ -13,11 +13,11 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } # Needed for various traits. In our case, `OnFinalize`. -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } # Needed for type-safe access to storage DB. frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } # `system` module provides us with all sorts of useful stuff and macros depend on it being around. @@ -25,9 +25,9 @@ frame-system = { version = "4.0.0-dev", default-features = false, path = "../sys frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-std = { version = "4.0.0-dev", path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-std = { version = "4.0.0", path = "../../primitives/std" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } [features] @@ -42,7 +42,7 @@ std = [ "frame-benchmarking/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "frame-system/runtime-benchmarks", ] diff --git a/frame/assets/README.md b/frame/assets/README.md index a99b60fa33d5..aae5244953e5 100644 --- a/frame/assets/README.md +++ b/frame/assets/README.md @@ -69,35 +69,43 @@ Import the Assets module and types and derive your runtime's configuration trait ```rust use pallet_assets as assets; -use frame_support::{decl_module, dispatch, ensure}; -use frame_system::ensure_signed; use sp_runtime::ArithmeticError; -pub trait Config: assets::Config { } +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; -decl_module! { - pub struct Module for enum Call where origin: T::Origin { - pub fn issue_token_airdrop(origin) -> dispatch::DispatchResult { - let sender = ensure_signed(origin).map_err(|e| e.as_str())?; + #[pallet::pallet] + pub struct Pallet(_); - const ACCOUNT_ALICE: u64 = 1; - const ACCOUNT_BOB: u64 = 2; - const COUNT_AIRDROP_RECIPIENTS: u64 = 2; - const TOKENS_FIXED_SUPPLY: u64 = 100; + #[pallet::config] + pub trait Config: frame_system::Config + assets::Config {} - ensure!(!COUNT_AIRDROP_RECIPIENTS.is_zero(), ArithmeticError::DivisionByZero); + #[pallet::call] + impl Pallet { + pub fn issue_token_airdrop(origin: OriginFor) -> DispatchResult { + let sender = ensure_signed(origin)?; - let asset_id = Self::next_asset_id(); + const ACCOUNT_ALICE: u64 = 1; + const ACCOUNT_BOB: u64 = 2; + const COUNT_AIRDROP_RECIPIENTS: u64 = 2; + const TOKENS_FIXED_SUPPLY: u64 = 100; - >::mutate(|asset_id| *asset_id += 1); - >::insert((asset_id, &ACCOUNT_ALICE), TOKENS_FIXED_SUPPLY / COUNT_AIRDROP_RECIPIENTS); - >::insert((asset_id, &ACCOUNT_BOB), TOKENS_FIXED_SUPPLY / COUNT_AIRDROP_RECIPIENTS); - >::insert(asset_id, TOKENS_FIXED_SUPPLY); + ensure!(!COUNT_AIRDROP_RECIPIENTS.is_zero(), ArithmeticError::DivisionByZero); - Self::deposit_event(RawEvent::Issued(asset_id, sender, TOKENS_FIXED_SUPPLY)); - Ok(()) - } - } + let asset_id = Self::next_asset_id(); + + >::mutate(|asset_id| *asset_id += 1); + >::insert((asset_id, &ACCOUNT_ALICE), TOKENS_FIXED_SUPPLY / COUNT_AIRDROP_RECIPIENTS); + >::insert((asset_id, &ACCOUNT_BOB), TOKENS_FIXED_SUPPLY / COUNT_AIRDROP_RECIPIENTS); + >::insert(asset_id, TOKENS_FIXED_SUPPLY); + + Self::deposit_event(Event::Issued(asset_id, sender, TOKENS_FIXED_SUPPLY)); + Ok(()) + } + } } ``` diff --git a/frame/assets/src/benchmarking.rs b/frame/assets/src/benchmarking.rs index 43eadffbe849..33de190a8e36 100644 --- a/frame/assets/src/benchmarking.rs +++ b/frame/assets/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,8 +21,7 @@ use super::*; use frame_benchmarking::{ - account, benchmarks_instance_pallet, impl_benchmark_test_suite, whitelist_account, - whitelisted_caller, + account, benchmarks_instance_pallet, whitelist_account, whitelisted_caller, }; use frame_support::{ dispatch::UnfilteredDispatchable, @@ -156,7 +155,7 @@ benchmarks_instance_pallet! { T::Currency::make_free_balance_be(&caller, DepositBalanceOf::::max_value()); }: _(SystemOrigin::Signed(caller.clone()), Default::default(), caller_lookup, 1u32.into()) verify { - assert_last_event::(Event::Created(Default::default(), caller.clone(), caller).into()); + assert_last_event::(Event::Created { asset_id: Default::default(), creator: caller.clone(), owner: caller }.into()); } force_create { @@ -164,7 +163,7 @@ benchmarks_instance_pallet! { let caller_lookup = T::Lookup::unlookup(caller.clone()); }: _(SystemOrigin::Root, Default::default(), caller_lookup, true, 1u32.into()) verify { - assert_last_event::(Event::ForceCreated(Default::default(), caller).into()); + assert_last_event::(Event::ForceCreated { asset_id: Default::default(), owner: caller }.into()); } destroy { @@ -178,7 +177,7 @@ benchmarks_instance_pallet! { let witness = Asset::::get(T::AssetId::default()).unwrap().destroy_witness(); }: _(SystemOrigin::Signed(caller), Default::default(), witness) verify { - assert_last_event::(Event::Destroyed(Default::default()).into()); + assert_last_event::(Event::Destroyed { asset_id: Default::default() }.into()); } mint { @@ -186,7 +185,7 @@ benchmarks_instance_pallet! { let amount = T::Balance::from(100u32); }: _(SystemOrigin::Signed(caller.clone()), Default::default(), caller_lookup, amount) verify { - assert_last_event::(Event::Issued(Default::default(), caller, amount).into()); + assert_last_event::(Event::Issued { asset_id: Default::default(), owner: caller, total_supply: amount }.into()); } burn { @@ -194,7 +193,7 @@ benchmarks_instance_pallet! { let (caller, caller_lookup) = create_default_minted_asset::(true, amount); }: _(SystemOrigin::Signed(caller.clone()), Default::default(), caller_lookup, amount) verify { - assert_last_event::(Event::Burned(Default::default(), caller, amount).into()); + assert_last_event::(Event::Burned { asset_id: Default::default(), owner: caller, balance: amount }.into()); } transfer { @@ -204,7 +203,7 @@ benchmarks_instance_pallet! { let target_lookup = T::Lookup::unlookup(target.clone()); }: _(SystemOrigin::Signed(caller.clone()), Default::default(), target_lookup, amount) verify { - assert_last_event::(Event::Transferred(Default::default(), caller, target, amount).into()); + assert_last_event::(Event::Transferred { asset_id: Default::default(), from: caller, to: target, amount }.into()); } transfer_keep_alive { @@ -216,7 +215,7 @@ benchmarks_instance_pallet! { }: _(SystemOrigin::Signed(caller.clone()), Default::default(), target_lookup, amount) verify { assert!(frame_system::Pallet::::account_exists(&caller)); - assert_last_event::(Event::Transferred(Default::default(), caller, target, amount).into()); + assert_last_event::(Event::Transferred { asset_id: Default::default(), from: caller, to: target, amount }.into()); } force_transfer { @@ -227,7 +226,7 @@ benchmarks_instance_pallet! { }: _(SystemOrigin::Signed(caller.clone()), Default::default(), caller_lookup, target_lookup, amount) verify { assert_last_event::( - Event::Transferred(Default::default(), caller, target, amount).into() + Event::Transferred { asset_id: Default::default(), from: caller, to: target, amount }.into() ); } @@ -235,7 +234,7 @@ benchmarks_instance_pallet! { let (caller, caller_lookup) = create_default_minted_asset::(true, 100u32.into()); }: _(SystemOrigin::Signed(caller.clone()), Default::default(), caller_lookup) verify { - assert_last_event::(Event::Frozen(Default::default(), caller).into()); + assert_last_event::(Event::Frozen { asset_id: Default::default(), who: caller }.into()); } thaw { @@ -247,14 +246,14 @@ benchmarks_instance_pallet! { )?; }: _(SystemOrigin::Signed(caller.clone()), Default::default(), caller_lookup) verify { - assert_last_event::(Event::Thawed(Default::default(), caller).into()); + assert_last_event::(Event::Thawed { asset_id: Default::default(), who: caller }.into()); } freeze_asset { let (caller, caller_lookup) = create_default_minted_asset::(true, 100u32.into()); }: _(SystemOrigin::Signed(caller.clone()), Default::default()) verify { - assert_last_event::(Event::AssetFrozen(Default::default()).into()); + assert_last_event::(Event::AssetFrozen { asset_id: Default::default() }.into()); } thaw_asset { @@ -265,7 +264,7 @@ benchmarks_instance_pallet! { )?; }: _(SystemOrigin::Signed(caller.clone()), Default::default()) verify { - assert_last_event::(Event::AssetThawed(Default::default()).into()); + assert_last_event::(Event::AssetThawed { asset_id: Default::default() }.into()); } transfer_ownership { @@ -274,7 +273,7 @@ benchmarks_instance_pallet! { let target_lookup = T::Lookup::unlookup(target.clone()); }: _(SystemOrigin::Signed(caller), Default::default(), target_lookup) verify { - assert_last_event::(Event::OwnerChanged(Default::default(), target).into()); + assert_last_event::(Event::OwnerChanged { asset_id: Default::default(), owner: target }.into()); } set_team { @@ -284,12 +283,12 @@ benchmarks_instance_pallet! { let target2 = T::Lookup::unlookup(account("target", 2, SEED)); }: _(SystemOrigin::Signed(caller), Default::default(), target0.clone(), target1.clone(), target2.clone()) verify { - assert_last_event::(Event::TeamChanged( - Default::default(), - account("target", 0, SEED), - account("target", 1, SEED), - account("target", 2, SEED), - ).into()); + assert_last_event::(Event::TeamChanged { + asset_id: Default::default(), + issuer: account("target", 0, SEED), + admin: account("target", 1, SEED), + freezer: account("target", 2, SEED), + }.into()); } set_metadata { @@ -305,7 +304,7 @@ benchmarks_instance_pallet! { }: _(SystemOrigin::Signed(caller), Default::default(), name.clone(), symbol.clone(), decimals) verify { let id = Default::default(); - assert_last_event::(Event::MetadataSet(id, name, symbol, decimals, false).into()); + assert_last_event::(Event::MetadataSet { asset_id: id, name, symbol, decimals, is_frozen: false }.into()); } clear_metadata { @@ -316,7 +315,7 @@ benchmarks_instance_pallet! { Assets::::set_metadata(origin, Default::default(), dummy.clone(), dummy, 12)?; }: _(SystemOrigin::Signed(caller), Default::default()) verify { - assert_last_event::(Event::MetadataCleared(Default::default()).into()); + assert_last_event::(Event::MetadataCleared { asset_id: Default::default() }.into()); } force_set_metadata { @@ -340,7 +339,7 @@ benchmarks_instance_pallet! { }: { call.dispatch_bypass_filter(origin)? } verify { let id = Default::default(); - assert_last_event::(Event::MetadataSet(id, name, symbol, decimals, false).into()); + assert_last_event::(Event::MetadataSet { asset_id: id, name, symbol, decimals, is_frozen: false }.into()); } force_clear_metadata { @@ -354,7 +353,7 @@ benchmarks_instance_pallet! { let call = Call::::force_clear_metadata { id: Default::default() }; }: { call.dispatch_bypass_filter(origin)? } verify { - assert_last_event::(Event::MetadataCleared(Default::default()).into()); + assert_last_event::(Event::MetadataCleared { asset_id: Default::default() }.into()); } force_asset_status { @@ -373,7 +372,7 @@ benchmarks_instance_pallet! { }; }: { call.dispatch_bypass_filter(origin)? } verify { - assert_last_event::(Event::AssetStatusChanged(Default::default()).into()); + assert_last_event::(Event::AssetStatusChanged { asset_id: Default::default() }.into()); } approve_transfer { @@ -386,7 +385,7 @@ benchmarks_instance_pallet! { let amount = 100u32.into(); }: _(SystemOrigin::Signed(caller.clone()), id, delegate_lookup, amount) verify { - assert_last_event::(Event::ApprovedTransfer(id, caller, delegate, amount).into()); + assert_last_event::(Event::ApprovedTransfer { asset_id: id, source: caller, delegate, amount }.into()); } transfer_approved { @@ -406,7 +405,7 @@ benchmarks_instance_pallet! { }: _(SystemOrigin::Signed(delegate.clone()), id, owner_lookup, dest_lookup, amount) verify { assert!(T::Currency::reserved_balance(&owner).is_zero()); - assert_event::(Event::Transferred(id, owner, dest, amount).into()); + assert_event::(Event::Transferred { asset_id: id, from: owner, to: dest, amount }.into()); } cancel_approval { @@ -421,7 +420,7 @@ benchmarks_instance_pallet! { Assets::::approve_transfer(origin, id, delegate_lookup.clone(), amount)?; }: _(SystemOrigin::Signed(caller.clone()), id, delegate_lookup) verify { - assert_last_event::(Event::ApprovalCancelled(id, caller, delegate).into()); + assert_last_event::(Event::ApprovalCancelled { asset_id: id, owner: caller, delegate }.into()); } force_cancel_approval { @@ -436,8 +435,8 @@ benchmarks_instance_pallet! { Assets::::approve_transfer(origin, id, delegate_lookup.clone(), amount)?; }: _(SystemOrigin::Signed(caller.clone()), id, caller_lookup, delegate_lookup) verify { - assert_last_event::(Event::ApprovalCancelled(id, caller, delegate).into()); + assert_last_event::(Event::ApprovalCancelled { asset_id: id, owner: caller, delegate }.into()); } -} -impl_benchmark_test_suite!(Assets, crate::mock::new_test_ext(), crate::mock::Test); + impl_benchmark_test_suite!(Assets, crate::mock::new_test_ext(), crate::mock::Test) +} diff --git a/frame/assets/src/extra_mutator.rs b/frame/assets/src/extra_mutator.rs index 8c601b746346..b72bfa86df5b 100644 --- a/frame/assets/src/extra_mutator.rs +++ b/frame/assets/src/extra_mutator.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -62,11 +62,11 @@ impl, I: 'static> ExtraMutator { id: T::AssetId, who: impl sp_std::borrow::Borrow, ) -> Option> { - if Account::::contains_key(id, who.borrow()) { + if let Some(a) = Account::::get(id, who.borrow()) { Some(ExtraMutator:: { id, who: who.borrow().clone(), - original: Account::::get(id, who.borrow()).extra, + original: a.extra, pending: None, }) } else { @@ -77,13 +77,8 @@ impl, I: 'static> ExtraMutator { /// Commit any changes to storage. pub fn commit(&mut self) -> Result<(), ()> { if let Some(extra) = self.pending.take() { - Account::::try_mutate_exists(self.id, self.who.borrow(), |maybe_account| { - if let Some(ref mut account) = maybe_account { - account.extra = extra; - Ok(()) - } else { - Err(()) - } + Account::::try_mutate(self.id, self.who.borrow(), |maybe_account| { + maybe_account.as_mut().ok_or(()).map(|account| account.extra = extra) }) } else { Ok(()) @@ -93,13 +88,11 @@ impl, I: 'static> ExtraMutator { /// Revert any changes, even those already committed by `self` and drop self. pub fn revert(mut self) -> Result<(), ()> { self.pending = None; - Account::::try_mutate_exists(self.id, self.who.borrow(), |maybe_account| { - if let Some(ref mut account) = maybe_account { - account.extra = self.original.clone(); - Ok(()) - } else { - Err(()) - } + Account::::try_mutate(self.id, self.who.borrow(), |maybe_account| { + maybe_account + .as_mut() + .ok_or(()) + .map(|account| account.extra = self.original.clone()) }) } } diff --git a/frame/assets/src/functions.rs b/frame/assets/src/functions.rs index ae31b8e39519..48a86ca3cfa0 100644 --- a/frame/assets/src/functions.rs +++ b/frame/assets/src/functions.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,15 @@ //! Functions for the Assets pallet. use super::*; +use frame_support::{traits::Get, BoundedVec}; + +#[must_use] +pub(super) enum DeadConsequence { + Remove, + Keep, +} + +use DeadConsequence::*; // The main implementation block for the module. impl, I: 'static> Pallet { @@ -31,47 +40,67 @@ impl, I: 'static> Pallet { ExtraMutator::maybe_new(id, who) } - /// Get the asset `id` balance of `who`. + /// Get the asset `id` balance of `who`, or zero if the asset-account doesn't exist. pub fn balance(id: T::AssetId, who: impl sp_std::borrow::Borrow) -> T::Balance { - Account::::get(id, who.borrow()).balance + Self::maybe_balance(id, who).unwrap_or_default() + } + + /// Get the asset `id` balance of `who` if the asset-account exists. + pub fn maybe_balance( + id: T::AssetId, + who: impl sp_std::borrow::Borrow, + ) -> Option { + Account::::get(id, who.borrow()).map(|a| a.balance) } /// Get the total supply of an asset `id`. pub fn total_supply(id: T::AssetId) -> T::Balance { - Asset::::get(id).map(|x| x.supply).unwrap_or_else(Zero::zero) + Self::maybe_total_supply(id).unwrap_or_default() + } + + /// Get the total supply of an asset `id` if the asset exists. + pub fn maybe_total_supply(id: T::AssetId) -> Option { + Asset::::get(id).map(|x| x.supply) } pub(super) fn new_account( who: &T::AccountId, d: &mut AssetDetails>, - ) -> Result { + maybe_deposit: Option>, + ) -> Result>, DispatchError> { let accounts = d.accounts.checked_add(1).ok_or(ArithmeticError::Overflow)?; - let is_sufficient = if d.is_sufficient { + let reason = if let Some(deposit) = maybe_deposit { + ExistenceReason::DepositHeld(deposit) + } else if d.is_sufficient { frame_system::Pallet::::inc_sufficients(who); d.sufficients += 1; - true + ExistenceReason::Sufficient } else { frame_system::Pallet::::inc_consumers(who).map_err(|_| Error::::NoProvider)?; - false + ExistenceReason::Consumer }; d.accounts = accounts; - Ok(is_sufficient) + Ok(reason) } pub(super) fn dead_account( - what: T::AssetId, who: &T::AccountId, d: &mut AssetDetails>, - sufficient: bool, - ) { - if sufficient { - d.sufficients = d.sufficients.saturating_sub(1); - frame_system::Pallet::::dec_sufficients(who); - } else { - frame_system::Pallet::::dec_consumers(who); + reason: &ExistenceReason>, + force: bool, + ) -> DeadConsequence { + match *reason { + ExistenceReason::Consumer => frame_system::Pallet::::dec_consumers(who), + ExistenceReason::Sufficient => { + d.sufficients = d.sufficients.saturating_sub(1); + frame_system::Pallet::::dec_sufficients(who); + }, + ExistenceReason::DepositRefunded => {}, + ExistenceReason::DepositHeld(_) if !force => return Keep, + ExistenceReason::DepositHeld(_) => {}, } d.accounts = d.accounts.saturating_sub(1); - T::Freezer::died(what, who) + Remove } pub(super) fn can_increase( @@ -86,15 +115,15 @@ impl, I: 'static> Pallet { if details.supply.checked_add(&amount).is_none() { return DepositConsequence::Overflow } - let account = Account::::get(id, who); - if account.balance.checked_add(&amount).is_none() { - return DepositConsequence::Overflow - } - if account.balance.is_zero() { + if let Some(balance) = Self::maybe_balance(id, who) { + if balance.checked_add(&amount).is_none() { + return DepositConsequence::Overflow + } + } else { if amount < details.min_balance { return DepositConsequence::BelowMinimum } - if !details.is_sufficient && frame_system::Pallet::::providers(who) == 0 { + if !details.is_sufficient && !frame_system::Pallet::::can_inc_consumer(who) { return DepositConsequence::CannotCreate } if details.is_sufficient && details.sufficients.checked_add(1).is_none() { @@ -123,7 +152,13 @@ impl, I: 'static> Pallet { if details.is_frozen { return Frozen } - let account = Account::::get(id, who); + if amount.is_zero() { + return Success + } + let account = match Account::::get(id, who) { + Some(a) => a, + None => return NoFunds, + }; if account.is_frozen { return Frozen } @@ -164,7 +199,7 @@ impl, I: 'static> Pallet { let details = Asset::::get(id).ok_or_else(|| Error::::Unknown)?; ensure!(!details.is_frozen, Error::::Frozen); - let account = Account::::get(id, who); + let account = Account::::get(id, who).ok_or(Error::::NoAccount)?; ensure!(!account.is_frozen, Error::::Frozen); let amount = if let Some(frozen) = T::Freezer::frozen_balance(id, who) { @@ -252,6 +287,50 @@ impl, I: 'static> Pallet { Ok((credit, maybe_burn)) } + /// Creates a account for `who` to hold asset `id` with a zero balance and takes a deposit. + pub(super) fn do_touch(id: T::AssetId, who: T::AccountId) -> DispatchResult { + ensure!(!Account::::contains_key(id, &who), Error::::AlreadyExists); + let deposit = T::AssetAccountDeposit::get(); + let mut details = Asset::::get(&id).ok_or(Error::::Unknown)?; + let reason = Self::new_account(&who, &mut details, Some(deposit))?; + T::Currency::reserve(&who, deposit)?; + Asset::::insert(&id, details); + Account::::insert( + id, + &who, + AssetAccountOf:: { + balance: Zero::zero(), + is_frozen: false, + reason, + extra: T::Extra::default(), + }, + ); + Ok(()) + } + + /// Returns a deposit, destroying an asset-account. + pub(super) fn do_refund(id: T::AssetId, who: T::AccountId, allow_burn: bool) -> DispatchResult { + let mut account = Account::::get(id, &who).ok_or(Error::::NoDeposit)?; + let deposit = account.reason.take_deposit().ok_or(Error::::NoDeposit)?; + let mut details = Asset::::get(&id).ok_or(Error::::Unknown)?; + + ensure!(account.balance.is_zero() || allow_burn, Error::::WouldBurn); + ensure!(!details.is_frozen, Error::::Frozen); + ensure!(!account.is_frozen, Error::::Frozen); + + T::Currency::unreserve(&who, deposit); + + if let Remove = Self::dead_account(&who, &mut details, &account.reason, false) { + Account::::remove(id, &who); + } else { + debug_assert!(false, "refund did not result in dead account?!"); + } + Asset::::insert(&id, details); + // Executing a hook here is safe, since it is not in a `mutate`. + T::Freezer::died(id, &who); + Ok(()) + } + /// Increases the asset `id` balance of `beneficiary` by `amount`. /// /// This alters the registered supply of the asset and emits an event. @@ -274,7 +353,11 @@ impl, I: 'static> Pallet { details.supply = details.supply.saturating_add(amount); Ok(()) })?; - Self::deposit_event(Event::Issued(id, beneficiary.clone(), amount)); + Self::deposit_event(Event::Issued { + asset_id: id, + owner: beneficiary.clone(), + total_supply: amount, + }); Ok(()) } @@ -302,13 +385,22 @@ impl, I: 'static> Pallet { check(details)?; - Account::::try_mutate(id, beneficiary, |t| -> DispatchResult { - let new_balance = t.balance.saturating_add(amount); - ensure!(new_balance >= details.min_balance, TokenError::BelowMinimum); - if t.balance.is_zero() { - t.sufficient = Self::new_account(beneficiary, details)?; + Account::::try_mutate(id, beneficiary, |maybe_account| -> DispatchResult { + match maybe_account { + Some(ref mut account) => { + account.balance.saturating_accrue(amount); + }, + maybe_account @ None => { + // Note this should never fail as it's already checked by `can_increase`. + ensure!(amount >= details.min_balance, TokenError::BelowMinimum); + *maybe_account = Some(AssetAccountOf:: { + balance: amount, + reason: Self::new_account(beneficiary, details, None)?, + is_frozen: false, + extra: T::Extra::default(), + }); + }, } - t.balance = new_balance; Ok(()) })?; Ok(()) @@ -341,7 +433,7 @@ impl, I: 'static> Pallet { Ok(()) })?; - Self::deposit_event(Event::Burned(id, target.clone(), actual)); + Self::deposit_event(Event::Burned { asset_id: id, owner: target.clone(), balance: actual }); Ok(actual) } @@ -368,31 +460,38 @@ impl, I: 'static> Pallet { } let actual = Self::prep_debit(id, target, amount, f)?; + let mut target_died: Option = None; Asset::::try_mutate(id, |maybe_details| -> DispatchResult { - let details = maybe_details.as_mut().ok_or(Error::::Unknown)?; + let mut details = maybe_details.as_mut().ok_or(Error::::Unknown)?; check(actual, details)?; - Account::::try_mutate_exists(id, target, |maybe_account| -> DispatchResult { - let mut account = maybe_account.take().unwrap_or_default(); + Account::::try_mutate(id, target, |maybe_account| -> DispatchResult { + let mut account = maybe_account.take().ok_or(Error::::NoAccount)?; debug_assert!(account.balance >= actual, "checked in prep; qed"); // Make the debit. account.balance = account.balance.saturating_sub(actual); - *maybe_account = if account.balance < details.min_balance { + if account.balance < details.min_balance { debug_assert!(account.balance.is_zero(), "checked in prep; qed"); - Self::dead_account(id, target, details, account.sufficient); - None - } else { - Some(account) + target_died = + Some(Self::dead_account(target, &mut details, &account.reason, false)); + if let Some(Remove) = target_died { + return Ok(()) + } }; + *maybe_account = Some(account); Ok(()) })?; Ok(()) })?; + // Execute hook outside of `mutate`. + if let Some(Remove) = target_died { + T::Freezer::died(id, target); + } Ok(actual) } @@ -412,17 +511,42 @@ impl, I: 'static> Pallet { maybe_need_admin: Option, f: TransferFlags, ) -> Result { + let (balance, died) = + Self::transfer_and_die(id, source, dest, amount, maybe_need_admin, f)?; + if let Some(Remove) = died { + T::Freezer::died(id, source); + } + Ok(balance) + } + + /// Same as `do_transfer` but it does not execute the `FrozenBalance::died` hook and + /// instead returns whether and how the `source` account died in this operation. + fn transfer_and_die( + id: T::AssetId, + source: &T::AccountId, + dest: &T::AccountId, + amount: T::Balance, + maybe_need_admin: Option, + f: TransferFlags, + ) -> Result<(T::Balance, Option), DispatchError> { // Early exist if no-op. if amount.is_zero() { - Self::deposit_event(Event::Transferred(id, source.clone(), dest.clone(), amount)); - return Ok(amount) + Self::deposit_event(Event::Transferred { + asset_id: id, + from: source.clone(), + to: dest.clone(), + amount, + }); + return Ok((amount, None)) } // Figure out the debit and credit, together with side-effects. let debit = Self::prep_debit(id, &source, amount, f.into())?; let (credit, maybe_burn) = Self::prep_credit(id, &dest, amount, debit, f.burn_dust)?; - let mut source_account = Account::::get(id, &source); + let mut source_account = + Account::::get(id, &source).ok_or(Error::::NoAccount)?; + let mut source_died: Option = None; Asset::::try_mutate(id, |maybe_details| -> DispatchResult { let details = maybe_details.as_mut().ok_or(Error::::Unknown)?; @@ -449,34 +573,50 @@ impl, I: 'static> Pallet { debug_assert!(source_account.balance >= debit, "checked in prep; qed"); source_account.balance = source_account.balance.saturating_sub(debit); - Account::::try_mutate(id, &dest, |a| -> DispatchResult { - // Calculate new balance; this will not saturate since it's already checked in prep. - debug_assert!(a.balance.checked_add(&credit).is_some(), "checked in prep; qed"); - let new_balance = a.balance.saturating_add(credit); - - // Create a new account if there wasn't one already. - if a.balance.is_zero() { - a.sufficient = Self::new_account(&dest, details)?; + Account::::try_mutate(id, &dest, |maybe_account| -> DispatchResult { + match maybe_account { + Some(ref mut account) => { + // Calculate new balance; this will not saturate since it's already checked + // in prep. + debug_assert!( + account.balance.checked_add(&credit).is_some(), + "checked in prep; qed" + ); + account.balance.saturating_accrue(credit); + }, + maybe_account @ None => { + *maybe_account = Some(AssetAccountOf:: { + balance: credit, + is_frozen: false, + reason: Self::new_account(&dest, details, None)?, + extra: T::Extra::default(), + }); + }, } - - a.balance = new_balance; Ok(()) })?; // Remove source account if it's now dead. if source_account.balance < details.min_balance { debug_assert!(source_account.balance.is_zero(), "checked in prep; qed"); - Self::dead_account(id, &source, details, source_account.sufficient); - Account::::remove(id, &source); - } else { - Account::::insert(id, &source, &source_account) + source_died = + Some(Self::dead_account(&source, details, &source_account.reason, false)); + if let Some(Remove) = source_died { + Account::::remove(id, &source); + return Ok(()) + } } - + Account::::insert(id, &source, &source_account); Ok(()) })?; - Self::deposit_event(Event::Transferred(id, source.clone(), dest.clone(), credit)); - Ok(credit) + Self::deposit_event(Event::Transferred { + asset_id: id, + from: source.clone(), + to: dest.clone(), + amount: credit, + }); + Ok((credit, source_died)) } /// Create a new asset without taking a deposit. @@ -513,7 +653,7 @@ impl, I: 'static> Pallet { is_frozen: false, }, ); - Self::deposit_event(Event::ForceCreated(id, owner)); + Self::deposit_event(Event::ForceCreated { asset_id: id, owner }); Ok(()) } @@ -529,37 +669,194 @@ impl, I: 'static> Pallet { witness: DestroyWitness, maybe_check_owner: Option, ) -> Result { - Asset::::try_mutate_exists(id, |maybe_details| { - let mut details = maybe_details.take().ok_or(Error::::Unknown)?; - if let Some(check_owner) = maybe_check_owner { - ensure!(details.owner == check_owner, Error::::NoPermission); - } - ensure!(details.accounts <= witness.accounts, Error::::BadWitness); - ensure!(details.sufficients <= witness.sufficients, Error::::BadWitness); - ensure!(details.approvals <= witness.approvals, Error::::BadWitness); + let mut dead_accounts: Vec = vec![]; - for (who, v) in Account::::drain_prefix(id) { - Self::dead_account(id, &who, &mut details, v.sufficient); - } - debug_assert_eq!(details.accounts, 0); - debug_assert_eq!(details.sufficients, 0); + let result_witness: DestroyWitness = Asset::::try_mutate_exists( + id, + |maybe_details| -> Result { + let mut details = maybe_details.take().ok_or(Error::::Unknown)?; + if let Some(check_owner) = maybe_check_owner { + ensure!(details.owner == check_owner, Error::::NoPermission); + } + ensure!(details.accounts <= witness.accounts, Error::::BadWitness); + ensure!(details.sufficients <= witness.sufficients, Error::::BadWitness); + ensure!(details.approvals <= witness.approvals, Error::::BadWitness); + + for (who, v) in Account::::drain_prefix(id) { + // We have to force this as it's destroying the entire asset class. + // This could mean that some accounts now have irreversibly reserved + // funds. + let _ = Self::dead_account(&who, &mut details, &v.reason, true); + dead_accounts.push(who); + } + debug_assert_eq!(details.accounts, 0); + debug_assert_eq!(details.sufficients, 0); - let metadata = Metadata::::take(&id); - T::Currency::unreserve( - &details.owner, - details.deposit.saturating_add(metadata.deposit), - ); + let metadata = Metadata::::take(&id); + T::Currency::unreserve( + &details.owner, + details.deposit.saturating_add(metadata.deposit), + ); + + for ((owner, _), approval) in Approvals::::drain_prefix((&id,)) { + T::Currency::unreserve(&owner, approval.deposit); + } + Self::deposit_event(Event::Destroyed { asset_id: id }); + + Ok(DestroyWitness { + accounts: details.accounts, + sufficients: details.sufficients, + approvals: details.approvals, + }) + }, + )?; + + // Execute hooks outside of `mutate`. + for who in dead_accounts { + T::Freezer::died(id, &who); + } + Ok(result_witness) + } - for ((owner, _), approval) in Approvals::::drain_prefix((&id,)) { - T::Currency::unreserve(&owner, approval.deposit); + /// Creates an approval from `owner` to spend `amount` of asset `id` tokens by 'delegate' + /// while reserving `T::ApprovalDeposit` from owner + /// + /// If an approval already exists, the new amount is added to such existing approval + pub(super) fn do_approve_transfer( + id: T::AssetId, + owner: &T::AccountId, + delegate: &T::AccountId, + amount: T::Balance, + ) -> DispatchResult { + let mut d = Asset::::get(id).ok_or(Error::::Unknown)?; + ensure!(!d.is_frozen, Error::::Frozen); + Approvals::::try_mutate( + (id, &owner, &delegate), + |maybe_approved| -> DispatchResult { + let mut approved = match maybe_approved.take() { + // an approval already exists and is being updated + Some(a) => a, + // a new approval is created + None => { + d.approvals.saturating_inc(); + Default::default() + }, + }; + let deposit_required = T::ApprovalDeposit::get(); + if approved.deposit < deposit_required { + T::Currency::reserve(&owner, deposit_required - approved.deposit)?; + approved.deposit = deposit_required; + } + approved.amount = approved.amount.saturating_add(amount); + *maybe_approved = Some(approved); + Ok(()) + }, + )?; + Asset::::insert(id, d); + Self::deposit_event(Event::ApprovedTransfer { + asset_id: id, + source: owner.clone(), + delegate: delegate.clone(), + amount, + }); + + Ok(()) + } + + /// Reduces the asset `id` balance of `owner` by some `amount` and increases the balance of + /// `dest` by (similar) amount, checking that 'delegate' has an existing approval from `owner` + /// to spend`amount`. + /// + /// Will fail if `amount` is greater than the approval from `owner` to 'delegate' + /// Will unreserve the deposit from `owner` if the entire approved `amount` is spent by + /// 'delegate' + pub(super) fn do_transfer_approved( + id: T::AssetId, + owner: &T::AccountId, + delegate: &T::AccountId, + destination: &T::AccountId, + amount: T::Balance, + ) -> DispatchResult { + let mut owner_died: Option = None; + + Approvals::::try_mutate_exists( + (id, &owner, delegate), + |maybe_approved| -> DispatchResult { + let mut approved = maybe_approved.take().ok_or(Error::::Unapproved)?; + let remaining = + approved.amount.checked_sub(&amount).ok_or(Error::::Unapproved)?; + + let f = TransferFlags { keep_alive: false, best_effort: false, burn_dust: false }; + owner_died = Self::transfer_and_die(id, &owner, &destination, amount, None, f)?.1; + + if remaining.is_zero() { + T::Currency::unreserve(&owner, approved.deposit); + Asset::::mutate(id, |maybe_details| { + if let Some(details) = maybe_details { + details.approvals.saturating_dec(); + } + }); + } else { + approved.amount = remaining; + *maybe_approved = Some(approved); + } + Ok(()) + }, + )?; + + // Execute hook outside of `mutate`. + if let Some(Remove) = owner_died { + T::Freezer::died(id, owner); + } + Ok(()) + } + + /// Do set metadata + pub(super) fn do_set_metadata( + id: T::AssetId, + from: &T::AccountId, + name: Vec, + symbol: Vec, + decimals: u8, + ) -> DispatchResult { + let bounded_name: BoundedVec = + name.clone().try_into().map_err(|_| Error::::BadMetadata)?; + let bounded_symbol: BoundedVec = + symbol.clone().try_into().map_err(|_| Error::::BadMetadata)?; + + let d = Asset::::get(id).ok_or(Error::::Unknown)?; + ensure!(from == &d.owner, Error::::NoPermission); + + Metadata::::try_mutate_exists(id, |metadata| { + ensure!(metadata.as_ref().map_or(true, |m| !m.is_frozen), Error::::NoPermission); + + let old_deposit = metadata.take().map_or(Zero::zero(), |m| m.deposit); + let new_deposit = T::MetadataDepositPerByte::get() + .saturating_mul(((name.len() + symbol.len()) as u32).into()) + .saturating_add(T::MetadataDepositBase::get()); + + if new_deposit > old_deposit { + T::Currency::reserve(from, new_deposit - old_deposit)?; + } else { + T::Currency::unreserve(from, old_deposit - new_deposit); } - Self::deposit_event(Event::Destroyed(id)); - Ok(DestroyWitness { - accounts: details.accounts, - sufficients: details.sufficients, - approvals: details.approvals, - }) + *metadata = Some(AssetMetadata { + deposit: new_deposit, + name: bounded_name, + symbol: bounded_symbol, + decimals, + is_frozen: false, + }); + + Self::deposit_event(Event::MetadataSet { + asset_id: id, + name, + symbol, + decimals, + is_frozen: false, + }); + Ok(()) }) } } diff --git a/frame/assets/src/impl_fungibles.rs b/frame/assets/src/impl_fungibles.rs index 25e18bfd437b..49caac83f4c4 100644 --- a/frame/assets/src/impl_fungibles.rs +++ b/frame/assets/src/impl_fungibles.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -60,6 +60,25 @@ impl, I: 'static> fungibles::Inspect<::AccountId } } +impl, I: 'static> fungibles::InspectMetadata<::AccountId> + for Pallet +{ + /// Return the name of an asset. + fn name(asset: &Self::AssetId) -> Vec { + Metadata::::get(asset).name.to_vec() + } + + /// Return the symbol of an asset. + fn symbol(asset: &Self::AssetId) -> Vec { + Metadata::::get(asset).symbol.to_vec() + } + + /// Return the decimals of an asset. + fn decimals(asset: &Self::AssetId) -> u8 { + Metadata::::get(asset).decimals + } +} + impl, I: 'static> fungibles::Mutate<::AccountId> for Pallet { fn mint_into( asset: Self::AssetId, @@ -174,3 +193,72 @@ impl, I: 'static> fungibles::Destroy for Pallet Self::do_destroy(id, witness, maybe_check_owner) } } + +impl, I: 'static> fungibles::metadata::Inspect<::AccountId> + for Pallet +{ + fn name(asset: T::AssetId) -> Vec { + Metadata::::get(asset).name.to_vec() + } + + fn symbol(asset: T::AssetId) -> Vec { + Metadata::::get(asset).symbol.to_vec() + } + + fn decimals(asset: T::AssetId) -> u8 { + Metadata::::get(asset).decimals + } +} + +impl, I: 'static> fungibles::metadata::Mutate<::AccountId> + for Pallet +{ + fn set( + asset: T::AssetId, + from: &::AccountId, + name: Vec, + symbol: Vec, + decimals: u8, + ) -> DispatchResult { + Self::do_set_metadata(asset, from, name, symbol, decimals) + } +} + +impl, I: 'static> fungibles::approvals::Inspect<::AccountId> + for Pallet +{ + // Check the amount approved to be spent by an owner to a delegate + fn allowance( + asset: T::AssetId, + owner: &::AccountId, + delegate: &::AccountId, + ) -> T::Balance { + Approvals::::get((asset, &owner, &delegate)) + .map(|x| x.amount) + .unwrap_or_else(Zero::zero) + } +} + +impl, I: 'static> fungibles::approvals::Mutate<::AccountId> + for Pallet +{ + fn approve( + asset: T::AssetId, + owner: &::AccountId, + delegate: &::AccountId, + amount: T::Balance, + ) -> DispatchResult { + Self::do_approve_transfer(asset, owner, delegate, amount) + } + + // Aprove spending tokens from a given account + fn transfer_from( + asset: T::AssetId, + owner: &::AccountId, + delegate: &::AccountId, + dest: &::AccountId, + amount: T::Balance, + ) -> DispatchResult { + Self::do_transfer_approved(asset, owner, delegate, dest, amount) + } +} diff --git a/frame/assets/src/impl_stored_map.rs b/frame/assets/src/impl_stored_map.rs index 4c1ff1a0c602..dfdcff37d1d6 100644 --- a/frame/assets/src/impl_stored_map.rs +++ b/frame/assets/src/impl_stored_map.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,11 +22,7 @@ use super::*; impl, I: 'static> StoredMap<(T::AssetId, T::AccountId), T::Extra> for Pallet { fn get(id_who: &(T::AssetId, T::AccountId)) -> T::Extra { let &(id, ref who) = id_who; - if Account::::contains_key(id, who) { - Account::::get(id, who).extra - } else { - Default::default() - } + Account::::get(id, who).map(|a| a.extra).unwrap_or_default() } fn try_mutate_exists>( @@ -34,13 +30,13 @@ impl, I: 'static> StoredMap<(T::AssetId, T::AccountId), T::Extra> f f: impl FnOnce(&mut Option) -> Result, ) -> Result { let &(id, ref who) = id_who; - let mut maybe_extra = Some(Account::::get(id, who).extra); + let mut maybe_extra = Account::::get(id, who).map(|a| a.extra); let r = f(&mut maybe_extra)?; // They want to write some value or delete it. // If the account existed and they want to write a value, then we write. // If the account didn't exist and they want to delete it, then we let it pass. // Otherwise, we fail. - Account::::try_mutate_exists(id, who, |maybe_account| { + Account::::try_mutate(id, who, |maybe_account| { if let Some(extra) = maybe_extra { // They want to write a value. Let this happen only if the account actually exists. if let Some(ref mut account) = maybe_account { diff --git a/frame/assets/src/lib.rs b/frame/assets/src/lib.rs index cf32e289446d..7b6d78cdff8f 100644 --- a/frame/assets/src/lib.rs +++ b/frame/assets/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -140,6 +140,15 @@ mod types; pub use types::*; use codec::HasCompact; +use scale_info::TypeInfo; +use sp_runtime::{ + traits::{ + AtLeast32BitUnsigned, Bounded, CheckedAdd, CheckedSub, Saturating, StaticLookup, Zero, + }, + ArithmeticError, TokenError, +}; +use sp_std::{borrow::Borrow, prelude::*}; + use frame_support::{ dispatch::{DispatchError, DispatchResult}, ensure, @@ -151,13 +160,6 @@ use frame_support::{ }, }; use frame_system::Config as SystemConfig; -use sp_runtime::{ - traits::{ - AtLeast32BitUnsigned, Bounded, CheckedAdd, CheckedSub, Saturating, StaticLookup, Zero, - }, - ArithmeticError, TokenError, -}; -use sp_std::{borrow::Borrow, convert::TryInto, prelude::*}; pub use pallet::*; pub use weights::WeightInfo; @@ -165,12 +167,11 @@ pub use weights::WeightInfo; #[frame_support::pallet] pub mod pallet { use super::*; - use frame_support::{dispatch::DispatchResult, pallet_prelude::*}; + use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] - #[pallet::generate_storage_info] pub struct Pallet(_); #[pallet::config] @@ -180,10 +181,24 @@ pub mod pallet { type Event: From> + IsType<::Event>; /// The units in which we record balances. - type Balance: Member + Parameter + AtLeast32BitUnsigned + Default + Copy + MaxEncodedLen; + type Balance: Member + + Parameter + + AtLeast32BitUnsigned + + Default + + Copy + + MaybeSerializeDeserialize + + MaxEncodedLen + + TypeInfo; /// Identifier for the class of asset. - type AssetId: Member + Parameter + Default + Copy + HasCompact + MaxEncodedLen; + type AssetId: Member + + Parameter + + Default + + Copy + + HasCompact + + MaybeSerializeDeserialize + + MaxEncodedLen + + TypeInfo; /// The currency mechanism. type Currency: ReservableCurrency; @@ -196,6 +211,11 @@ pub mod pallet { #[pallet::constant] type AssetDeposit: Get>; + /// The amount of funds that must be reserved for a non-provider asset account to be + /// maintained. + #[pallet::constant] + type AssetAccountDeposit: Get>; + /// The basic amount of funds that must be reserved when adding metadata to your asset. #[pallet::constant] type MetadataDepositBase: Get>; @@ -236,15 +256,15 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn account)] - /// The number of units of assets held by any given account. + /// The holdings of a specific account for a specific asset. pub(super) type Account, I: 'static = ()> = StorageDoubleMap< _, Blake2_128Concat, T::AssetId, Blake2_128Concat, T::AccountId, - AssetBalance, - ValueQuery, + AssetAccountOf, + OptionQuery, GetDefault, ConstU32<300_000>, >; @@ -278,58 +298,164 @@ pub mod pallet { ConstU32<300_000>, >; + #[pallet::genesis_config] + pub struct GenesisConfig, I: 'static = ()> { + /// Genesis assets: id, owner, is_sufficient, min_balance + pub assets: Vec<(T::AssetId, T::AccountId, bool, T::Balance)>, + /// Genesis metadata: id, name, symbol, decimals + pub metadata: Vec<(T::AssetId, Vec, Vec, u8)>, + /// Genesis accounts: id, account_id, balance + pub accounts: Vec<(T::AssetId, T::AccountId, T::Balance)>, + } + + #[cfg(feature = "std")] + impl, I: 'static> Default for GenesisConfig { + fn default() -> Self { + Self { + assets: Default::default(), + metadata: Default::default(), + accounts: Default::default(), + } + } + } + + #[pallet::genesis_build] + impl, I: 'static> GenesisBuild for GenesisConfig { + fn build(&self) { + for (id, owner, is_sufficient, min_balance) in &self.assets { + assert!(!Asset::::contains_key(id), "Asset id already in use"); + assert!(!min_balance.is_zero(), "Min balance should not be zero"); + Asset::::insert( + id, + AssetDetails { + owner: owner.clone(), + issuer: owner.clone(), + admin: owner.clone(), + freezer: owner.clone(), + supply: Zero::zero(), + deposit: Zero::zero(), + min_balance: *min_balance, + is_sufficient: *is_sufficient, + accounts: 0, + sufficients: 0, + approvals: 0, + is_frozen: false, + }, + ); + } + + for (id, name, symbol, decimals) in &self.metadata { + assert!(Asset::::contains_key(id), "Asset does not exist"); + + let bounded_name: BoundedVec = + name.clone().try_into().expect("asset name is too long"); + let bounded_symbol: BoundedVec = + symbol.clone().try_into().expect("asset symbol is too long"); + + let metadata = AssetMetadata { + deposit: Zero::zero(), + name: bounded_name, + symbol: bounded_symbol, + decimals: *decimals, + is_frozen: false, + }; + Metadata::::insert(id, metadata); + } + + for (id, account_id, amount) in &self.accounts { + let result = >::increase_balance( + *id, + account_id, + *amount, + |details| -> DispatchResult { + debug_assert!( + T::Balance::max_value() - details.supply >= *amount, + "checked in prep; qed" + ); + details.supply = details.supply.saturating_add(*amount); + Ok(()) + }, + ); + assert!(result.is_ok()); + } + } + } + #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event, I: 'static = ()> { - /// Some asset class was created. \[asset_id, creator, owner\] - Created(T::AssetId, T::AccountId, T::AccountId), - /// Some assets were issued. \[asset_id, owner, total_supply\] - Issued(T::AssetId, T::AccountId, T::Balance), - /// Some assets were transferred. \[asset_id, from, to, amount\] - Transferred(T::AssetId, T::AccountId, T::AccountId, T::Balance), - /// Some assets were destroyed. \[asset_id, owner, balance\] - Burned(T::AssetId, T::AccountId, T::Balance), - /// The management team changed \[asset_id, issuer, admin, freezer\] - TeamChanged(T::AssetId, T::AccountId, T::AccountId, T::AccountId), - /// The owner changed \[asset_id, owner\] - OwnerChanged(T::AssetId, T::AccountId), - /// Some account `who` was frozen. \[asset_id, who\] - Frozen(T::AssetId, T::AccountId), - /// Some account `who` was thawed. \[asset_id, who\] - Thawed(T::AssetId, T::AccountId), - /// Some asset `asset_id` was frozen. \[asset_id\] - AssetFrozen(T::AssetId), - /// Some asset `asset_id` was thawed. \[asset_id\] - AssetThawed(T::AssetId), + /// Some asset class was created. + Created { asset_id: T::AssetId, creator: T::AccountId, owner: T::AccountId }, + /// Some assets were issued. + Issued { asset_id: T::AssetId, owner: T::AccountId, total_supply: T::Balance }, + /// Some assets were transferred. + Transferred { + asset_id: T::AssetId, + from: T::AccountId, + to: T::AccountId, + amount: T::Balance, + }, + /// Some assets were destroyed. + Burned { asset_id: T::AssetId, owner: T::AccountId, balance: T::Balance }, + /// The management team changed. + TeamChanged { + asset_id: T::AssetId, + issuer: T::AccountId, + admin: T::AccountId, + freezer: T::AccountId, + }, + /// The owner changed. + OwnerChanged { asset_id: T::AssetId, owner: T::AccountId }, + /// Some account `who` was frozen. + Frozen { asset_id: T::AssetId, who: T::AccountId }, + /// Some account `who` was thawed. + Thawed { asset_id: T::AssetId, who: T::AccountId }, + /// Some asset `asset_id` was frozen. + AssetFrozen { asset_id: T::AssetId }, + /// Some asset `asset_id` was thawed. + AssetThawed { asset_id: T::AssetId }, /// An asset class was destroyed. - Destroyed(T::AssetId), - /// Some asset class was force-created. \[asset_id, owner\] - ForceCreated(T::AssetId, T::AccountId), - /// New metadata has been set for an asset. \[asset_id, name, symbol, decimals, is_frozen\] - MetadataSet(T::AssetId, Vec, Vec, u8, bool), - /// Metadata has been cleared for an asset. \[asset_id\] - MetadataCleared(T::AssetId), + Destroyed { asset_id: T::AssetId }, + /// Some asset class was force-created. + ForceCreated { asset_id: T::AssetId, owner: T::AccountId }, + /// New metadata has been set for an asset. + MetadataSet { + asset_id: T::AssetId, + name: Vec, + symbol: Vec, + decimals: u8, + is_frozen: bool, + }, + /// Metadata has been cleared for an asset. + MetadataCleared { asset_id: T::AssetId }, /// (Additional) funds have been approved for transfer to a destination account. - /// \[asset_id, source, delegate, amount\] - ApprovedTransfer(T::AssetId, T::AccountId, T::AccountId, T::Balance), + ApprovedTransfer { + asset_id: T::AssetId, + source: T::AccountId, + delegate: T::AccountId, + amount: T::Balance, + }, /// An approval for account `delegate` was cancelled by `owner`. - /// \[id, owner, delegate\] - ApprovalCancelled(T::AssetId, T::AccountId, T::AccountId), + ApprovalCancelled { asset_id: T::AssetId, owner: T::AccountId, delegate: T::AccountId }, /// An `amount` was transferred in its entirety from `owner` to `destination` by /// the approved `delegate`. - /// \[id, owner, delegate, destination\] - TransferredApproved(T::AssetId, T::AccountId, T::AccountId, T::AccountId, T::Balance), + TransferredApproved { + asset_id: T::AssetId, + owner: T::AccountId, + delegate: T::AccountId, + destination: T::AccountId, + amount: T::Balance, + }, /// An asset has had its attributes changed by the `Force` origin. - /// \[id\] - AssetStatusChanged(T::AssetId), + AssetStatusChanged { asset_id: T::AssetId }, } #[pallet::error] pub enum Error { /// Account balance must be greater than or equal to the transfer amount. BalanceLow, - /// Balance should be non-zero. - BalanceZero, + /// The account to alter does not exist. + NoAccount, /// The signing account has no permission to do the operation. NoPermission, /// The given asset ID is unknown. @@ -342,8 +468,9 @@ pub mod pallet { BadWitness, /// Minimum balance should be non-zero. MinBalanceZero, - /// No provider reference exists to allow a non-zero balance of a non-self-sufficient - /// asset. + /// Unable to increment the consumer reference counters on the account. Either no provider + /// reference exists to allow a non-zero balance of a non-self-sufficient asset, or the + /// maximum number of consumers has been reached. NoProvider, /// Invalid metadata given. BadMetadata, @@ -351,6 +478,12 @@ pub mod pallet { Unapproved, /// The source account would not survive the transfer and it needs to stay alive. WouldDie, + /// The asset-account already exists. + AlreadyExists, + /// The asset-account doesn't have an associated deposit. + NoDeposit, + /// The operation would result in funds being burned. + WouldBurn, } #[pallet::call] @@ -407,7 +540,7 @@ pub mod pallet { is_frozen: false, }, ); - Self::deposit_event(Event::Created(id, owner, admin)); + Self::deposit_event(Event::Created { asset_id: id, creator: owner, owner: admin }); Ok(()) } @@ -513,7 +646,7 @@ pub mod pallet { /// /// Origin must be Signed and the sender should be the Manager of the asset `id`. /// - /// Bails with `BalanceZero` if the `who` is already dead. + /// Bails with `NoAccount` if the `who` is already dead. /// /// - `id`: The identifier of the asset to have some amount burned. /// - `who`: The account to be debited from. @@ -659,11 +792,13 @@ pub mod pallet { let d = Asset::::get(id).ok_or(Error::::Unknown)?; ensure!(&origin == &d.freezer, Error::::NoPermission); let who = T::Lookup::lookup(who)?; - ensure!(Account::::contains_key(id, &who), Error::::BalanceZero); - Account::::mutate(id, &who, |a| a.is_frozen = true); + Account::::try_mutate(id, &who, |maybe_account| -> DispatchResult { + maybe_account.as_mut().ok_or(Error::::NoAccount)?.is_frozen = true; + Ok(()) + })?; - Self::deposit_event(Event::::Frozen(id, who)); + Self::deposit_event(Event::::Frozen { asset_id: id, who }); Ok(()) } @@ -688,11 +823,13 @@ pub mod pallet { let details = Asset::::get(id).ok_or(Error::::Unknown)?; ensure!(&origin == &details.admin, Error::::NoPermission); let who = T::Lookup::lookup(who)?; - ensure!(Account::::contains_key(id, &who), Error::::BalanceZero); - Account::::mutate(id, &who, |a| a.is_frozen = false); + Account::::try_mutate(id, &who, |maybe_account| -> DispatchResult { + maybe_account.as_mut().ok_or(Error::::NoAccount)?.is_frozen = false; + Ok(()) + })?; - Self::deposit_event(Event::::Thawed(id, who)); + Self::deposit_event(Event::::Thawed { asset_id: id, who }); Ok(()) } @@ -718,7 +855,7 @@ pub mod pallet { d.is_frozen = true; - Self::deposit_event(Event::::AssetFrozen(id)); + Self::deposit_event(Event::::AssetFrozen { asset_id: id }); Ok(()) }) } @@ -745,7 +882,7 @@ pub mod pallet { d.is_frozen = false; - Self::deposit_event(Event::::AssetThawed(id)); + Self::deposit_event(Event::::AssetThawed { asset_id: id }); Ok(()) }) } @@ -784,7 +921,7 @@ pub mod pallet { details.owner = owner.clone(); - Self::deposit_event(Event::OwnerChanged(id, owner)); + Self::deposit_event(Event::OwnerChanged { asset_id: id, owner }); Ok(()) }) } @@ -822,7 +959,7 @@ pub mod pallet { details.admin = admin.clone(); details.freezer = freezer.clone(); - Self::deposit_event(Event::TeamChanged(id, issuer, admin, freezer)); + Self::deposit_event(Event::TeamChanged { asset_id: id, issuer, admin, freezer }); Ok(()) }) } @@ -852,43 +989,7 @@ pub mod pallet { decimals: u8, ) -> DispatchResult { let origin = ensure_signed(origin)?; - - let bounded_name: BoundedVec = - name.clone().try_into().map_err(|_| Error::::BadMetadata)?; - let bounded_symbol: BoundedVec = - symbol.clone().try_into().map_err(|_| Error::::BadMetadata)?; - - let d = Asset::::get(id).ok_or(Error::::Unknown)?; - ensure!(&origin == &d.owner, Error::::NoPermission); - - Metadata::::try_mutate_exists(id, |metadata| { - ensure!( - metadata.as_ref().map_or(true, |m| !m.is_frozen), - Error::::NoPermission - ); - - let old_deposit = metadata.take().map_or(Zero::zero(), |m| m.deposit); - let new_deposit = T::MetadataDepositPerByte::get() - .saturating_mul(((name.len() + symbol.len()) as u32).into()) - .saturating_add(T::MetadataDepositBase::get()); - - if new_deposit > old_deposit { - T::Currency::reserve(&origin, new_deposit - old_deposit)?; - } else { - T::Currency::unreserve(&origin, old_deposit - new_deposit); - } - - *metadata = Some(AssetMetadata { - deposit: new_deposit, - name: bounded_name, - symbol: bounded_symbol, - decimals, - is_frozen: false, - }); - - Self::deposit_event(Event::MetadataSet(id, name, symbol, decimals, false)); - Ok(()) - }) + Self::do_set_metadata(id, &origin, name, symbol, decimals) } /// Clear the metadata for an asset. @@ -915,7 +1016,7 @@ pub mod pallet { Metadata::::try_mutate_exists(id, |metadata| { let deposit = metadata.take().ok_or(Error::::Unknown)?.deposit; T::Currency::unreserve(&d.owner, deposit); - Self::deposit_event(Event::MetadataCleared(id)); + Self::deposit_event(Event::MetadataCleared { asset_id: id }); Ok(()) }) } @@ -962,7 +1063,13 @@ pub mod pallet { is_frozen, }); - Self::deposit_event(Event::MetadataSet(id, name, symbol, decimals, is_frozen)); + Self::deposit_event(Event::MetadataSet { + asset_id: id, + name, + symbol, + decimals, + is_frozen, + }); Ok(()) }) } @@ -989,7 +1096,7 @@ pub mod pallet { Metadata::::try_mutate_exists(id, |metadata| { let deposit = metadata.take().ok_or(Error::::Unknown)?.deposit; T::Currency::unreserve(&d.owner, deposit); - Self::deposit_event(Event::MetadataCleared(id)); + Self::deposit_event(Event::MetadataCleared { asset_id: id }); Ok(()) }) } @@ -1041,7 +1148,7 @@ pub mod pallet { asset.is_frozen = is_frozen; *maybe_asset = Some(asset); - Self::deposit_event(Event::AssetStatusChanged(id)); + Self::deposit_event(Event::AssetStatusChanged { asset_id: id }); Ok(()) }) } @@ -1075,35 +1182,7 @@ pub mod pallet { ) -> DispatchResult { let owner = ensure_signed(origin)?; let delegate = T::Lookup::lookup(delegate)?; - - let mut d = Asset::::get(id).ok_or(Error::::Unknown)?; - ensure!(!d.is_frozen, Error::::Frozen); - Approvals::::try_mutate( - (id, &owner, &delegate), - |maybe_approved| -> DispatchResult { - let mut approved = match maybe_approved.take() { - // an approval already exists and is being updated - Some(a) => a, - // a new approval is created - None => { - d.approvals.saturating_inc(); - Default::default() - }, - }; - let deposit_required = T::ApprovalDeposit::get(); - if approved.deposit < deposit_required { - T::Currency::reserve(&owner, deposit_required - approved.deposit)?; - approved.deposit = deposit_required; - } - approved.amount = approved.amount.saturating_add(amount); - *maybe_approved = Some(approved); - Ok(()) - }, - )?; - Asset::::insert(id, d); - Self::deposit_event(Event::ApprovedTransfer(id, owner, delegate, amount)); - - Ok(()) + Self::do_approve_transfer(id, &owner, &delegate, amount) } /// Cancel all of some asset approved for delegated transfer by a third-party account. @@ -1135,7 +1214,7 @@ pub mod pallet { d.approvals.saturating_dec(); Asset::::insert(id, d); - Self::deposit_event(Event::ApprovalCancelled(id, owner, delegate)); + Self::deposit_event(Event::ApprovalCancelled { asset_id: id, owner, delegate }); Ok(()) } @@ -1177,7 +1256,7 @@ pub mod pallet { d.approvals.saturating_dec(); Asset::::insert(id, d); - Self::deposit_event(Event::ApprovalCancelled(id, owner, delegate)); + Self::deposit_event(Event::ApprovalCancelled { asset_id: id, owner, delegate }); Ok(()) } @@ -1210,33 +1289,38 @@ pub mod pallet { let delegate = ensure_signed(origin)?; let owner = T::Lookup::lookup(owner)?; let destination = T::Lookup::lookup(destination)?; + Self::do_transfer_approved(id, &owner, &delegate, &destination, amount) + } - Approvals::::try_mutate_exists( - (id, &owner, delegate), - |maybe_approved| -> DispatchResult { - let mut approved = maybe_approved.take().ok_or(Error::::Unapproved)?; - let remaining = - approved.amount.checked_sub(&amount).ok_or(Error::::Unapproved)?; - - let f = - TransferFlags { keep_alive: false, best_effort: false, burn_dust: false }; - Self::do_transfer(id, &owner, &destination, amount, None, f)?; - - if remaining.is_zero() { - T::Currency::unreserve(&owner, approved.deposit); - Asset::::mutate(id, |maybe_details| { - if let Some(details) = maybe_details { - details.approvals.saturating_dec(); - } - }); - } else { - approved.amount = remaining; - *maybe_approved = Some(approved); - } - Ok(()) - }, - )?; - Ok(()) + /// Create an asset account for non-provider assets. + /// + /// A deposit will be taken from the signer account. + /// + /// - `origin`: Must be Signed; the signer account must have sufficient funds for a deposit + /// to be taken. + /// - `id`: The identifier of the asset for the account to be created. + /// + /// Emits `Touched` event when successful. + #[pallet::weight(T::WeightInfo::mint())] + pub fn touch(origin: OriginFor, #[pallet::compact] id: T::AssetId) -> DispatchResult { + Self::do_touch(id, ensure_signed(origin)?) + } + + /// Return the deposit (if any) of an asset account. + /// + /// The origin must be Signed. + /// + /// - `id`: The identifier of the asset for the account to be created. + /// - `allow_burn`: If `true` then assets may be destroyed in order to complete the refund. + /// + /// Emits `Refunded` event when successful. + #[pallet::weight(T::WeightInfo::mint())] + pub fn refund( + origin: OriginFor, + #[pallet::compact] id: T::AssetId, + allow_burn: bool, + ) -> DispatchResult { + Self::do_refund(id, ensure_signed(origin)?, allow_burn) } } } diff --git a/frame/assets/src/mock.rs b/frame/assets/src/mock.rs index 1b2602792d84..67690e2b28ec 100644 --- a/frame/assets/src/mock.rs +++ b/frame/assets/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,10 @@ use super::*; use crate as pallet_assets; -use frame_support::{construct_runtime, parameter_types}; +use frame_support::{ + construct_runtime, + traits::{ConstU32, ConstU64, GenesisBuild}, +}; use sp_core::H256; use sp_runtime::{ testing::Header, @@ -42,9 +45,6 @@ construct_runtime!( } ); -parameter_types! { - pub const BlockHashCount: u64 = 250; -} impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; type BlockWeights = (); @@ -59,7 +59,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type DbWeight = (); type Version = (); type PalletInfo = PalletInfo; @@ -69,17 +69,14 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); -} - -parameter_types! { - pub const ExistentialDeposit: u64 = 1; + type MaxConsumers = ConstU32<2>; } impl pallet_balances::Config for Test { type Balance = u64; type DustRemoval = (); type Event = Event; - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); type MaxLocks = (); @@ -87,25 +84,18 @@ impl pallet_balances::Config for Test { type ReserveIdentifier = [u8; 8]; } -parameter_types! { - pub const AssetDeposit: u64 = 1; - pub const ApprovalDeposit: u64 = 1; - pub const StringLimit: u32 = 50; - pub const MetadataDepositBase: u64 = 1; - pub const MetadataDepositPerByte: u64 = 1; -} - impl Config for Test { type Event = Event; type Balance = u64; type AssetId = u32; type Currency = Balances; type ForceOrigin = frame_system::EnsureRoot; - type AssetDeposit = AssetDeposit; - type MetadataDepositBase = MetadataDepositBase; - type MetadataDepositPerByte = MetadataDepositPerByte; - type ApprovalDeposit = ApprovalDeposit; - type StringLimit = StringLimit; + type AssetDeposit = ConstU64<1>; + type AssetAccountDeposit = ConstU64<10>; + type MetadataDepositBase = ConstU64<1>; + type MetadataDepositPerByte = ConstU64<1>; + type ApprovalDeposit = ConstU64<1>; + type StringLimit = ConstU32<50>; type Freezer = TestFreezer; type WeightInfo = (); type Extra = (); @@ -130,23 +120,50 @@ impl FrozenBalance for TestFreezer { fn died(asset: u32, who: &u64) { HOOKS.with(|h| h.borrow_mut().push(Hook::Died(asset, who.clone()))); + // Sanity check: dead accounts have no balance. + assert!(Assets::balance(asset, *who).is_zero()); } } pub(crate) fn set_frozen_balance(asset: u32, who: u64, amount: u64) { FROZEN.with(|f| f.borrow_mut().insert((asset, who), amount)); } + pub(crate) fn clear_frozen_balance(asset: u32, who: u64) { FROZEN.with(|f| f.borrow_mut().remove(&(asset, who))); } + pub(crate) fn hooks() -> Vec { HOOKS.with(|h| h.borrow().clone()) } -pub(crate) fn new_test_ext() -> sp_io::TestExternalities { - let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); +pub(crate) fn take_hooks() -> Vec { + HOOKS.with(|h| h.take()) +} - let mut ext = sp_io::TestExternalities::new(t); +pub(crate) fn new_test_ext() -> sp_io::TestExternalities { + let mut storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); + + let config: pallet_assets::GenesisConfig = pallet_assets::GenesisConfig { + assets: vec![ + // id, owner, is_sufficient, min_balance + (999, 0, true, 1), + ], + metadata: vec![ + // id, name, symbol, decimals + (999, "Token Name".into(), "TOKEN".into(), 10), + ], + accounts: vec![ + // id, account_id, balance + (999, 1, 100), + ], + }; + + config.assimilate_storage(&mut storage).unwrap(); + + let mut ext: sp_io::TestExternalities = storage.into(); + // Clear thread local vars for https://github.com/paritytech/substrate/issues/10479. + ext.execute_with(|| take_hooks()); ext.execute_with(|| System::set_block_number(1)); ext } diff --git a/frame/assets/src/tests.rs b/frame/assets/src/tests.rs index aab534a6e4ef..7430b742e7d2 100644 --- a/frame/assets/src/tests.rs +++ b/frame/assets/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,6 +34,112 @@ fn basic_minting_should_work() { }); } +#[test] +fn minting_too_many_insufficient_assets_fails() { + new_test_ext().execute_with(|| { + assert_ok!(Assets::force_create(Origin::root(), 0, 1, false, 1)); + assert_ok!(Assets::force_create(Origin::root(), 1, 1, false, 1)); + assert_ok!(Assets::force_create(Origin::root(), 2, 1, false, 1)); + Balances::make_free_balance_be(&1, 100); + assert_ok!(Assets::mint(Origin::signed(1), 0, 1, 100)); + assert_ok!(Assets::mint(Origin::signed(1), 1, 1, 100)); + assert_noop!(Assets::mint(Origin::signed(1), 2, 1, 100), TokenError::CannotCreate); + + Balances::make_free_balance_be(&2, 1); + assert_ok!(Assets::transfer(Origin::signed(1), 0, 2, 100)); + assert_ok!(Assets::mint(Origin::signed(1), 2, 1, 100)); + }); +} + +#[test] +fn minting_insufficient_asset_with_deposit_should_work_when_consumers_exhausted() { + new_test_ext().execute_with(|| { + assert_ok!(Assets::force_create(Origin::root(), 0, 1, false, 1)); + assert_ok!(Assets::force_create(Origin::root(), 1, 1, false, 1)); + assert_ok!(Assets::force_create(Origin::root(), 2, 1, false, 1)); + Balances::make_free_balance_be(&1, 100); + assert_ok!(Assets::mint(Origin::signed(1), 0, 1, 100)); + assert_ok!(Assets::mint(Origin::signed(1), 1, 1, 100)); + assert_noop!(Assets::mint(Origin::signed(1), 2, 1, 100), TokenError::CannotCreate); + + assert_ok!(Assets::touch(Origin::signed(1), 2)); + assert_eq!(Balances::reserved_balance(&1), 10); + + assert_ok!(Assets::mint(Origin::signed(1), 2, 1, 100)); + }); +} + +#[test] +fn minting_insufficient_assets_with_deposit_without_consumer_should_work() { + new_test_ext().execute_with(|| { + assert_ok!(Assets::force_create(Origin::root(), 0, 1, false, 1)); + assert_noop!(Assets::mint(Origin::signed(1), 0, 1, 100), TokenError::CannotCreate); + Balances::make_free_balance_be(&1, 100); + assert_ok!(Assets::touch(Origin::signed(1), 0)); + assert_ok!(Assets::mint(Origin::signed(1), 0, 1, 100)); + assert_eq!(Balances::reserved_balance(&1), 10); + assert_eq!(System::consumers(&1), 0); + }); +} + +#[test] +fn refunding_asset_deposit_with_burn_should_work() { + new_test_ext().execute_with(|| { + assert_ok!(Assets::force_create(Origin::root(), 0, 1, false, 1)); + Balances::make_free_balance_be(&1, 100); + assert_ok!(Assets::touch(Origin::signed(1), 0)); + assert_ok!(Assets::mint(Origin::signed(1), 0, 1, 100)); + assert_ok!(Assets::refund(Origin::signed(1), 0, true)); + assert_eq!(Balances::reserved_balance(&1), 0); + assert_eq!(Assets::balance(1, 0), 0); + }); +} + +#[test] +fn refunding_asset_deposit_with_burn_disallowed_should_fail() { + new_test_ext().execute_with(|| { + assert_ok!(Assets::force_create(Origin::root(), 0, 1, false, 1)); + Balances::make_free_balance_be(&1, 100); + assert_ok!(Assets::touch(Origin::signed(1), 0)); + assert_ok!(Assets::mint(Origin::signed(1), 0, 1, 100)); + assert_noop!(Assets::refund(Origin::signed(1), 0, false), Error::::WouldBurn); + }); +} + +#[test] +fn refunding_asset_deposit_without_burn_should_work() { + new_test_ext().execute_with(|| { + assert_ok!(Assets::force_create(Origin::root(), 0, 1, false, 1)); + assert_noop!(Assets::mint(Origin::signed(1), 0, 1, 100), TokenError::CannotCreate); + Balances::make_free_balance_be(&1, 100); + assert_ok!(Assets::touch(Origin::signed(1), 0)); + assert_ok!(Assets::mint(Origin::signed(1), 0, 1, 100)); + Balances::make_free_balance_be(&2, 100); + assert_ok!(Assets::transfer(Origin::signed(1), 0, 2, 100)); + assert_eq!(Assets::balance(0, 2), 100); + assert_eq!(Assets::balance(0, 1), 0); + assert_eq!(Balances::reserved_balance(&1), 10); + assert_ok!(Assets::refund(Origin::signed(1), 0, false)); + assert_eq!(Balances::reserved_balance(&1), 0); + assert_eq!(Assets::balance(1, 0), 0); + }); +} + +/// Refunding reaps an account and calls the `FrozenBalance::died` hook. +#[test] +fn refunding_calls_died_hook() { + new_test_ext().execute_with(|| { + assert_ok!(Assets::force_create(Origin::root(), 0, 1, false, 1)); + Balances::make_free_balance_be(&1, 100); + assert_ok!(Assets::touch(Origin::signed(1), 0)); + assert_ok!(Assets::mint(Origin::signed(1), 0, 1, 100)); + assert_ok!(Assets::refund(Origin::signed(1), 0, true)); + + assert_eq!(Asset::::get(0).unwrap().accounts, 0); + assert_eq!(hooks(), vec![Hook::Died(0, 1)]); + }); +} + #[test] fn approval_lifecycle_works() { new_test_ext().execute_with(|| { @@ -298,19 +404,32 @@ fn min_balance_should_work() { ); // When deducting from an account to below minimum, it should be reaped. + // Death by `transfer`. assert_ok!(Assets::transfer(Origin::signed(1), 0, 2, 91)); - assert!(Assets::balance(0, 1).is_zero()); + assert!(Assets::maybe_balance(0, 1).is_none()); assert_eq!(Assets::balance(0, 2), 100); assert_eq!(Asset::::get(0).unwrap().accounts, 1); + assert_eq!(take_hooks(), vec![Hook::Died(0, 1)]); + // Death by `force_transfer`. assert_ok!(Assets::force_transfer(Origin::signed(1), 0, 2, 1, 91)); - assert!(Assets::balance(0, 2).is_zero()); + assert!(Assets::maybe_balance(0, 2).is_none()); assert_eq!(Assets::balance(0, 1), 100); assert_eq!(Asset::::get(0).unwrap().accounts, 1); + assert_eq!(take_hooks(), vec![Hook::Died(0, 2)]); + // Death by `burn`. assert_ok!(Assets::burn(Origin::signed(1), 0, 1, 91)); - assert!(Assets::balance(0, 1).is_zero()); + assert!(Assets::maybe_balance(0, 1).is_none()); assert_eq!(Asset::::get(0).unwrap().accounts, 0); + assert_eq!(take_hooks(), vec![Hook::Died(0, 1)]); + + // Death by `transfer_approved`. + assert_ok!(Assets::mint(Origin::signed(1), 0, 1, 100)); + Balances::make_free_balance_be(&1, 1); + assert_ok!(Assets::approve_transfer(Origin::signed(1), 0, 2, 100)); + assert_ok!(Assets::transfer_approved(Origin::signed(2), 0, 1, 3, 91)); + assert_eq!(take_hooks(), vec![Hook::Died(0, 1)]); }); } @@ -357,6 +476,7 @@ fn transferring_enough_to_kill_source_when_keep_alive_should_fail() { assert_ok!(Assets::transfer_keep_alive(Origin::signed(1), 0, 2, 90)); assert_eq!(Assets::balance(0, 1), 10); assert_eq!(Assets::balance(0, 2), 90); + assert!(hooks().is_empty()); }); } @@ -488,7 +608,7 @@ fn transferring_amount_more_than_available_balance_should_not_work() { assert_eq!(Assets::balance(0, 2), 50); assert_ok!(Assets::burn(Origin::signed(1), 0, 1, u64::MAX)); assert_eq!(Assets::balance(0, 1), 0); - assert_noop!(Assets::transfer(Origin::signed(1), 0, 1, 50), Error::::BalanceLow); + assert_noop!(Assets::transfer(Origin::signed(1), 0, 1, 50), Error::::NoAccount); assert_noop!(Assets::transfer(Origin::signed(2), 0, 1, 51), Error::::BalanceLow); }); } @@ -500,7 +620,12 @@ fn transferring_less_than_one_unit_is_fine() { assert_ok!(Assets::mint(Origin::signed(1), 0, 1, 100)); assert_eq!(Assets::balance(0, 1), 100); assert_ok!(Assets::transfer(Origin::signed(1), 0, 2, 0)); - System::assert_last_event(mock::Event::Assets(crate::Event::Transferred(0, 1, 2, 0))); + System::assert_last_event(mock::Event::Assets(crate::Event::Transferred { + asset_id: 0, + from: 1, + to: 2, + amount: 0, + })); }); } @@ -531,7 +656,7 @@ fn burning_asset_balance_with_zero_balance_does_nothing() { assert_ok!(Assets::force_create(Origin::root(), 0, 1, true, 1)); assert_ok!(Assets::mint(Origin::signed(1), 0, 1, 100)); assert_eq!(Assets::balance(0, 2), 0); - assert_ok!(Assets::burn(Origin::signed(1), 0, 2, u64::MAX)); + assert_noop!(Assets::burn(Origin::signed(1), 0, 2, u64::MAX), Error::::NoAccount); assert_eq!(Assets::balance(0, 2), 0); assert_eq!(Assets::total_supply(0), 100); }); @@ -588,6 +713,24 @@ fn set_metadata_should_work() { }); } +/// Destroying an asset calls the `FrozenBalance::died` hooks of all accounts. +#[test] +fn destroy_calls_died_hooks() { + new_test_ext().execute_with(|| { + assert_ok!(Assets::force_create(Origin::root(), 0, 1, true, 50)); + // Create account 1 and 2. + assert_ok!(Assets::mint(Origin::signed(1), 0, 1, 100)); + assert_ok!(Assets::mint(Origin::signed(1), 0, 2, 100)); + // Destroy the asset. + let w = Asset::::get(0).unwrap().destroy_witness(); + assert_ok!(Assets::destroy(Origin::signed(1), 0, w)); + + // Asset is gone and accounts 1 and 2 died. + assert!(Asset::::get(0).is_none()); + assert_eq!(hooks(), vec![Hook::Died(0, 1), Hook::Died(0, 2)]); + }) +} + #[test] fn freezer_should_work() { new_test_ext().execute_with(|| { @@ -683,7 +826,7 @@ fn force_metadata_should_work() { ); // string length limit check - let limit = StringLimit::get() as usize; + let limit = 50usize; assert_noop!( Assets::force_set_metadata( Origin::root(), @@ -784,3 +927,47 @@ fn balance_conversion_should_work() { ); }); } + +#[test] +fn assets_from_genesis_should_exist() { + new_test_ext().execute_with(|| { + assert!(Asset::::contains_key(999)); + assert!(Metadata::::contains_key(999)); + assert_eq!(Assets::balance(999, 1), 100); + assert_eq!(Assets::total_supply(999), 100); + }); +} + +#[test] +fn querying_name_symbol_and_decimals_should_work() { + new_test_ext().execute_with(|| { + use frame_support::traits::tokens::fungibles::metadata::Inspect; + assert_ok!(Assets::force_create(Origin::root(), 0, 1, true, 1)); + assert_ok!(Assets::force_set_metadata( + Origin::root(), + 0, + vec![0u8; 10], + vec![1u8; 10], + 12, + false + )); + assert_eq!(Assets::name(0), vec![0u8; 10]); + assert_eq!(Assets::symbol(0), vec![1u8; 10]); + assert_eq!(Assets::decimals(0), 12); + }); +} + +#[test] +fn querying_allowance_should_work() { + new_test_ext().execute_with(|| { + use frame_support::traits::tokens::fungibles::approvals::{Inspect, Mutate}; + assert_ok!(Assets::force_create(Origin::root(), 0, 1, true, 1)); + assert_ok!(Assets::mint(Origin::signed(1), 0, 1, 100)); + Balances::make_free_balance_be(&1, 1); + assert_ok!(Assets::approve(0, &1, &2, 50)); + assert_eq!(Assets::allowance(0, &1, &2), 50); + // Transfer asset 0, from owner 1 and delegate 2 to destination 3 + assert_ok!(Assets::transfer_from(0, &1, &2, &3, 50)); + assert_eq!(Assets::allowance(0, &1, &2), 0); + }); +} diff --git a/frame/assets/src/types.rs b/frame/assets/src/types.rs index d3f349b9620d..7dacb469cf13 100644 --- a/frame/assets/src/types.rs +++ b/frame/assets/src/types.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,14 +18,16 @@ //! Various basic types for use in the assets pallet. use super::*; -use frame_support::pallet_prelude::*; -use scale_info::TypeInfo; - -use frame_support::traits::{fungible, tokens::BalanceConversion}; +use frame_support::{ + pallet_prelude::*, + traits::{fungible, tokens::BalanceConversion}, +}; use sp_runtime::{traits::Convert, FixedPointNumber, FixedPointOperand, FixedU128}; pub(super) type DepositBalanceOf = <>::Currency as Currency<::AccountId>>::Balance; +pub(super) type AssetAccountOf = + AssetAccount<>::Balance, DepositBalanceOf, >::Extra>; #[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, MaxEncodedLen, TypeInfo)] pub struct AssetDetails { @@ -76,14 +78,47 @@ pub struct Approval { pub(super) deposit: DepositBalance, } -#[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, Default, MaxEncodedLen, TypeInfo)] -pub struct AssetBalance { +#[test] +fn ensure_bool_decodes_to_consumer_or_sufficient() { + assert_eq!(false.encode(), ExistenceReason::<()>::Consumer.encode()); + assert_eq!(true.encode(), ExistenceReason::<()>::Sufficient.encode()); +} + +#[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, MaxEncodedLen, TypeInfo)] +pub enum ExistenceReason { + #[codec(index = 0)] + Consumer, + #[codec(index = 1)] + Sufficient, + #[codec(index = 2)] + DepositHeld(Balance), + #[codec(index = 3)] + DepositRefunded, +} + +impl ExistenceReason { + pub(crate) fn take_deposit(&mut self) -> Option { + if !matches!(self, ExistenceReason::DepositHeld(_)) { + return None + } + if let ExistenceReason::DepositHeld(deposit) = + sp_std::mem::replace(self, ExistenceReason::DepositRefunded) + { + return Some(deposit) + } else { + return None + } + } +} + +#[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, MaxEncodedLen, TypeInfo)] +pub struct AssetAccount { /// The balance. pub balance: Balance, /// Whether the account is frozen. pub(super) is_frozen: bool, - /// `true` if this balance gave the account a self-sufficient reference. - pub(super) sufficient: bool, + /// The reason for the existence of the account. + pub(super) reason: ExistenceReason, /// Additional "sidecar" data, in case some other pallet wants to use this storage item. pub(super) extra: Extra, } @@ -122,20 +157,24 @@ pub struct DestroyWitness { /// `minimum_balance` of the asset. This is additive - the `minimum_balance` of the asset must be /// met *and then* anything here in addition. pub trait FrozenBalance { - /// Return the frozen balance. Under normal behaviour, this amount should always be - /// withdrawable. + /// Return the frozen balance. /// - /// In reality, the balance of every account must be at least the sum of this (if `Some`) and - /// the asset's minimum_balance, since there may be complications to destroying an asset's - /// account completely. + /// Generally, the balance of every account must be at least the sum of this (if `Some`) and + /// the asset's `minimum_balance` (the latter since there may be complications to destroying an + /// asset's account completely). /// - /// If `None` is returned, then nothing special is enforced. + /// Under normal behaviour, the account balance should not go below the sum of this (if `Some`) + /// and the asset's minimum balance. However, the account balance may reasonably begin below + /// this sum (e.g. if less than the sum had ever been transfered into the account). + /// + /// In special cases (privileged intervention) the account balance may also go below the sum. /// - /// If any operation ever breaks this requirement (which will only happen through some sort of - /// privileged intervention), then `melted` is called to do any cleanup. + /// If `None` is returned, then nothing special is enforced. fn frozen_balance(asset: AssetId, who: &AccountId) -> Option; - /// Called when an account has been removed. + /// Called after an account has been removed. + /// + /// NOTE: It is possible that the asset does no longer exist when this hook is called. fn died(asset: AssetId, who: &AccountId); } diff --git a/frame/assets/src/weights.rs b/frame/assets/src/weights.rs index 912ebcf7e851..a1cd77faee82 100644 --- a/frame/assets/src/weights.rs +++ b/frame/assets/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_assets //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-30, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/assets/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -75,13 +76,13 @@ pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { // Storage: Assets Asset (r:1 w:1) fn create() -> Weight { - (41_651_000 as Weight) + (22_102_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) fn force_create() -> Weight { - (21_378_000 as Weight) + (12_124_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -92,12 +93,12 @@ impl WeightInfo for SubstrateWeight { // Storage: Assets Approvals (r:501 w:500) fn destroy(c: u32, s: u32, a: u32, ) -> Weight { (0 as Weight) - // Standard Error: 32_000 - .saturating_add((21_163_000 as Weight).saturating_mul(c as Weight)) - // Standard Error: 32_000 - .saturating_add((26_932_000 as Weight).saturating_mul(s as Weight)) - // Standard Error: 329_000 - .saturating_add((29_714_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 34_000 + .saturating_add((14_683_000 as Weight).saturating_mul(c as Weight)) + // Standard Error: 34_000 + .saturating_add((17_080_000 as Weight).saturating_mul(s as Weight)) + // Standard Error: 342_000 + .saturating_add((16_533_000 as Weight).saturating_mul(a as Weight)) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().reads((2 as Weight).saturating_mul(c as Weight))) .saturating_add(T::DbWeight::get().reads((2 as Weight).saturating_mul(s as Weight))) @@ -110,14 +111,14 @@ impl WeightInfo for SubstrateWeight { // Storage: Assets Asset (r:1 w:1) // Storage: Assets Account (r:1 w:1) fn mint() -> Weight { - (47_913_000 as Weight) + (26_632_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Assets Asset (r:1 w:1) // Storage: Assets Account (r:1 w:1) fn burn() -> Weight { - (55_759_000 as Weight) + (30_048_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -125,7 +126,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Assets Account (r:2 w:2) // Storage: System Account (r:1 w:1) fn transfer() -> Weight { - (83_205_000 as Weight) + (44_000_000 as Weight) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } @@ -133,7 +134,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Assets Account (r:2 w:2) // Storage: System Account (r:1 w:1) fn transfer_keep_alive() -> Weight { - (70_665_000 as Weight) + (37_286_000 as Weight) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } @@ -141,93 +142,95 @@ impl WeightInfo for SubstrateWeight { // Storage: Assets Account (r:2 w:2) // Storage: System Account (r:1 w:1) fn force_transfer() -> Weight { - (81_458_000 as Weight) + (44_120_000 as Weight) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } // Storage: Assets Asset (r:1 w:0) // Storage: Assets Account (r:1 w:1) fn freeze() -> Weight { - (32_845_000 as Weight) + (18_309_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:0) // Storage: Assets Account (r:1 w:1) fn thaw() -> Weight { - (33_303_000 as Weight) + (18_290_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) fn freeze_asset() -> Weight { - (23_434_000 as Weight) + (14_744_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) fn thaw_asset() -> Weight { - (24_173_000 as Weight) + (14_833_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) // Storage: Assets Metadata (r:1 w:0) fn transfer_ownership() -> Weight { - (27_466_000 as Weight) + (16_654_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) fn set_team() -> Weight { - (24_608_000 as Weight) + (15_351_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:0) // Storage: Assets Metadata (r:1 w:1) fn set_metadata(n: u32, s: u32, ) -> Weight { - (49_515_000 as Weight) - // Standard Error: 1_000 - .saturating_add((1_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 1_000 - .saturating_add((6_000 as Weight).saturating_mul(s as Weight)) + (27_588_000 as Weight) + // Standard Error: 0 + .saturating_add((6_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 0 + .saturating_add((3_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:0) // Storage: Assets Metadata (r:1 w:1) fn clear_metadata() -> Weight { - (48_163_000 as Weight) + (27_710_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:0) // Storage: Assets Metadata (r:1 w:1) - fn force_set_metadata(_n: u32, s: u32, ) -> Weight { - (26_722_000 as Weight) + fn force_set_metadata(n: u32, s: u32, ) -> Weight { + (15_345_000 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(n as Weight)) // Standard Error: 0 - .saturating_add((5_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:0) // Storage: Assets Metadata (r:1 w:1) fn force_clear_metadata() -> Weight { - (47_923_000 as Weight) + (27_552_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) fn force_asset_status() -> Weight { - (23_081_000 as Weight) + (13_755_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) // Storage: Assets Approvals (r:1 w:1) fn approve_transfer() -> Weight { - (56_998_000 as Weight) + (30_831_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -236,21 +239,21 @@ impl WeightInfo for SubstrateWeight { // Storage: Assets Account (r:2 w:2) // Storage: System Account (r:1 w:1) fn transfer_approved() -> Weight { - (107_171_000 as Weight) + (56_267_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } // Storage: Assets Asset (r:1 w:1) // Storage: Assets Approvals (r:1 w:1) fn cancel_approval() -> Weight { - (57_358_000 as Weight) + (31_964_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Assets Asset (r:1 w:1) // Storage: Assets Approvals (r:1 w:1) fn force_cancel_approval() -> Weight { - (58_330_000 as Weight) + (31_806_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -260,13 +263,13 @@ impl WeightInfo for SubstrateWeight { impl WeightInfo for () { // Storage: Assets Asset (r:1 w:1) fn create() -> Weight { - (41_651_000 as Weight) + (22_102_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) fn force_create() -> Weight { - (21_378_000 as Weight) + (12_124_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -277,12 +280,12 @@ impl WeightInfo for () { // Storage: Assets Approvals (r:501 w:500) fn destroy(c: u32, s: u32, a: u32, ) -> Weight { (0 as Weight) - // Standard Error: 32_000 - .saturating_add((21_163_000 as Weight).saturating_mul(c as Weight)) - // Standard Error: 32_000 - .saturating_add((26_932_000 as Weight).saturating_mul(s as Weight)) - // Standard Error: 329_000 - .saturating_add((29_714_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 34_000 + .saturating_add((14_683_000 as Weight).saturating_mul(c as Weight)) + // Standard Error: 34_000 + .saturating_add((17_080_000 as Weight).saturating_mul(s as Weight)) + // Standard Error: 342_000 + .saturating_add((16_533_000 as Weight).saturating_mul(a as Weight)) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().reads((2 as Weight).saturating_mul(c as Weight))) .saturating_add(RocksDbWeight::get().reads((2 as Weight).saturating_mul(s as Weight))) @@ -295,14 +298,14 @@ impl WeightInfo for () { // Storage: Assets Asset (r:1 w:1) // Storage: Assets Account (r:1 w:1) fn mint() -> Weight { - (47_913_000 as Weight) + (26_632_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Assets Asset (r:1 w:1) // Storage: Assets Account (r:1 w:1) fn burn() -> Weight { - (55_759_000 as Weight) + (30_048_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -310,7 +313,7 @@ impl WeightInfo for () { // Storage: Assets Account (r:2 w:2) // Storage: System Account (r:1 w:1) fn transfer() -> Weight { - (83_205_000 as Weight) + (44_000_000 as Weight) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } @@ -318,7 +321,7 @@ impl WeightInfo for () { // Storage: Assets Account (r:2 w:2) // Storage: System Account (r:1 w:1) fn transfer_keep_alive() -> Weight { - (70_665_000 as Weight) + (37_286_000 as Weight) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } @@ -326,93 +329,95 @@ impl WeightInfo for () { // Storage: Assets Account (r:2 w:2) // Storage: System Account (r:1 w:1) fn force_transfer() -> Weight { - (81_458_000 as Weight) + (44_120_000 as Weight) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } // Storage: Assets Asset (r:1 w:0) // Storage: Assets Account (r:1 w:1) fn freeze() -> Weight { - (32_845_000 as Weight) + (18_309_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:0) // Storage: Assets Account (r:1 w:1) fn thaw() -> Weight { - (33_303_000 as Weight) + (18_290_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) fn freeze_asset() -> Weight { - (23_434_000 as Weight) + (14_744_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) fn thaw_asset() -> Weight { - (24_173_000 as Weight) + (14_833_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) // Storage: Assets Metadata (r:1 w:0) fn transfer_ownership() -> Weight { - (27_466_000 as Weight) + (16_654_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) fn set_team() -> Weight { - (24_608_000 as Weight) + (15_351_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:0) // Storage: Assets Metadata (r:1 w:1) fn set_metadata(n: u32, s: u32, ) -> Weight { - (49_515_000 as Weight) - // Standard Error: 1_000 - .saturating_add((1_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 1_000 - .saturating_add((6_000 as Weight).saturating_mul(s as Weight)) + (27_588_000 as Weight) + // Standard Error: 0 + .saturating_add((6_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 0 + .saturating_add((3_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:0) // Storage: Assets Metadata (r:1 w:1) fn clear_metadata() -> Weight { - (48_163_000 as Weight) + (27_710_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:0) // Storage: Assets Metadata (r:1 w:1) - fn force_set_metadata(_n: u32, s: u32, ) -> Weight { - (26_722_000 as Weight) + fn force_set_metadata(n: u32, s: u32, ) -> Weight { + (15_345_000 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(n as Weight)) // Standard Error: 0 - .saturating_add((5_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:0) // Storage: Assets Metadata (r:1 w:1) fn force_clear_metadata() -> Weight { - (47_923_000 as Weight) + (27_552_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) fn force_asset_status() -> Weight { - (23_081_000 as Weight) + (13_755_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Assets Asset (r:1 w:1) // Storage: Assets Approvals (r:1 w:1) fn approve_transfer() -> Weight { - (56_998_000 as Weight) + (30_831_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -421,21 +426,21 @@ impl WeightInfo for () { // Storage: Assets Account (r:2 w:2) // Storage: System Account (r:1 w:1) fn transfer_approved() -> Weight { - (107_171_000 as Weight) + (56_267_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } // Storage: Assets Asset (r:1 w:1) // Storage: Assets Approvals (r:1 w:1) fn cancel_approval() -> Weight { - (57_358_000 as Weight) + (31_964_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Assets Asset (r:1 w:1) // Storage: Assets Approvals (r:1 w:1) fn force_cancel_approval() -> Weight { - (58_330_000 as Weight) + (31_806_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } diff --git a/frame/atomic-swap/Cargo.toml b/frame/atomic-swap/Cargo.toml index 53a8c3a81165..361b7d5833e6 100644 --- a/frame/atomic-swap/Cargo.toml +++ b/frame/atomic-swap/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-atomic-swap" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME atomic swap pallet" readme = "README.md" @@ -13,14 +13,14 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } [dev-dependencies] pallet-balances = { version = "4.0.0-dev", path = "../balances" } diff --git a/frame/atomic-swap/src/lib.rs b/frame/atomic-swap/src/lib.rs index 9cf92c3bd233..e57496954a57 100644 --- a/frame/atomic-swap/src/lib.rs +++ b/frame/atomic-swap/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -45,6 +45,7 @@ mod tests; use codec::{Decode, Encode}; use frame_support::{ dispatch::DispatchResult, + pallet_prelude::MaxEncodedLen, traits::{BalanceStatus, Currency, Get, ReservableCurrency}, weights::Weight, RuntimeDebugNoBound, @@ -59,8 +60,9 @@ use sp_std::{ }; /// Pending atomic swap operation. -#[derive(Clone, Eq, PartialEq, RuntimeDebugNoBound, Encode, Decode, TypeInfo)] +#[derive(Clone, Eq, PartialEq, RuntimeDebugNoBound, Encode, Decode, TypeInfo, MaxEncodedLen)] #[scale_info(skip_type_params(T))] +#[codec(mel_bound())] pub struct PendingSwap { /// Source of the swap. pub source: T::AccountId, @@ -93,8 +95,9 @@ pub trait SwapAction { } /// A swap action that only allows transferring balances. -#[derive(Clone, RuntimeDebug, Eq, PartialEq, Encode, Decode, TypeInfo)] +#[derive(Clone, RuntimeDebug, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen)] #[scale_info(skip_type_params(C))] +#[codec(mel_bound())] pub struct BalanceSwapAction> { value: >::Balance, _marker: PhantomData, @@ -165,7 +168,7 @@ pub mod pallet { /// The overarching event type. type Event: From> + IsType<::Event>; /// Swap action. - type SwapAction: SwapAction + Parameter; + type SwapAction: SwapAction + Parameter + MaxEncodedLen; /// Limit of proof size. /// /// Atomic swap is only atomic if once the proof is revealed, both parties can submit the @@ -192,6 +195,9 @@ pub mod pallet { Blake2_128Concat, HashedProof, PendingSwap, + OptionQuery, + GetDefault, + ConstU32<300_000>, >; #[pallet::error] @@ -218,13 +224,12 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// Swap created. \[account, proof, swap\] - NewSwap(T::AccountId, HashedProof, PendingSwap), + /// Swap created. + NewSwap { account: T::AccountId, proof: HashedProof, swap: PendingSwap }, /// Swap claimed. The last parameter indicates whether the execution succeeds. - /// \[account, proof, success\] - SwapClaimed(T::AccountId, HashedProof, bool), - /// Swap cancelled. \[account, proof\] - SwapCancelled(T::AccountId, HashedProof), + SwapClaimed { account: T::AccountId, proof: HashedProof, success: bool }, + /// Swap cancelled. + SwapCancelled { account: T::AccountId, proof: HashedProof }, } /// Old name generated by `decl_event`. @@ -268,7 +273,7 @@ pub mod pallet { }; PendingSwaps::::insert(target.clone(), hashed_proof.clone(), swap.clone()); - Self::deposit_event(Event::NewSwap(target, hashed_proof, swap)); + Self::deposit_event(Event::NewSwap { account: target, proof: hashed_proof, swap }); Ok(()) } @@ -304,7 +309,11 @@ pub mod pallet { PendingSwaps::::remove(target.clone(), hashed_proof.clone()); - Self::deposit_event(Event::SwapClaimed(target, hashed_proof, succeeded)); + Self::deposit_event(Event::SwapClaimed { + account: target, + proof: hashed_proof, + success: succeeded, + }); Ok(()) } @@ -333,7 +342,7 @@ pub mod pallet { swap.action.cancel(&swap.source); PendingSwaps::::remove(&target, hashed_proof.clone()); - Self::deposit_event(Event::SwapCancelled(target, hashed_proof)); + Self::deposit_event(Event::SwapCancelled { account: target, proof: hashed_proof }); Ok(()) } diff --git a/frame/atomic-swap/src/tests.rs b/frame/atomic-swap/src/tests.rs index a76d0f20ffa3..2352e7852d09 100644 --- a/frame/atomic-swap/src/tests.rs +++ b/frame/atomic-swap/src/tests.rs @@ -3,7 +3,10 @@ use super::*; use crate as pallet_atomic_swap; -use frame_support::parameter_types; +use frame_support::{ + parameter_types, + traits::{ConstU32, ConstU64}, +}; use sp_core::H256; use sp_runtime::{ testing::Header, @@ -26,7 +29,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -45,7 +47,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -54,10 +56,9 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } -parameter_types! { - pub const ExistentialDeposit: u64 = 1; -} + impl pallet_balances::Config for Test { type MaxLocks = (); type MaxReserves = (); @@ -65,18 +66,15 @@ impl pallet_balances::Config for Test { type Balance = u64; type DustRemoval = (); type Event = Event; - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } -parameter_types! { - pub const ProofLimit: u32 = 1024; - pub const ExpireDuration: u64 = 100; -} + impl Config for Test { type Event = Event; type SwapAction = BalanceSwapAction; - type ProofLimit = ProofLimit; + type ProofLimit = ConstU32<1024>; } const A: u64 = 1; diff --git a/frame/aura/Cargo.toml b/frame/aura/Cargo.toml index 8f5c42bc3c46..96cfade56cc0 100644 --- a/frame/aura/Cargo.toml +++ b/frame/aura/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-aura" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME AURA consensus pallet" readme = "README.md" @@ -13,19 +13,19 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } -codec = { package = "parity-scale-codec", version = "2.2.0", default-features = false, features = ["derive", "max-encoded-len"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../../primitives/application-crypto" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } sp-consensus-aura = { version = "0.10.0-dev", path = "../../primitives/consensus/aura", default-features = false } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } pallet-timestamp = { version = "4.0.0-dev", default-features = false, path = "../timestamp" } [dev-dependencies] -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } [features] default = ["std"] diff --git a/frame/aura/src/lib.rs b/frame/aura/src/lib.rs index e8b68f928e08..0f474770017d 100644 --- a/frame/aura/src/lib.rs +++ b/frame/aura/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -49,7 +49,7 @@ use sp_runtime::{ traits::{IsMember, Member, SaturatedConversion, Saturating, Zero}, RuntimeAppPublic, }; -use sp_std::{convert::TryFrom, vec::Vec}; +use sp_std::prelude::*; pub mod migrations; mod mock; @@ -69,7 +69,6 @@ pub mod pallet { type AuthorityId: Member + Parameter + RuntimeAppPublic - + Default + MaybeSerializeDeserialize + MaxEncodedLen; /// The maximum number of authorities that the pallet can hold. @@ -82,7 +81,6 @@ pub mod pallet { } #[pallet::pallet] - #[pallet::generate_storage_info] pub struct Pallet(sp_std::marker::PhantomData); #[pallet::hooks] @@ -151,11 +149,11 @@ impl Pallet { fn change_authorities(new: WeakBoundedVec) { >::put(&new); - let log: DigestItem = DigestItem::Consensus( + let log = DigestItem::Consensus( AURA_ENGINE_ID, ConsensusLog::AuthoritiesChange(new.into_inner()).encode(), ); - >::deposit_log(log.into()); + >::deposit_log(log); } fn initialize_authorities(authorities: &[T::AuthorityId]) { @@ -221,13 +219,13 @@ impl OneSessionHandler for Pallet { } } - fn on_disabled(i: usize) { - let log: DigestItem = DigestItem::Consensus( + fn on_disabled(i: u32) { + let log = DigestItem::Consensus( AURA_ENGINE_ID, ConsensusLog::::OnDisabled(i as AuthorityIndex).encode(), ); - >::deposit_log(log.into()); + >::deposit_log(log); } } diff --git a/frame/aura/src/migrations.rs b/frame/aura/src/migrations.rs index e194c17406b6..d9b7cb8872a8 100644 --- a/frame/aura/src/migrations.rs +++ b/frame/aura/src/migrations.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/aura/src/mock.rs b/frame/aura/src/mock.rs index 4418d9e85ae2..636a28692ba2 100644 --- a/frame/aura/src/mock.rs +++ b/frame/aura/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,7 @@ use crate as pallet_aura; use frame_support::{ parameter_types, - traits::{DisabledValidators, GenesisBuild}, + traits::{ConstU32, ConstU64, DisabledValidators, GenesisBuild}, }; use sp_consensus_aura::{ed25519::AuthorityId, AuthorityIndex}; use sp_core::H256; @@ -48,10 +48,8 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); - pub const MinimumPeriod: u64 = 1; } impl frame_system::Config for Test { @@ -69,7 +67,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -78,19 +76,16 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } impl pallet_timestamp::Config for Test { type Moment = u64; type OnTimestampSet = Aura; - type MinimumPeriod = MinimumPeriod; + type MinimumPeriod = ConstU64<1>; type WeightInfo = (); } -parameter_types! { - pub const MaxAuthorities: u32 = 10; -} - thread_local! { static DISABLED_VALIDATORS: RefCell> = RefCell::new(Default::default()); } @@ -117,7 +112,7 @@ impl DisabledValidators for MockDisabledValidators { impl pallet_aura::Config for Test { type AuthorityId = AuthorityId; type DisabledValidators = MockDisabledValidators; - type MaxAuthorities = MaxAuthorities; + type MaxAuthorities = ConstU32<10>; } pub fn new_test_ext(authorities: Vec) -> sp_io::TestExternalities { diff --git a/frame/aura/src/tests.rs b/frame/aura/src/tests.rs index 596858aac7c9..ce09f85678c0 100644 --- a/frame/aura/src/tests.rs +++ b/frame/aura/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,6 @@ use crate::mock::{new_test_ext, Aura, MockDisabledValidators, System}; use codec::Encode; use frame_support::traits::OnInitialize; -use frame_system::InitKind; use sp_consensus_aura::{Slot, AURA_ENGINE_ID}; use sp_runtime::{Digest, DigestItem}; @@ -45,7 +44,8 @@ fn disabled_validators_cannot_author_blocks() { let pre_digest = Digest { logs: vec![DigestItem::PreRuntime(AURA_ENGINE_ID, slot.encode())] }; - System::initialize(&42, &System::parent_hash(), &pre_digest, InitKind::Full); + System::reset_events(); + System::initialize(&42, &System::parent_hash(), &pre_digest); // let's disable the validator MockDisabledValidators::disable_validator(1); diff --git a/frame/authority-discovery/Cargo.toml b/frame/authority-discovery/Cargo.toml index 80a320c31e77..d379bdda6cf8 100644 --- a/frame/authority-discovery/Cargo.toml +++ b/frame/authority-discovery/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-authority-discovery" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for authority discovery" readme = "README.md" @@ -14,22 +14,22 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sp-authority-discovery = { version = "4.0.0-dev", default-features = false, path = "../../primitives/authority-discovery" } -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../../primitives/application-crypto" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } pallet-session = { version = "4.0.0-dev", features = [ "historical", ], path = "../session", default-features = false } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } [features] default = ["std"] diff --git a/frame/authority-discovery/src/lib.rs b/frame/authority-discovery/src/lib.rs index d093b1533c69..a56d8e785f6a 100644 --- a/frame/authority-discovery/src/lib.rs +++ b/frame/authority-discovery/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,8 +30,6 @@ use frame_support::{ use sp_authority_discovery::AuthorityId; use sp_std::prelude::*; -use core::convert::TryFrom; - pub use pallet::*; #[frame_support::pallet] @@ -41,7 +39,6 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] - #[pallet::generate_storage_info] pub struct Pallet(_); #[pallet::config] @@ -63,17 +60,12 @@ pub mod pallet { pub(super) type NextKeys = StorageValue<_, WeakBoundedVec, ValueQuery>; + #[cfg_attr(feature = "std", derive(Default))] #[pallet::genesis_config] pub struct GenesisConfig { pub keys: Vec, } - #[cfg(feature = "std")] - impl Default for GenesisConfig { - fn default() -> Self { - Self { keys: Default::default() } - } - } #[pallet::genesis_build] impl GenesisBuild for GenesisConfig { fn build(&self) { @@ -166,7 +158,7 @@ impl OneSessionHandler for Pallet { } } - fn on_disabled(_i: usize) { + fn on_disabled(_i: u32) { // ignore } } @@ -175,7 +167,10 @@ impl OneSessionHandler for Pallet { mod tests { use super::*; use crate as pallet_authority_discovery; - use frame_support::{parameter_types, traits::GenesisBuild}; + use frame_support::{ + parameter_types, + traits::{ConstU32, ConstU64, GenesisBuild}, + }; use sp_application_crypto::Pair; use sp_authority_discovery::AuthorityPair; use sp_core::{crypto::key_types, H256}; @@ -203,11 +198,10 @@ mod tests { parameter_types! { pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(33); - pub const MaxAuthorities: u32 = 100; } impl Config for Test { - type MaxAuthorities = MaxAuthorities; + type MaxAuthorities = ConstU32<100>; } impl pallet_session::Config for Test { @@ -218,7 +212,6 @@ mod tests { type Event = Event; type ValidatorId = AuthorityId; type ValidatorIdOf = ConvertInto; - type DisabledValidatorsThreshold = DisabledValidatorsThreshold; type NextSessionRotation = pallet_session::PeriodicSessions; type WeightInfo = (); } @@ -233,8 +226,6 @@ mod tests { parameter_types! { pub const Period: BlockNumber = 1; pub const Offset: BlockNumber = 0; - pub const UncleGenerations: u64 = 0; - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -254,7 +245,7 @@ mod tests { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -263,6 +254,7 @@ mod tests { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } pub struct TestSessionHandler; @@ -276,7 +268,7 @@ mod tests { ) { } - fn on_disabled(_validator_index: usize) {} + fn on_disabled(_validator_index: u32) {} fn on_genesis_session(_validators: &[(AuthorityId, Ks)]) {} } diff --git a/frame/authorship/Cargo.toml b/frame/authorship/Cargo.toml index 120b72f8e651..4557ee65f1a6 100644 --- a/frame/authorship/Cargo.toml +++ b/frame/authorship/Cargo.toml @@ -3,9 +3,9 @@ name = "pallet-authorship" version = "4.0.0-dev" description = "Block and Uncle Author tracking for the FRAME" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,20 +13,20 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } sp-authorship = { version = "4.0.0-dev", default-features = false, path = "../../primitives/authorship" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -impl-trait-for-tuples = "0.2.1" +impl-trait-for-tuples = "0.2.2" [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } [features] default = ["std"] diff --git a/frame/authorship/src/lib.rs b/frame/authorship/src/lib.rs index 5d36adabe888..6b72d6ac5d28 100644 --- a/frame/authorship/src/lib.rs +++ b/frame/authorship/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -156,6 +156,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); #[pallet::hooks] @@ -170,7 +171,9 @@ pub mod pallet { >::put(false); - T::EventHandler::note_author(Self::author()); + if let Some(author) = Self::author() { + T::EventHandler::note_author(author); + } 0 } @@ -300,20 +303,18 @@ impl Pallet { /// /// This is safe to invoke in `on_initialize` implementations, as well /// as afterwards. - pub fn author() -> T::AccountId { + pub fn author() -> Option { // Check the memoized storage value. if let Some(author) = >::get() { - return author + return Some(author) } let digest = >::digest(); let pre_runtime_digests = digest.logs.iter().filter_map(|d| d.as_pre_runtime()); - if let Some(author) = T::FindAuthor::find_author(pre_runtime_digests) { - >::put(&author); - author - } else { - Default::default() - } + T::FindAuthor::find_author(pre_runtime_digests).map(|a| { + >::put(&a); + a + }) } fn verify_and_import_uncles(new_uncles: Vec) -> dispatch::DispatchResult { @@ -329,14 +330,13 @@ impl Pallet { UncleEntryItem::InclusionHeight(_) => None, UncleEntryItem::Uncle(h, _) => Some(h), }); - let author = Self::verify_uncle(&uncle, prev_uncles, &mut acc)?; + let maybe_author = Self::verify_uncle(&uncle, prev_uncles, &mut acc)?; let hash = uncle.hash(); - T::EventHandler::note_uncle( - author.clone().unwrap_or_default(), - now - uncle.number().clone(), - ); - uncles.push(UncleEntryItem::Uncle(hash, author)); + if let Some(author) = maybe_author.clone() { + T::EventHandler::note_uncle(author, now - uncle.number().clone()); + } + uncles.push(UncleEntryItem::Uncle(hash, maybe_author)); } >::put(&uncles); @@ -406,7 +406,11 @@ impl Pallet { mod tests { use super::*; use crate as pallet_authorship; - use frame_support::{parameter_types, ConsensusEngineId}; + use frame_support::{ + parameter_types, + traits::{ConstU32, ConstU64}, + ConsensusEngineId, + }; use sp_core::H256; use sp_runtime::{ generic::DigestItem, @@ -429,7 +433,6 @@ mod tests { ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -449,7 +452,7 @@ mod tests { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -458,15 +461,12 @@ mod tests { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); - } - - parameter_types! { - pub const UncleGenerations: u64 = 5; + type MaxConsumers = ConstU32<16>; } impl pallet::Config for Test { type FindAuthor = AuthorGiven; - type UncleGenerations = UncleGenerations; + type UncleGenerations = ConstU64<5>; type FilterUncle = SealVerify; type EventHandler = (); } @@ -480,9 +480,9 @@ mod tests { where I: 'a + IntoIterator, { - for (id, data) in digests { + for (id, mut data) in digests { if id == TEST_ID { - return u64::decode(&mut &data[..]).ok() + return u64::decode(&mut data).ok() } } @@ -500,9 +500,9 @@ mod tests { let author = AuthorGiven::find_author(pre_runtime_digests).ok_or_else(|| "no author")?; - for (id, seal) in seals { + for (id, mut seal) in seals { if id == TEST_ID { - match u64::decode(&mut &seal[..]) { + match u64::decode(&mut seal) { Err(_) => return Err("wrong seal"), Ok(a) => { if a != author { @@ -597,7 +597,8 @@ mod tests { }; let initialize_block = |number, hash: H256| { - System::initialize(&number, &hash, &Default::default(), Default::default()) + System::reset_events(); + System::initialize(&number, &hash, &Default::default()) }; for number in 1..8 { @@ -690,9 +691,10 @@ mod tests { seal_header(create_header(1, Default::default(), [1; 32].into()), author); header.digest_mut().pop(); // pop the seal off. - System::initialize(&1, &Default::default(), header.digest(), Default::default()); + System::reset_events(); + System::initialize(&1, &Default::default(), header.digest()); - assert_eq!(Authorship::author(), author); + assert_eq!(Authorship::author(), Some(author)); }); } diff --git a/frame/babe/Cargo.toml b/frame/babe/Cargo.toml index d95f1419fd03..a13a2d71f25d 100644 --- a/frame/babe/Cargo.toml +++ b/frame/babe/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-babe" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Consensus extension module for BABE consensus. Collects on-chain randomness from VRF outputs and manages epoch transitions." readme = "README.md" @@ -13,22 +13,22 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } pallet-authorship = { version = "4.0.0-dev", default-features = false, path = "../authorship" } pallet-session = { version = "4.0.0-dev", default-features = false, path = "../session" } pallet-timestamp = { version = "4.0.0-dev", default-features = false, path = "../timestamp" } -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../../primitives/application-crypto" } sp-consensus-babe = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/babe" } sp-consensus-vrf = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/vrf" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-session = { version = "4.0.0-dev", default-features = false, path = "../../primitives/session" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } log = { version = "0.4.14", default-features = false } [dev-dependencies] @@ -36,7 +36,7 @@ pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-offences = { version = "4.0.0-dev", path = "../offences" } pallet-staking = { version = "4.0.0-dev", path = "../staking" } pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } frame-election-provider-support = { version = "4.0.0-dev", path = "../election-provider-support" } [features] @@ -60,5 +60,5 @@ std = [ "sp-std/std", "log/std", ] -runtime-benchmarks = ["frame-benchmarking"] +runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/babe/src/benchmarking.rs b/frame/babe/src/benchmarking.rs index 372dfa532a89..ac7ab28b5164 100644 --- a/frame/babe/src/benchmarking.rs +++ b/frame/babe/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -63,6 +63,12 @@ benchmarks! { } verify { assert!(sp_consensus_babe::check_equivocation_proof::
(equivocation_proof2)); } + + impl_benchmark_test_suite!( + Pallet, + crate::mock::new_test_ext(3), + crate::mock::Test, + ) } #[cfg(test)] @@ -70,12 +76,6 @@ mod tests { use super::*; use crate::mock::*; - frame_benchmarking::impl_benchmark_test_suite!( - Pallet, - crate::mock::new_test_ext(3), - crate::mock::Test, - ); - #[test] fn test_generate_equivocation_report_blob() { let (pairs, mut ext) = new_test_ext_with_pairs(3); diff --git a/frame/babe/src/default_weights.rs b/frame/babe/src/default_weights.rs index 20ac9b961fc8..57c74323b793 100644 --- a/frame/babe/src/default_weights.rs +++ b/frame/babe/src/default_weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/babe/src/equivocation.rs b/frame/babe/src/equivocation.rs index 2397918d1ef1..df46f3544b38 100644 --- a/frame/babe/src/equivocation.rs +++ b/frame/babe/src/equivocation.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -176,7 +176,7 @@ where } fn block_author() -> Option { - Some(>::author()) + >::author() } } diff --git a/frame/babe/src/lib.rs b/frame/babe/src/lib.rs index 4ccfdf6c13fe..87ae762707cc 100644 --- a/frame/babe/src/lib.rs +++ b/frame/babe/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,7 +31,7 @@ use frame_support::{ weights::{Pays, Weight}, BoundedVec, WeakBoundedVec, }; -use sp_application_crypto::{Public, TryFrom}; +use sp_application_crypto::ByteArray; use sp_runtime::{ generic::DigestItem, traits::{IsMember, One, SaturatedConversion, Saturating, Zero}, @@ -115,7 +115,6 @@ pub mod pallet { /// The BABE Pallet #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] - #[pallet::generate_storage_info] pub struct Pallet(_); #[pallet::config] @@ -248,7 +247,7 @@ pub mod pallet { /// Randomness under construction. /// - /// We make a tradeoff between storage accesses and list length. + /// We make a trade-off between storage accesses and list length. /// We store the under-construction randomness in segments of up to /// `UNDER_CONSTRUCTION_SEGMENT_LENGTH`. /// @@ -310,19 +309,13 @@ pub mod pallet { #[pallet::storage] pub(super) type NextEpochConfig = StorageValue<_, BabeEpochConfiguration>; + #[cfg_attr(feature = "std", derive(Default))] #[pallet::genesis_config] pub struct GenesisConfig { pub authorities: Vec<(AuthorityId, BabeAuthorityWeight)>, pub epoch_config: Option, } - #[cfg(feature = "std")] - impl Default for GenesisConfig { - fn default() -> Self { - GenesisConfig { authorities: Default::default(), epoch_config: Default::default() } - } - } - #[pallet::genesis_build] impl GenesisBuild for GenesisConfig { fn build(&self) { @@ -633,8 +626,8 @@ impl Pallet { } fn deposit_consensus(new: U) { - let log: DigestItem = DigestItem::Consensus(BABE_ENGINE_ID, new.encode()); - >::deposit_log(log.into()) + let log = DigestItem::Consensus(BABE_ENGINE_ID, new.encode()); + >::deposit_log(log) } fn deposit_randomness(randomness: &schnorrkel::Randomness) { @@ -926,8 +919,8 @@ impl OneSessionHandler for Pallet { Self::enact_epoch_change(bounded_authorities, next_bounded_authorities) } - fn on_disabled(i: usize) { - Self::deposit_consensus(ConsensusLog::OnDisabled(i as u32)) + fn on_disabled(i: u32) { + Self::deposit_consensus(ConsensusLog::OnDisabled(i)) } } diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index b504a26f6042..37d8e9e37a5f 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,14 +19,12 @@ use crate::{self as pallet_babe, Config, CurrentSlot}; use codec::Encode; -use frame_election_provider_support::onchain; +use frame_election_provider_support::{onchain, SequentialPhragmen}; use frame_support::{ parameter_types, - traits::{GenesisBuild, KeyOwnerProofSystem, OnInitialize}, + traits::{ConstU128, ConstU32, ConstU64, GenesisBuild, KeyOwnerProofSystem, OnInitialize}, }; -use frame_system::InitKind; use pallet_session::historical as pallet_session_historical; -use pallet_staking::EraIndex; use sp_consensus_babe::{AuthorityId, AuthorityPair, Slot}; use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof}; use sp_core::{ @@ -41,7 +39,7 @@ use sp_runtime::{ traits::{Header as _, IdentityLookup, OpaqueKeys}, Perbill, }; -use sp_staking::SessionIndex; +use sp_staking::{EraIndex, SessionIndex}; type DummyValidatorId = u64; @@ -67,8 +65,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; - pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(16); pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -89,7 +85,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; type OnNewAccount = (); @@ -97,6 +93,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } impl frame_system::offchain::SendTransactionTypes for Test @@ -122,7 +119,6 @@ impl pallet_session::Config for Test { type SessionManager = pallet_session::historical::NoteHistoricalRoot; type SessionHandler = ::KeyTypeIdProviders; type Keys = MockSessionKeys; - type DisabledValidatorsThreshold = DisabledValidatorsThreshold; type WeightInfo = (); } @@ -131,32 +127,20 @@ impl pallet_session::historical::Config for Test { type FullIdentificationOf = pallet_staking::ExposureOf; } -parameter_types! { - pub const UncleGenerations: u64 = 0; -} - impl pallet_authorship::Config for Test { type FindAuthor = pallet_session::FindAccountFromAuthorIndex; - type UncleGenerations = UncleGenerations; + type UncleGenerations = ConstU64<0>; type FilterUncle = (); type EventHandler = (); } -parameter_types! { - pub const MinimumPeriod: u64 = 1; -} - impl pallet_timestamp::Config for Test { type Moment = u64; type OnTimestampSet = Babe; - type MinimumPeriod = MinimumPeriod; + type MinimumPeriod = ConstU64<1>; type WeightInfo = (); } -parameter_types! { - pub const ExistentialDeposit: u128 = 1; -} - impl pallet_balances::Config for Test { type MaxLocks = (); type MaxReserves = (); @@ -164,7 +148,7 @@ impl pallet_balances::Config for Test { type Balance = u128; type DustRemoval = (); type Event = Event; - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU128<1>; type AccountStore = System; type WeightInfo = (); } @@ -184,20 +168,19 @@ parameter_types! { pub const SessionsPerEra: SessionIndex = 3; pub const BondingDuration: EraIndex = 3; pub const SlashDeferDuration: EraIndex = 0; - pub const AttestationPeriod: u64 = 100; pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; - pub const MaxNominatorRewardedPerValidator: u32 = 64; - pub const ElectionLookahead: u64 = 0; - pub const StakingUnsignedPriority: u64 = u64::MAX / 2; + pub const OffendingValidatorsThreshold: Perbill = Perbill::from_percent(16); } -impl onchain::Config for Test { - type Accuracy = Perbill; +pub struct OnChainSeqPhragmen; +impl onchain::ExecutionConfig for OnChainSeqPhragmen { + type System = Test; + type Solver = SequentialPhragmen; type DataProvider = Staking; } impl pallet_staking::Config for Test { - const MAX_NOMINATIONS: u32 = 16; + type MaxNominations = ConstU32<16>; type RewardRemainder = (); type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; type Event = Event; @@ -211,12 +194,15 @@ impl pallet_staking::Config for Test { type SessionInterface = Self; type UnixTime = pallet_timestamp::Pallet; type EraPayout = pallet_staking::ConvertCurve; - type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type MaxNominatorRewardedPerValidator = ConstU32<64>; + type OffendingValidatorsThreshold = OffendingValidatorsThreshold; type NextNewSession = Session; - type ElectionProvider = onchain::OnChainSequentialPhragmen; + type ElectionProvider = onchain::UnboundedExecution; type GenesisElectionProvider = Self::ElectionProvider; + type VoterList = pallet_staking::UseNominatorsAndValidatorsMap; + type MaxUnlockingChunks = ConstU32<32>; + type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig; type WeightInfo = (); - type SortedListProvider = pallet_staking::UseNominatorsMap; } impl pallet_offences::Config for Test { @@ -227,15 +213,13 @@ impl pallet_offences::Config for Test { parameter_types! { pub const EpochDuration: u64 = 3; - pub const ExpectedBlockTime: u64 = 1; pub const ReportLongevity: u64 = BondingDuration::get() as u64 * SessionsPerEra::get() as u64 * EpochDuration::get(); - pub const MaxAuthorities: u32 = 10; } impl Config for Test { type EpochDuration = EpochDuration; - type ExpectedBlockTime = ExpectedBlockTime; + type ExpectedBlockTime = ConstU64<1>; type EpochChangeTrigger = crate::ExternalTrigger; type DisabledValidators = Session; @@ -253,7 +237,7 @@ impl Config for Test { super::EquivocationHandler; type WeightInfo = (); - type MaxAuthorities = MaxAuthorities; + type MaxAuthorities = ConstU32<10>; } pub fn go_to_block(n: u64, s: u64) { @@ -272,7 +256,8 @@ pub fn go_to_block(n: u64, s: u64) { let pre_digest = make_secondary_plain_pre_digest(0, s.into()); - System::initialize(&n, &parent_hash, &pre_digest, InitKind::Full); + System::reset_events(); + System::initialize(&n, &parent_hash, &pre_digest); Babe::on_initialize(n); Session::on_initialize(n); @@ -438,7 +423,8 @@ pub fn generate_equivocation_proof( let make_header = || { let parent_hash = System::parent_hash(); let pre_digest = make_secondary_plain_pre_digest(offender_authority_index, slot); - System::initialize(¤t_block, &parent_hash, &pre_digest, InitKind::Full); + System::reset_events(); + System::initialize(¤t_block, &parent_hash, &pre_digest); System::set_block_number(current_block); Timestamp::set_timestamp(current_block); System::finalize() diff --git a/frame/babe/src/randomness.rs b/frame/babe/src/randomness.rs index 7d1862905021..7be27f568e9f 100644 --- a/frame/babe/src/randomness.rs +++ b/frame/babe/src/randomness.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/babe/src/tests.rs b/frame/babe/src/tests.rs index 34d861d5d97f..65c9de85586e 100644 --- a/frame/babe/src/tests.rs +++ b/frame/babe/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -67,7 +67,8 @@ fn first_block_epoch_zero_start() { let pre_digest = make_primary_pre_digest(0, genesis_slot, first_vrf.clone(), vrf_proof); assert_eq!(Babe::genesis_slot(), Slot::from(0)); - System::initialize(&1, &Default::default(), &pre_digest, Default::default()); + System::reset_events(); + System::initialize(&1, &Default::default(), &pre_digest); // see implementation of the function for details why: we issue an // epoch-change digest but don't do it via the normal session mechanism. @@ -112,7 +113,8 @@ fn author_vrf_output_for_primary() { let (vrf_output, vrf_proof, vrf_randomness) = make_vrf_output(genesis_slot, &pairs[0]); let primary_pre_digest = make_primary_pre_digest(0, genesis_slot, vrf_output, vrf_proof); - System::initialize(&1, &Default::default(), &primary_pre_digest, Default::default()); + System::reset_events(); + System::initialize(&1, &Default::default(), &primary_pre_digest); Babe::do_initialize(1); assert_eq!(Babe::author_vrf_randomness(), Some(vrf_randomness)); @@ -133,7 +135,8 @@ fn author_vrf_output_for_secondary_vrf() { let secondary_vrf_pre_digest = make_secondary_vrf_pre_digest(0, genesis_slot, vrf_output, vrf_proof); - System::initialize(&1, &Default::default(), &secondary_vrf_pre_digest, Default::default()); + System::reset_events(); + System::initialize(&1, &Default::default(), &secondary_vrf_pre_digest); Babe::do_initialize(1); assert_eq!(Babe::author_vrf_randomness(), Some(vrf_randomness)); @@ -150,12 +153,8 @@ fn no_author_vrf_output_for_secondary_plain() { let genesis_slot = Slot::from(10); let secondary_plain_pre_digest = make_secondary_plain_pre_digest(0, genesis_slot); - System::initialize( - &1, - &Default::default(), - &secondary_plain_pre_digest, - Default::default(), - ); + System::reset_events(); + System::initialize(&1, &Default::default(), &secondary_plain_pre_digest); assert_eq!(Babe::author_vrf_randomness(), None); Babe::do_initialize(1); diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index 860a6edc4214..87b8d7939e12 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-bags-list" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet bags list" readme = "README.md" @@ -14,12 +14,12 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] # parity -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } # primitives -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } # FRAME frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } @@ -32,14 +32,14 @@ log = { version = "0.4.14", default-features = false } # Optional imports for benchmarking frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking", optional = true, default-features = false } pallet-balances = { version = "4.0.0-dev", path = "../balances", optional = true, default-features = false } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core", optional = true, default-features = false } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io", optional = true, default-features = false } -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing", optional = true, default-features = false } +sp-core = { version = "6.0.0", path = "../../primitives/core", optional = true, default-features = false } +sp-io = { version = "6.0.0", path = "../../primitives/io", optional = true, default-features = false } +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing", optional = true, default-features = false } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core"} -sp-io = { version = "4.0.0-dev", path = "../../primitives/io"} -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } +sp-core = { version = "6.0.0", path = "../../primitives/core"} +sp-io = { version = "6.0.0", path = "../../primitives/io"} +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } frame-election-provider-support = { version = "4.0.0-dev", path = "../election-provider-support", features = ["runtime-benchmarks"] } frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } @@ -56,11 +56,17 @@ std = [ "log/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "sp-core", "sp-io", "pallet-balances", "sp-tracing", "frame-election-provider-support/runtime-benchmarks", ] - +fuzz = [ + "sp-core", + "sp-io", + "pallet-balances", + "sp-tracing", +] +try-runtime = [ "frame-support/try-runtime" ] diff --git a/frame/bags-list/fuzzer/.gitignore b/frame/bags-list/fuzzer/.gitignore new file mode 100644 index 000000000000..3ebcb104d4a5 --- /dev/null +++ b/frame/bags-list/fuzzer/.gitignore @@ -0,0 +1,2 @@ +hfuzz_target +hfuzz_workspace diff --git a/frame/bags-list/fuzzer/Cargo.toml b/frame/bags-list/fuzzer/Cargo.toml new file mode 100644 index 000000000000..510000f631ad --- /dev/null +++ b/frame/bags-list/fuzzer/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "pallet-bags-list-fuzzer" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Fuzzer for FRAME pallet bags list" +readme = "README.md" +publish = false + +[dependencies] +honggfuzz = "0.5" +rand = { version = "0.8", features = ["std", "small_rng"] } + +pallet-bags-list = { version = "4.0.0-dev", features = ["fuzz"], path = ".." } +frame-election-provider-support = { version = "4.0.0-dev", path = "../../election-provider-support", features = ["runtime-benchmarks"] } + +[[bin]] +name = "bags-list" +path = "src/main.rs" diff --git a/frame/bags-list/fuzzer/src/main.rs b/frame/bags-list/fuzzer/src/main.rs new file mode 100644 index 000000000000..6f538eb28e7e --- /dev/null +++ b/frame/bags-list/fuzzer/src/main.rs @@ -0,0 +1,87 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Running +//! Running this fuzzer can be done with `cargo hfuzz run bags-list`. `honggfuzz` CLI options can +//! be used by setting `HFUZZ_RUN_ARGS`, such as `-n 4` to use 4 threads. +//! +//! # Debugging a panic +//! Once a panic is found, it can be debugged with +//! `cargo hfuzz run-debug fixed_point hfuzz_workspace/bags_list/*.fuzz`. +//! +//! # More information +//! More information about `honggfuzz` can be found +//! [here](https://docs.rs/honggfuzz/). + +use frame_election_provider_support::{SortedListProvider, VoteWeight}; +use honggfuzz::fuzz; +use pallet_bags_list::mock::{AccountId, BagsList, ExtBuilder}; + +const ID_RANGE: AccountId = 25_000; + +/// Actions of a `SortedListProvider` that we fuzz. +enum Action { + Insert, + Update, + Remove, +} + +impl From for Action { + fn from(v: u32) -> Self { + let num_variants = Self::Remove as u32 + 1; + match v % num_variants { + _x if _x == Action::Insert as u32 => Action::Insert, + _x if _x == Action::Update as u32 => Action::Update, + _x if _x == Action::Remove as u32 => Action::Remove, + _ => unreachable!(), + } + } +} + +fn main() { + ExtBuilder::default().build_and_execute(|| loop { + fuzz!(|data: (AccountId, VoteWeight, u32)| { + let (account_id_seed, vote_weight, action_seed) = data; + + let id = account_id_seed % ID_RANGE; + let action = Action::from(action_seed); + + match action { + Action::Insert => { + if BagsList::on_insert(id.clone(), vote_weight).is_err() { + // this was a duplicate id, which is ok. We can just update it. + BagsList::on_update(&id, vote_weight); + } + assert!(BagsList::contains(&id)); + }, + Action::Update => { + let already_contains = BagsList::contains(&id); + BagsList::on_update(&id, vote_weight); + if already_contains { + assert!(BagsList::contains(&id)); + } + }, + Action::Remove => { + BagsList::on_remove(&id); + assert!(!BagsList::contains(&id)); + }, + } + + assert!(BagsList::sanity_check().is_ok()); + }) + }); +} diff --git a/frame/bags-list/remote-tests/Cargo.toml b/frame/bags-list/remote-tests/Cargo.toml new file mode 100644 index 000000000000..e81c4f1a8d4e --- /dev/null +++ b/frame/bags-list/remote-tests/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "pallet-bags-list-remote-tests" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet bags list remote test" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +# frame +pallet-staking = { path = "../../staking", version = "4.0.0-dev" } +pallet-bags-list = { path = "../../bags-list", version = "4.0.0-dev" } +frame-election-provider-support = { path = "../../election-provider-support", version = "4.0.0-dev" } +frame-system = { path = "../../system", version = "4.0.0-dev" } +frame-support = { path = "../../support", version = "4.0.0-dev" } + +# core +sp-storage = { path = "../../../primitives/storage", version = "6.0.0"} +sp-core = { path = "../../../primitives/core", version = "6.0.0"} +sp-tracing = { path = "../../../primitives/tracing", version = "5.0.0"} +sp-runtime = { path = "../../../primitives/runtime", version = "6.0.0"} +sp-std = { path = "../../../primitives/std", version = "4.0.0" } + +# utils +remote-externalities = { path = "../../../utils/frame/remote-externalities", version = "0.10.0-dev" } + +# others +log = "0.4.14" +tokio = { version = "1", features = ["macros"] } diff --git a/frame/bags-list/remote-tests/src/lib.rs b/frame/bags-list/remote-tests/src/lib.rs new file mode 100644 index 000000000000..caf7a2a547e0 --- /dev/null +++ b/frame/bags-list/remote-tests/src/lib.rs @@ -0,0 +1,147 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Utilities for remote-testing pallet-bags-list. + +use frame_election_provider_support::ScoreProvider; +use sp_std::prelude::*; + +/// A common log target to use. +pub const LOG_TARGET: &'static str = "runtime::bags-list::remote-tests"; + +pub mod migration; +pub mod sanity_check; +pub mod snapshot; + +/// A wrapper for a runtime that the functions of this crate expect. +/// +/// For example, this can be the `Runtime` type of the Polkadot runtime. +pub trait RuntimeT: + pallet_staking::Config + pallet_bags_list::Config + frame_system::Config +{ +} +impl RuntimeT for T {} + +fn percent(portion: u32, total: u32) -> f64 { + (portion as f64 / total as f64) * 100f64 +} + +/// Display the number of nodes in each bag, while identifying those that need a rebag. +pub fn display_and_check_bags(currency_unit: u64, currency_name: &'static str) { + use frame_election_provider_support::SortedListProvider; + use frame_support::traits::Get; + + let min_nominator_bond = >::get(); + log::info!(target: LOG_TARGET, "min nominator bond is {:?}", min_nominator_bond); + + let voter_list_count = ::VoterList::count(); + + // go through every bag to track the total number of voters within bags and log some info about + // how voters are distributed within the bags. + let mut seen_in_bags = 0; + let mut rebaggable = 0; + let mut active_bags = 0; + for vote_weight_thresh in ::BagThresholds::get() { + let vote_weight_thresh_u64: u64 = (*vote_weight_thresh) + .try_into() + .map_err(|_| "runtime must configure score to at most u64 to use this test") + .unwrap(); + // threshold in terms of UNITS (e.g. KSM, DOT etc) + let vote_weight_thresh_as_unit = vote_weight_thresh_u64 as f64 / currency_unit as f64; + let pretty_thresh = format!("Threshold: {}. {}", vote_weight_thresh_as_unit, currency_name); + + let bag = match pallet_bags_list::Pallet::::list_bags_get(*vote_weight_thresh) { + Some(bag) => bag, + None => { + log::info!(target: LOG_TARGET, "{} NO VOTERS.", pretty_thresh); + continue + }, + }; + + active_bags += 1; + + for id in bag.std_iter().map(|node| node.std_id().clone()) { + let vote_weight = ::ScoreProvider::score(&id); + let vote_weight_thresh_u64: u64 = (*vote_weight_thresh) + .try_into() + .map_err(|_| "runtime must configure score to at most u64 to use this test") + .unwrap(); + let vote_weight_as_balance: pallet_staking::BalanceOf = + vote_weight_thresh_u64.try_into().map_err(|_| "can't convert").unwrap(); + + if vote_weight_as_balance < min_nominator_bond { + log::trace!( + target: LOG_TARGET, + "⚠️ {} Account found below min bond: {:?}.", + pretty_thresh, + id + ); + } + + let node = + pallet_bags_list::Node::::get(&id).expect("node in bag must exist."); + if node.is_misplaced(vote_weight) { + rebaggable += 1; + let notional_bag = pallet_bags_list::notional_bag_for::(vote_weight); + let notional_bag_as_u64: u64 = notional_bag + .try_into() + .map_err(|_| "runtime must configure score to at most u64 to use this test") + .unwrap(); + log::trace!( + target: LOG_TARGET, + "Account {:?} can be rebagged from {:?} to {:?}", + id, + vote_weight_thresh_as_unit, + notional_bag_as_u64 as f64 / currency_unit as f64 + ); + } + } + + // update our overall counter + let voters_in_bag = bag.std_iter().count() as u32; + seen_in_bags += voters_in_bag; + + // percentage of all nominators + let percent_of_voters = percent(voters_in_bag, voter_list_count); + + log::info!( + target: LOG_TARGET, + "{} Nominators: {} [%{:.3}]", + pretty_thresh, + voters_in_bag, + percent_of_voters, + ); + } + + if seen_in_bags != voter_list_count { + log::error!( + target: LOG_TARGET, + "bags list population ({}) not on par whoever is voter_list ({})", + seen_in_bags, + voter_list_count, + ) + } + + log::info!( + target: LOG_TARGET, + "a total of {} nodes are in {} active bags [{} total bags], {} of which can be rebagged.", + voter_list_count, + active_bags, + ::BagThresholds::get().len(), + rebaggable, + ); +} diff --git a/frame/bags-list/remote-tests/src/migration.rs b/frame/bags-list/remote-tests/src/migration.rs new file mode 100644 index 000000000000..c4cd73c45d37 --- /dev/null +++ b/frame/bags-list/remote-tests/src/migration.rs @@ -0,0 +1,63 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test to check the migration of the voter bag. + +use crate::{RuntimeT, LOG_TARGET}; +use frame_support::traits::PalletInfoAccess; +use pallet_staking::Nominators; +use remote_externalities::{Builder, Mode, OnlineConfig}; +use sp_runtime::{traits::Block as BlockT, DeserializeOwned}; + +/// Test voter bags migration. `currency_unit` is the number of planks per the the runtimes `UNITS` +/// (i.e. number of decimal places per DOT, KSM etc) +pub async fn execute( + currency_unit: u64, + currency_name: &'static str, + ws_url: String, +) { + let mut ext = Builder::::new() + .mode(Mode::Online(OnlineConfig { + transport: ws_url.to_string().into(), + pallets: vec![pallet_staking::Pallet::::name().to_string()], + ..Default::default() + })) + .build() + .await + .unwrap(); + + ext.execute_with(|| { + // get the nominator & validator count prior to migrating; these should be invariant. + let pre_migrate_nominator_count = >::iter().count() as u32; + log::info!(target: LOG_TARGET, "Nominator count: {}", pre_migrate_nominator_count); + + use frame_election_provider_support::SortedListProvider; + // run the actual migration + let moved = ::VoterList::unsafe_regenerate( + pallet_staking::Nominators::::iter().map(|(n, _)| n), + pallet_staking::Pallet::::weight_of_fn(), + ); + log::info!(target: LOG_TARGET, "Moved {} nominators", moved); + + let voter_list_len = ::VoterList::iter().count() as u32; + let voter_list_count = ::VoterList::count(); + // and confirm it is equal to the length of the `VoterList`. + assert_eq!(pre_migrate_nominator_count, voter_list_len); + assert_eq!(pre_migrate_nominator_count, voter_list_count); + + crate::display_and_check_bags::(currency_unit, currency_name); + }); +} diff --git a/frame/bags-list/remote-tests/src/sanity_check.rs b/frame/bags-list/remote-tests/src/sanity_check.rs new file mode 100644 index 000000000000..1027efb8539e --- /dev/null +++ b/frame/bags-list/remote-tests/src/sanity_check.rs @@ -0,0 +1,52 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test to execute the sanity-check of the voter bag. + +use frame_election_provider_support::SortedListProvider; +use frame_support::{ + storage::generator::StorageMap, + traits::{Get, PalletInfoAccess}, +}; +use remote_externalities::{Builder, Mode, OnlineConfig}; +use sp_runtime::{traits::Block as BlockT, DeserializeOwned}; + +/// Execute the sanity check of the bags-list. +pub async fn execute( + currency_unit: u64, + currency_name: &'static str, + ws_url: String, +) { + let mut ext = Builder::::new() + .mode(Mode::Online(OnlineConfig { + transport: ws_url.to_string().into(), + pallets: vec![pallet_bags_list::Pallet::::name().to_string()], + ..Default::default() + })) + .inject_hashed_prefix(&>::prefix_hash()) + .inject_hashed_prefix(&>::prefix_hash()) + .build() + .await + .unwrap(); + + ext.execute_with(|| { + sp_core::crypto::set_default_ss58_version(Runtime::SS58Prefix::get().try_into().unwrap()); + pallet_bags_list::Pallet::::sanity_check().unwrap(); + log::info!(target: crate::LOG_TARGET, "executed bags-list sanity check with no errors."); + + crate::display_and_check_bags::(currency_unit, currency_name); + }); +} diff --git a/frame/bags-list/remote-tests/src/snapshot.rs b/frame/bags-list/remote-tests/src/snapshot.rs new file mode 100644 index 000000000000..408f5f2bd8aa --- /dev/null +++ b/frame/bags-list/remote-tests/src/snapshot.rs @@ -0,0 +1,86 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test to execute the snapshot using the voter bag. + +use frame_election_provider_support::SortedListProvider; +use frame_support::traits::PalletInfoAccess; +use remote_externalities::{Builder, Mode, OnlineConfig}; +use sp_runtime::{traits::Block as BlockT, DeserializeOwned}; + +/// Execute create a snapshot from pallet-staking. +pub async fn execute( + voter_limit: Option, + currency_unit: u64, + ws_url: String, +) { + use frame_support::storage::generator::StorageMap; + + let mut ext = Builder::::new() + .mode(Mode::Online(OnlineConfig { + transport: ws_url.to_string().into(), + // NOTE: we don't scrape pallet-staking, this kinda ensures that the source of the data + // is bags-list. + pallets: vec![pallet_bags_list::Pallet::::name().to_string()], + at: None, + ..Default::default() + })) + .inject_hashed_prefix(&>::prefix_hash()) + .inject_hashed_prefix(&>::prefix_hash()) + .inject_hashed_prefix(&>::map_storage_final_prefix()) + .inject_hashed_prefix(&>::map_storage_final_prefix()) + .inject_hashed_key(&>::counter_storage_final_key()) + .inject_hashed_key(&>::counter_storage_final_key()) + .build() + .await + .unwrap(); + + ext.execute_with(|| { + use frame_election_provider_support::ElectionDataProvider; + log::info!( + target: crate::LOG_TARGET, + "{} nodes in bags list.", + ::VoterList::count(), + ); + + let voters = + as ElectionDataProvider>::electing_voters(voter_limit) + .unwrap(); + + let mut voters_nominator_only = voters + .iter() + .filter(|(v, _, _)| pallet_staking::Nominators::::contains_key(v)) + .cloned() + .collect::>(); + voters_nominator_only.sort_by_key(|(_, w, _)| *w); + + let currency_unit = currency_unit as f64; + let min_voter = voters_nominator_only + .first() + .map(|(x, y, _)| (x.clone(), *y as f64 / currency_unit)); + let max_voter = voters_nominator_only + .last() + .map(|(x, y, _)| (x.clone(), *y as f64 / currency_unit)); + log::info!( + target: crate::LOG_TARGET, + "a snapshot with limit {:?} has been created, {} voters are taken. min nominator: {:?}, max: {:?}", + voter_limit, + voters.len(), + min_voter, + max_voter + ); + }); +} diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index a820eeba13b1..bcfd1e3392b0 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,10 +19,11 @@ use super::*; use crate::list::List; -use frame_benchmarking::{account, whitelisted_caller}; -use frame_election_provider_support::VoteWeightProvider; +use frame_benchmarking::{account, whitelist_account, whitelisted_caller}; +use frame_election_provider_support::ScoreProvider; use frame_support::{assert_ok, traits::Get}; use frame_system::RawOrigin as SystemOrigin; +use sp_runtime::traits::One; frame_benchmarking::benchmarks! { rebag_non_terminal { @@ -35,7 +36,8 @@ frame_benchmarking::benchmarks! { // node in the destination in addition to the work we do otherwise. (2 W/R) // clear any pre-existing storage. - List::::clear(None); + // NOTE: safe to call outside block production + List::::unsafe_clear(); // define our origin and destination thresholds. let origin_bag_thresh = T::BagThresholds::get()[0]; @@ -43,21 +45,21 @@ frame_benchmarking::benchmarks! { // seed items in the origin bag. let origin_head: T::AccountId = account("origin_head", 0, 0); - assert_ok!(List::::insert(origin_head.clone(), origin_bag_thresh)); + assert_ok!(List::::insert(origin_head.clone(), origin_bag_thresh)); let origin_middle: T::AccountId = account("origin_middle", 0, 0); // the node we rebag (_R_) - assert_ok!(List::::insert(origin_middle.clone(), origin_bag_thresh)); + assert_ok!(List::::insert(origin_middle.clone(), origin_bag_thresh)); let origin_tail: T::AccountId = account("origin_tail", 0, 0); - assert_ok!(List::::insert(origin_tail.clone(), origin_bag_thresh)); + assert_ok!(List::::insert(origin_tail.clone(), origin_bag_thresh)); // seed items in the destination bag. let dest_head: T::AccountId = account("dest_head", 0, 0); - assert_ok!(List::::insert(dest_head.clone(), dest_bag_thresh)); + assert_ok!(List::::insert(dest_head.clone(), dest_bag_thresh)); // the bags are in the expected state after initial setup. assert_eq!( - List::::get_bags(), + List::::get_bags(), vec![ (origin_bag_thresh, vec![origin_head.clone(), origin_middle.clone(), origin_tail.clone()]), (dest_bag_thresh, vec![dest_head.clone()]) @@ -66,12 +68,12 @@ frame_benchmarking::benchmarks! { let caller = whitelisted_caller(); // update the weight of `origin_middle` to guarantee it will be rebagged into the destination. - T::VoteWeightProvider::set_vote_weight_of(&origin_middle, dest_bag_thresh); + T::ScoreProvider::set_score_of(&origin_middle, dest_bag_thresh); }: rebag(SystemOrigin::Signed(caller), origin_middle.clone()) verify { // check the bags have updated as expected. assert_eq!( - List::::get_bags(), + List::::get_bags(), vec![ ( origin_bag_thresh, @@ -94,7 +96,8 @@ frame_benchmarking::benchmarks! { // node in the destination in addition to the work we do otherwise. (2 W/R) // clear any pre-existing storage. - List::::clear(None); + // NOTE: safe to call outside block production + List::::unsafe_clear(); // define our origin and destination thresholds. let origin_bag_thresh = T::BagThresholds::get()[0]; @@ -102,18 +105,18 @@ frame_benchmarking::benchmarks! { // seed items in the origin bag. let origin_head: T::AccountId = account("origin_head", 0, 0); - assert_ok!(List::::insert(origin_head.clone(), origin_bag_thresh)); + assert_ok!(List::::insert(origin_head.clone(), origin_bag_thresh)); let origin_tail: T::AccountId = account("origin_tail", 0, 0); // the node we rebag (_R_) - assert_ok!(List::::insert(origin_tail.clone(), origin_bag_thresh)); + assert_ok!(List::::insert(origin_tail.clone(), origin_bag_thresh)); // seed items in the destination bag. let dest_head: T::AccountId = account("dest_head", 0, 0); - assert_ok!(List::::insert(dest_head.clone(), dest_bag_thresh)); + assert_ok!(List::::insert(dest_head.clone(), dest_bag_thresh)); // the bags are in the expected state after initial setup. assert_eq!( - List::::get_bags(), + List::::get_bags(), vec![ (origin_bag_thresh, vec![origin_head.clone(), origin_tail.clone()]), (dest_bag_thresh, vec![dest_head.clone()]) @@ -122,23 +125,64 @@ frame_benchmarking::benchmarks! { let caller = whitelisted_caller(); // update the weight of `origin_tail` to guarantee it will be rebagged into the destination. - T::VoteWeightProvider::set_vote_weight_of(&origin_tail, dest_bag_thresh); + T::ScoreProvider::set_score_of(&origin_tail, dest_bag_thresh); }: rebag(SystemOrigin::Signed(caller), origin_tail.clone()) verify { // check the bags have updated as expected. assert_eq!( - List::::get_bags(), + List::::get_bags(), vec![ (origin_bag_thresh, vec![origin_head.clone()]), (dest_bag_thresh, vec![dest_head.clone(), origin_tail.clone()]) ] ); } + + put_in_front_of { + // The most expensive case for `put_in_front_of`: + // + // - both heavier's `prev` and `next` are nodes that will need to be read and written. + // - `lighter` is the bag's `head`, so the bag will need to be read and written. + + // clear any pre-existing storage. + // NOTE: safe to call outside block production + List::::unsafe_clear(); + + let bag_thresh = T::BagThresholds::get()[0]; + + // insert the nodes in order + let lighter: T::AccountId = account("lighter", 0, 0); + assert_ok!(List::::insert(lighter.clone(), bag_thresh)); + + let heavier_prev: T::AccountId = account("heavier_prev", 0, 0); + assert_ok!(List::::insert(heavier_prev.clone(), bag_thresh)); + + let heavier: T::AccountId = account("heavier", 0, 0); + assert_ok!(List::::insert(heavier.clone(), bag_thresh)); + + let heavier_next: T::AccountId = account("heavier_next", 0, 0); + assert_ok!(List::::insert(heavier_next.clone(), bag_thresh)); + + T::ScoreProvider::set_score_of(&lighter, bag_thresh - One::one()); + T::ScoreProvider::set_score_of(&heavier, bag_thresh); + + assert_eq!( + List::::iter().map(|n| n.id().clone()).collect::>(), + vec![lighter.clone(), heavier_prev.clone(), heavier.clone(), heavier_next.clone()] + ); + + whitelist_account!(heavier); + }: _(SystemOrigin::Signed(heavier.clone()), lighter.clone()) + verify { + assert_eq!( + List::::iter().map(|n| n.id().clone()).collect::>(), + vec![heavier, lighter, heavier_prev, heavier_next] + ) + } } -use frame_benchmarking::impl_benchmark_test_suite; -impl_benchmark_test_suite!( +frame_benchmarking::impl_benchmark_test_suite!( Pallet, - crate::mock::ExtBuilder::default().build(), - crate::mock::Runtime, + crate::mock::ExtBuilder::default().skip_genesis_ids().build(), + crate::mock::Runtime ); diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 4202a4d49989..94553433e230 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,60 +17,63 @@ //! # Bags-List Pallet //! -//! A semi-sorted list, where items hold an `AccountId` based on some `VoteWeight`. The `AccountId` -//! (`id` for short) might be synonym to a `voter` or `nominator` in some context, and `VoteWeight` -//! signifies the chance of each id being included in the final [`VoteWeightProvider::iter`]. +//! A semi-sorted list, where items hold an `AccountId` based on some `Score`. The +//! `AccountId` (`id` for short) might be synonym to a `voter` or `nominator` in some context, and +//! `Score` signifies the chance of each id being included in the final +//! [`SortedListProvider::iter`]. //! -//! It implements [`sp_election_provider_support::SortedListProvider`] to provide a semi-sorted list -//! of accounts to another pallet. It needs some other pallet to give it some information about the -//! weights of accounts via [`sp_election_provider_support::VoteWeightProvider`]. +//! It implements [`frame_election_provider_support::SortedListProvider`] to provide a semi-sorted +//! list of accounts to another pallet. It needs some other pallet to give it some information about +//! the weights of accounts via [`frame_election_provider_support::ScoreProvider`]. //! //! This pallet is not configurable at genesis. Whoever uses it should call appropriate functions of -//! the `SortedListProvider` (e.g. `on_insert`, or `regenerate`) at their genesis. +//! the `SortedListProvider` (e.g. `on_insert`, or `unsafe_regenerate`) at their genesis. //! //! # Goals //! //! The data structure exposed by this pallet aims to be optimized for: //! //! - insertions and removals. -//! - iteration over the top* N items by weight, where the precise ordering of items doesn't +//! - iteration over the top* N items by score, where the precise ordering of items doesn't //! particularly matter. //! //! # Details //! -//! - items are kept in bags, which are delineated by their range of weight (See [`BagThresholds`]). +//! - items are kept in bags, which are delineated by their range of score (See +//! [`Config::BagThresholds`]). //! - for iteration, bags are chained together from highest to lowest and elements within the bag //! are iterated from head to tail. //! - items within a bag are iterated in order of insertion. Thus removing an item and re-inserting //! it will worsen its position in list iteration; this reduces incentives for some types of spam //! that involve consistently removing and inserting for better position. Further, ordering //! granularity is thus dictated by range between each bag threshold. -//! - if an item's weight changes to a value no longer within the range of its current bag the -//! item's position will need to be updated by an external actor with rebag (update), or removal -//! and insertion. +//! - if an item's score changes to a value no longer within the range of its current bag the item's +//! position will need to be updated by an external actor with rebag (update), or removal and +//! insertion. #![cfg_attr(not(feature = "std"), no_std)] -use frame_election_provider_support::{SortedListProvider, VoteWeight, VoteWeightProvider}; +use codec::FullCodec; +use frame_election_provider_support::{ScoreProvider, SortedListProvider}; use frame_system::ensure_signed; +use sp_runtime::traits::{AtLeast32BitUnsigned, Bounded}; use sp_std::prelude::*; #[cfg(any(feature = "runtime-benchmarks", test))] mod benchmarks; mod list; -#[cfg(test)] -mod mock; +pub mod migrations; +#[cfg(any(test, feature = "fuzz"))] +pub mod mock; #[cfg(test)] mod tests; pub mod weights; +pub use list::{notional_bag_for, Bag, List, ListError, Node}; pub use pallet::*; pub use weights::WeightInfo; -pub use list::Error; -use list::List; - pub(crate) const LOG_TARGET: &'static str = "runtime::bags_list"; // syntactic sugar for logging. @@ -92,39 +95,38 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(crate) trait Store)] - #[pallet::generate_storage_info] - pub struct Pallet(_); + pub struct Pallet(_); #[pallet::config] - pub trait Config: frame_system::Config { + pub trait Config: frame_system::Config { /// The overarching event type. - type Event: From> + IsType<::Event>; + type Event: From> + IsType<::Event>; /// Weight information for extrinsics in this pallet. type WeightInfo: weights::WeightInfo; - /// Something that provides the weights of ids. - type VoteWeightProvider: VoteWeightProvider; + /// Something that provides the scores of ids. + type ScoreProvider: ScoreProvider; /// The list of thresholds separating the various bags. /// - /// Ids are separated into unsorted bags according to their vote weight. This specifies the - /// thresholds separating the bags. An id's bag is the largest bag for which the id's weight + /// Ids are separated into unsorted bags according to their score. This specifies the + /// thresholds separating the bags. An id's bag is the largest bag for which the id's score /// is less than or equal to its upper threshold. /// /// When ids are iterated, higher bags are iterated completely before lower bags. This means - /// that iteration is _semi-sorted_: ids of higher weight tend to come before ids of lower - /// weight, but peer ids within a particular bag are sorted in insertion order. + /// that iteration is _semi-sorted_: ids of higher score tend to come before ids of lower + /// score, but peer ids within a particular bag are sorted in insertion order. /// /// # Expressing the constant /// /// This constant must be sorted in strictly increasing order. Duplicate items are not /// permitted. /// - /// There is an implied upper limit of `VoteWeight::MAX`; that value does not need to be + /// There is an implied upper limit of `Score::MAX`; that value does not need to be /// specified within the bag. For any two threshold lists, if one ends with - /// `VoteWeight::MAX`, the other one does not, and they are otherwise equal, the two lists - /// will behave identically. + /// `Score::MAX`, the other one does not, and they are otherwise equal, the two + /// lists will behave identically. /// /// # Calculation /// @@ -142,46 +144,68 @@ pub mod pallet { /// the procedure given above, then the constant ratio is equal to 2. /// - If `BagThresholds::get().len() == 200`, and the thresholds are determined according to /// the procedure given above, then the constant ratio is approximately equal to 1.248. - /// - If the threshold list begins `[1, 2, 3, ...]`, then an id with weight 0 or 1 will fall - /// into bag 0, an id with weight 2 will fall into bag 1, etc. + /// - If the threshold list begins `[1, 2, 3, ...]`, then an id with score 0 or 1 will fall + /// into bag 0, an id with score 2 will fall into bag 1, etc. /// /// # Migration /// /// In the event that this list ever changes, a copy of the old bags list must be retained. /// With that `List::migrate` can be called, which will perform the appropriate migration. #[pallet::constant] - type BagThresholds: Get<&'static [VoteWeight]>; + type BagThresholds: Get<&'static [Self::Score]>; + + /// The type used to dictate a node position relative to other nodes. + type Score: Clone + + Default + + PartialEq + + Eq + + Ord + + PartialOrd + + sp_std::fmt::Debug + + Copy + + AtLeast32BitUnsigned + + Bounded + + TypeInfo + + FullCodec + + MaxEncodedLen; } - /// How many ids are registered. - // NOTE: This is merely a counter for `ListNodes`. It should someday be replaced by the - // `CountedMaop` storage. - #[pallet::storage] - pub(crate) type CounterForListNodes = StorageValue<_, u32, ValueQuery>; - /// A single node, within some bag. /// /// Nodes store links forward and back within their respective bags. #[pallet::storage] - pub(crate) type ListNodes = StorageMap<_, Twox64Concat, T::AccountId, list::Node>; + pub(crate) type ListNodes, I: 'static = ()> = + CountedStorageMap<_, Twox64Concat, T::AccountId, list::Node>; /// A bag stored in storage. /// /// Stores a `Bag` struct, which stores head and tail pointers to itself. #[pallet::storage] - pub(crate) type ListBags = StorageMap<_, Twox64Concat, VoteWeight, list::Bag>; + pub(crate) type ListBags, I: 'static = ()> = + StorageMap<_, Twox64Concat, T::Score, list::Bag>; #[pallet::event] #[pallet::generate_deposit(pub(crate) fn deposit_event)] - pub enum Event { - /// Moved an account from one bag to another. \[who, from, to\]. - Rebagged(T::AccountId, VoteWeight, VoteWeight), + pub enum Event, I: 'static = ()> { + /// Moved an account from one bag to another. + Rebagged { who: T::AccountId, from: T::Score, to: T::Score }, + } + + #[pallet::error] + #[cfg_attr(test, derive(PartialEq))] + pub enum Error { + /// Attempted to place node in front of a node in another bag. + NotInSameBag, + /// Id not found in list. + IdNotFound, + /// An Id does not have a greater score than another Id. + NotHeavier, } #[pallet::call] - impl Pallet { + impl, I: 'static> Pallet { /// Declare that some `dislocated` account has, through rewards or penalties, sufficiently - /// changed its weight that it should properly fall into a different bag than its current + /// changed its score that it should properly fall into a different bag than its current /// one. /// /// Anyone can call this function about any potentially dislocated account. @@ -191,14 +215,28 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::rebag_non_terminal().max(T::WeightInfo::rebag_terminal()))] pub fn rebag(origin: OriginFor, dislocated: T::AccountId) -> DispatchResult { ensure_signed(origin)?; - let current_weight = T::VoteWeightProvider::vote_weight(&dislocated); - let _ = Pallet::::do_rebag(&dislocated, current_weight); + let current_score = T::ScoreProvider::score(&dislocated); + let _ = Pallet::::do_rebag(&dislocated, current_score); Ok(()) } + + /// Move the caller's Id directly in front of `lighter`. + /// + /// The dispatch origin for this call must be _Signed_ and can only be called by the Id of + /// the account going in front of `lighter`. + /// + /// Only works if + /// - both nodes are within the same bag, + /// - and `origin` has a greater `Score` than `lighter`. + #[pallet::weight(T::WeightInfo::put_in_front_of())] + pub fn put_in_front_of(origin: OriginFor, lighter: T::AccountId) -> DispatchResult { + let heavier = ensure_signed(origin)?; + List::::put_in_front_of(&lighter, &heavier).map_err(Into::into) + } } #[pallet::hooks] - impl Hooks> for Pallet { + impl, I: 'static> Hooks> for Pallet { fn integrity_test() { // ensure they are strictly increasing, this also implies that duplicates are detected. assert!( @@ -209,68 +247,76 @@ pub mod pallet { } } -impl Pallet { +impl, I: 'static> Pallet { /// Move an account from one bag to another, depositing an event on success. /// /// If the account changed bags, returns `Some((from, to))`. - pub fn do_rebag( - account: &T::AccountId, - new_weight: VoteWeight, - ) -> Option<(VoteWeight, VoteWeight)> { + pub fn do_rebag(account: &T::AccountId, new_weight: T::Score) -> Option<(T::Score, T::Score)> { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. - let maybe_movement = list::Node::::get(&account) + let maybe_movement = list::Node::::get(&account) .and_then(|node| List::update_position_for(node, new_weight)); if let Some((from, to)) = maybe_movement { - Self::deposit_event(Event::::Rebagged(account.clone(), from, to)); + Self::deposit_event(Event::::Rebagged { who: account.clone(), from, to }); }; maybe_movement } /// Equivalent to `ListBags::get`, but public. Useful for tests in outside of this crate. #[cfg(feature = "std")] - pub fn list_bags_get(weight: VoteWeight) -> Option> { - ListBags::get(weight) + pub fn list_bags_get(score: T::Score) -> Option> { + ListBags::get(score) } } -impl SortedListProvider for Pallet { - type Error = Error; +impl, I: 'static> SortedListProvider for Pallet { + type Error = ListError; + type Score = T::Score; fn iter() -> Box> { - Box::new(List::::iter().map(|n| n.id().clone())) + Box::new(List::::iter().map(|n| n.id().clone())) + } + + fn iter_from( + start: &T::AccountId, + ) -> Result>, Self::Error> { + let iter = List::::iter_from(start)?; + Ok(Box::new(iter.map(|n| n.id().clone()))) } fn count() -> u32 { - CounterForListNodes::::get() + ListNodes::::count() } fn contains(id: &T::AccountId) -> bool { - List::::contains(id) + List::::contains(id) } - fn on_insert(id: T::AccountId, weight: VoteWeight) -> Result<(), Error> { - List::::insert(id, weight) + fn on_insert(id: T::AccountId, score: T::Score) -> Result<(), ListError> { + List::::insert(id, score) } - fn on_update(id: &T::AccountId, new_weight: VoteWeight) { - Pallet::::do_rebag(id, new_weight); + fn on_update(id: &T::AccountId, new_score: T::Score) { + Pallet::::do_rebag(id, new_score); } fn on_remove(id: &T::AccountId) { - List::::remove(id) + List::::remove(id) } - fn regenerate( + fn unsafe_regenerate( all: impl IntoIterator, - weight_of: Box VoteWeight>, + score_of: Box T::Score>, ) -> u32 { - List::::regenerate(all, weight_of) + // NOTE: This call is unsafe for the same reason as SortedListProvider::unsafe_regenerate. + // I.e. because it can lead to many storage accesses. + // So it is ok to call it as caller must ensure the conditions. + List::::unsafe_regenerate(all, score_of) } #[cfg(feature = "std")] fn sanity_check() -> Result<(), &'static str> { - List::::sanity_check() + List::::sanity_check() } #[cfg(not(feature = "std"))] @@ -278,18 +324,21 @@ impl SortedListProvider for Pallet { Ok(()) } - fn clear(maybe_count: Option) -> u32 { - List::::clear(maybe_count) + fn unsafe_clear() { + // NOTE: This call is unsafe for the same reason as SortedListProvider::unsafe_clear. + // I.e. because it can lead to many storage accesses. + // So it is ok to call it as caller must ensure the conditions. + List::::unsafe_clear() } #[cfg(feature = "runtime-benchmarks")] - fn weight_update_worst_case(who: &T::AccountId, is_increase: bool) -> VoteWeight { + fn score_update_worst_case(who: &T::AccountId, is_increase: bool) -> Self::Score { use frame_support::traits::Get as _; let thresholds = T::BagThresholds::get(); - let node = list::Node::::get(who).unwrap(); + let node = list::Node::::get(who).unwrap(); let current_bag_idx = thresholds .iter() - .chain(sp_std::iter::once(&VoteWeight::MAX)) + .chain(sp_std::iter::once(&T::Score::max_value())) .position(|w| w == &node.bag_upper()) .unwrap(); diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 3f55f2227191..db8c06a38d67 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,8 +17,8 @@ //! Implementation of a "bags list": a semi-sorted list where ordering granularity is dictated by //! configurable thresholds that delineate the boundaries of bags. It uses a pattern of composite -//! data structures, where multiple storage items are masked by one outer API. See [`ListNodes`], -//! [`CounterForListNodes`] and [`ListBags`] for more information. +//! data structures, where multiple storage items are masked by one outer API. See +//! [`crate::ListNodes`], [`crate::ListBags`] for more information. //! //! The outer API of this module is the [`List`] struct. It wraps all acceptable operations on top //! of the aggregate linked list. All operations with the bags list should happen through this @@ -26,9 +26,10 @@ use crate::Config; use codec::{Decode, Encode, MaxEncodedLen}; -use frame_election_provider_support::{VoteWeight, VoteWeightProvider}; +use frame_election_provider_support::ScoreProvider; use frame_support::{traits::Get, DefaultNoBound}; use scale_info::TypeInfo; +use sp_runtime::traits::{Bounded, Zero}; use sp_std::{ boxed::Box, collections::{btree_map::BTreeMap, btree_set::BTreeSet}, @@ -38,56 +39,55 @@ use sp_std::{ }; #[derive(Debug, PartialEq, Eq)] -pub enum Error { +pub enum ListError { /// A duplicate id has been detected. Duplicate, + /// Given node id was not found. + NodeNotFound, } #[cfg(test)] mod tests; -/// Given a certain vote weight, to which bag does it belong to? +/// Given a certain score, to which bag does it belong to? /// /// Bags are identified by their upper threshold; the value returned by this function is guaranteed /// to be a member of `T::BagThresholds`. /// -/// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this -/// function behaves as if it does. -pub(crate) fn notional_bag_for(weight: VoteWeight) -> VoteWeight { +/// Note that even if the thresholds list does not have `T::Score::max_value()` as its final member, +/// this function behaves as if it does. +pub fn notional_bag_for, I: 'static>(score: T::Score) -> T::Score { let thresholds = T::BagThresholds::get(); - let idx = thresholds.partition_point(|&threshold| weight > threshold); - thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) + let idx = thresholds.partition_point(|&threshold| score > threshold); + thresholds.get(idx).copied().unwrap_or(T::Score::max_value()) } /// The **ONLY** entry point of this module. All operations to the bags-list should happen through /// this interface. It is forbidden to access other module members directly. // -// Data structure providing efficient mostly-accurate selection of the top N id by `VoteWeight`. +// Data structure providing efficient mostly-accurate selection of the top N id by `Score`. // // It's implemented as a set of linked lists. Each linked list comprises a bag of ids of -// arbitrary and unbounded length, all having a vote weight within a particular constant range. +// arbitrary and unbounded length, all having a score within a particular constant range. // This structure means that ids can be added and removed in `O(1)` time. // // Iteration is accomplished by chaining the iteration of each bag, from greatest to least. While -// the users within any particular bag are sorted in an entirely arbitrary order, the overall vote -// weight decreases as successive bags are reached. This means that it is valid to truncate +// the users within any particular bag are sorted in an entirely arbitrary order, the overall score +// decreases as successive bags are reached. This means that it is valid to truncate // iteration at any desired point; only those ids in the lowest bag can be excluded. This // satisfies both the desire for fairness and the requirement for efficiency. -pub struct List(PhantomData); - -impl List { - /// Remove all data associated with the list from storage. Parameter `items` is the number of - /// items to clear from the list. WARNING: `None` will clear all items and should generally not - /// be used in production as it could lead to an infinite number of storage accesses. - pub(crate) fn clear(maybe_count: Option) -> u32 { - crate::ListBags::::remove_all(maybe_count); - crate::ListNodes::::remove_all(maybe_count); - if let Some(count) = maybe_count { - crate::CounterForListNodes::::mutate(|items| *items - count); - count - } else { - crate::CounterForListNodes::::take() - } +pub struct List, I: 'static = ()>(PhantomData<(T, I)>); + +impl, I: 'static> List { + /// Remove all data associated with the list from storage. + /// + /// ## WARNING + /// + /// this function should generally not be used in production as it could lead to a very large + /// number of storage accesses. + pub(crate) fn unsafe_clear() { + crate::ListBags::::remove_all(None); + crate::ListNodes::::remove_all(); } /// Regenerate all of the data from the given ids. @@ -99,12 +99,15 @@ impl List { /// pallet using this `List`. /// /// Returns the number of ids migrated. - pub fn regenerate( + pub fn unsafe_regenerate( all: impl IntoIterator, - weight_of: Box VoteWeight>, + score_of: Box T::Score>, ) -> u32 { - Self::clear(None); - Self::insert_many(all, weight_of) + // NOTE: This call is unsafe for the same reason as SortedListProvider::unsafe_regenerate. + // I.e. because it can lead to many storage accesses. + // So it is ok to call it as caller must ensure the conditions. + Self::unsafe_clear(); + Self::insert_many(all, score_of) } /// Migrate the list from one set of thresholds to another. @@ -127,7 +130,7 @@ impl List { /// - ids whose bags change at all are implicitly rebagged into the appropriate bag in the new /// threshold set. #[allow(dead_code)] - pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { + pub fn migrate(old_thresholds: &[T::Score]) -> u32 { let new_thresholds = T::BagThresholds::get(); if new_thresholds == old_thresholds { return 0 @@ -135,11 +138,13 @@ impl List { // we can't check all preconditions, but we can check one debug_assert!( - crate::ListBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), + crate::ListBags::::iter() + .all(|(threshold, _)| old_thresholds.contains(&threshold)), "not all `bag_upper` currently in storage are members of `old_thresholds`", ); debug_assert!( - crate::ListNodes::::iter().all(|(_, node)| old_thresholds.contains(&node.bag_upper)), + crate::ListNodes::::iter() + .all(|(_, node)| old_thresholds.contains(&node.bag_upper)), "not all `node.bag_upper` currently in storage are members of `old_thresholds`", ); @@ -158,7 +163,7 @@ impl List { let affected_bag = { // this recreates `notional_bag_for` logic, but with the old thresholds. let idx = old_thresholds.partition_point(|&threshold| inserted_bag > threshold); - old_thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) + old_thresholds.get(idx).copied().unwrap_or(T::Score::max_value()) }; if !affected_old_bags.insert(affected_bag) { // If the previous threshold list was [10, 20], and we insert [3, 5], then there's @@ -166,7 +171,7 @@ impl List { continue } - if let Some(bag) = Bag::::get(affected_bag) { + if let Some(bag) = Bag::::get(affected_bag) { affected_accounts.extend(bag.iter().map(|node| node.id)); } } @@ -178,17 +183,17 @@ impl List { continue } - if let Some(bag) = Bag::::get(removed_bag) { + if let Some(bag) = Bag::::get(removed_bag) { affected_accounts.extend(bag.iter().map(|node| node.id)); } } // migrate the voters whose bag has changed let num_affected = affected_accounts.len() as u32; - let weight_of = T::VoteWeightProvider::vote_weight; + let score_of = T::ScoreProvider::score; let _removed = Self::remove_many(&affected_accounts); debug_assert_eq!(_removed, num_affected); - let _inserted = Self::insert_many(affected_accounts.into_iter(), weight_of); + let _inserted = Self::insert_many(affected_accounts.into_iter(), score_of); debug_assert_eq!(_inserted, num_affected); // we couldn't previously remove the old bags because both insertion and removal assume that @@ -199,10 +204,10 @@ impl List { // lookups. for removed_bag in removed_bags { debug_assert!( - !crate::ListNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), + !crate::ListNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), "no id should be present in a removed bag", ); - crate::ListBags::::remove(removed_bag); + crate::ListBags::::remove(removed_bag); } debug_assert_eq!(Self::sanity_check(), Ok(())); @@ -212,14 +217,14 @@ impl List { /// Returns `true` if the list contains `id`, otherwise returns `false`. pub(crate) fn contains(id: &T::AccountId) -> bool { - crate::ListNodes::::contains_key(id) + crate::ListNodes::::contains_key(id) } /// Iterate over all nodes in all bags in the list. /// /// Full iteration can be expensive; it's recommended to limit the number of items with /// `.take(n)`. - pub(crate) fn iter() -> impl Iterator> { + pub(crate) fn iter() -> impl Iterator> { // We need a touch of special handling here: because we permit `T::BagThresholds` to // omit the final bound, we need to ensure that we explicitly include that threshold in the // list. @@ -228,29 +233,58 @@ impl List { // easier; they can just configure `type BagThresholds = ()`. let thresholds = T::BagThresholds::get(); let iter = thresholds.iter().copied(); - let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { + let iter: Box> = if thresholds.last() == + Some(&T::Score::max_value()) + { // in the event that they included it, we can just pass the iterator through unchanged. Box::new(iter.rev()) } else { // otherwise, insert it here. - Box::new(iter.chain(iter::once(VoteWeight::MAX)).rev()) + Box::new(iter.chain(iter::once(T::Score::max_value())).rev()) }; iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) } + /// Same as `iter`, but we start from a specific node. + /// + /// All items after this node are returned, excluding `start` itself. + pub(crate) fn iter_from( + start: &T::AccountId, + ) -> Result>, ListError> { + // We chain two iterators: + // 1. from the given `start` till the end of the bag + // 2. all the bags that come after `start`'s bag. + + let start_node = Node::::get(start).ok_or(ListError::NodeNotFound)?; + let start_node_upper = start_node.bag_upper; + let start_bag = sp_std::iter::successors(start_node.next(), |prev| prev.next()); + + let thresholds = T::BagThresholds::get(); + let idx = thresholds.partition_point(|&threshold| start_node_upper > threshold); + let leftover_bags = thresholds + .into_iter() + .take(idx) + .copied() + .rev() + .filter_map(Bag::get) + .flat_map(|bag| bag.iter()); + + Ok(start_bag.chain(leftover_bags)) + } + /// Insert several ids into the appropriate bags in the list. Continues with insertions /// if duplicates are detected. /// /// Returns the final count of number of ids inserted. fn insert_many( ids: impl IntoIterator, - weight_of: impl Fn(&T::AccountId) -> VoteWeight, + score_of: impl Fn(&T::AccountId) -> T::Score, ) -> u32 { let mut count = 0; ids.into_iter().for_each(|v| { - let weight = weight_of(&v); - if Self::insert(v, weight).is_ok() { + let score = score_of(&v); + if Self::insert(v, score).is_ok() { count += 1; } }); @@ -261,30 +295,27 @@ impl List { /// Insert a new id into the appropriate bag in the list. /// /// Returns an error if the list already contains `id`. - pub(crate) fn insert(id: T::AccountId, weight: VoteWeight) -> Result<(), Error> { + pub(crate) fn insert(id: T::AccountId, score: T::Score) -> Result<(), ListError> { if Self::contains(&id) { - return Err(Error::Duplicate) + return Err(ListError::Duplicate) } - let bag_weight = notional_bag_for::(weight); - let mut bag = Bag::::get_or_make(bag_weight); + let bag_score = notional_bag_for::(score); + let mut bag = Bag::::get_or_make(bag_score); // unchecked insertion is okay; we just got the correct `notional_bag_for`. bag.insert_unchecked(id.clone()); // new inserts are always the tail, so we must write the bag. bag.put(); - crate::CounterForListNodes::::mutate(|prev_count| { - *prev_count = prev_count.saturating_add(1) - }); - crate::log!( debug, - "inserted {:?} with weight {} into bag {:?}, new count is {}", + "inserted {:?} with score {:? + } into bag {:?}, new count is {}", id, - weight, - bag_weight, - crate::CounterForListNodes::::get(), + score, + bag_score, + crate::ListNodes::::count(), ); Ok(()) @@ -305,7 +336,7 @@ impl List { let mut count = 0; for id in ids.into_iter() { - let node = match Node::::get(id) { + let node = match Node::::get(id) { Some(node) => node, None => continue, }; @@ -318,7 +349,7 @@ impl List { // this node is a head or tail, so the bag needs to be updated let bag = bags .entry(node.bag_upper) - .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); + .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); // node.bag_upper must be correct, therefore this bag will contain this node. bag.remove_node_unchecked(&node); } @@ -331,10 +362,6 @@ impl List { bag.put(); } - crate::CounterForListNodes::::mutate(|prev_count| { - *prev_count = prev_count.saturating_sub(count) - }); - count } @@ -349,17 +376,17 @@ impl List { /// [`self.insert`]. However, given large quantities of nodes to move, it may be more efficient /// to call [`self.remove_many`] followed by [`self.insert_many`]. pub(crate) fn update_position_for( - node: Node, - new_weight: VoteWeight, - ) -> Option<(VoteWeight, VoteWeight)> { - node.is_misplaced(new_weight).then(move || { + node: Node, + new_score: T::Score, + ) -> Option<(T::Score, T::Score)> { + node.is_misplaced(new_score).then(move || { let old_bag_upper = node.bag_upper; if !node.is_terminal() { // this node is not a head or a tail, so we can just cut it out of the list. update // and put the prev and next of this node, we do `node.put` inside `insert_note`. node.excise(); - } else if let Some(mut bag) = Bag::::get(node.bag_upper) { + } else if let Some(mut bag) = Bag::::get(node.bag_upper) { // this is a head or tail, so the bag must be updated. bag.remove_node_unchecked(&node); bag.put(); @@ -373,8 +400,8 @@ impl List { } // put the node into the appropriate new bag. - let new_bag_upper = notional_bag_for::(new_weight); - let mut bag = Bag::::get_or_make(new_bag_upper); + let new_bag_upper = notional_bag_for::(new_score); + let mut bag = Bag::::get_or_make(new_bag_upper); // prev, next, and bag_upper of the node are updated inside `insert_node`, also // `node.put` is in there. bag.insert_node_unchecked(node); @@ -384,15 +411,92 @@ impl List { }) } + /// Put `heavier_id` to the position directly in front of `lighter_id`. Both ids must be in the + /// same bag and the `score_of` `lighter_id` must be less than that of `heavier_id`. + pub(crate) fn put_in_front_of( + lighter_id: &T::AccountId, + heavier_id: &T::AccountId, + ) -> Result<(), crate::pallet::Error> { + use crate::pallet; + use frame_support::ensure; + + let lighter_node = Node::::get(&lighter_id).ok_or(pallet::Error::IdNotFound)?; + let heavier_node = Node::::get(&heavier_id).ok_or(pallet::Error::IdNotFound)?; + + ensure!(lighter_node.bag_upper == heavier_node.bag_upper, pallet::Error::NotInSameBag); + + // this is the most expensive check, so we do it last. + ensure!( + T::ScoreProvider::score(&heavier_id) > T::ScoreProvider::score(&lighter_id), + pallet::Error::NotHeavier + ); + + // remove the heavier node from this list. Note that this removes the node from storage and + // decrements the node counter. + Self::remove(&heavier_id); + + // re-fetch `lighter_node` from storage since it may have been updated when `heavier_node` + // was removed. + let lighter_node = Node::::get(&lighter_id).ok_or_else(|| { + debug_assert!(false, "id that should exist cannot be found"); + crate::log!(warn, "id that should exist cannot be found"); + pallet::Error::IdNotFound + })?; + + // insert `heavier_node` directly in front of `lighter_node`. This will update both nodes + // in storage and update the node counter. + Self::insert_at_unchecked(lighter_node, heavier_node); + + Ok(()) + } + + /// Insert `node` directly in front of `at`. + /// + /// WARNINGS: + /// - this is a naive function in that it does not check if `node` belongs to the same bag as + /// `at`. It is expected that the call site will check preconditions. + /// - this will panic if `at.bag_upper` is not a bag that already exists in storage. + fn insert_at_unchecked(mut at: Node, mut node: Node) { + // connect `node` to its new `prev`. + node.prev = at.prev.clone(); + if let Some(mut prev) = at.prev() { + prev.next = Some(node.id().clone()); + prev.put() + } + + // connect `node` and `at`. + node.next = Some(at.id().clone()); + at.prev = Some(node.id().clone()); + + if node.is_terminal() { + // `node` is the new head, so we make sure the bag is updated. Note, + // since `node` is always in front of `at` we know that 1) there is always at least 2 + // nodes in the bag, and 2) only `node` could be the head and only `at` could be the + // tail. + let mut bag = Bag::::get(at.bag_upper) + .expect("given nodes must always have a valid bag. qed."); + + if node.prev == None { + bag.head = Some(node.id().clone()) + } + + bag.put() + }; + + // write the updated nodes to storage. + at.put(); + node.put(); + } + /// Sanity check the list. /// /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) /// is being used, after all other staking data (such as counter) has been updated. It checks: /// /// * there are no duplicate ids, - /// * length of this list is in sync with `CounterForListNodes`, - /// * and sanity-checks all bags. This will cascade down all the checks and makes sure all bags - /// are checked per *any* update to `List`. + /// * length of this list is in sync with `ListNodes::count()`, + /// * and sanity-checks all bags and nodes. This will cascade down all the checks and makes sure + /// all bags and nodes are checked per *any* update to `List`. #[cfg(feature = "std")] pub(crate) fn sanity_check() -> Result<(), &'static str> { use frame_support::ensure; @@ -403,8 +507,8 @@ impl List { ); let iter_count = Self::iter().count() as u32; - let stored_count = crate::CounterForListNodes::::get(); - let nodes_count = crate::ListNodes::::iter().count() as u32; + let stored_count = crate::ListNodes::::count(); + let nodes_count = crate::ListNodes::::iter().count() as u32; ensure!(iter_count == stored_count, "iter_count != stored_count"); ensure!(stored_count == nodes_count, "stored_count != nodes_count"); @@ -412,15 +516,15 @@ impl List { let active_bags = { let thresholds = T::BagThresholds::get().iter().copied(); - let thresholds: Vec = if thresholds.clone().last() == Some(VoteWeight::MAX) { - // in the event that they included it, we don't need to make any changes - // Box::new(thresholds.collect() - thresholds.collect() - } else { - // otherwise, insert it here. - thresholds.chain(iter::once(VoteWeight::MAX)).collect() - }; - thresholds.into_iter().filter_map(|t| Bag::::get(t)) + let thresholds: Vec = + if thresholds.clone().last() == Some(T::Score::max_value()) { + // in the event that they included it, we don't need to make any changes + thresholds.collect() + } else { + // otherwise, insert it here. + thresholds.chain(iter::once(T::Score::max_value())).collect() + }; + thresholds.into_iter().filter_map(|t| Bag::::get(t)) }; let _ = active_bags.clone().map(|b| b.sanity_check()).collect::>()?; @@ -433,7 +537,7 @@ impl List { // check that all nodes are sane. We check the `ListNodes` storage item directly in case we // have some "stale" nodes that are not in a bag. - for (_id, node) in crate::ListNodes::::iter() { + for (_id, node) in crate::ListNodes::::iter() { node.sanity_check()? } @@ -448,27 +552,30 @@ impl List { /// Returns the nodes of all non-empty bags. For testing and benchmarks. #[cfg(any(feature = "std", feature = "runtime-benchmarks"))] #[allow(dead_code)] - pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { + pub(crate) fn get_bags() -> Vec<(T::Score, Vec)> { use frame_support::traits::Get as _; let thresholds = T::BagThresholds::get(); let iter = thresholds.iter().copied(); - let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { + let iter: Box> = if thresholds.last() == + Some(&T::Score::max_value()) + { // in the event that they included it, we can just pass the iterator through unchanged. Box::new(iter) } else { // otherwise, insert it here. - Box::new(iter.chain(sp_std::iter::once(VoteWeight::MAX))) + Box::new(iter.chain(sp_std::iter::once(T::Score::max_value()))) }; iter.filter_map(|t| { - Bag::::get(t).map(|bag| (t, bag.iter().map(|n| n.id().clone()).collect::>())) + Bag::::get(t) + .map(|bag| (t, bag.iter().map(|n| n.id().clone()).collect::>())) }) .collect::>() } } -/// A Bag is a doubly-linked list of ids, where each id is mapped to a [`ListNode`]. +/// A Bag is a doubly-linked list of ids, where each id is mapped to a [`Node`]. /// /// Note that we maintain both head and tail pointers. While it would be possible to get away with /// maintaining only a head pointer and cons-ing elements onto the front of the list, it's more @@ -476,38 +583,40 @@ impl List { /// iteration so that there's no incentive to churn ids positioning to improve the chances of /// appearing within the ids set. #[derive(DefaultNoBound, Encode, Decode, MaxEncodedLen, TypeInfo)] -#[codec(mel_bound(T: Config))] -#[scale_info(skip_type_params(T))] +#[codec(mel_bound())] +#[scale_info(skip_type_params(T, I))] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] -pub struct Bag { +pub struct Bag, I: 'static = ()> { head: Option, tail: Option, #[codec(skip)] - bag_upper: VoteWeight, + bag_upper: T::Score, + #[codec(skip)] + _phantom: PhantomData, } -impl Bag { +impl, I: 'static> Bag { #[cfg(test)] pub(crate) fn new( head: Option, tail: Option, - bag_upper: VoteWeight, + bag_upper: T::Score, ) -> Self { - Self { head, tail, bag_upper } + Self { head, tail, bag_upper, _phantom: PhantomData } } - /// Get a bag by its upper vote weight. - pub(crate) fn get(bag_upper: VoteWeight) -> Option> { - crate::ListBags::::try_get(bag_upper).ok().map(|mut bag| { + /// Get a bag by its upper score. + pub(crate) fn get(bag_upper: T::Score) -> Option> { + crate::ListBags::::try_get(bag_upper).ok().map(|mut bag| { bag.bag_upper = bag_upper; bag }) } - /// Get a bag by its upper vote weight or make it, appropriately initialized. Does not check if + /// Get a bag by its upper score or make it, appropriately initialized. Does not check if /// if `bag_upper` is a valid threshold. - fn get_or_make(bag_upper: VoteWeight) -> Bag { + fn get_or_make(bag_upper: T::Score) -> Bag { Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) } @@ -519,24 +628,24 @@ impl Bag { /// Put the bag back into storage. fn put(self) { if self.is_empty() { - crate::ListBags::::remove(self.bag_upper); + crate::ListBags::::remove(self.bag_upper); } else { - crate::ListBags::::insert(self.bag_upper, self); + crate::ListBags::::insert(self.bag_upper, self); } } /// Get the head node in this bag. - fn head(&self) -> Option> { + fn head(&self) -> Option> { self.head.as_ref().and_then(|id| Node::get(id)) } /// Get the tail node in this bag. - fn tail(&self) -> Option> { + fn tail(&self) -> Option> { self.tail.as_ref().and_then(|id| Node::get(id)) } /// Iterate over the nodes in this bag. - pub(crate) fn iter(&self) -> impl Iterator> { + pub(crate) fn iter(&self) -> impl Iterator> { sp_std::iter::successors(self.head(), |prev| prev.next()) } @@ -551,7 +660,13 @@ impl Bag { // insert_node will overwrite `prev`, `next` and `bag_upper` to the proper values. As long // as this bag is the correct one, we're good. All calls to this must come after getting the // correct [`notional_bag_for`]. - self.insert_node_unchecked(Node:: { id, prev: None, next: None, bag_upper: 0 }); + self.insert_node_unchecked(Node:: { + id, + prev: None, + next: None, + bag_upper: Zero::zero(), + _phantom: PhantomData, + }); } /// Insert a node into this bag. @@ -561,7 +676,7 @@ impl Bag { /// /// Storage note: this modifies storage, but only for the node. You still need to call /// `self.put()` after use. - fn insert_node_unchecked(&mut self, mut node: Node) { + fn insert_node_unchecked(&mut self, mut node: Node) { if let Some(tail) = &self.tail { if *tail == node.id { // this should never happen, but this check prevents one path to a worst case @@ -605,7 +720,7 @@ impl Bag { /// /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call /// `self.put()` and `ListNodes::remove(id)` to update storage for the bag and `node`. - fn remove_node_unchecked(&mut self, node: &Node) { + fn remove_node_unchecked(&mut self, node: &Node) { // reassign neighboring nodes. node.excise(); @@ -666,38 +781,40 @@ impl Bag { /// Iterate over the nodes in this bag (public for tests). #[cfg(feature = "std")] #[allow(dead_code)] - pub fn std_iter(&self) -> impl Iterator> { + pub fn std_iter(&self) -> impl Iterator> { sp_std::iter::successors(self.head(), |prev| prev.next()) } /// Check if the bag contains a node with `id`. #[cfg(feature = "std")] fn contains(&self, id: &T::AccountId) -> bool { - self.iter().find(|n| n.id() == id).is_some() + self.iter().any(|n| n.id() == id) } } /// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. #[derive(Encode, Decode, MaxEncodedLen, TypeInfo)] -#[codec(mel_bound(T: Config))] -#[scale_info(skip_type_params(T))] +#[codec(mel_bound())] +#[scale_info(skip_type_params(T, I))] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] -pub struct Node { +pub struct Node, I: 'static = ()> { id: T::AccountId, prev: Option, next: Option, - bag_upper: VoteWeight, + bag_upper: T::Score, + #[codec(skip)] + _phantom: PhantomData, } -impl Node { +impl, I: 'static> Node { /// Get a node by id. - pub(crate) fn get(id: &T::AccountId) -> Option> { - crate::ListNodes::::try_get(id).ok() + pub fn get(id: &T::AccountId) -> Option> { + crate::ListNodes::::try_get(id).ok() } /// Put the node back into storage. fn put(self) { - crate::ListNodes::::insert(self.id.clone(), self); + crate::ListNodes::::insert(self.id.clone(), self); } /// Update neighboring nodes to point to reach other. @@ -721,22 +838,22 @@ impl Node { /// /// It is naive because it does not check if the node has first been removed from its bag. fn remove_from_storage_unchecked(&self) { - crate::ListNodes::::remove(&self.id) + crate::ListNodes::::remove(&self.id) } /// Get the previous node in the bag. - fn prev(&self) -> Option> { + fn prev(&self) -> Option> { self.prev.as_ref().and_then(|id| Node::get(id)) } /// Get the next node in the bag. - fn next(&self) -> Option> { + fn next(&self) -> Option> { self.next.as_ref().and_then(|id| Node::get(id)) } /// `true` when this voter is in the wrong bag. - pub(crate) fn is_misplaced(&self, current_weight: VoteWeight) -> bool { - notional_bag_for::(current_weight) != self.bag_upper + pub fn is_misplaced(&self, current_score: T::Score) -> bool { + notional_bag_for::(current_score) != self.bag_upper } /// `true` when this voter is a bag head or tail. @@ -759,13 +876,13 @@ impl Node { /// The bag this nodes belongs to (public for benchmarks). #[cfg(feature = "runtime-benchmarks")] #[allow(dead_code)] - pub fn bag_upper(&self) -> VoteWeight { + pub fn bag_upper(&self) -> T::Score { self.bag_upper } #[cfg(feature = "std")] fn sanity_check(&self) -> Result<(), &'static str> { - let expected_bag = Bag::::get(self.bag_upper).ok_or("bag not found for node")?; + let expected_bag = Bag::::get(self.bag_upper).ok_or("bag not found for node")?; let id = self.id(); @@ -774,10 +891,13 @@ impl Node { "node does not exist in the expected bag" ); + let non_terminal_check = !self.is_terminal() && + expected_bag.head.as_ref() != Some(id) && + expected_bag.tail.as_ref() != Some(id); + let terminal_check = + expected_bag.head.as_ref() == Some(id) || expected_bag.tail.as_ref() == Some(id); frame_support::ensure!( - !self.is_terminal() || - expected_bag.head.as_ref() == Some(id) || - expected_bag.tail.as_ref() == Some(id), + non_terminal_check || terminal_check, "a terminal node is neither its bag head or tail" ); diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 14802bac9d1d..c8e233f1e62c 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,18 +18,24 @@ use super::*; use crate::{ mock::{test_utils::*, *}, - CounterForListNodes, ListBags, ListNodes, + ListBags, ListNodes, }; -use frame_election_provider_support::SortedListProvider; +use frame_election_provider_support::{SortedListProvider, VoteWeight}; use frame_support::{assert_ok, assert_storage_noop}; #[test] fn basic_setup_works() { ExtBuilder::default().build_and_execute(|| { // syntactic sugar to create a raw node - let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; + let node = |id, prev, next, bag_upper| Node:: { + id, + prev, + next, + bag_upper, + _phantom: PhantomData, + }; - assert_eq!(CounterForListNodes::::get(), 4); + assert_eq!(ListNodes::::count(), 4); assert_eq!(ListNodes::::iter().count(), 4); assert_eq!(ListBags::::iter().count(), 2); @@ -38,11 +44,11 @@ fn basic_setup_works() { // the state of the bags is as expected assert_eq!( ListBags::::get(10).unwrap(), - Bag:: { head: Some(1), tail: Some(1), bag_upper: 0 } + Bag:: { head: Some(1), tail: Some(1), bag_upper: 0, _phantom: PhantomData } ); assert_eq!( ListBags::::get(1_000).unwrap(), - Bag:: { head: Some(2), tail: Some(4), bag_upper: 0 } + Bag:: { head: Some(2), tail: Some(4), bag_upper: 0, _phantom: PhantomData } ); assert_eq!(ListNodes::::get(2).unwrap(), node(2, None, Some(3), 1_000)); @@ -65,24 +71,24 @@ fn basic_setup_works() { #[test] fn notional_bag_for_works() { // under a threshold gives the next threshold. - assert_eq!(notional_bag_for::(0), 10); - assert_eq!(notional_bag_for::(9), 10); + assert_eq!(notional_bag_for::(0), 10); + assert_eq!(notional_bag_for::(9), 10); // at a threshold gives that threshold. - assert_eq!(notional_bag_for::(10), 10); + assert_eq!(notional_bag_for::(10), 10); // above the threshold, gives the next threshold. - assert_eq!(notional_bag_for::(11), 20); + assert_eq!(notional_bag_for::(11), 20); let max_explicit_threshold = *::BagThresholds::get().last().unwrap(); assert_eq!(max_explicit_threshold, 10_000); - // if the max explicit threshold is less than VoteWeight::MAX, + // if the max explicit threshold is less than T::Value::max_value(), assert!(VoteWeight::MAX > max_explicit_threshold); - // then anything above it will belong to the VoteWeight::MAX bag. - assert_eq!(notional_bag_for::(max_explicit_threshold), max_explicit_threshold); - assert_eq!(notional_bag_for::(max_explicit_threshold + 1), VoteWeight::MAX); + // then anything above it will belong to the T::Value::max_value() bag. + assert_eq!(notional_bag_for::(max_explicit_threshold), max_explicit_threshold); + assert_eq!(notional_bag_for::(max_explicit_threshold + 1), VoteWeight::MAX); } #[test] @@ -242,17 +248,17 @@ mod list { // then assert_storage_noop!(assert_eq!( List::::insert(3, 20).unwrap_err(), - Error::Duplicate + ListError::Duplicate )); }); } #[test] fn remove_works() { - use crate::{CounterForListNodes, ListBags, ListNodes}; + use crate::{ListBags, ListNodes}; let ensure_left = |id, counter| { assert!(!ListNodes::::contains_key(id)); - assert_eq!(CounterForListNodes::::get(), counter); + assert_eq!(ListNodes::::count(), counter); assert_eq!(ListNodes::::iter().count() as u32, counter); }; @@ -357,10 +363,19 @@ mod list { assert_eq!(List::::sanity_check(), Err("duplicate identified")); }); - // ensure count is in sync with `CounterForListNodes`. + // ensure count is in sync with `ListNodes::count()`. ExtBuilder::default().build_and_execute_no_post_check(|| { - crate::CounterForListNodes::::mutate(|counter| *counter += 1); - assert_eq!(crate::CounterForListNodes::::get(), 5); + assert_eq!(crate::ListNodes::::count(), 4); + // we do some wacky stuff here to get access to the counter, since it is (reasonably) + // not exposed as mutable in any sense. + frame_support::generate_storage_alias!( + BagsList, + CounterForListNodes + => Value + ); + CounterForListNodes::mutate(|counter| *counter += 1); + assert_eq!(crate::ListNodes::::count(), 5); + assert_eq!(List::::sanity_check(), Err("iter_count != stored_count")); }); } @@ -374,6 +389,157 @@ mod list { assert!(non_existent_ids.iter().all(|id| !List::::contains(id))); }) } + + #[test] + #[should_panic = "given nodes must always have a valid bag. qed."] + fn put_in_front_of_panics_if_bag_not_found() { + ExtBuilder::default().skip_genesis_ids().build_and_execute_no_post_check(|| { + let node_10_no_bag = Node:: { + id: 10, + prev: None, + next: None, + bag_upper: 15, + _phantom: PhantomData, + }; + let node_11_no_bag = Node:: { + id: 11, + prev: None, + next: None, + bag_upper: 15, + _phantom: PhantomData, + }; + + // given + ListNodes::::insert(10, node_10_no_bag); + ListNodes::::insert(11, node_11_no_bag); + StakingMock::set_score_of(&10, 14); + StakingMock::set_score_of(&11, 15); + assert!(!ListBags::::contains_key(15)); + assert_eq!(List::::get_bags(), vec![]); + + // then .. this panics + let _ = List::::put_in_front_of(&10, &11); + }); + } + + #[test] + fn insert_at_unchecked_at_is_only_node() { + // Note that this `insert_at_unchecked` test should fail post checks because node 42 does + // not get re-assigned the correct bagu pper. This is because `insert_at_unchecked` assumes + // both nodes are already in the same bag with the correct bag upper. + ExtBuilder::default().build_and_execute_no_post_check(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + // implicitly also test that `node`'s `prev`/`next` are correctly re-assigned. + let node_42 = Node:: { + id: 42, + prev: Some(1), + next: Some(2), + bag_upper: 1_000, + _phantom: PhantomData, + }; + assert!(!crate::ListNodes::::contains_key(42)); + + let node_1 = crate::ListNodes::::get(&1).unwrap(); + + // when + List::::insert_at_unchecked(node_1, node_42); + + // then + assert_eq!( + List::::get_bags(), + vec![(10, vec![42, 1]), (1_000, vec![2, 3, 4])] + ); + }) + } + + #[test] + fn insert_at_unchecked_at_is_head() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + // implicitly also test that `node`'s `prev`/`next` are correctly re-assigned. + let node_42 = Node:: { + id: 42, + prev: Some(4), + next: None, + bag_upper: 1_000, + _phantom: PhantomData, + }; + assert!(!crate::ListNodes::::contains_key(42)); + + let node_2 = crate::ListNodes::::get(&2).unwrap(); + + // when + List::::insert_at_unchecked(node_2, node_42); + + // then + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![42, 2, 3, 4])] + ); + }) + } + + #[test] + fn insert_at_unchecked_at_is_non_terminal() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + // implicitly also test that `node`'s `prev`/`next` are correctly re-assigned. + let node_42 = Node:: { + id: 42, + prev: None, + next: Some(2), + bag_upper: 1_000, + _phantom: PhantomData, + }; + assert!(!crate::ListNodes::::contains_key(42)); + + let node_3 = crate::ListNodes::::get(&3).unwrap(); + + // when + List::::insert_at_unchecked(node_3, node_42); + + // then + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 42, 3, 4])] + ); + }) + } + + #[test] + fn insert_at_unchecked_at_is_tail() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + // implicitly also test that `node`'s `prev`/`next` are correctly re-assigned. + let node_42 = Node:: { + id: 42, + prev: Some(42), + next: Some(42), + bag_upper: 1_000, + _phantom: PhantomData, + }; + assert!(!crate::ListNodes::::contains_key(42)); + + let node_4 = crate::ListNodes::::get(&4).unwrap(); + + // when + List::::insert_at_unchecked(node_4, node_42); + + // then + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 42, 4])] + ); + }) + } } mod bags { @@ -386,7 +552,7 @@ mod bags { let bag = Bag::::get(bag_upper).unwrap(); let bag_ids = bag.iter().map(|n| *n.id()).collect::>(); - assert_eq!(bag, Bag:: { head, tail, bag_upper }); + assert_eq!(bag, Bag:: { head, tail, bag_upper, _phantom: PhantomData }); assert_eq!(bag_ids, ids); }; @@ -417,7 +583,13 @@ mod bags { #[test] fn insert_node_sets_proper_bag() { ExtBuilder::default().build_and_execute_no_post_check(|| { - let node = |id, bag_upper| Node:: { id, prev: None, next: None, bag_upper }; + let node = |id, bag_upper| Node:: { + id, + prev: None, + next: None, + bag_upper, + _phantom: PhantomData, + }; assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); @@ -426,7 +598,7 @@ mod bags { assert_eq!( ListNodes::::get(&42).unwrap(), - Node { bag_upper: 10, prev: Some(1), next: None, id: 42 } + Node { bag_upper: 10, prev: Some(1), next: None, id: 42, _phantom: PhantomData } ); }); } @@ -434,7 +606,13 @@ mod bags { #[test] fn insert_node_happy_paths_works() { ExtBuilder::default().build_and_execute_no_post_check(|| { - let node = |id, bag_upper| Node:: { id, prev: None, next: None, bag_upper }; + let node = |id, bag_upper| Node:: { + id, + prev: None, + next: None, + bag_upper, + _phantom: PhantomData, + }; // when inserting into a bag with 1 node let mut bag_10 = Bag::::get(10).unwrap(); @@ -455,15 +633,26 @@ mod bags { assert_eq!(bag_as_ids(&bag_20), vec![62]); // when inserting a node pointing to the accounts not in the bag - let node_61 = - Node:: { id: 61, prev: Some(21), next: Some(101), bag_upper: 20 }; + let node_61 = Node:: { + id: 61, + prev: Some(21), + next: Some(101), + bag_upper: 20, + _phantom: PhantomData, + }; bag_20.insert_node_unchecked(node_61); // then ids are in order assert_eq!(bag_as_ids(&bag_20), vec![62, 61]); // and when the node is re-fetched all the info is correct assert_eq!( Node::::get(&61).unwrap(), - Node:: { id: 61, prev: Some(62), next: None, bag_upper: 20 } + Node:: { + id: 61, + prev: Some(62), + next: None, + bag_upper: 20, + _phantom: PhantomData, + } ); // state of all bags is as expected @@ -478,7 +667,13 @@ mod bags { // Document improper ways `insert_node` may be getting used. #[test] fn insert_node_bad_paths_documented() { - let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; + let node = |id, prev, next, bag_upper| Node:: { + id, + prev, + next, + bag_upper, + _phantom: PhantomData, + }; ExtBuilder::default().build_and_execute_no_post_check(|| { // when inserting a node with both prev & next pointing at an account in an incorrect // bag. @@ -531,7 +726,10 @@ mod bags { ); // ^^^ despite being the bags head, it has a prev - assert_eq!(bag_1000, Bag { head: Some(2), tail: Some(2), bag_upper: 1_000 }) + assert_eq!( + bag_1000, + Bag { head: Some(2), tail: Some(2), bag_upper: 1_000, _phantom: PhantomData } + ) }); } @@ -543,7 +741,13 @@ mod bags { )] fn insert_node_duplicate_tail_panics_with_debug_assert() { ExtBuilder::default().build_and_execute(|| { - let node = |id, prev, next, bag_upper| Node:: { id, prev, next, bag_upper }; + let node = |id, prev, next, bag_upper| Node:: { + id, + prev, + next, + bag_upper, + _phantom: PhantomData, + }; // given assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])],); @@ -675,6 +879,7 @@ mod bags { prev: None, next: Some(3), bag_upper: 10, // should be 1_000 + _phantom: PhantomData, }; let mut bag_1000 = Bag::::get(1_000).unwrap(); diff --git a/frame/bags-list/src/migrations.rs b/frame/bags-list/src/migrations.rs new file mode 100644 index 000000000000..696733e8c7ba --- /dev/null +++ b/frame/bags-list/src/migrations.rs @@ -0,0 +1,49 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The migrations of this pallet. + +use frame_support::traits::OnRuntimeUpgrade; + +/// A struct that does not migration, but only checks that the counter prefix exists and is correct. +pub struct CheckCounterPrefix(sp_std::marker::PhantomData); +impl OnRuntimeUpgrade for CheckCounterPrefix { + fn on_runtime_upgrade() -> frame_support::weights::Weight { + 0 + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result<(), &'static str> { + use frame_support::ensure; + // The old explicit storage item. + frame_support::generate_storage_alias!(BagsList, CounterForListNodes => Value); + + // ensure that a value exists in the counter struct. + ensure!( + crate::ListNodes::::count() == CounterForListNodes::get().unwrap(), + "wrong list node counter" + ); + + crate::log!( + info, + "checked bags-list prefix to be correct and have {} nodes", + crate::ListNodes::::count() + ); + + Ok(()) + } +} diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index a6ab35896b1e..fce143105417 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,28 +21,28 @@ use super::*; use crate::{self as bags_list}; use frame_election_provider_support::VoteWeight; use frame_support::parameter_types; +use std::collections::HashMap; pub type AccountId = u32; pub type Balance = u32; parameter_types! { + // Set the vote weight for any id who's weight has _not_ been set with `set_score_of`. pub static NextVoteWeight: VoteWeight = 0; + pub static NextVoteWeightMap: HashMap = Default::default(); } pub struct StakingMock; -impl frame_election_provider_support::VoteWeightProvider for StakingMock { - fn vote_weight(id: &AccountId) -> VoteWeight { - match id { - 710 => 15, - 711 => 16, - 712 => 2_000, // special cases used for migrate test - _ => NextVoteWeight::get(), - } +impl frame_election_provider_support::ScoreProvider for StakingMock { + type Score = VoteWeight; + + fn score(id: &AccountId) -> Self::Score { + *NextVoteWeightMap::get().get(id).unwrap_or(&NextVoteWeight::get()) } + #[cfg(any(feature = "runtime-benchmarks", test))] - fn set_vote_weight_of(_: &AccountId, weight: VoteWeight) { - // we don't really keep a mapping, just set weight for everyone. - NextVoteWeight::set(weight) + fn set_score_of(id: &AccountId, weight: Self::Score) { + NEXT_VOTE_WEIGHT_MAP.with(|m| m.borrow_mut().insert(id.clone(), weight)); } } @@ -70,6 +70,7 @@ impl frame_system::Config for Runtime { type OnKilledAccount = (); type SystemWeightInfo = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } parameter_types! { @@ -80,7 +81,8 @@ impl bags_list::Config for Runtime { type Event = Event; type WeightInfo = (); type BagThresholds = BagThresholds; - type VoteWeightProvider = StakingMock; + type ScoreProvider = StakingMock; + type Score = VoteWeight; } type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; @@ -101,12 +103,21 @@ pub(crate) const GENESIS_IDS: [(AccountId, VoteWeight); 4] = [(1, 10), (2, 1_000), (3, 1_000), (4, 1_000)]; #[derive(Default)] -pub(crate) struct ExtBuilder { +pub struct ExtBuilder { ids: Vec<(AccountId, VoteWeight)>, + skip_genesis_ids: bool, } impl ExtBuilder { + /// Skip adding the default genesis ids to the list. + #[cfg(test)] + pub(crate) fn skip_genesis_ids(mut self) -> Self { + self.skip_genesis_ids = true; + self + } + /// Add some AccountIds to insert into `List`. + #[cfg(test)] pub(crate) fn add_ids(mut self, ids: Vec<(AccountId, VoteWeight)>) -> Self { self.ids = ids; self @@ -116,28 +127,37 @@ impl ExtBuilder { sp_tracing::try_init_simple(); let storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let ids_with_weight: Vec<_> = if self.skip_genesis_ids { + self.ids.iter().collect() + } else { + GENESIS_IDS.iter().chain(self.ids.iter()).collect() + }; + let mut ext = sp_io::TestExternalities::from(storage); ext.execute_with(|| { - for (id, weight) in GENESIS_IDS.iter().chain(self.ids.iter()) { + for (id, weight) in ids_with_weight { frame_support::assert_ok!(List::::insert(*id, *weight)); + StakingMock::set_score_of(id, *weight); } }); ext } - pub(crate) fn build_and_execute(self, test: impl FnOnce() -> ()) { + pub fn build_and_execute(self, test: impl FnOnce() -> ()) { self.build().execute_with(|| { test(); List::::sanity_check().expect("Sanity check post condition failed") }) } + #[cfg(test)] pub(crate) fn build_and_execute_no_post_check(self, test: impl FnOnce() -> ()) { self.build().execute_with(test) } } +#[cfg(test)] pub(crate) mod test_utils { use super::*; use list::Bag; diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index e94017730668..941623229dc2 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,10 +15,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use frame_support::{assert_ok, assert_storage_noop, traits::IntegrityTest}; +use frame_support::{assert_noop, assert_ok, assert_storage_noop, traits::IntegrityTest}; use super::*; -use frame_election_provider_support::SortedListProvider; +use frame_election_provider_support::{SortedListProvider, VoteWeight}; use list::Bag; use mock::{test_utils::*, *}; @@ -35,7 +35,7 @@ mod pallet { ); // when increasing vote weight to the level of non-existent bag - NextVoteWeight::set(2_000); + StakingMock::set_score_of(&42, 2_000); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // then a new bag is created and the id moves into it @@ -45,7 +45,7 @@ mod pallet { ); // when decreasing weight within the range of the current bag - NextVoteWeight::set(1001); + StakingMock::set_score_of(&42, 1_001); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // then the id does not move @@ -55,7 +55,7 @@ mod pallet { ); // when reducing weight to the level of a non-existent bag - NextVoteWeight::set(30); + StakingMock::set_score_of(&42, 30); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // then a new bag is created and the id moves into it @@ -65,7 +65,7 @@ mod pallet { ); // when increasing weight to the level of a pre-existing bag - NextVoteWeight::set(500); + StakingMock::set_score_of(&42, 500); assert_ok!(BagsList::rebag(Origin::signed(0), 42)); // then the id moves into that bag @@ -85,7 +85,7 @@ mod pallet { assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // when - NextVoteWeight::set(10); + StakingMock::set_score_of(&4, 10); assert_ok!(BagsList::rebag(Origin::signed(0), 4)); // then @@ -93,6 +93,7 @@ mod pallet { assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(2), Some(3), 1_000)); // when + StakingMock::set_score_of(&3, 10); assert_ok!(BagsList::rebag(Origin::signed(0), 3)); // then @@ -103,6 +104,7 @@ mod pallet { assert_eq!(get_list_as_ids(), vec![2u32, 1, 4, 3]); // when + StakingMock::set_score_of(&2, 10); assert_ok!(BagsList::rebag(Origin::signed(0), 2)); // then @@ -117,7 +119,7 @@ mod pallet { fn rebag_head_works() { ExtBuilder::default().build_and_execute(|| { // when - NextVoteWeight::set(10); + StakingMock::set_score_of(&2, 10); assert_ok!(BagsList::rebag(Origin::signed(0), 2)); // then @@ -125,6 +127,7 @@ mod pallet { assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(3), Some(4), 1_000)); // when + StakingMock::set_score_of(&3, 10); assert_ok!(BagsList::rebag(Origin::signed(0), 3)); // then @@ -132,6 +135,7 @@ mod pallet { assert_eq!(Bag::::get(1_000).unwrap(), Bag::new(Some(4), Some(4), 1_000)); // when + StakingMock::set_score_of(&4, 10); assert_ok!(BagsList::rebag(Origin::signed(0), 4)); // then @@ -196,6 +200,249 @@ mod pallet { assert_storage_noop!(assert!(BagsList::rebag(Origin::signed(0), 10).is_ok())); }) } + + #[test] + fn put_in_front_of_two_node_bag_heavier_is_tail() { + ExtBuilder::default() + .skip_genesis_ids() + .add_ids(vec![(10, 15), (11, 16)]) + .build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(20, vec![10, 11])]); + + // when + assert_ok!(BagsList::put_in_front_of(Origin::signed(11), 10)); + + // then + assert_eq!(List::::get_bags(), vec![(20, vec![11, 10])]); + }); + } + + #[test] + fn put_in_front_of_two_node_bag_heavier_is_head() { + ExtBuilder::default() + .skip_genesis_ids() + .add_ids(vec![(11, 16), (10, 15)]) + .build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(20, vec![11, 10])]); + + // when + assert_ok!(BagsList::put_in_front_of(Origin::signed(11), 10)); + + // then + assert_eq!(List::::get_bags(), vec![(20, vec![11, 10])]); + }); + } + + #[test] + fn put_in_front_of_non_terminal_nodes_heavier_behind() { + ExtBuilder::default().add_ids(vec![(5, 1_000)]).build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 5])]); + + StakingMock::set_score_of(&3, 999); + + // when + assert_ok!(BagsList::put_in_front_of(Origin::signed(4), 3)); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 4, 3, 5])]); + }); + } + + #[test] + fn put_in_front_of_non_terminal_nodes_heavier_in_front() { + ExtBuilder::default() + .add_ids(vec![(5, 1_000), (6, 1_000)]) + .build_and_execute(|| { + // given + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 3, 4, 5, 6])] + ); + + StakingMock::set_score_of(&5, 999); + + // when + assert_ok!(BagsList::put_in_front_of(Origin::signed(3), 5)); + + // then + assert_eq!( + List::::get_bags(), + vec![(10, vec![1]), (1_000, vec![2, 4, 3, 5, 6])] + ); + }); + } + + #[test] + fn put_in_front_of_lighter_is_head_heavier_is_non_terminal() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + StakingMock::set_score_of(&2, 999); + + // when + assert_ok!(BagsList::put_in_front_of(Origin::signed(3), 2)); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 2, 4])]); + }); + } + + #[test] + fn put_in_front_of_heavier_is_tail_lighter_is_non_terminal() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + StakingMock::set_score_of(&3, 999); + + // when + assert_ok!(BagsList::put_in_front_of(Origin::signed(4), 3)); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 4, 3])]); + }); + } + + #[test] + fn put_in_front_of_heavier_is_tail_lighter_is_head() { + ExtBuilder::default().add_ids(vec![(5, 1_000)]).build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 5])]); + + StakingMock::set_score_of(&2, 999); + + // when + assert_ok!(BagsList::put_in_front_of(Origin::signed(5), 2)); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![5, 2, 3, 4])]); + }); + } + + #[test] + fn put_in_front_of_heavier_is_head_lighter_is_not_terminal() { + ExtBuilder::default().add_ids(vec![(5, 1_000)]).build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 5])]); + + StakingMock::set_score_of(&4, 999); + + // when + BagsList::put_in_front_of(Origin::signed(2), 4).unwrap(); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 2, 4, 5])]); + }); + } + + #[test] + fn put_in_front_of_lighter_is_tail_heavier_is_not_terminal() { + ExtBuilder::default().add_ids(vec![(5, 900)]).build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4, 5])]); + + // when + BagsList::put_in_front_of(Origin::signed(3), 5).unwrap(); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 4, 3, 5])]); + }); + } + + #[test] + fn put_in_front_of_lighter_is_tail_heavier_is_head() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + StakingMock::set_score_of(&4, 999); + + // when + BagsList::put_in_front_of(Origin::signed(2), 4).unwrap(); + + // then + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 2, 4])]); + }); + } + + #[test] + fn put_in_front_of_errors_if_heavier_is_less_than_lighter() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + StakingMock::set_score_of(&3, 999); + + // then + assert_noop!( + BagsList::put_in_front_of(Origin::signed(3), 2), + crate::pallet::Error::::NotHeavier + ); + }); + } + + #[test] + fn put_in_front_of_errors_if_heavier_is_equal_weight_to_lighter() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + // then + assert_noop!( + BagsList::put_in_front_of(Origin::signed(3), 4), + crate::pallet::Error::::NotHeavier + ); + }); + } + + #[test] + fn put_in_front_of_errors_if_nodes_not_found() { + // `heavier` not found + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + assert!(!ListNodes::::contains_key(5)); + + // then + assert_noop!( + BagsList::put_in_front_of(Origin::signed(5), 4), + crate::pallet::Error::::IdNotFound + ); + }); + + // `lighter` not found + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + assert!(!ListNodes::::contains_key(5)); + + // then + assert_noop!( + BagsList::put_in_front_of(Origin::signed(4), 5), + crate::pallet::Error::::IdNotFound + ); + }); + } + + #[test] + fn put_in_front_of_errors_if_nodes_not_in_same_bag() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); + + // then + assert_noop!( + BagsList::put_in_front_of(Origin::signed(4), 1), + crate::pallet::Error::::NotInSameBag + ); + }); + } } mod sorted_list_provider { @@ -211,6 +458,27 @@ mod sorted_list_provider { }); } + #[test] + fn iter_from_works() { + ExtBuilder::default().add_ids(vec![(5, 5), (6, 15)]).build_and_execute(|| { + // given + assert_eq!( + List::::get_bags(), + vec![(10, vec![1, 5]), (20, vec![6]), (1000, vec![2, 3, 4])] + ); + + assert_eq!(BagsList::iter_from(&2).unwrap().collect::>(), vec![3, 4, 6, 1, 5]); + assert_eq!(BagsList::iter_from(&3).unwrap().collect::>(), vec![4, 6, 1, 5]); + assert_eq!(BagsList::iter_from(&4).unwrap().collect::>(), vec![6, 1, 5]); + assert_eq!(BagsList::iter_from(&6).unwrap().collect::>(), vec![1, 5]); + assert_eq!(BagsList::iter_from(&1).unwrap().collect::>(), vec![5]); + assert!(BagsList::iter_from(&5).unwrap().collect::>().is_empty()); + assert!(BagsList::iter_from(&7).is_err()); + + assert_storage_noop!(assert!(BagsList::iter_from(&8).is_err())); + }); + } + #[test] fn count_works() { ExtBuilder::default().build_and_execute(|| { @@ -271,7 +539,7 @@ mod sorted_list_provider { // then assert_storage_noop!(assert_eq!( BagsList::on_insert(3, 20).unwrap_err(), - Error::Duplicate + ListError::Duplicate )); }); } @@ -340,7 +608,7 @@ mod sorted_list_provider { let ensure_left = |id, counter| { assert!(!ListNodes::::contains_key(id)); assert_eq!(BagsList::count(), counter); - assert_eq!(CounterForListNodes::::get(), counter); + assert_eq!(ListNodes::::count(), counter); assert_eq!(ListNodes::::iter().count() as u32, counter); }; diff --git a/frame/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs index 95d3dfa6eb98..f35c0aaee6d6 100644 --- a/frame/bags-list/src/weights.rs +++ b/frame/bags-list/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_bags_list //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-09-15, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-30, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/bags-list/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -47,6 +48,7 @@ use sp_std::marker::PhantomData; pub trait WeightInfo { fn rebag_non_terminal() -> Weight; fn rebag_terminal() -> Weight; + fn put_in_front_of() -> Weight; } /// Weights for pallet_bags_list using the Substrate node and recommended hardware. @@ -57,7 +59,7 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListNodes (r:4 w:4) // Storage: BagsList ListBags (r:1 w:1) fn rebag_non_terminal() -> Weight { - (74_175_000 as Weight) + (46_965_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } @@ -66,10 +68,20 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListNodes (r:3 w:3) // Storage: BagsList ListBags (r:2 w:2) fn rebag_terminal() -> Weight { - (73_305_000 as Weight) + (45_952_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } + // Storage: BagsList ListNodes (r:4 w:4) + // Storage: Staking Bonded (r:2 w:0) + // Storage: Staking Ledger (r:2 w:0) + // Storage: BagsList CounterForListNodes (r:1 w:1) + // Storage: BagsList ListBags (r:1 w:1) + fn put_in_front_of() -> Weight { + (52_928_000 as Weight) + .saturating_add(T::DbWeight::get().reads(10 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) + } } // For backwards compatibility and tests @@ -79,7 +91,7 @@ impl WeightInfo for () { // Storage: BagsList ListNodes (r:4 w:4) // Storage: BagsList ListBags (r:1 w:1) fn rebag_non_terminal() -> Weight { - (74_175_000 as Weight) + (46_965_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } @@ -88,8 +100,18 @@ impl WeightInfo for () { // Storage: BagsList ListNodes (r:3 w:3) // Storage: BagsList ListBags (r:2 w:2) fn rebag_terminal() -> Weight { - (73_305_000 as Weight) + (45_952_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } + // Storage: BagsList ListNodes (r:4 w:4) + // Storage: Staking Bonded (r:2 w:0) + // Storage: Staking Ledger (r:2 w:0) + // Storage: BagsList CounterForListNodes (r:1 w:1) + // Storage: BagsList ListBags (r:1 w:1) + fn put_in_front_of() -> Weight { + (52_928_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(10 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + } } diff --git a/frame/balances/Cargo.toml b/frame/balances/Cargo.toml index 2263387d6d8e..a0de11638e66 100644 --- a/frame/balances/Cargo.toml +++ b/frame/balances/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-balances" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet to manage balances" readme = "README.md" @@ -13,18 +13,18 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.2.0", default-features = false, features = ["derive", "max-encoded-len"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } log = { version = "0.4.14", default-features = false } [dev-dependencies] -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } pallet-transaction-payment = { version = "4.0.0-dev", path = "../transaction-payment" } [features] @@ -39,5 +39,5 @@ std = [ "frame-system/std", "log/std", ] -runtime-benchmarks = ["frame-benchmarking"] +runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/balances/src/benchmarking.rs b/frame/balances/src/benchmarking.rs index 06d202ea3700..4a874e4ffa1d 100644 --- a/frame/balances/src/benchmarking.rs +++ b/frame/balances/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,9 +21,7 @@ use super::*; -use frame_benchmarking::{ - account, benchmarks_instance_pallet, impl_benchmark_test_suite, whitelisted_caller, -}; +use frame_benchmarking::{account, benchmarks_instance_pallet, whitelisted_caller}; use frame_system::RawOrigin; use sp_runtime::traits::Bounded; @@ -215,10 +213,10 @@ benchmarks_instance_pallet! { assert!(Balances::::reserved_balance(&user).is_zero()); assert_eq!(Balances::::free_balance(&user), balance); } -} -impl_benchmark_test_suite!( - Balances, - crate::tests_composite::ExtBuilder::default().build(), - crate::tests_composite::Test, -); + impl_benchmark_test_suite!( + Balances, + crate::tests_composite::ExtBuilder::default().build(), + crate::tests_composite::Test, + ) +} diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index afd2331c8e3c..6bf37dfda037 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -170,7 +170,7 @@ use frame_support::{ pallet_prelude::DispatchResult, traits::{ tokens::{fungible, BalanceStatus as Status, DepositConsequence, WithdrawConsequence}, - Currency, ExistenceRequirement, + Currency, DefensiveSaturating, ExistenceRequirement, ExistenceRequirement::{AllowDeath, KeepAlive}, Get, Imbalance, LockIdentifier, LockableCurrency, NamedReservableCurrency, OnUnbalanced, ReservableCurrency, SignedImbalance, StoredMap, TryDrop, WithdrawReasons, @@ -242,7 +242,6 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] - #[pallet::generate_storage_info] pub struct Pallet(PhantomData<(T, I)>); #[pallet::call] @@ -250,7 +249,6 @@ pub mod pallet { /// Transfer some liquid free balance to another account. /// /// `transfer` will set the `FreeBalance` of the sender and receiver. - /// It will decrease the total issuance of the system by the `TransferFee`. /// If the sender's account is below the existential deposit as a result /// of the transfer, the account will be reaped. /// @@ -271,8 +269,6 @@ pub mod pallet { /// - `transfer_keep_alive` works the same way as `transfer`, but has an additional check /// that the transfer will not kill the origin account. /// --------------------------------- - /// - Base Weight: 73.64 µs, worst case scenario (account created, account removed) - /// - DB Weight: 1 Read and 1 Write to destination account /// - Origin account is already in memory, so no DB operations for them. /// # #[pallet::weight(T::WeightInfo::transfer())] @@ -295,21 +291,11 @@ pub mod pallet { /// Set the balances of a given account. /// /// This will alter `FreeBalance` and `ReservedBalance` in storage. it will - /// also decrease the total issuance of the system (`TotalIssuance`). + /// also alter the total issuance of the system (`TotalIssuance`) appropriately. /// If the new free or reserved balance is below the existential deposit, /// it will reset the account nonce (`frame_system::AccountNonce`). /// /// The dispatch origin for this call is `root`. - /// - /// # - /// - Independent of the arguments. - /// - Contains a limited number of reads and writes. - /// --------------------- - /// - Base Weight: - /// - Creating: 27.56 µs - /// - Killing: 35.11 µs - /// - DB Weight: 1 Read, 1 Write to `who` - /// # #[pallet::weight( T::WeightInfo::set_balance_creating() // Creates a new account. .max(T::WeightInfo::set_balance_killing()) // Kills an existing account. @@ -328,25 +314,32 @@ pub mod pallet { let new_free = if wipeout { Zero::zero() } else { new_free }; let new_reserved = if wipeout { Zero::zero() } else { new_reserved }; - let (free, reserved) = Self::mutate_account(&who, |account| { - if new_free > account.free { - mem::drop(PositiveImbalance::::new(new_free - account.free)); - } else if new_free < account.free { - mem::drop(NegativeImbalance::::new(account.free - new_free)); - } - - if new_reserved > account.reserved { - mem::drop(PositiveImbalance::::new(new_reserved - account.reserved)); - } else if new_reserved < account.reserved { - mem::drop(NegativeImbalance::::new(account.reserved - new_reserved)); - } + // First we try to modify the account's balance to the forced balance. + let (old_free, old_reserved) = Self::mutate_account(&who, |account| { + let old_free = account.free; + let old_reserved = account.reserved; account.free = new_free; account.reserved = new_reserved; - (account.free, account.reserved) + (old_free, old_reserved) })?; - Self::deposit_event(Event::BalanceSet(who, free, reserved)); + + // This will adjust the total issuance, which was not done by the `mutate_account` + // above. + if new_free > old_free { + mem::drop(PositiveImbalance::::new(new_free - old_free)); + } else if new_free < old_free { + mem::drop(NegativeImbalance::::new(old_free - new_free)); + } + + if new_reserved > old_reserved { + mem::drop(PositiveImbalance::::new(new_reserved - old_reserved)); + } else if new_reserved < old_reserved { + mem::drop(NegativeImbalance::::new(old_reserved - new_reserved)); + } + + Self::deposit_event(Event::BalanceSet { who, free: new_free, reserved: new_reserved }); Ok(().into()) } @@ -381,11 +374,6 @@ pub mod pallet { /// 99% of the time you want [`transfer`] instead. /// /// [`transfer`]: struct.Pallet.html#method.transfer - /// # - /// - Cheaper than transfer because account cannot be killed. - /// - Base Weight: 51.4 µs - /// - DB Weight: 1 Read and 1 Write to dest (sender is in overlay already) - /// # #[pallet::weight(T::WeightInfo::transfer_keep_alive())] pub fn transfer_keep_alive( origin: OriginFor, @@ -426,12 +414,7 @@ pub mod pallet { let reducible_balance = Self::reducible_balance(&transactor, keep_alive); let dest = T::Lookup::lookup(dest)?; let keep_alive = if keep_alive { KeepAlive } else { AllowDeath }; - >::transfer( - &transactor, - &dest, - reducible_balance, - keep_alive.into(), - )?; + >::transfer(&transactor, &dest, reducible_balance, keep_alive)?; Ok(()) } @@ -454,25 +437,33 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event, I: 'static = ()> { - /// An account was created with some free balance. \[account, free_balance\] - Endowed(T::AccountId, T::Balance), + /// An account was created with some free balance. + Endowed { account: T::AccountId, free_balance: T::Balance }, /// An account was removed whose balance was non-zero but below ExistentialDeposit, - /// resulting in an outright loss. \[account, balance\] - DustLost(T::AccountId, T::Balance), - /// Transfer succeeded. \[from, to, value\] - Transfer(T::AccountId, T::AccountId, T::Balance), - /// A balance was set by root. \[who, free, reserved\] - BalanceSet(T::AccountId, T::Balance, T::Balance), - /// Some amount was deposited (e.g. for transaction fees). \[who, deposit\] - Deposit(T::AccountId, T::Balance), - /// Some balance was reserved (moved from free to reserved). \[who, value\] - Reserved(T::AccountId, T::Balance), - /// Some balance was unreserved (moved from reserved to free). \[who, value\] - Unreserved(T::AccountId, T::Balance), + /// resulting in an outright loss. + DustLost { account: T::AccountId, amount: T::Balance }, + /// Transfer succeeded. + Transfer { from: T::AccountId, to: T::AccountId, amount: T::Balance }, + /// A balance was set by root. + BalanceSet { who: T::AccountId, free: T::Balance, reserved: T::Balance }, + /// Some balance was reserved (moved from free to reserved). + Reserved { who: T::AccountId, amount: T::Balance }, + /// Some balance was unreserved (moved from reserved to free). + Unreserved { who: T::AccountId, amount: T::Balance }, /// Some balance was moved from the reserve of the first account to the second account. /// Final argument indicates the destination balance type. - /// \[from, to, balance, destination_status\] - ReserveRepatriated(T::AccountId, T::AccountId, T::Balance, Status), + ReserveRepatriated { + from: T::AccountId, + to: T::AccountId, + amount: T::Balance, + destination_status: Status, + }, + /// Some amount was deposited (e.g. for transaction fees). + Deposit { who: T::AccountId, amount: T::Balance }, + /// Some amount was withdrawn from the account (e.g. for transaction fees). + Withdraw { who: T::AccountId, amount: T::Balance }, + /// Some amount was removed from the account (e.g. for misbehavior). + Slashed { who: T::AccountId, amount: T::Balance }, } /// Old name generated by `decl_event`. @@ -504,8 +495,29 @@ pub mod pallet { #[pallet::getter(fn total_issuance)] pub type TotalIssuance, I: 'static = ()> = StorageValue<_, T::Balance, ValueQuery>; - /// The balance of an account. + /// The Balances pallet example of storing the balance of an account. + /// + /// # Example + /// + /// ```nocompile + /// impl pallet_balances::Config for Runtime { + /// type AccountStore = StorageMapShim, frame_system::Provider, AccountId, Self::AccountData> + /// } + /// ``` /// + /// You can also store the balance of an account in the `System` pallet. + /// + /// # Example + /// + /// ```nocompile + /// impl pallet_balances::Config for Runtime { + /// type AccountStore = System + /// } + /// ``` + /// + /// But this comes with tradeoffs, storing account balances in the system pallet stores + /// `frame_system` data alongside the account data contrary to storing account balances in the + /// `Balances` pallet, which uses a `StorageMap` to store balances data only. /// NOTE: This is only used in the case that this pallet is used to store balances. #[pallet::storage] pub type Account, I: 'static = ()> = StorageMap< @@ -628,7 +640,7 @@ pub enum Reasons { impl From for Reasons { fn from(r: WithdrawReasons) -> Reasons { - if r == WithdrawReasons::from(WithdrawReasons::TRANSACTION_PAYMENT) { + if r == WithdrawReasons::TRANSACTION_PAYMENT { Reasons::Fee } else if r.contains(WithdrawReasons::TRANSACTION_PAYMENT) { Reasons::All @@ -736,7 +748,7 @@ pub struct DustCleaner, I: 'static = ()>( impl, I: 'static> Drop for DustCleaner { fn drop(&mut self) { if let Some((who, dust)) = self.0.take() { - Pallet::::deposit_event(Event::DustLost(who, dust.peek())); + Pallet::::deposit_event(Event::DustLost { account: who, amount: dust.peek() }); T::DustRemoval::on_unbalanced(dust); } } @@ -933,7 +945,7 @@ impl, I: 'static> Pallet { }); result.map(|(maybe_endowed, maybe_dust, result)| { if let Some(endowed) = maybe_endowed { - Self::deposit_event(Event::Endowed(who.clone(), endowed)); + Self::deposit_event(Event::Endowed { account: who.clone(), free_balance: endowed }); } let dust_cleaner = DustCleaner(maybe_dust.map(|dust| (who.clone(), dust))); (result, dust_cleaner) @@ -980,7 +992,7 @@ impl, I: 'static> Pallet { } else { Locks::::insert(who, bounded_locks); if !existed { - if system::Pallet::::inc_consumers(who).is_err() { + if system::Pallet::::inc_consumers_without_limit(who).is_err() { // No providers for the locks. This is impossible under normal circumstances // since the funds that are under the lock will themselves be stored in the // account and therefore will need a reference. @@ -1045,12 +1057,12 @@ impl, I: 'static> Pallet { }, )?; - Self::deposit_event(Event::ReserveRepatriated( - slashed.clone(), - beneficiary.clone(), - actual, - status, - )); + Self::deposit_event(Event::ReserveRepatriated { + from: slashed.clone(), + to: beneficiary.clone(), + amount: actual, + destination_status: status, + }); Ok(actual) } } @@ -1103,6 +1115,7 @@ impl, I: 'static> fungible::Mutate for Pallet { Ok(()) })?; TotalIssuance::::mutate(|t| *t += amount); + Self::deposit_event(Event::Deposit { who: who.clone(), amount }); Ok(()) } @@ -1123,6 +1136,7 @@ impl, I: 'static> fungible::Mutate for Pallet { }, )?; TotalIssuance::::mutate(|t| *t -= actual); + Self::deposit_event(Event::Withdraw { who: who.clone(), amount }); Ok(actual) } } @@ -1141,7 +1155,14 @@ impl, I: 'static> fungible::Transfer for Pallet impl, I: 'static> fungible::Unbalanced for Pallet { fn set_balance(who: &T::AccountId, amount: Self::Balance) -> DispatchResult { - Self::mutate_account(who, |account| account.free = amount)?; + Self::mutate_account(who, |account| { + account.free = amount; + Self::deposit_event(Event::BalanceSet { + who: who.clone(), + free: account.free, + reserved: account.reserved, + }); + })?; Ok(()) } @@ -1503,7 +1524,7 @@ where .map_err(|_| Error::::LiquidityRestrictions)?; // TODO: This is over-conservative. There may now be other providers, and - // this pallet may not even be a provider. + // this pallet may not even be a provider. let allow_death = existence_requirement == ExistenceRequirement::AllowDeath; let allow_death = allow_death && system::Pallet::::can_dec_provider(transactor); @@ -1520,7 +1541,11 @@ where )?; // Emit transfer event. - Self::deposit_event(Event::Transfer(transactor.clone(), dest.clone(), value)); + Self::deposit_event(Event::Transfer { + from: transactor.clone(), + to: dest.clone(), + amount: value, + }); Ok(()) } @@ -1583,7 +1608,13 @@ where } }, ) { - Ok(r) => return r, + Ok((imbalance, not_slashed)) => { + Self::deposit_event(Event::Slashed { + who: who.clone(), + amount: value.saturating_sub(not_slashed), + }); + return (imbalance, not_slashed) + }, Err(_) => (), } } @@ -1608,6 +1639,7 @@ where |account, is_new| -> Result { ensure!(!is_new, Error::::DeadAccount); account.free = account.free.checked_add(&value).ok_or(ArithmeticError::Overflow)?; + Self::deposit_event(Event::Deposit { who: who.clone(), amount: value }); Ok(PositiveImbalance::new(value)) }, ) @@ -1640,6 +1672,7 @@ where None => return Ok(Self::PositiveImbalance::zero()), }; + Self::deposit_event(Event::Deposit { who: who.clone(), amount: value }); Ok(PositiveImbalance::new(value)) }, ) @@ -1677,6 +1710,7 @@ where account.free = new_free_account; + Self::deposit_event(Event::Withdraw { who: who.clone(), amount: value }); Ok(NegativeImbalance::new(value)) }, ) @@ -1709,6 +1743,11 @@ where SignedImbalance::Negative(NegativeImbalance::new(account.free - value)) }; account.free = value; + Self::deposit_event(Event::BalanceSet { + who: who.clone(), + free: account.free, + reserved: account.reserved, + }); Ok(imbalance) }, ) @@ -1752,7 +1791,7 @@ where Self::ensure_can_withdraw(&who, value.clone(), WithdrawReasons::RESERVE, account.free) })?; - Self::deposit_event(Event::Reserved(who.clone(), value)); + Self::deposit_event(Event::Reserved { who: who.clone(), amount: value }); Ok(()) } @@ -1772,7 +1811,7 @@ where account.reserved -= actual; // defensive only: this can never fail since total issuance which is at least // free+reserved fits into the same data type. - account.free = account.free.saturating_add(actual); + account.free = account.free.defensive_saturating_add(actual); actual }) { Ok(x) => x, @@ -1784,7 +1823,7 @@ where }, }; - Self::deposit_event(Event::Unreserved(who.clone(), actual.clone())); + Self::deposit_event(Event::Unreserved { who: who.clone(), amount: actual.clone() }); value - actual } @@ -1824,7 +1863,13 @@ where // underflow should never happen, but it if does, there's nothing to be done here. (NegativeImbalance::new(actual), value - actual) }) { - Ok(r) => return r, + Ok((imbalance, not_slashed)) => { + Self::deposit_event(Event::Slashed { + who: who.clone(), + amount: value.saturating_sub(not_slashed), + }); + return (imbalance, not_slashed) + }, Err(_) => (), } } @@ -1879,7 +1924,7 @@ where match reserves.binary_search_by_key(id, |data| data.id) { Ok(index) => { // this add can't overflow but just to be defensive. - reserves[index].amount = reserves[index].amount.saturating_add(value); + reserves[index].amount = reserves[index].amount.defensive_saturating_add(value); }, Err(index) => { reserves @@ -1912,8 +1957,8 @@ where let remain = >::unreserve(who, to_change); - // remain should always be zero but just to be defensive here - let actual = to_change.saturating_sub(remain); + // remain should always be zero but just to be defensive here. + let actual = to_change.defensive_saturating_sub(remain); // `actual <= to_change` and `to_change <= amount`; qed; reserves[index].amount -= actual; @@ -1959,12 +2004,13 @@ where let (imb, remain) = >::slash_reserved(who, to_change); - // remain should always be zero but just to be defensive here - let actual = to_change.saturating_sub(remain); + // remain should always be zero but just to be defensive here. + let actual = to_change.defensive_saturating_sub(remain); // `actual <= to_change` and `to_change <= amount`; qed; reserves[index].amount -= actual; + Self::deposit_event(Event::Slashed { who: who.clone(), amount: actual }); (imb, value - actual) }, Err(_) => (NegativeImbalance::zero(), value), @@ -2018,12 +2064,12 @@ where )?; // remain should always be zero but just to be defensive - // here - let actual = to_change.saturating_sub(remain); + // here. + let actual = to_change.defensive_saturating_sub(remain); // this add can't overflow but just to be defensive. reserves[index].amount = - reserves[index].amount.saturating_add(actual); + reserves[index].amount.defensive_saturating_add(actual); Ok(actual) }, @@ -2038,7 +2084,7 @@ where // remain should always be zero but just to be defensive // here - let actual = to_change.saturating_sub(remain); + let actual = to_change.defensive_saturating_sub(remain); reserves .try_insert( @@ -2061,7 +2107,7 @@ where )?; // remain should always be zero but just to be defensive here - to_change.saturating_sub(remain) + to_change.defensive_saturating_sub(remain) }; // `actual <= to_change` and `to_change <= amount`; qed; diff --git a/frame/balances/src/tests.rs b/frame/balances/src/tests.rs index a08643821eba..8f5470ae3cac 100644 --- a/frame/balances/src/tests.rs +++ b/frame/balances/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -314,6 +314,7 @@ macro_rules! decl_tests { <$ext_builder>::default().monied(true).build().execute_with(|| { assert_eq!(Balances::total_balance(&1), 10); assert_ok!(Balances::deposit_into_existing(&1, 10).map(drop)); + System::assert_last_event(Event::Balances(crate::Event::Deposit { who: 1, amount: 10 })); assert_eq!(Balances::total_balance(&1), 20); assert_eq!(>::get(), 120); }); @@ -341,6 +342,7 @@ macro_rules! decl_tests { fn balance_works() { <$ext_builder>::default().build().execute_with(|| { let _ = Balances::deposit_creating(&1, 42); + System::assert_has_event(Event::Balances(crate::Event::Deposit { who: 1, amount: 42 })); assert_eq!(Balances::free_balance(1), 42); assert_eq!(Balances::reserved_balance(1), 0); assert_eq!(Balances::total_balance(&1), 42); @@ -435,6 +437,19 @@ macro_rules! decl_tests { }); } + #[test] + fn withdrawing_balance_should_work() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&2, 111); + let _ = Balances::withdraw( + &2, 11, WithdrawReasons::TRANSFER, ExistenceRequirement::KeepAlive + ); + System::assert_last_event(Event::Balances(crate::Event::Withdraw { who: 2, amount: 11 })); + assert_eq!(Balances::free_balance(2), 100); + assert_eq!(>::get(), 100); + }); + } + #[test] fn slashing_incomplete_balance_should_work() { <$ext_builder>::default().build().execute_with(|| { @@ -490,7 +505,7 @@ macro_rules! decl_tests { assert_ok!(Balances::reserve(&1, 110)); assert_ok!(Balances::repatriate_reserved(&1, &2, 41, Status::Free), 0); System::assert_last_event( - Event::Balances(crate::Event::ReserveRepatriated(1, 2, 41, Status::Free)) + Event::Balances(crate::Event::ReserveRepatriated { from: 1, to: 2, amount: 41, destination_status: Status::Free }) ); assert_eq!(Balances::reserved_balance(1), 69); assert_eq!(Balances::free_balance(1), 0); @@ -709,18 +724,18 @@ macro_rules! decl_tests { System::set_block_number(2); assert_ok!(Balances::reserve(&1, 10)); - System::assert_last_event(Event::Balances(crate::Event::Reserved(1, 10))); + System::assert_last_event(Event::Balances(crate::Event::Reserved { who: 1, amount: 10 })); System::set_block_number(3); assert!(Balances::unreserve(&1, 5).is_zero()); - System::assert_last_event(Event::Balances(crate::Event::Unreserved(1, 5))); + System::assert_last_event(Event::Balances(crate::Event::Unreserved { who: 1, amount: 5 })); System::set_block_number(4); assert_eq!(Balances::unreserve(&1, 6), 1); // should only unreserve 5 - System::assert_last_event(Event::Balances(crate::Event::Unreserved(1, 5))); + System::assert_last_event(Event::Balances(crate::Event::Unreserved { who: 1, amount: 5 })); }); } @@ -735,9 +750,9 @@ macro_rules! decl_tests { assert_eq!( events(), [ - Event::System(system::Event::NewAccount(1)), - Event::Balances(crate::Event::Endowed(1, 100)), - Event::Balances(crate::Event::BalanceSet(1, 100, 0)), + Event::System(system::Event::NewAccount { account: 1 }), + Event::Balances(crate::Event::Endowed { account: 1, free_balance: 100 }), + Event::Balances(crate::Event::BalanceSet { who: 1, free: 100, reserved: 0 }), ] ); @@ -747,8 +762,9 @@ macro_rules! decl_tests { assert_eq!( events(), [ - Event::System(system::Event::KilledAccount(1)), - Event::Balances(crate::Event::DustLost(1, 99)), + Event::System(system::Event::KilledAccount { account: 1 }), + Event::Balances(crate::Event::DustLost { account: 1, amount: 99 }), + Event::Balances(crate::Event::Slashed { who: 1, amount: 1 }), ] ); }); @@ -765,9 +781,9 @@ macro_rules! decl_tests { assert_eq!( events(), [ - Event::System(system::Event::NewAccount(1)), - Event::Balances(crate::Event::Endowed(1, 100)), - Event::Balances(crate::Event::BalanceSet(1, 100, 0)), + Event::System(system::Event::NewAccount { account: 1 }), + Event::Balances(crate::Event::Endowed { account: 1, free_balance: 100 }), + Event::Balances(crate::Event::BalanceSet { who: 1, free: 100, reserved: 0 }), ] ); @@ -777,7 +793,8 @@ macro_rules! decl_tests { assert_eq!( events(), [ - Event::System(system::Event::KilledAccount(1)) + Event::System(system::Event::KilledAccount { account: 1 }), + Event::Balances(crate::Event::Slashed { who: 1, amount: 100 }), ] ); }); @@ -797,6 +814,7 @@ macro_rules! decl_tests { assert_eq!(Balances::slash(&1, 900), (NegativeImbalance::new(900), 0)); // Account is still alive assert!(System::account_exists(&1)); + System::assert_last_event(Event::Balances(crate::Event::Slashed { who: 1, amount: 900 })); // SCENARIO: Slash will kill account because not enough balance left. assert_ok!(Balances::set_balance(Origin::root(), 1, 1_000, 0)); @@ -1221,5 +1239,29 @@ macro_rules! decl_tests { assert_eq!(Balances::free_balance(&3), 25); }); } + + #[test] + fn set_balance_handles_killing_account() { + <$ext_builder>::default().build().execute_with(|| { + let _ = Balances::deposit_creating(&1, 111); + assert_ok!(frame_system::Pallet::::inc_consumers(&1)); + assert_noop!( + Balances::set_balance(Origin::root(), 1, 0, 0), + DispatchError::ConsumerRemaining, + ); + }); + } + + #[test] + fn set_balance_handles_total_issuance() { + <$ext_builder>::default().build().execute_with(|| { + let old_total_issuance = Balances::total_issuance(); + assert_ok!(Balances::set_balance(Origin::root(), 1337, 69, 42)); + assert_eq!(Balances::total_issuance(), old_total_issuance + 69 + 42); + assert_eq!(Balances::total_balance(&1337), 69 + 42); + assert_eq!(Balances::free_balance(&1337), 69); + assert_eq!(Balances::reserved_balance(&1337), 42); + }); + } } } diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs index f6faebed3931..4a2cc1d91936 100644 --- a/frame/balances/src/tests_composite.rs +++ b/frame/balances/src/tests_composite.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,6 +22,7 @@ use crate::{self as pallet_balances, decl_tests, Config, Pallet}; use frame_support::{ parameter_types, + traits::{ConstU32, ConstU64, ConstU8}, weights::{DispatchInfo, IdentityFee, Weight}, }; use pallet_transaction_payment::CurrencyAdapter; @@ -44,7 +45,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); pub static ExistentialDeposit: u64 = 0; @@ -64,7 +64,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = super::AccountData; @@ -73,21 +73,17 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } -parameter_types! { - pub const TransactionByteFee: u64 = 1; -} + impl pallet_transaction_payment::Config for Test { type OnChargeTransaction = CurrencyAdapter, ()>; - type TransactionByteFee = TransactionByteFee; + type OperationalFeeMultiplier = ConstU8<5>; type WeightToFee = IdentityFee; + type LengthToFee = IdentityFee; type FeeMultiplierUpdate = (); } -parameter_types! { - pub const MaxReserves: u32 = 2; -} - impl Config for Test { type Balance = u64; type DustRemoval = (); @@ -95,7 +91,7 @@ impl Config for Test { type ExistentialDeposit = ExistentialDeposit; type AccountStore = frame_system::Pallet; type MaxLocks = (); - type MaxReserves = MaxReserves; + type MaxReserves = ConstU32<2>; type ReserveIdentifier = [u8; 8]; type WeightInfo = (); } diff --git a/frame/balances/src/tests_local.rs b/frame/balances/src/tests_local.rs index d8c07aa9c42e..cfc7f84ab3a3 100644 --- a/frame/balances/src/tests_local.rs +++ b/frame/balances/src/tests_local.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,7 @@ use crate::{self as pallet_balances, decl_tests, Config, Pallet}; use frame_support::{ parameter_types, - traits::StorageMapShim, + traits::{ConstU32, ConstU64, ConstU8, StorageMapShim}, weights::{DispatchInfo, IdentityFee, Weight}, }; use pallet_transaction_payment::CurrencyAdapter; @@ -46,7 +46,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); pub static ExistentialDeposit: u64 = 0; @@ -66,7 +65,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -75,20 +74,17 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } -parameter_types! { - pub const TransactionByteFee: u64 = 1; -} + impl pallet_transaction_payment::Config for Test { type OnChargeTransaction = CurrencyAdapter, ()>; - type TransactionByteFee = TransactionByteFee; + type OperationalFeeMultiplier = ConstU8<5>; type WeightToFee = IdentityFee; + type LengthToFee = IdentityFee; type FeeMultiplierUpdate = (); } -parameter_types! { - pub const MaxLocks: u32 = 50; - pub const MaxReserves: u32 = 2; -} + impl Config for Test { type Balance = u64; type DustRemoval = (); @@ -96,8 +92,8 @@ impl Config for Test { type ExistentialDeposit = ExistentialDeposit; type AccountStore = StorageMapShim, system::Provider, u64, super::AccountData>; - type MaxLocks = MaxLocks; - type MaxReserves = MaxReserves; + type MaxLocks = ConstU32<50>; + type MaxReserves = ConstU32<2>; type ReserveIdentifier = [u8; 8]; type WeightInfo = (); } @@ -161,9 +157,9 @@ fn emit_events_with_no_existential_deposit_suicide_with_dust() { assert_eq!( events(), [ - Event::System(system::Event::NewAccount(1)), - Event::Balances(crate::Event::Endowed(1, 100)), - Event::Balances(crate::Event::BalanceSet(1, 100, 0)), + Event::System(system::Event::NewAccount { account: 1 }), + Event::Balances(crate::Event::Endowed { account: 1, free_balance: 100 }), + Event::Balances(crate::Event::BalanceSet { who: 1, free: 100, reserved: 0 }), ] ); @@ -171,7 +167,7 @@ fn emit_events_with_no_existential_deposit_suicide_with_dust() { assert_eq!(res, (NegativeImbalance::new(98), 0)); // no events - assert_eq!(events(), []); + assert_eq!(events(), [Event::Balances(crate::Event::Slashed { who: 1, amount: 98 })]); let res = Balances::slash(&1, 1); assert_eq!(res, (NegativeImbalance::new(1), 0)); @@ -179,8 +175,9 @@ fn emit_events_with_no_existential_deposit_suicide_with_dust() { assert_eq!( events(), [ - Event::System(system::Event::KilledAccount(1)), - Event::Balances(crate::Event::DustLost(1, 1)), + Event::System(system::Event::KilledAccount { account: 1 }), + Event::Balances(crate::Event::DustLost { account: 1, amount: 1 }), + Event::Balances(crate::Event::Slashed { who: 1, amount: 1 }) ] ); }); diff --git a/frame/balances/src/tests_reentrancy.rs b/frame/balances/src/tests_reentrancy.rs index 9c7ba3e1ec82..7037e9615afd 100644 --- a/frame/balances/src/tests_reentrancy.rs +++ b/frame/balances/src/tests_reentrancy.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2020 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,11 @@ #![cfg(test)] use crate::{self as pallet_balances, Config, Pallet}; -use frame_support::{parameter_types, traits::StorageMapShim, weights::IdentityFee}; +use frame_support::{ + parameter_types, + traits::{ConstU32, ConstU64, ConstU8, StorageMapShim}, + weights::IdentityFee, +}; use pallet_transaction_payment::CurrencyAdapter; use sp_core::H256; use sp_io; @@ -48,7 +52,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); pub static ExistentialDeposit: u64 = 0; @@ -68,7 +71,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -77,14 +80,14 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } -parameter_types! { - pub const TransactionByteFee: u64 = 1; -} + impl pallet_transaction_payment::Config for Test { type OnChargeTransaction = CurrencyAdapter, ()>; - type TransactionByteFee = TransactionByteFee; + type OperationalFeeMultiplier = ConstU8<5>; type WeightToFee = IdentityFee; + type LengthToFee = IdentityFee; type FeeMultiplierUpdate = (); } @@ -94,10 +97,7 @@ impl OnUnbalanced> for OnDustRemoval { assert_ok!(Balances::resolve_into_existing(&1, amount)); } } -parameter_types! { - pub const MaxLocks: u32 = 50; - pub const MaxReserves: u32 = 2; -} + impl Config for Test { type Balance = u64; type DustRemoval = OnDustRemoval; @@ -105,8 +105,8 @@ impl Config for Test { type ExistentialDeposit = ExistentialDeposit; type AccountStore = StorageMapShim, system::Provider, u64, super::AccountData>; - type MaxLocks = MaxLocks; - type MaxReserves = MaxReserves; + type MaxLocks = ConstU32<50>; + type MaxReserves = ConstU32<2>; type ReserveIdentifier = [u8; 8]; type WeightInfo = (); } @@ -165,11 +165,18 @@ fn transfer_dust_removal_tst1_should_work() { assert_eq!(Balances::free_balance(&1), 1050); // Verify the events - // Number of events expected is 8 - assert_eq!(System::events().len(), 11); - - System::assert_has_event(Event::Balances(crate::Event::Transfer(2, 3, 450))); - System::assert_has_event(Event::Balances(crate::Event::DustLost(2, 50))); + assert_eq!(System::events().len(), 12); + + System::assert_has_event(Event::Balances(crate::Event::Transfer { + from: 2, + to: 3, + amount: 450, + })); + System::assert_has_event(Event::Balances(crate::Event::DustLost { + account: 2, + amount: 50, + })); + System::assert_has_event(Event::Balances(crate::Event::Deposit { who: 1, amount: 50 })); }); } @@ -193,11 +200,18 @@ fn transfer_dust_removal_tst2_should_work() { assert_eq!(Balances::free_balance(&1), 1500); // Verify the events - // Number of events expected is 8 - assert_eq!(System::events().len(), 9); + assert_eq!(System::events().len(), 10); - System::assert_has_event(Event::Balances(crate::Event::Transfer(2, 1, 450))); - System::assert_has_event(Event::Balances(crate::Event::DustLost(2, 50))); + System::assert_has_event(Event::Balances(crate::Event::Transfer { + from: 2, + to: 1, + amount: 450, + })); + System::assert_has_event(Event::Balances(crate::Event::DustLost { + account: 2, + amount: 50, + })); + System::assert_has_event(Event::Balances(crate::Event::Deposit { who: 1, amount: 50 })); }); } @@ -230,16 +244,20 @@ fn repatriating_reserved_balance_dust_removal_should_work() { assert_eq!(Balances::free_balance(1), 1500); // Verify the events - // Number of events expected is 10 - assert_eq!(System::events().len(), 10); + assert_eq!(System::events().len(), 11); + + System::assert_has_event(Event::Balances(crate::Event::ReserveRepatriated { + from: 2, + to: 1, + amount: 450, + destination_status: Status::Free, + })); - System::assert_has_event(Event::Balances(crate::Event::ReserveRepatriated( - 2, - 1, - 450, - Status::Free, - ))); + System::assert_has_event(Event::Balances(crate::Event::DustLost { + account: 2, + amount: 50, + })); - System::assert_last_event(Event::Balances(crate::Event::DustLost(2, 50))); + System::assert_last_event(Event::Balances(crate::Event::Deposit { who: 1, amount: 50 })); }); } diff --git a/frame/balances/src/weights.rs b/frame/balances/src/weights.rs index 6f333bfc0500..435438055b33 100644 --- a/frame/balances/src/weights.rs +++ b/frame/balances/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_balances //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-09-13, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-30, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/balances/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -59,43 +60,43 @@ pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { // Storage: System Account (r:1 w:1) fn transfer() -> Weight { - (70_952_000 as Weight) + (34_200_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: System Account (r:1 w:1) fn transfer_keep_alive() -> Weight { - (54_410_000 as Weight) + (27_263_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: System Account (r:1 w:1) fn set_balance_creating() -> Weight { - (29_176_000 as Weight) + (17_425_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: System Account (r:1 w:1) fn set_balance_killing() -> Weight { - (35_214_000 as Weight) + (19_979_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: System Account (r:2 w:2) fn force_transfer() -> Weight { - (71_780_000 as Weight) + (34_783_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: System Account (r:1 w:1) fn transfer_all() -> Weight { - (66_475_000 as Weight) + (31_620_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: System Account (r:1 w:1) fn force_unreserve() -> Weight { - (27_766_000 as Weight) + (15_750_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -105,43 +106,43 @@ impl WeightInfo for SubstrateWeight { impl WeightInfo for () { // Storage: System Account (r:1 w:1) fn transfer() -> Weight { - (70_952_000 as Weight) + (34_200_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: System Account (r:1 w:1) fn transfer_keep_alive() -> Weight { - (54_410_000 as Weight) + (27_263_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: System Account (r:1 w:1) fn set_balance_creating() -> Weight { - (29_176_000 as Weight) + (17_425_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: System Account (r:1 w:1) fn set_balance_killing() -> Weight { - (35_214_000 as Weight) + (19_979_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: System Account (r:2 w:2) fn force_transfer() -> Weight { - (71_780_000 as Weight) + (34_783_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: System Account (r:1 w:1) fn transfer_all() -> Weight { - (66_475_000 as Weight) + (31_620_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: System Account (r:1 w:1) fn force_unreserve() -> Weight { - (27_766_000 as Weight) + (15_750_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } diff --git a/frame/beefy-mmr/Cargo.toml b/frame/beefy-mmr/Cargo.toml index 3d4a9a72ddf8..19703fa79863 100644 --- a/frame/beefy-mmr/Cargo.toml +++ b/frame/beefy-mmr/Cargo.toml @@ -2,17 +2,18 @@ name = "pallet-beefy-mmr" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" description = "BEEFY + MMR runtime utilities" +repository = "https://github.com/paritytech/substrate" [dependencies] hex = { version = "0.4", optional = true } -codec = { version = "2.2.0", package = "parity-scale-codec", default-features = false, features = ["derive"] } -libsecp256k1 = { version = "0.7.0", default-features = false } +codec = { version = "3.0.0", package = "parity-scale-codec", default-features = false, features = ["derive"] } +k256 = { version = "0.10.2", default-features = false, features = ["arithmetic"] } log = { version = "0.4.13", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.130", optional = true } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +serde = { version = "1.0.136", optional = true } frame-support = { version = "4.0.0-dev", path = "../support", default-features = false } frame-system = { version = "4.0.0-dev", path = "../system", default-features = false } @@ -20,10 +21,10 @@ pallet-mmr = { version = "4.0.0-dev", path = "../merkle-mountain-range", default pallet-mmr-primitives = { version = "4.0.0-dev", path = "../merkle-mountain-range/primitives", default-features = false } pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core", default-features = false } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io", default-features = false } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime", default-features = false } -sp-std = { version = "4.0.0-dev", path = "../../primitives/std", default-features = false } +sp-core = { version = "6.0.0", path = "../../primitives/core", default-features = false } +sp-io = { version = "6.0.0", path = "../../primitives/io", default-features = false } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime", default-features = false } +sp-std = { version = "4.0.0", path = "../../primitives/std", default-features = false } beefy-merkle-tree = { version = "4.0.0-dev", path = "./primitives", default-features = false } beefy-primitives = { version = "4.0.0-dev", path = "../../primitives/beefy", default-features = false } @@ -42,7 +43,7 @@ std = [ "frame-support/std", "frame-system/std", "hex", - "libsecp256k1/std", + "k256/std", "log/std", "pallet-beefy/std", "pallet-mmr-primitives/std", diff --git a/frame/beefy-mmr/primitives/Cargo.toml b/frame/beefy-mmr/primitives/Cargo.toml index d5dcc0eed335..b54ac225e781 100644 --- a/frame/beefy-mmr/primitives/Cargo.toml +++ b/frame/beefy-mmr/primitives/Cargo.toml @@ -2,8 +2,9 @@ name = "beefy-merkle-tree" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" +repository = "https://github.com/paritytech/substrate" description = "A no-std/Substrate compatible library to construct binary merkle tree." [dependencies] @@ -17,7 +18,7 @@ hex = "0.4" hex-literal = "0.3" [features] -debug = ["hex", "log"] +debug = ["hex", "hex/std", "log"] default = ["std", "debug", "keccak"] keccak = ["tiny-keccak"] std = [] diff --git a/frame/beefy-mmr/primitives/src/lib.rs b/frame/beefy-mmr/primitives/src/lib.rs index 4d4d4e8721ac..04fa11760765 100644 --- a/frame/beefy-mmr/primitives/src/lib.rs +++ b/frame/beefy-mmr/primitives/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -44,7 +44,8 @@ pub type Hash = [u8; 32]; /// Generic hasher trait. /// /// Implement the function to support custom way of hashing data. -/// The implementation must return a [Hash] type, so only 32-byte output hashes are supported. +/// The implementation must return a [Hash](type@Hash) type, so only 32-byte output hashes are +/// supported. pub trait Hasher { /// Hash given arbitrary-length piece of data. fn hash(data: &[u8]) -> Hash; @@ -173,7 +174,7 @@ impl Visitor for () { /// /// # Panic /// -/// The function will panic if given [`leaf_index`] is greater than the number of leaves. +/// The function will panic if given `leaf_index` is greater than the number of leaves. pub fn merkle_proof(leaves: I, leaf_index: usize) -> MerkleProof where H: Hasher, diff --git a/frame/beefy-mmr/src/lib.rs b/frame/beefy-mmr/src/lib.rs index 001831639b16..9ee7bd770f64 100644 --- a/frame/beefy-mmr/src/lib.rs +++ b/frame/beefy-mmr/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,17 +29,16 @@ //! The MMR leaf contains: //! 1. Block number and parent block hash. //! 2. Merkle Tree Root Hash of next BEEFY validator set. -//! 3. Merkle Tree Root Hash of current parachain heads state. +//! 3. Arbitrary extra leaf data to be used by downstream pallets to include custom data. //! //! and thanks to versioning can be easily updated in the future. -use sp_runtime::traits::{Convert, Hash}; +use sp_runtime::traits::{Convert, Hash, Member}; use sp_std::prelude::*; -use beefy_primitives::mmr::{BeefyNextAuthoritySet, MmrLeaf, MmrLeafVersion}; +use beefy_primitives::mmr::{BeefyDataProvider, BeefyNextAuthoritySet, MmrLeaf, MmrLeafVersion}; use pallet_mmr::primitives::LeafDataProvider; -use codec::Encode; use frame_support::traits::Get; pub use pallet::*; @@ -72,42 +71,24 @@ where pub struct BeefyEcdsaToEthereum; impl Convert> for BeefyEcdsaToEthereum { fn convert(a: beefy_primitives::crypto::AuthorityId) -> Vec { - use sp_core::crypto::Public; - let compressed_key = a.as_slice(); - - libsecp256k1::PublicKey::parse_slice( - compressed_key, - Some(libsecp256k1::PublicKeyFormat::Compressed), - ) - // uncompress the key - .map(|pub_key| pub_key.serialize().to_vec()) - // now convert to ETH address - .map(|uncompressed| sp_io::hashing::keccak_256(&uncompressed[1..])[12..].to_vec()) - .map_err(|_| { - log::error!(target: "runtime::beefy", "Invalid BEEFY PublicKey format!"); - }) - .unwrap_or_default() + use k256::{elliptic_curve::sec1::ToEncodedPoint, PublicKey}; + use sp_core::crypto::ByteArray; + + PublicKey::from_sec1_bytes(a.as_slice()) + .map(|pub_key| { + // uncompress the key + let uncompressed = pub_key.to_encoded_point(false); + // convert to ETH address + sp_io::hashing::keccak_256(&uncompressed.as_bytes()[1..])[12..].to_vec() + }) + .map_err(|_| { + log::error!(target: "runtime::beefy", "Invalid BEEFY PublicKey format!"); + }) + .unwrap_or_default() } } type MerkleRootOf = ::Hash; -type ParaId = u32; -type ParaHead = Vec; - -/// A type that is able to return current list of parachain heads that end up in the MMR leaf. -pub trait ParachainHeadsProvider { - /// Return a list of tuples containing a `ParaId` and Parachain Header data (ParaHead). - /// - /// The returned data does not have to be sorted. - fn parachain_heads() -> Vec<(ParaId, ParaHead)>; -} - -/// A default implementation for runtimes without parachains. -impl ParachainHeadsProvider for () { - fn parachain_heads() -> Vec<(ParaId, ParaHead)> { - Default::default() - } -} #[frame_support::pallet] pub mod pallet { @@ -139,17 +120,16 @@ pub mod pallet { /// efficiency reasons. type BeefyAuthorityToMerkleLeaf: Convert<::BeefyId, Vec>; - /// Retrieve a list of current parachain heads. - /// - /// The trait is implemented for `paras` module, but since not all chains might have - /// parachains, and we want to keep the MMR leaf structure uniform, it's possible to use - /// `()` as well to simply put dummy data to the leaf. - type ParachainHeads: ParachainHeadsProvider; + /// The type expected for the leaf extra data + type LeafExtra: Member + codec::FullCodec; + + /// Retrieve arbitrary data that should be added to the mmr leaf + type BeefyDataProvider: BeefyDataProvider; } /// Details of next BEEFY authority set. /// - /// This storage entry is used as cache for calls to [`update_beefy_next_authority_set`]. + /// This storage entry is used as cache for calls to `update_beefy_next_authority_set`. #[pallet::storage] #[pallet::getter(fn beefy_next_authorities)] pub type BeefyNextAuthorities = @@ -164,13 +144,14 @@ where ::BlockNumber, ::Hash, MerkleRootOf, + T::LeafExtra, >; fn leaf_data() -> Self::LeafData { MmrLeaf { version: T::LeafVersion::get(), parent_number_and_hash: frame_system::Pallet::::leaf_data(), - parachain_heads: Pallet::::parachain_heads_merkle_root(), + leaf_extra: T::BeefyDataProvider::extra_data(), beefy_next_authority_set: Pallet::::update_beefy_next_authority_set(), } } @@ -189,23 +170,6 @@ impl Pallet where MerkleRootOf: From + Into, { - /// Returns latest root hash of a merkle tree constructed from all active parachain headers. - /// - /// The leafs are sorted by `ParaId` to allow more efficient lookups and non-existence proofs. - /// - /// NOTE this does not include parathreads - only parachains are part of the merkle tree. - /// - /// NOTE This is an initial and inefficient implementation, which re-constructs - /// the merkle tree every block. Instead we should update the merkle root in - /// [Self::on_initialize] call of this pallet and update the merkle tree efficiently (use - /// on-chain storage to persist inner nodes). - fn parachain_heads_merkle_root() -> MerkleRootOf { - let mut para_heads = T::ParachainHeads::parachain_heads(); - para_heads.sort(); - let para_heads = para_heads.into_iter().map(|pair| pair.encode()); - beefy_merkle_tree::merkle_root::(para_heads).into() - } - /// Returns details of the next BEEFY authority set. /// /// Details contain authority set id, authority set length and a merkle root, diff --git a/frame/beefy-mmr/src/mock.rs b/frame/beefy-mmr/src/mock.rs index 4c9e103eb7b8..f6a35f68a4a1 100644 --- a/frame/beefy-mmr/src/mock.rs +++ b/frame/beefy-mmr/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,8 +18,11 @@ use std::vec; use beefy_primitives::mmr::MmrLeafVersion; +use codec::Encode; use frame_support::{ - construct_runtime, parameter_types, sp_io::TestExternalities, traits::GenesisBuild, + construct_runtime, parameter_types, + sp_io::TestExternalities, + traits::{ConstU16, ConstU32, ConstU64, GenesisBuild}, BasicExternalities, }; use sp_core::{Hasher, H256}; @@ -28,12 +31,13 @@ use sp_runtime::{ impl_opaque_keys, testing::Header, traits::{BlakeTwo256, ConvertInto, IdentityLookup, Keccak256, OpaqueKeys}, - Perbill, }; use crate as pallet_beefy_mmr; -pub use beefy_primitives::{crypto::AuthorityId as BeefyId, ConsensusLog, BEEFY_ENGINE_ID}; +pub use beefy_primitives::{ + crypto::AuthorityId as BeefyId, mmr::BeefyDataProvider, ConsensusLog, BEEFY_ENGINE_ID, +}; impl_opaque_keys! { pub struct MockSessionKeys { @@ -58,11 +62,6 @@ construct_runtime!( } ); -parameter_types! { - pub const BlockHashCount: u64 = 250; - pub const SS58Prefix: u8 = 42; -} - impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; type BlockWeights = (); @@ -78,33 +77,27 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); type OnNewAccount = (); type OnKilledAccount = (); type SystemWeightInfo = (); - type SS58Prefix = SS58Prefix; + type SS58Prefix = ConstU16<42>; type OnSetCode = (); -} - -parameter_types! { - pub const Period: u64 = 1; - pub const Offset: u64 = 0; - pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(33); + type MaxConsumers = ConstU32<16>; } impl pallet_session::Config for Test { type Event = Event; type ValidatorId = u64; type ValidatorIdOf = ConvertInto; - type ShouldEndSession = pallet_session::PeriodicSessions; - type NextSessionRotation = pallet_session::PeriodicSessions; + type ShouldEndSession = pallet_session::PeriodicSessions, ConstU64<0>>; + type NextSessionRotation = pallet_session::PeriodicSessions, ConstU64<0>>; type SessionManager = MockSessionManager; type SessionHandler = ::KeyTypeIdProviders; type Keys = MockSessionKeys; - type DisabledValidatorsThreshold = DisabledValidatorsThreshold; type WeightInfo = (); } @@ -112,6 +105,7 @@ pub type MmrLeaf = beefy_primitives::mmr::MmrLeaf< ::BlockNumber, ::Hash, ::Hash, + Vec, >; impl pallet_mmr::Config for Test { @@ -141,13 +135,20 @@ impl pallet_beefy_mmr::Config for Test { type BeefyAuthorityToMerkleLeaf = pallet_beefy_mmr::BeefyEcdsaToEthereum; - type ParachainHeads = DummyParaHeads; + type LeafExtra = Vec; + + type BeefyDataProvider = DummyDataProvider; } -pub struct DummyParaHeads; -impl pallet_beefy_mmr::ParachainHeadsProvider for DummyParaHeads { - fn parachain_heads() -> Vec<(pallet_beefy_mmr::ParaId, pallet_beefy_mmr::ParaHead)> { - vec![(15, vec![1, 2, 3]), (5, vec![4, 5, 6])] +pub struct DummyDataProvider; +impl BeefyDataProvider> for DummyDataProvider { + fn extra_data() -> Vec { + let mut col = vec![(15, vec![1, 2, 3]), (5, vec![4, 5, 6])]; + col.sort(); + beefy_merkle_tree::merkle_root::, _, _>( + col.into_iter().map(|pair| pair.encode()), + ) + .to_vec() } } @@ -168,9 +169,12 @@ impl pallet_session::SessionManager for MockSessionManager { // Note, that we can't use `UintAuthorityId` here. Reason is that the implementation // of `to_public_key()` assumes, that a public key is 32 bytes long. This is true for -// ed25519 and sr25519 but *not* for ecdsa. An ecdsa public key is 33 bytes. +// ed25519 and sr25519 but *not* for ecdsa. A compressed ecdsa public key is 33 bytes, +// with the first one containing information to reconstruct the uncompressed key. pub fn mock_beefy_id(id: u8) -> BeefyId { - let buf: [u8; 33] = [id; 33]; + let mut buf: [u8; 33] = [id; 33]; + // Set to something valid. + buf[0] = 0x02; let pk = Public::from_raw(buf); BeefyId::from(pk) } diff --git a/frame/beefy-mmr/src/tests.rs b/frame/beefy-mmr/src/tests.rs index 7c70766623b4..fd383adb1d4a 100644 --- a/frame/beefy-mmr/src/tests.rs +++ b/frame/beefy-mmr/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,7 +40,7 @@ fn init_block(block: u64) { BeefyMmr::on_initialize(block); } -pub fn beefy_log(log: ConsensusLog) -> DigestItem { +pub fn beefy_log(log: ConsensusLog) -> DigestItem { DigestItem::Consensus(BEEFY_ENGINE_ID, log.encode()) } @@ -71,7 +71,7 @@ fn should_contain_mmr_digest() { assert_eq!( System::digest().logs, vec![beefy_log(ConsensusLog::MmrRoot( - hex!("f3e3afbfa69e89cd1e99f8d3570155962f3346d1d8758dc079be49ef70387758").into() + hex!("fa0275b19b2565089f7e2377ee73b9050e8d53bce108ef722a3251fd9d371d4b").into() ))] ); @@ -82,14 +82,13 @@ fn should_contain_mmr_digest() { System::digest().logs, vec![ beefy_log(ConsensusLog::MmrRoot( - hex!("f3e3afbfa69e89cd1e99f8d3570155962f3346d1d8758dc079be49ef70387758").into() + hex!("fa0275b19b2565089f7e2377ee73b9050e8d53bce108ef722a3251fd9d371d4b").into() + )), + beefy_log(ConsensusLog::AuthoritiesChange( + ValidatorSet::new(vec![mock_beefy_id(3), mock_beefy_id(4),], 1,).unwrap() )), - beefy_log(ConsensusLog::AuthoritiesChange(ValidatorSet { - validators: vec![mock_beefy_id(3), mock_beefy_id(4),], - id: 1, - })), beefy_log(ConsensusLog::MmrRoot( - hex!("7d4ae4524bae75d52b63f08eab173b0c263eb95ae2c55c3a1d871241bd0cc559").into() + hex!("85554fa7d4e863cce3cdce668c1ae82c0174ad37f8d1399284018bec9f9971c3").into() )), ] ); @@ -112,13 +111,11 @@ fn should_contain_valid_leaf_data() { beefy_next_authority_set: BeefyNextAuthoritySet { id: 1, len: 2, - root: hex!("01b1a742589773fc054c8f5021a456316ffcec0370b25678b0696e116d1ef9ae") + root: hex!("176e73f1bf656478b728e28dd1a7733c98621b8acf830bff585949763dca7a96") .into(), }, - parachain_heads: hex!( - "ed893c8f8cc87195a5d4d2805b011506322036bcace79642aa3e94ab431e442e" - ) - .into(), + leaf_extra: hex!("55b8e9e1cc9f0db7776fac0ca66318ef8acfb8ec26db11e373120583e07ee648") + .to_vec(), } ); @@ -139,10 +136,8 @@ fn should_contain_valid_leaf_data() { root: hex!("9c6b2c1b0d0b25a008e6c882cc7b415f309965c72ad2b944ac0931048ca31cd5") .into(), }, - parachain_heads: hex!( - "ed893c8f8cc87195a5d4d2805b011506322036bcace79642aa3e94ab431e442e" - ) - .into(), + leaf_extra: hex!("55b8e9e1cc9f0db7776fac0ca66318ef8acfb8ec26db11e373120583e07ee648") + .to_vec() } ); } diff --git a/frame/beefy/Cargo.toml b/frame/beefy/Cargo.toml index e5af666e7ca5..6ed9e7375bfe 100644 --- a/frame/beefy/Cargo.toml +++ b/frame/beefy/Cargo.toml @@ -2,27 +2,29 @@ name = "pallet-beefy" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" +repository = "https://github.com/paritytech/substrate" +description = "BEEFY FRAME pallet" [dependencies] -codec = { version = "2.2.0", package = "parity-scale-codec", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.130", optional = true } +codec = { version = "3.0.0", package = "parity-scale-codec", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +serde = { version = "1.0.136", optional = true } frame-support = { version = "4.0.0-dev", path = "../support", default-features = false } frame-system = { version = "4.0.0-dev", path = "../system", default-features = false } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime", default-features = false } -sp-std = { version = "4.0.0-dev", path = "../../primitives/std", default-features = false } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime", default-features = false } +sp-std = { version = "4.0.0", path = "../../primitives/std", default-features = false } pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false } beefy-primitives = { version = "4.0.0-dev", path = "../../primitives/beefy", default-features = false } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } sp-staking = { version = "4.0.0-dev", path = "../../primitives/staking" } [features] diff --git a/frame/beefy/src/lib.rs b/frame/beefy/src/lib.rs index 32f313337343..744a06561e8c 100644 --- a/frame/beefy/src/lib.rs +++ b/frame/beefy/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -47,10 +47,11 @@ pub mod pallet { #[pallet::config] pub trait Config: frame_system::Config { /// Authority identifier type - type BeefyId: Member + Parameter + RuntimeAppPublic + Default + MaybeSerializeDeserialize; + type BeefyId: Member + Parameter + RuntimeAppPublic + MaybeSerializeDeserialize; } #[pallet::pallet] + #[pallet::without_storage_info] pub struct Pallet(PhantomData); #[pallet::hooks] @@ -97,23 +98,25 @@ pub mod pallet { impl Pallet { /// Return the current active BEEFY validator set. - pub fn validator_set() -> ValidatorSet { - ValidatorSet:: { validators: Self::authorities(), id: Self::validator_set_id() } + pub fn validator_set() -> Option> { + let validators: Vec = Self::authorities(); + let id: beefy_primitives::ValidatorSetId = Self::validator_set_id(); + ValidatorSet::::new(validators, id) } fn change_authorities(new: Vec, queued: Vec) { - // As in GRANDPA, we trigger a validator set change only if the the validator - // set has actually changed. - if new != Self::authorities() { - >::put(&new); - - let next_id = Self::validator_set_id() + 1u64; - >::put(next_id); - - let log: DigestItem = DigestItem::Consensus( + // Always issue a change if `session` says that the validators have changed. + // Even if their session keys are the same as before, the underlying economic + // identities have changed. Furthermore, the digest below is used to signal + // BEEFY mandatory blocks. + >::put(&new); + + let next_id = Self::validator_set_id() + 1u64; + >::put(next_id); + if let Some(validator_set) = ValidatorSet::::new(new, next_id) { + let log = DigestItem::Consensus( BEEFY_ENGINE_ID, - ConsensusLog::AuthoritiesChange(ValidatorSet { validators: new, id: next_id }) - .encode(), + ConsensusLog::AuthoritiesChange(validator_set).encode(), ); >::deposit_log(log); } @@ -162,8 +165,8 @@ impl OneSessionHandler for Pallet { } } - fn on_disabled(i: usize) { - let log: DigestItem = DigestItem::Consensus( + fn on_disabled(i: u32) { + let log = DigestItem::Consensus( BEEFY_ENGINE_ID, ConsensusLog::::OnDisabled(i as AuthorityIndex).encode(), ); diff --git a/frame/beefy/src/mock.rs b/frame/beefy/src/mock.rs index baa2fae746fe..5fc04f7cbd1d 100644 --- a/frame/beefy/src/mock.rs +++ b/frame/beefy/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,9 @@ use std::vec; use frame_support::{ - construct_runtime, parameter_types, sp_io::TestExternalities, traits::GenesisBuild, + construct_runtime, parameter_types, + sp_io::TestExternalities, + traits::{ConstU16, ConstU32, ConstU64, GenesisBuild}, BasicExternalities, }; use sp_core::H256; @@ -55,11 +57,6 @@ construct_runtime!( } ); -parameter_types! { - pub const BlockHashCount: u64 = 250; - pub const SS58Prefix: u8 = 42; -} - impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; type BlockWeights = (); @@ -75,15 +72,16 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); type OnNewAccount = (); type OnKilledAccount = (); type SystemWeightInfo = (); - type SS58Prefix = SS58Prefix; + type SS58Prefix = ConstU16<42>; type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl pallet_beefy::Config for Test { @@ -91,8 +89,6 @@ impl pallet_beefy::Config for Test { } parameter_types! { - pub const Period: u64 = 1; - pub const Offset: u64 = 0; pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(33); } @@ -100,12 +96,11 @@ impl pallet_session::Config for Test { type Event = Event; type ValidatorId = u64; type ValidatorIdOf = ConvertInto; - type ShouldEndSession = pallet_session::PeriodicSessions; - type NextSessionRotation = pallet_session::PeriodicSessions; + type ShouldEndSession = pallet_session::PeriodicSessions, ConstU64<0>>; + type NextSessionRotation = pallet_session::PeriodicSessions, ConstU64<0>>; type SessionManager = MockSessionManager; type SessionHandler = ::KeyTypeIdProviders; type Keys = MockSessionKeys; - type DisabledValidatorsThreshold = DisabledValidatorsThreshold; type WeightInfo = (); } @@ -127,9 +122,12 @@ impl pallet_session::SessionManager for MockSessionManager { // Note, that we can't use `UintAuthorityId` here. Reason is that the implementation // of `to_public_key()` assumes, that a public key is 32 bytes long. This is true for -// ed25519 and sr25519 but *not* for ecdsa. An ecdsa public key is 33 bytes. +// ed25519 and sr25519 but *not* for ecdsa. A compressed ecdsa public key is 33 bytes, +// with the first one containing information to reconstruct the uncompressed key. pub fn mock_beefy_id(id: u8) -> BeefyId { - let buf: [u8; 33] = [id; 33]; + let mut buf: [u8; 33] = [id; 33]; + // Set to something valid. + buf[0] = 0x02; let pk = Public::from_raw(buf); BeefyId::from(pk) } diff --git a/frame/beefy/src/tests.rs b/frame/beefy/src/tests.rs index 24f9acaf76bf..7acb40f200df 100644 --- a/frame/beefy/src/tests.rs +++ b/frame/beefy/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,6 @@ use std::vec; use beefy_primitives::ValidatorSet; use codec::Encode; -use sp_core::H256; use sp_runtime::DigestItem; use frame_support::traits::OnInitialize; @@ -32,7 +31,7 @@ fn init_block(block: u64) { Session::on_initialize(block); } -pub fn beefy_log(log: ConsensusLog) -> DigestItem { +pub fn beefy_log(log: ConsensusLog) -> DigestItem { DigestItem::Consensus(BEEFY_ENGINE_ID, log.encode()) } @@ -71,10 +70,9 @@ fn session_change_updates_authorities() { assert!(1 == Beefy::validator_set_id()); - let want = beefy_log(ConsensusLog::AuthoritiesChange(ValidatorSet { - validators: vec![mock_beefy_id(3), mock_beefy_id(4)], - id: 1, - })); + let want = beefy_log(ConsensusLog::AuthoritiesChange( + ValidatorSet::new(vec![mock_beefy_id(3), mock_beefy_id(4)], 1).unwrap(), + )); let log = System::digest().logs[0].clone(); @@ -110,11 +108,11 @@ fn validator_set_at_genesis() { let want = vec![mock_beefy_id(1), mock_beefy_id(2)]; new_test_ext(vec![1, 2, 3, 4]).execute_with(|| { - let vs = Beefy::validator_set(); + let vs = Beefy::validator_set().unwrap(); - assert_eq!(vs.id, 0u64); - assert_eq!(vs.validators[0], want[0]); - assert_eq!(vs.validators[1], want[1]); + assert_eq!(vs.id(), 0u64); + assert_eq!(vs.validators()[0], want[0]); + assert_eq!(vs.validators()[1], want[1]); }); } @@ -125,18 +123,18 @@ fn validator_set_updates_work() { new_test_ext(vec![1, 2, 3, 4]).execute_with(|| { init_block(1); - let vs = Beefy::validator_set(); + let vs = Beefy::validator_set().unwrap(); - assert_eq!(vs.id, 0u64); - assert_eq!(want[0], vs.validators[0]); - assert_eq!(want[1], vs.validators[1]); + assert_eq!(vs.id(), 0u64); + assert_eq!(want[0], vs.validators()[0]); + assert_eq!(want[1], vs.validators()[1]); init_block(2); - let vs = Beefy::validator_set(); + let vs = Beefy::validator_set().unwrap(); - assert_eq!(vs.id, 1u64); - assert_eq!(want[2], vs.validators[0]); - assert_eq!(want[3], vs.validators[1]); + assert_eq!(vs.id(), 1u64); + assert_eq!(want[2], vs.validators()[0]); + assert_eq!(want[3], vs.validators()[1]); }); } diff --git a/frame/benchmarking/Cargo.toml b/frame/benchmarking/Cargo.toml index ea690d966c97..ce301ff9034a 100644 --- a/frame/benchmarking/Cargo.toml +++ b/frame/benchmarking/Cargo.toml @@ -2,9 +2,9 @@ name = "frame-benchmarking" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Macro for benchmarking a FRAME runtime." readme = "README.md" @@ -13,28 +13,32 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -linregress = { version = "0.4.3", optional = true } +linregress = { version = "0.4.4", optional = true } paste = "1.0" -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } sp-api = { version = "4.0.0-dev", path = "../../primitives/api", default-features = false } -sp-runtime-interface = { version = "4.0.0-dev", path = "../../primitives/runtime-interface", default-features = false } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime", default-features = false } -sp-std = { version = "4.0.0-dev", path = "../../primitives/std", default-features = false } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io", default-features = false } -sp-storage = { version = "4.0.0-dev", path = "../../primitives/storage", default-features = false } +sp-runtime-interface = { version = "6.0.0", path = "../../primitives/runtime-interface", default-features = false } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime", default-features = false } +sp-std = { version = "4.0.0", path = "../../primitives/std", default-features = false } +sp-io = { version = "6.0.0", path = "../../primitives/io", default-features = false } +sp-application-crypto = { version = "6.0.0", path = "../../primitives/application-crypto", default-features = false } +sp-storage = { version = "6.0.0", path = "../../primitives/storage", default-features = false } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } log = { version = "0.4.14", default-features = false } +serde = { version = "1.0.136", optional = true } [dev-dependencies] -hex-literal = "0.3.1" +hex-literal = "0.3.4" +sp-keystore = { version = "0.12.0", path = "../../primitives/keystore" } [features] default = ["std"] std = [ "codec/std", "scale-info/std", + "serde", "sp-runtime-interface/std", "sp-runtime/std", "sp-api/std", @@ -44,3 +48,4 @@ std = [ "linregress", "log/std", ] +runtime-benchmarks = [] diff --git a/frame/benchmarking/src/analysis.rs b/frame/benchmarking/src/analysis.rs index 2bb20ebe2e7f..52baec80e62e 100644 --- a/frame/benchmarking/src/analysis.rs +++ b/frame/benchmarking/src/analysis.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,6 @@ //! Tools for analyzing the benchmark results. use crate::BenchmarkResult; -use core::convert::TryFrom; use linregress::{FormulaRegressionBuilder, RegressionDataBuilder}; use std::collections::BTreeMap; @@ -200,7 +199,7 @@ impl Analysis { } pub fn min_squares_iqr(r: &Vec, selector: BenchmarkSelector) -> Option { - if r[0].components.is_empty() { + if r[0].components.is_empty() || r.len() <= 2 { return Self::median_value(r, selector) } diff --git a/frame/benchmarking/src/baseline.rs b/frame/benchmarking/src/baseline.rs new file mode 100644 index 000000000000..1ceb9a4f8904 --- /dev/null +++ b/frame/benchmarking/src/baseline.rs @@ -0,0 +1,213 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! A set of benchmarks which can establish a global baseline for all other +//! benchmarking. + +#![cfg(feature = "runtime-benchmarks")] + +use crate::benchmarks; +use codec::Encode; +use frame_system::Pallet as System; +use sp_application_crypto::KeyTypeId; +use sp_runtime::{ + traits::{AppVerify, Hash}, + RuntimeAppPublic, +}; +use sp_std::prelude::*; + +pub const TEST_KEY_TYPE_ID: KeyTypeId = KeyTypeId(*b"test"); + +mod app_sr25519 { + use super::TEST_KEY_TYPE_ID; + use sp_application_crypto::{app_crypto, sr25519}; + app_crypto!(sr25519, TEST_KEY_TYPE_ID); +} + +type SignerId = app_sr25519::Public; + +pub struct Pallet(System); +pub trait Config: frame_system::Config {} + +benchmarks! { + addition { + let i in 0 .. 1_000_000; + let mut start = 0; + }: { + (0..i).for_each(|_| start += 1); + } verify { + assert_eq!(start, i); + } + + subtraction { + let i in 0 .. 1_000_000; + let mut start = u32::MAX; + }: { + (0..i).for_each(|_| start -= 1); + } verify { + assert_eq!(start, u32::MAX - i); + } + + multiplication { + let i in 0 .. 1_000_000; + let mut out = 0; + }: { + (1..=i).for_each(|j| out = 2 * j); + } verify { + assert_eq!(out, 2 * i); + } + + division { + let i in 0 .. 1_000_000; + let mut out = 0; + }: { + (0..=i).for_each(|j| out = j / 2); + } verify { + assert_eq!(out, i / 2); + } + + hashing { + let i in 0 .. 100; + let mut hash = T::Hash::default(); + }: { + (0..=100_000u32).for_each(|j| hash = T::Hashing::hash(&j.to_be_bytes())); + } verify { + assert!(hash != T::Hash::default()); + } + + sr25519_verification { + let i in 1 .. 100; + + let public = SignerId::generate_pair(None); + + let sigs_count: u8 = i.try_into().unwrap(); + let msg_and_sigs: Vec<_> = (0..sigs_count).map(|j| { + let msg = vec![j, j]; + (msg.clone(), public.sign(&msg).unwrap()) + }) + .collect(); + }: { + msg_and_sigs.iter().for_each(|(msg, sig)| { + assert!(sig.verify(&msg[..], &public)); + }); + } + + #[skip_meta] + storage_read { + let i in 0 .. 1_000; + let mut people = Vec::new(); + (0..i).for_each(|j| { + let hash = T::Hashing::hash(&j.to_be_bytes()).encode(); + frame_support::storage::unhashed::put(&hash, &hash); + people.push(hash); + }); + }: { + people.iter().for_each(|hash| { + // This does a storage read + let value = frame_support::storage::unhashed::get(hash); + assert_eq!(value, Some(hash.to_vec())); + }); + } + + #[skip_meta] + storage_write { + let i in 0 .. 1_000; + let mut hashes = Vec::new(); + (0..i).for_each(|j| { + let hash = T::Hashing::hash(&j.to_be_bytes()); + hashes.push(hash.encode()); + }); + }: { + hashes.iter().for_each(|hash| { + // This does a storage write + frame_support::storage::unhashed::put(hash, hash); + }); + } verify { + hashes.iter().for_each(|hash| { + let value = frame_support::storage::unhashed::get(hash); + assert_eq!(value, Some(hash.to_vec())); + }); + } + + impl_benchmark_test_suite!( + Pallet, + crate::baseline::mock::new_test_ext(), + crate::baseline::mock::Test, + ); +} + +#[cfg(test)] +pub mod mock { + use sp_runtime::{testing::H256, traits::IdentityLookup}; + + type AccountId = u64; + type AccountIndex = u32; + type BlockNumber = u64; + + type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; + type Block = frame_system::mocking::MockBlock; + + frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + } + ); + + impl frame_system::Config for Test { + type BaseCallFilter = frame_support::traits::Everything; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Origin = Origin; + type Index = AccountIndex; + type BlockNumber = BlockNumber; + type Call = Call; + type Hash = H256; + type Hashing = ::sp_runtime::traits::BlakeTwo256; + type AccountId = AccountId; + type Lookup = IdentityLookup; + type Header = sp_runtime::testing::Header; + type Event = Event; + type BlockHashCount = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; + } + + impl super::Config for Test {} + + pub fn new_test_ext() -> sp_io::TestExternalities { + use sp_keystore::{testing::KeyStore, KeystoreExt, SyncCryptoStorePtr}; + use sp_std::sync::Arc; + + let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let mut ext = sp_io::TestExternalities::new(t); + ext.register_extension(KeystoreExt(Arc::new(KeyStore::new()) as SyncCryptoStorePtr)); + + ext + } +} diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index 4a6c5e15ae20..ca836e431e5e 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,6 +27,8 @@ mod tests; mod tests_instance; mod utils; +pub mod baseline; + #[cfg(feature = "std")] pub use analysis::{Analysis, AnalysisChoice, BenchmarkSelector, RegressionModel}; #[doc(hidden)] @@ -40,8 +42,9 @@ pub use sp_io::storage::root as storage_root; #[doc(hidden)] pub use sp_runtime::traits::Zero; #[doc(hidden)] -pub use sp_std::{self, boxed::Box, prelude::Vec, str, vec}; +pub use sp_runtime::StateVersion; #[doc(hidden)] +pub use sp_std::{self, boxed::Box, prelude::Vec, str, vec}; pub use sp_storage::TrackedStorageKey; pub use utils::*; @@ -155,6 +158,12 @@ macro_rules! whitelist { /// benchmark just like a regular benchmark, but only testing at the lowest and highest values for /// each component. The function will return `Ok(())` if the benchmarks return no errors. /// +/// It is also possible to generate one #[test] function per benchmark by calling the +/// `impl_benchmark_test_suite` macro inside the `benchmarks` block. The functions will be named +/// `bench_` and can be run via `cargo test`. +/// You will see one line of output per benchmark. This approach will give you more understandable +/// error messages and allows for parallel benchmark execution. +/// /// You can optionally add a `verify` code block at the end of a benchmark to test any final state /// of your benchmark in a unit test. For example: /// @@ -174,7 +183,8 @@ macro_rules! whitelist { /// /// These `verify` blocks will not affect your benchmark results! /// -/// You can construct benchmark tests like so: +/// You can construct benchmark by using the `impl_benchmark_test_suite` macro or +/// by manually implementing them like so: /// /// ```ignore /// #[test] @@ -193,6 +203,7 @@ macro_rules! benchmarks { $( $rest:tt )* ) => { $crate::benchmarks_iter!( + { } { } { } ( ) @@ -212,6 +223,7 @@ macro_rules! benchmarks_instance { $( $rest:tt )* ) => { $crate::benchmarks_iter!( + { } { I: Instance } { } ( ) @@ -231,6 +243,7 @@ macro_rules! benchmarks_instance_pallet { $( $rest:tt )* ) => { $crate::benchmarks_iter!( + { } { I: 'static } { } ( ) @@ -244,8 +257,60 @@ macro_rules! benchmarks_instance_pallet { #[macro_export] #[doc(hidden)] macro_rules! benchmarks_iter { + // detect and extract `impl_benchmark_test_suite` call: + // - with a semi-colon + ( + { } + { $( $instance:ident: $instance_bound:tt )? } + { $( $where_clause:tt )* } + ( $( $names:tt )* ) + ( $( $names_extra:tt )* ) + ( $( $names_skip_meta:tt )* ) + impl_benchmark_test_suite!( + $bench_module:ident, + $new_test_ext:expr, + $test:path + $(, $( $args:tt )* )?); + $( $rest:tt )* + ) => { + $crate::benchmarks_iter! { + { $bench_module, $new_test_ext, $test $(, $( $args )* )? } + { $( $instance: $instance_bound )? } + { $( $where_clause )* } + ( $( $names )* ) + ( $( $names_extra )* ) + ( $( $names_skip_meta )* ) + $( $rest )* + } + }; + // - without a semicolon + ( + { } + { $( $instance:ident: $instance_bound:tt )? } + { $( $where_clause:tt )* } + ( $( $names:tt )* ) + ( $( $names_extra:tt )* ) + ( $( $names_skip_meta:tt )* ) + impl_benchmark_test_suite!( + $bench_module:ident, + $new_test_ext:expr, + $test:path + $(, $( $args:tt )* )?) + $( $rest:tt )* + ) => { + $crate::benchmarks_iter! { + { $bench_module, $new_test_ext, $test $(, $( $args )* )? } + { $( $instance: $instance_bound )? } + { $( $where_clause )* } + ( $( $names )* ) + ( $( $names_extra )* ) + ( $( $names_skip_meta )* ) + $( $rest )* + } + }; // detect and extract where clause: ( + { $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? } { $( $instance:ident: $instance_bound:tt )? } { $( $where_clause:tt )* } ( $( $names:tt )* ) @@ -255,6 +320,7 @@ macro_rules! benchmarks_iter { $( $rest:tt )* ) => { $crate::benchmarks_iter! { + { $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? } { $( $instance: $instance_bound)? } { $( $where_bound )* } ( $( $names )* ) @@ -265,6 +331,7 @@ macro_rules! benchmarks_iter { }; // detect and extract `#[skip_meta]` tag: ( + { $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? } { $( $instance:ident: $instance_bound:tt )? } { $( $where_clause:tt )* } ( $( $names:tt )* ) @@ -275,6 +342,7 @@ macro_rules! benchmarks_iter { $( $rest:tt )* ) => { $crate::benchmarks_iter! { + { $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? } { $( $instance: $instance_bound )? } { $( $where_clause )* } ( $( $names )* ) @@ -284,8 +352,9 @@ macro_rules! benchmarks_iter { $( $rest )* } }; - // detect and extract `#[extra] tag: + // detect and extract `#[extra]` tag: ( + { $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? } { $( $instance:ident: $instance_bound:tt )? } { $( $where_clause:tt )* } ( $( $names:tt )* ) @@ -296,6 +365,7 @@ macro_rules! benchmarks_iter { $( $rest:tt )* ) => { $crate::benchmarks_iter! { + { $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? } { $( $instance: $instance_bound )? } { $( $where_clause )* } ( $( $names )* ) @@ -307,6 +377,7 @@ macro_rules! benchmarks_iter { }; // mutation arm: ( + { $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? } { $( $instance:ident: $instance_bound:tt )? } { $( $where_clause:tt )* } ( $( $names:tt )* ) // This contains $( $( { $instance } )? $name:ident )* @@ -317,6 +388,7 @@ macro_rules! benchmarks_iter { $( $rest:tt )* ) => { $crate::benchmarks_iter! { + { $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? } { $( $instance: $instance_bound )? } { $( $where_clause )* } ( $( $names )* ) @@ -329,6 +401,7 @@ macro_rules! benchmarks_iter { }; // mutation arm: ( + { $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? } { $( $instance:ident: $instance_bound:tt )? } { $( $where_clause:tt )* } ( $( $names:tt )* ) @@ -340,6 +413,7 @@ macro_rules! benchmarks_iter { ) => { $crate::paste::paste! { $crate::benchmarks_iter! { + { $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? } { $( $instance: $instance_bound )? } { $( $where_clause )* } ( $( $names )* ) @@ -373,6 +447,7 @@ macro_rules! benchmarks_iter { }; // iteration arm: ( + { $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? } { $( $instance:ident: $instance_bound:tt )? } { $( $where_clause:tt )* } ( $( $names:tt )* ) @@ -400,6 +475,7 @@ macro_rules! benchmarks_iter { ); $crate::benchmarks_iter!( + { $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? } { $( $instance: $instance_bound )? } { $( $where_clause )* } ( $( $names )* { $( $instance )? } $name ) @@ -408,8 +484,40 @@ macro_rules! benchmarks_iter { $( $rest )* ); }; - // iteration-exit arm + // iteration-exit arm which generates a #[test] function for each case. + ( + { $bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )? } + { $( $instance:ident: $instance_bound:tt )? } + { $( $where_clause:tt )* } + ( $( $names:tt )* ) + ( $( $names_extra:tt )* ) + ( $( $names_skip_meta:tt )* ) + ) => { + $crate::selected_benchmark!( + { $( $where_clause)* } + { $( $instance: $instance_bound )? } + $( $names )* + ); + $crate::impl_benchmark!( + { $( $where_clause )* } + { $( $instance: $instance_bound )? } + ( $( $names )* ) + ( $( $names_extra ),* ) + ( $( $names_skip_meta ),* ) + ); + $crate::impl_test_function!( + ( $( $names )* ) + ( $( $names_extra )* ) + ( $( $names_skip_meta )* ) + $bench_module, + $new_test_ext, + $test + $(, $( $args )* )? + ); + }; + // iteration-exit arm which doesn't generate a #[test] function for all cases. ( + { } { $( $instance:ident: $instance_bound:tt )? } { $( $where_clause:tt )* } ( $( $names:tt )* ) @@ -431,6 +539,7 @@ macro_rules! benchmarks_iter { }; // add verify block to _() format ( + { $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? } { $( $instance:ident: $instance_bound:tt )? } { $( $where_clause:tt )* } ( $( $names:tt )* ) @@ -440,6 +549,7 @@ macro_rules! benchmarks_iter { $( $rest:tt )* ) => { $crate::benchmarks_iter! { + { $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? } { $( $instance: $instance_bound )? } { $( $where_clause )* } ( $( $names )* ) @@ -452,6 +562,7 @@ macro_rules! benchmarks_iter { }; // add verify block to name() format ( + { $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? } { $( $instance:ident: $instance_bound:tt )? } { $( $where_clause:tt )* } ( $( $names:tt )* ) @@ -461,6 +572,7 @@ macro_rules! benchmarks_iter { $( $rest:tt )* ) => { $crate::benchmarks_iter! { + { $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? } { $( $instance: $instance_bound )? } { $( $where_clause )* } ( $( $names )* ) @@ -473,6 +585,7 @@ macro_rules! benchmarks_iter { }; // add verify block to {} format ( + { $($bench_module:ident, $new_test_ext:expr, $test:path $(, $( $args:tt )* )?)? } { $( $instance:ident: $instance_bound:tt )? } { $( $where_clause:tt )* } ( $( $names:tt )* ) @@ -482,6 +595,7 @@ macro_rules! benchmarks_iter { $( $rest:tt )* ) => { $crate::benchmarks_iter!( + { $($bench_module, $new_test_ext, $test $(, $( $args )* )?)? } { $( $instance: $instance_bound )? } { $( $where_clause )* } ( $( $names )* ) @@ -695,6 +809,100 @@ macro_rules! benchmark_backend { }; } +// Creates #[test] functions for the given bench cases. +#[macro_export] +#[doc(hidden)] +macro_rules! impl_bench_case_tests { + ( + { $module:ident, $new_test_exec:expr, $exec_name:ident, $test:path, $extra:expr } + { $( $names_extra:tt )* } + $( { $( $bench_inst:ident )? } $bench:ident )* + ) + => { + $crate::impl_bench_name_tests!( + $module, $new_test_exec, $exec_name, $test, $extra, + { $( $names_extra )* }, + $( { $bench } )+ + ); + } +} + +// Creates a #[test] function for the given bench name. +#[macro_export] +#[doc(hidden)] +macro_rules! impl_bench_name_tests { + // recursion anchor + ( + $module:ident, $new_test_exec:expr, $exec_name:ident, $test:path, $extra:expr, + { $( $names_extra:tt )* }, + { $name:ident } + ) => { + $crate::paste::paste! { + #[test] + fn [] () { + $new_test_exec.$exec_name(|| { + // Skip all #[extra] benchmarks if $extra is false. + if !($extra) { + let disabled = $crate::vec![ $( stringify!($names_extra).as_ref() ),* ]; + if disabled.contains(&stringify!($name)) { + $crate::log::error!( + "INFO: extra benchmark skipped - {}", + stringify!($name), + ); + return (); + } + } + + // Same per-case logic as when all cases are run in the + // same function. + match std::panic::catch_unwind(|| { + $module::<$test>::[< test_benchmark_ $name >] () + }) { + Err(err) => { + panic!("{}: {:?}", stringify!($name), err); + }, + Ok(Err(err)) => { + match err { + $crate::BenchmarkError::Stop(err) => { + panic!("{}: {:?}", stringify!($name), err); + }, + $crate::BenchmarkError::Override(_) => { + // This is still considered a success condition. + $crate::log::error!( + "WARNING: benchmark error overrided - {}", + stringify!($name), + ); + }, + $crate::BenchmarkError::Skip => { + // This is considered a success condition. + $crate::log::error!( + "WARNING: benchmark error skipped - {}", + stringify!($name), + ); + } + } + }, + Ok(Ok(())) => (), + } + }); + } + } + }; + // recursion tail + ( + $module:ident, $new_test_exec:expr, $exec_name:ident, $test:path, $extra:expr, + { $( $names_extra:tt )* }, + { $name:ident } $( { $rest:ident } )+ + ) => { + // car + $crate::impl_bench_name_tests!($module, $new_test_exec, $exec_name, $test, $extra, + { $( $names_extra )* }, { $name }); + // cdr + $crate::impl_bench_name_tests!($module, $new_test_exec, $exec_name, $test, $extra, + { $( $names_extra )* }, $( { $rest } )+); + }; +} + // Creates a `SelectedBenchmark` enum implementing `BenchmarkingSetup`. // // Every variant must implement [`BenchmarkingSetup`]. @@ -872,7 +1080,7 @@ macro_rules! impl_benchmark { // Time the storage root recalculation. let start_storage_root = $crate::benchmarking::current_time(); - $crate::storage_root(); + $crate::storage_root($crate::StateVersion::V1); let finish_storage_root = $crate::benchmarking::current_time(); let elapsed_storage_root = finish_storage_root - start_storage_root; @@ -1010,7 +1218,7 @@ macro_rules! impl_benchmark_test { /// This creates a test suite which runs the module's benchmarks. /// -/// When called in `pallet_example` as +/// When called in `pallet_example_basic` as /// /// ```rust,ignore /// impl_benchmark_test_suite!(Pallet, crate::tests::new_test_ext(), crate::tests::Test); @@ -1030,13 +1238,54 @@ macro_rules! impl_benchmark_test { /// new_test_ext().execute_with(|| { /// assert_ok!(test_benchmark_accumulate_dummy::()); /// assert_ok!(test_benchmark_set_dummy::()); -/// assert_ok!(test_benchmark_another_set_dummy::()); /// assert_ok!(test_benchmark_sort_vector::()); /// }); /// } /// } /// ``` /// +/// When called inside the `benchmarks` macro of the `pallet_example_basic` as +/// +/// ```rust,ignore +/// benchmarks! { +/// // Benchmarks omitted for brevity +/// +/// impl_benchmark_test_suite!(Pallet, crate::tests::new_test_ext(), crate::tests::Test); +/// } +/// ``` +/// +/// It expands to the equivalent of: +/// +/// ```rust,ignore +/// #[cfg(test)] +/// mod benchmarking { +/// use super::*; +/// use crate::tests::{new_test_ext, Test}; +/// use frame_support::assert_ok; +/// +/// #[test] +/// fn bench_accumulate_dummy() { +/// new_test_ext().execute_with(|| { +/// assert_ok!(test_benchmark_accumulate_dummy::()); +/// }) +/// } +/// +/// #[test] +/// fn bench_set_dummy() { +/// new_test_ext().execute_with(|| { +/// assert_ok!(test_benchmark_set_dummy::()); +/// }) +/// } +/// +/// #[test] +/// fn bench_sort_vector() { +/// new_test_ext().execute_with(|| { +/// assert_ok!(test_benchmark_sort_vector::()); +/// }) +/// } +/// } +/// ``` +/// /// ## Arguments /// /// The first argument, `module`, must be the path to this crate's module. @@ -1109,16 +1358,50 @@ macro_rules! impl_benchmark_test { // just iterate over the `Benchmarking::benchmarks` list to run the actual implementations. #[macro_export] macro_rules! impl_benchmark_test_suite { + ( + $bench_module:ident, + $new_test_ext:expr, + $test:path + $(, $( $rest:tt )* )? + ) => { + $crate::impl_test_function!( + () + () + () + $bench_module, + $new_test_ext, + $test + $(, $( $rest )* )? + ); + } +} + +// Takes all arguments from `impl_benchmark_test_suite` and three additional arguments. +// +// Can be configured to generate one #[test] fn per bench case or +// one #[test] fn for all bench cases. +// This depends on whether or not the first argument contains a non-empty list of bench names. +#[macro_export] +#[doc(hidden)] +macro_rules! impl_test_function { // user might or might not have set some keyword arguments; set the defaults // // The weird syntax indicates that `rest` comes only after a comma, which is otherwise optional ( + ( $( $names:tt )* ) + ( $( $names_extra:tt )* ) + ( $( $names_skip_meta:tt )* ) + $bench_module:ident, $new_test_ext:expr, $test:path $(, $( $rest:tt )* )? ) => { - $crate::impl_benchmark_test_suite!( + $crate::impl_test_function!( + @cases: + ( $( $names )* ) + ( $( $names_extra )* ) + ( $( $names_skip_meta )* ) @selected: $bench_module, $new_test_ext, @@ -1132,6 +1415,10 @@ macro_rules! impl_benchmark_test_suite { }; // pick off the benchmarks_path keyword argument ( + @cases: + ( $( $names:tt )* ) + ( $( $names_extra:tt )* ) + ( $( $names_skip_meta:tt )* ) @selected: $bench_module:ident, $new_test_ext:expr, @@ -1143,7 +1430,11 @@ macro_rules! impl_benchmark_test_suite { benchmarks_path = $benchmarks_path:ident $(, $( $rest:tt )* )? ) => { - $crate::impl_benchmark_test_suite!( + $crate::impl_test_function!( + @cases: + ( $( $names )* ) + ( $( $names_extra )* ) + ( $( $names_skip_meta )* ) @selected: $bench_module, $new_test_ext, @@ -1157,6 +1448,10 @@ macro_rules! impl_benchmark_test_suite { }; // pick off the extra keyword argument ( + @cases: + ( $( $names:tt )* ) + ( $( $names_extra:tt )* ) + ( $( $names_skip_meta:tt )* ) @selected: $bench_module:ident, $new_test_ext:expr, @@ -1168,7 +1463,11 @@ macro_rules! impl_benchmark_test_suite { extra = $extra:expr $(, $( $rest:tt )* )? ) => { - $crate::impl_benchmark_test_suite!( + $crate::impl_test_function!( + @cases: + ( $( $names )* ) + ( $( $names_extra )* ) + ( $( $names_skip_meta )* ) @selected: $bench_module, $new_test_ext, @@ -1182,6 +1481,10 @@ macro_rules! impl_benchmark_test_suite { }; // pick off the exec_name keyword argument ( + @cases: + ( $( $names:tt )* ) + ( $( $names_extra:tt )* ) + ( $( $names_skip_meta:tt )* ) @selected: $bench_module:ident, $new_test_ext:expr, @@ -1193,7 +1496,11 @@ macro_rules! impl_benchmark_test_suite { exec_name = $exec_name:ident $(, $( $rest:tt )* )? ) => { - $crate::impl_benchmark_test_suite!( + $crate::impl_test_function!( + @cases: + ( $( $names )* ) + ( $( $names_extra )* ) + ( $( $names_skip_meta )* ) @selected: $bench_module, $new_test_ext, @@ -1205,8 +1512,34 @@ macro_rules! impl_benchmark_test_suite { $( $( $rest )* )? ); }; - // all options set; nothing else in user-provided keyword arguments + // iteration-exit arm which generates a #[test] function for each case. + ( + @cases: + ( $( $names:tt )+ ) + ( $( $names_extra:tt )* ) + ( $( $names_skip_meta:tt )* ) + @selected: + $bench_module:ident, + $new_test_ext:expr, + $test:path, + benchmarks_path = $path_to_benchmarks_invocation:ident, + extra = $extra:expr, + exec_name = $exec_name:ident, + @user: + $(,)? + ) => { + $crate::impl_bench_case_tests!( + { $bench_module, $new_test_ext, $exec_name, $test, $extra } + { $( $names_extra:tt )* } + $($names)+ + ); + }; + // iteration-exit arm which generates one #[test] function for all cases. ( + @cases: + () + () + () @selected: $bench_module:ident, $new_test_ext:expr, @@ -1368,7 +1701,6 @@ pub fn show_benchmark_debug_info( /// type Council2 = TechnicalCommittee; /// add_benchmark!(params, batches, pallet_collective, Council2); // pallet_collective_council_2.rs /// ``` - #[macro_export] macro_rules! add_benchmark { ( $params:ident, $batches:ident, $name:path, $( $location:tt )* ) => ( @@ -1437,6 +1769,20 @@ macro_rules! add_benchmark { ) } +/// Callback for `define_benchmarks` to call `add_benchmark`. +#[macro_export] +macro_rules! cb_add_benchmarks { + // anchor + ( $params:ident, $batches:ident, [ $name:path, $( $location:tt )* ] ) => { + add_benchmark!( $params, $batches, $name, $( $location )* ); + }; + // recursion tail + ( $params:ident, $batches:ident, [ $name:path, $( $location:tt )* ] $([ $names:path, $( $locations:tt )* ])+ ) => { + cb_add_benchmarks!( $params, $batches, [ $name, $( $location )* ] ); + cb_add_benchmarks!( $params, $batches, $([ $names, $( $locations )* ])+ ); + } +} + /// This macro allows users to easily generate a list of benchmarks for the pallets configured /// in the runtime. /// @@ -1456,7 +1802,6 @@ macro_rules! add_benchmark { /// ``` /// /// This should match what exists with the `add_benchmark!` macro. - #[macro_export] macro_rules! list_benchmark { ( $list:ident, $extra:ident, $name:path, $( $location:tt )* ) => ( @@ -1471,3 +1816,54 @@ macro_rules! list_benchmark { $list.push(pallet_benchmarks) ) } + +/// Callback for `define_benchmarks` to call `list_benchmark`. +#[macro_export] +macro_rules! cb_list_benchmarks { + // anchor + ( $list:ident, $extra:ident, [ $name:path, $( $location:tt )* ] ) => { + list_benchmark!( $list, $extra, $name, $( $location )* ); + }; + // recursion tail + ( $list:ident, $extra:ident, [ $name:path, $( $location:tt )* ] $([ $names:path, $( $locations:tt )* ])+ ) => { + cb_list_benchmarks!( $list, $extra, [ $name, $( $location )* ] ); + cb_list_benchmarks!( $list, $extra, $([ $names, $( $locations )* ])+ ); + } +} + +/// Defines pallet configs that `add_benchmarks` and `list_benchmarks` use. +/// Should be preferred instead of having a repetitive list of configs +/// in `add_benchmark` and `list_benchmark`. + +#[macro_export] +macro_rules! define_benchmarks { + ( $([ $names:path, $( $locations:tt )* ])* ) => { + /// Calls `list_benchmark` with all configs from `define_benchmarks` + /// and passes the first two parameters on. + /// + /// Use as: + /// ```ignore + /// list_benchmarks!(list, extra); + /// ``` + #[macro_export] + macro_rules! list_benchmarks { + ( $list:ident, $extra:ident ) => { + cb_list_benchmarks!( $list, $extra, $([ $names, $( $locations )* ])+ ); + } + } + + /// Calls `add_benchmark` with all configs from `define_benchmarks` + /// and passes the first two parameters on. + /// + /// Use as: + /// ```ignore + /// add_benchmarks!(params, batches); + /// ``` + #[macro_export] + macro_rules! add_benchmarks { + ( $params:ident, $batches:ident ) => { + cb_add_benchmarks!( $params, $batches, $([ $names, $( $locations )* ])+ ); + } + } + } +} diff --git a/frame/benchmarking/src/tests.rs b/frame/benchmarking/src/tests.rs index a2cf381e6ecf..06f2b5bdc491 100644 --- a/frame/benchmarking/src/tests.rs +++ b/frame/benchmarking/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ #![cfg(test)] use super::*; -use frame_support::parameter_types; +use frame_support::{parameter_types, traits::ConstU32}; use sp_runtime::{ testing::{Header, H256}, traits::{BlakeTwo256, IdentityLookup}, @@ -108,17 +108,16 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } parameter_types! { - pub const LowerBound: u32 = 1; - pub const UpperBound: u32 = 100; pub const MaybeItem: Option = None; } impl pallet_test::Config for Test { - type LowerBound = LowerBound; - type UpperBound = UpperBound; + type LowerBound = ConstU32<1>; + type UpperBound = ConstU32<100>; type MaybeItem = MaybeItem; } diff --git a/frame/benchmarking/src/tests_instance.rs b/frame/benchmarking/src/tests_instance.rs index caccebd39c70..ef8351d37e95 100644 --- a/frame/benchmarking/src/tests_instance.rs +++ b/frame/benchmarking/src/tests_instance.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ #![cfg(test)] use super::*; -use frame_support::parameter_types; +use frame_support::traits::ConstU32; use sp_runtime::{ testing::{Header, H256}, traits::{BlakeTwo256, IdentityLookup}, @@ -110,17 +110,13 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); -} - -parameter_types! { - pub const LowerBound: u32 = 1; - pub const UpperBound: u32 = 100; + type MaxConsumers = frame_support::traits::ConstU32<16>; } impl pallet_test::Config for Test { type Event = Event; - type LowerBound = LowerBound; - type UpperBound = UpperBound; + type LowerBound = ConstU32<1>; + type UpperBound = ConstU32<100>; } impl pallet_test::OtherConfig for Test { @@ -173,11 +169,11 @@ mod benchmarks { } verify { ensure!(m[0] == 0, "You forgot to sort!") } - } - crate::impl_benchmark_test_suite!( - Pallet, - crate::tests_instance::new_test_ext(), - crate::tests_instance::Test - ); + impl_benchmark_test_suite!( + Pallet, + crate::tests_instance::new_test_ext(), + crate::tests_instance::Test + ) + } } diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs index c24ad2f64e18..8c642f74358d 100644 --- a/frame/benchmarking/src/utils.rs +++ b/frame/benchmarking/src/utils.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,11 +22,15 @@ use frame_support::{ pallet_prelude::*, traits::StorageInfo, }; +#[cfg(feature = "std")] +use serde::Serialize; use sp_io::hashing::blake2_256; +use sp_runtime::traits::TrailingZeroInput; use sp_std::{prelude::Box, vec::Vec}; use sp_storage::TrackedStorageKey; /// An alphabet of possible parameters to use for benchmarking. +#[cfg_attr(feature = "std", derive(Serialize))] #[derive(Encode, Decode, Clone, Copy, PartialEq, Debug)] #[allow(missing_docs)] #[allow(non_camel_case_types)] @@ -67,13 +71,17 @@ impl std::fmt::Display for BenchmarkParameter { } /// The results of a single of benchmark. +#[cfg_attr(feature = "std", derive(Serialize))] #[derive(Encode, Decode, Clone, PartialEq, Debug)] pub struct BenchmarkBatch { /// The pallet containing this benchmark. + #[cfg_attr(feature = "std", serde(with = "serde_as_str"))] pub pallet: Vec, /// The instance of this pallet being benchmarked. + #[cfg_attr(feature = "std", serde(with = "serde_as_str"))] pub instance: Vec, /// The extrinsic (or benchmark name) of this benchmark. + #[cfg_attr(feature = "std", serde(with = "serde_as_str"))] pub benchmark: Vec, /// The results from this benchmark. pub results: Vec, @@ -81,13 +89,17 @@ pub struct BenchmarkBatch { // TODO: could probably make API cleaner here. /// The results of a single of benchmark, where time and db results are separated. +#[cfg_attr(feature = "std", derive(Serialize))] #[derive(Encode, Decode, Clone, PartialEq, Debug)] pub struct BenchmarkBatchSplitResults { /// The pallet containing this benchmark. + #[cfg_attr(feature = "std", serde(with = "serde_as_str"))] pub pallet: Vec, /// The instance of this pallet being benchmarked. + #[cfg_attr(feature = "std", serde(with = "serde_as_str"))] pub instance: Vec, /// The extrinsic (or benchmark name) of this benchmark. + #[cfg_attr(feature = "std", serde(with = "serde_as_str"))] pub benchmark: Vec, /// The extrinsic timing results from this benchmark. pub time_results: Vec, @@ -98,6 +110,7 @@ pub struct BenchmarkBatchSplitResults { /// Result from running benchmarks on a FRAME pallet. /// Contains duration of the function call in nanoseconds along with the benchmark parameters /// used for that benchmark result. +#[cfg_attr(feature = "std", derive(Serialize))] #[derive(Encode, Decode, Default, Clone, PartialEq, Debug)] pub struct BenchmarkResult { pub components: Vec<(BenchmarkParameter, u32)>, @@ -108,6 +121,7 @@ pub struct BenchmarkResult { pub writes: u32, pub repeat_writes: u32, pub proof_size: u32, + #[cfg_attr(feature = "std", serde(skip_serializing))] pub keys: Vec<(Vec, u32, u32, bool)>, } @@ -117,6 +131,18 @@ impl BenchmarkResult { } } +/// Helper module to make serde serialize `Vec` as strings. +#[cfg(feature = "std")] +mod serde_as_str { + pub fn serialize(value: &Vec, serializer: S) -> Result + where + S: serde::Serializer, + { + let s = std::str::from_utf8(value).map_err(serde::ser::Error::custom)?; + serializer.collect_str(s) + } +} + /// Possible errors returned from the benchmarking pipeline. #[derive(Clone, PartialEq, Debug)] pub enum BenchmarkError { @@ -321,17 +347,14 @@ pub trait BenchmarkingSetup { } /// Grab an account, seeded by a name and index. -pub fn account( - name: &'static str, - index: u32, - seed: u32, -) -> AccountId { +pub fn account(name: &'static str, index: u32, seed: u32) -> AccountId { let entropy = (name, index, seed).using_encoded(blake2_256); - AccountId::decode(&mut &entropy[..]).unwrap_or_default() + Decode::decode(&mut TrailingZeroInput::new(entropy.as_ref())) + .expect("infinite length input; no invalid inputs for type; qed") } /// This caller account is automatically whitelisted for DB reads/writes by the benchmarking macro. -pub fn whitelisted_caller() -> AccountId { +pub fn whitelisted_caller() -> AccountId { account::("whitelisted_caller", 0, 0) } diff --git a/frame/benchmarking/src/weights.rs b/frame/benchmarking/src/weights.rs new file mode 100644 index 000000000000..a49204fbf0b9 --- /dev/null +++ b/frame/benchmarking/src/weights.rs @@ -0,0 +1,134 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for frame_benchmarking +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2022-01-30, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 + +// Executed Command: +// ./target/production/substrate +// benchmark +// --chain=dev +// --steps=50 +// --repeat=20 +// --pallet=frame_benchmarking +// --extrinsic=* +// --execution=wasm +// --wasm-execution=compiled +// --heap-pages=4096 +// --output=./frame/benchmarking/src/weights.rs +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use sp_std::marker::PhantomData; + +/// Weight functions needed for frame_benchmarking. +pub trait WeightInfo { + fn addition(i: u32, ) -> Weight; + fn subtraction(i: u32, ) -> Weight; + fn multiplication(i: u32, ) -> Weight; + fn division(i: u32, ) -> Weight; + fn hashing(i: u32, ) -> Weight; + fn sr25519_verification(i: u32, ) -> Weight; + fn storage_read(i: u32, ) -> Weight; + fn storage_write(i: u32, ) -> Weight; +} + +/// Weights for frame_benchmarking using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + fn addition(_i: u32, ) -> Weight { + (106_000 as Weight) + } + fn subtraction(_i: u32, ) -> Weight { + (111_000 as Weight) + } + fn multiplication(_i: u32, ) -> Weight { + (119_000 as Weight) + } + fn division(_i: u32, ) -> Weight { + (111_000 as Weight) + } + fn hashing(_i: u32, ) -> Weight { + (21_203_386_000 as Weight) + } + fn sr25519_verification(i: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 6_000 + .saturating_add((47_077_000 as Weight).saturating_mul(i as Weight)) + } + // Storage: Skipped Metadata (r:0 w:0) + fn storage_read(i: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 3_000 + .saturating_add((1_963_000 as Weight).saturating_mul(i as Weight)) + .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(i as Weight))) + } + // Storage: Skipped Metadata (r:0 w:0) + fn storage_write(i: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 0 + .saturating_add((364_000 as Weight).saturating_mul(i as Weight)) + .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(i as Weight))) + } +} + +// For backwards compatibility and tests +impl WeightInfo for () { + fn addition(_i: u32, ) -> Weight { + (106_000 as Weight) + } + fn subtraction(_i: u32, ) -> Weight { + (111_000 as Weight) + } + fn multiplication(_i: u32, ) -> Weight { + (119_000 as Weight) + } + fn division(_i: u32, ) -> Weight { + (111_000 as Weight) + } + fn hashing(_i: u32, ) -> Weight { + (21_203_386_000 as Weight) + } + fn sr25519_verification(i: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 6_000 + .saturating_add((47_077_000 as Weight).saturating_mul(i as Weight)) + } + // Storage: Skipped Metadata (r:0 w:0) + fn storage_read(i: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 3_000 + .saturating_add((1_963_000 as Weight).saturating_mul(i as Weight)) + .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(i as Weight))) + } + // Storage: Skipped Metadata (r:0 w:0) + fn storage_write(i: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 0 + .saturating_add((364_000 as Weight).saturating_mul(i as Weight)) + .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(i as Weight))) + } +} diff --git a/frame/bounties/Cargo.toml b/frame/bounties/Cargo.toml index 93a7ababb2eb..a158895a252b 100644 --- a/frame/bounties/Cargo.toml +++ b/frame/bounties/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-bounties" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet to manage bounties" readme = "README.md" @@ -13,17 +13,17 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } pallet-treasury = { version = "4.0.0-dev", default-features = false, path = "../treasury" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io", default-features = false } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core", default-features = false } +sp-io = { version = "6.0.0", path = "../../primitives/io", default-features = false } +sp-core = { version = "6.0.0", path = "../../primitives/core", default-features = false } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } log = { version = "0.4.14", default-features = false } @@ -45,7 +45,7 @@ std = [ "log/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", ] diff --git a/frame/bounties/src/benchmarking.rs b/frame/bounties/src/benchmarking.rs index 1aa1eabdb517..04adacf6e4ae 100644 --- a/frame/bounties/src/benchmarking.rs +++ b/frame/bounties/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use super::*; -use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite, whitelisted_caller}; +use frame_benchmarking::{account, benchmarks, whitelisted_caller}; use frame_system::RawOrigin; use sp_runtime::traits::Bounded; @@ -33,7 +33,8 @@ const SEED: u32 = 0; // Create bounties that are approved for use in `on_initialize`. fn create_approved_bounties(n: u32) -> Result<(), &'static str> { for i in 0..n { - let (caller, _curator, _fee, value, reason) = setup_bounty::(i, MAX_BYTES); + let (caller, _curator, _fee, value, reason) = + setup_bounty::(i, T::MaximumReasonLength::get()); Bounties::::propose_bounty(RawOrigin::Signed(caller).into(), value, reason)?; let bounty_id = BountyCount::::get() - 1; Bounties::::approve_bounty(RawOrigin::Root.into(), bounty_id)?; @@ -50,7 +51,8 @@ fn setup_bounty( let caller = account("caller", u, SEED); let value: BalanceOf = T::BountyValueMinimum::get().saturating_mul(100u32.into()); let fee = value / 2u32.into(); - let deposit = T::BountyDepositBase::get() + T::DataDepositPerByte::get() * MAX_BYTES.into(); + let deposit = T::BountyDepositBase::get() + + T::DataDepositPerByte::get() * T::MaximumReasonLength::get().into(); let _ = T::Currency::make_free_balance_be(&caller, deposit); let curator = account("curator", u, SEED); let _ = T::Currency::make_free_balance_be(&curator, fee / 2u32.into()); @@ -60,7 +62,7 @@ fn setup_bounty( fn create_bounty( ) -> Result<(::Source, BountyIndex), &'static str> { - let (caller, curator, fee, value, reason) = setup_bounty::(0, MAX_BYTES); + let (caller, curator, fee, value, reason) = setup_bounty::(0, T::MaximumReasonLength::get()); let curator_lookup = T::Lookup::unlookup(curator.clone()); Bounties::::propose_bounty(RawOrigin::Signed(caller).into(), value, reason)?; let bounty_id = BountyCount::::get() - 1; @@ -81,24 +83,22 @@ fn assert_last_event(generic_event: ::Event) { frame_system::Pallet::::assert_last_event(generic_event.into()); } -const MAX_BYTES: u32 = 16384; - benchmarks! { propose_bounty { - let d in 0 .. MAX_BYTES; + let d in 0 .. T::MaximumReasonLength::get(); let (caller, curator, fee, value, description) = setup_bounty::(0, d); }: _(RawOrigin::Signed(caller), value, description) approve_bounty { - let (caller, curator, fee, value, reason) = setup_bounty::(0, MAX_BYTES); + let (caller, curator, fee, value, reason) = setup_bounty::(0, T::MaximumReasonLength::get()); Bounties::::propose_bounty(RawOrigin::Signed(caller).into(), value, reason)?; let bounty_id = BountyCount::::get() - 1; }: _(RawOrigin::Root, bounty_id) propose_curator { setup_pot_account::(); - let (caller, curator, fee, value, reason) = setup_bounty::(0, MAX_BYTES); + let (caller, curator, fee, value, reason) = setup_bounty::(0, T::MaximumReasonLength::get()); let curator_lookup = T::Lookup::unlookup(curator.clone()); Bounties::::propose_bounty(RawOrigin::Signed(caller).into(), value, reason)?; let bounty_id = BountyCount::::get() - 1; @@ -118,7 +118,7 @@ benchmarks! { accept_curator { setup_pot_account::(); - let (caller, curator, fee, value, reason) = setup_bounty::(0, MAX_BYTES); + let (caller, curator, fee, value, reason) = setup_bounty::(0, T::MaximumReasonLength::get()); let curator_lookup = T::Lookup::unlookup(curator.clone()); Bounties::::propose_bounty(RawOrigin::Signed(caller).into(), value, reason)?; let bounty_id = BountyCount::::get() - 1; @@ -172,7 +172,7 @@ benchmarks! { let bounty_id = BountyCount::::get() - 1; }: close_bounty(RawOrigin::Root, bounty_id) verify { - assert_last_event::(Event::BountyCanceled(bounty_id).into()) + assert_last_event::(Event::BountyCanceled { index: bounty_id }.into()) } extend_bounty_expiry { @@ -184,7 +184,7 @@ benchmarks! { let curator = T::Lookup::lookup(curator_lookup).map_err(<&str>::from)?; }: _(RawOrigin::Signed(curator), bounty_id, Vec::new()) verify { - assert_last_event::(Event::BountyExtended(bounty_id).into()) + assert_last_event::(Event::BountyExtended { index: bounty_id }.into()) } spend_funds { @@ -207,8 +207,8 @@ benchmarks! { verify { ensure!(budget_remaining < BalanceOf::::max_value(), "Budget not used"); ensure!(missed_any == false, "Missed some"); - assert_last_event::(Event::BountyBecameActive(b - 1).into()) + assert_last_event::(Event::BountyBecameActive { index: b - 1 }.into()) } -} -impl_benchmark_test_suite!(Bounties, crate::tests::new_test_ext(), crate::tests::Test); + impl_benchmark_test_suite!(Bounties, crate::tests::new_test_ext(), crate::tests::Test) +} diff --git a/frame/bounties/src/lib.rs b/frame/bounties/src/lib.rs index 69380502bad3..98f2da305a06 100644 --- a/frame/bounties/src/lib.rs +++ b/frame/bounties/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -107,7 +107,7 @@ type PositiveImbalanceOf = pallet_treasury::PositiveImbalanceOf; pub type BountyIndex = u32; /// A bounty proposal. -#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo)] +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub struct Bounty { /// The account proposing it. proposer: AccountId, @@ -123,8 +123,17 @@ pub struct Bounty { status: BountyStatus, } +impl + Bounty +{ + /// Getter for bounty status, to be used for child bounties. + pub fn get_status(&self) -> BountyStatus { + self.status.clone() + } +} + /// The status of a bounty proposal. -#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo)] +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub enum BountyStatus { /// The bounty is proposed and waiting for approval. Proposed, @@ -156,6 +165,15 @@ pub enum BountyStatus { }, } +/// The child-bounty manager. +pub trait ChildBountyManager { + /// Get the active child-bounties for a parent bounty. + fn child_bounties_count(bounty_id: BountyIndex) -> BountyIndex; + + /// Get total curator fees of children-bounty curators. + fn children_curator_fees(bounty_id: BountyIndex) -> Balance; +} + #[frame_support::pallet] pub mod pallet { use super::*; @@ -178,10 +196,20 @@ pub mod pallet { #[pallet::constant] type BountyUpdatePeriod: Get; - /// Percentage of the curator fee that will be reserved upfront as deposit for bounty - /// curator. + /// The curator deposit is calculated as a percentage of the curator fee. + /// + /// This deposit has optional upper and lower bounds with `CuratorDepositMax` and + /// `CuratorDepositMin`. + #[pallet::constant] + type CuratorDepositMultiplier: Get; + + /// Maximum amount of funds that should be placed in a deposit for making a proposal. + #[pallet::constant] + type CuratorDepositMax: Get>>; + + /// Minimum amount of funds that should be placed in a deposit for making a proposal. #[pallet::constant] - type BountyCuratorDeposit: Get; + type CuratorDepositMin: Get>>; /// Minimum value for a bounty. #[pallet::constant] @@ -195,11 +223,16 @@ pub mod pallet { type Event: From> + IsType<::Event>; /// Maximum acceptable reason length. + /// + /// Benchmarks depend on this value, be sure to update weights file when changing this value #[pallet::constant] type MaximumReasonLength: Get; /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; + + /// The child-bounty manager. + type ChildBountyManager: ChildBountyManager>; } #[pallet::error] @@ -223,25 +256,29 @@ pub mod pallet { PendingPayout, /// The bounties cannot be claimed/closed because it's still in the countdown period. Premature, + /// The bounty cannot be closed because it has active child-bounties. + HasActiveChildBounty, + /// Too many approvals are already queued. + TooManyQueued, } #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// New bounty proposal. \[index\] - BountyProposed(BountyIndex), - /// A bounty proposal was rejected; funds were slashed. \[index, bond\] - BountyRejected(BountyIndex, BalanceOf), - /// A bounty proposal is funded and became active. \[index\] - BountyBecameActive(BountyIndex), - /// A bounty is awarded to a beneficiary. \[index, beneficiary\] - BountyAwarded(BountyIndex, T::AccountId), - /// A bounty is claimed by beneficiary. \[index, payout, beneficiary\] - BountyClaimed(BountyIndex, BalanceOf, T::AccountId), - /// A bounty is cancelled. \[index\] - BountyCanceled(BountyIndex), - /// A bounty expiry is extended. \[index\] - BountyExtended(BountyIndex), + /// New bounty proposal. + BountyProposed { index: BountyIndex }, + /// A bounty proposal was rejected; funds were slashed. + BountyRejected { index: BountyIndex, bond: BalanceOf }, + /// A bounty proposal is funded and became active. + BountyBecameActive { index: BountyIndex }, + /// A bounty is awarded to a beneficiary. + BountyAwarded { index: BountyIndex, beneficiary: T::AccountId }, + /// A bounty is claimed by beneficiary. + BountyClaimed { index: BountyIndex, payout: BalanceOf, beneficiary: T::AccountId }, + /// A bounty is cancelled. + BountyCanceled { index: BountyIndex }, + /// A bounty expiry is extended. + BountyExtended { index: BountyIndex }, } /// Number of bounty proposals that have been made. @@ -262,12 +299,14 @@ pub mod pallet { /// The description of each bounty. #[pallet::storage] #[pallet::getter(fn bounty_descriptions)] - pub type BountyDescriptions = StorageMap<_, Twox64Concat, BountyIndex, Vec>; + pub type BountyDescriptions = + StorageMap<_, Twox64Concat, BountyIndex, BoundedVec>; /// Bounty indices that have been approved but not yet funded. #[pallet::storage] #[pallet::getter(fn bounty_approvals)] - pub type BountyApprovals = StorageValue<_, Vec, ValueQuery>; + pub type BountyApprovals = + StorageValue<_, BoundedVec, ValueQuery>; #[pallet::call] impl Pallet { @@ -315,7 +354,8 @@ pub mod pallet { bounty.status = BountyStatus::Approved; - BountyApprovals::::append(bounty_id); + BountyApprovals::::try_append(bounty_id) + .map_err(|()| Error::::TooManyQueued)?; Ok(()) })?; @@ -428,6 +468,7 @@ pub mod pallet { let err_amount = T::Currency::unreserve(&curator, bounty.curator_deposit); debug_assert!(err_amount.is_zero()); + bounty.curator_deposit = Zero::zero(); // Continue to change bounty status below... } }, @@ -471,7 +512,7 @@ pub mod pallet { BountyStatus::CuratorProposed { ref curator } => { ensure!(signer == *curator, Error::::RequireCurator); - let deposit = T::BountyCuratorDeposit::get() * bounty.fee; + let deposit = Self::calculate_curator_deposit(&bounty.fee); T::Currency::reserve(curator, deposit)?; bounty.curator_deposit = deposit; @@ -510,6 +551,13 @@ pub mod pallet { Bounties::::try_mutate_exists(bounty_id, |maybe_bounty| -> DispatchResult { let mut bounty = maybe_bounty.as_mut().ok_or(Error::::InvalidIndex)?; + + // Ensure no active child-bounties before processing the call. + ensure!( + T::ChildBountyManager::child_bounties_count(bounty_id) == 0, + Error::::HasActiveChildBounty + ); + match &bounty.status { BountyStatus::Active { curator, .. } => { ensure!(signer == *curator, Error::::RequireCurator); @@ -526,7 +574,7 @@ pub mod pallet { Ok(()) })?; - Self::deposit_event(Event::::BountyAwarded(bounty_id, beneficiary)); + Self::deposit_event(Event::::BountyAwarded { index: bounty_id, beneficiary }); Ok(()) } @@ -561,7 +609,15 @@ pub mod pallet { let payout = balance.saturating_sub(fee); let err_amount = T::Currency::unreserve(&curator, bounty.curator_deposit); debug_assert!(err_amount.is_zero()); - let res = T::Currency::transfer(&bounty_account, &curator, fee, AllowDeath); // should not fail + + // Get total child-bounties curator fees, and subtract it from master curator + // fee. + let children_fee = T::ChildBountyManager::children_curator_fees(bounty_id); + debug_assert!(children_fee <= fee); + + let final_fee = fee.saturating_sub(children_fee); + let res = + T::Currency::transfer(&bounty_account, &curator, final_fee, AllowDeath); // should not fail debug_assert!(res.is_ok()); let res = T::Currency::transfer(&bounty_account, &beneficiary, payout, AllowDeath); // should not fail @@ -571,7 +627,11 @@ pub mod pallet { BountyDescriptions::::remove(bounty_id); - Self::deposit_event(Event::::BountyClaimed(bounty_id, payout, beneficiary)); + Self::deposit_event(Event::::BountyClaimed { + index: bounty_id, + payout, + beneficiary, + }); Ok(()) } else { Err(Error::::UnexpectedStatus.into()) @@ -603,6 +663,12 @@ pub mod pallet { |maybe_bounty| -> DispatchResultWithPostInfo { let bounty = maybe_bounty.as_ref().ok_or(Error::::InvalidIndex)?; + // Ensure no active child-bounties before processing the call. + ensure!( + T::ChildBountyManager::child_bounties_count(bounty_id) == 0, + Error::::HasActiveChildBounty + ); + match &bounty.status { BountyStatus::Proposed => { // The reject origin would like to cancel a proposed bounty. @@ -612,7 +678,10 @@ pub mod pallet { T::OnSlash::on_unbalanced(imbalance); *maybe_bounty = None; - Self::deposit_event(Event::::BountyRejected(bounty_id, value)); + Self::deposit_event(Event::::BountyRejected { + index: bounty_id, + bond: value, + }); // Return early, nothing else to do. return Ok( Some(::WeightInfo::close_bounty_proposed()).into() @@ -656,7 +725,7 @@ pub mod pallet { debug_assert!(res.is_ok()); *maybe_bounty = None; - Self::deposit_event(Event::::BountyCanceled(bounty_id)); + Self::deposit_event(Event::::BountyCanceled { index: bounty_id }); Ok(Some(::WeightInfo::close_bounty_active()).into()) }, ) @@ -696,14 +765,26 @@ pub mod pallet { Ok(()) })?; - Self::deposit_event(Event::::BountyExtended(bounty_id)); + Self::deposit_event(Event::::BountyExtended { index: bounty_id }); Ok(()) } } } impl Pallet { - // Add public immutables and private mutables. + pub fn calculate_curator_deposit(fee: &BalanceOf) -> BalanceOf { + let mut deposit = T::CuratorDepositMultiplier::get() * *fee; + + if let Some(max_deposit) = T::CuratorDepositMax::get() { + deposit = deposit.min(max_deposit) + } + + if let Some(min_deposit) = T::CuratorDepositMin::get() { + deposit = deposit.max(min_deposit) + } + + deposit + } /// The account ID of the treasury pot. /// @@ -725,17 +806,15 @@ impl Pallet { description: Vec, value: BalanceOf, ) -> DispatchResult { - ensure!( - description.len() <= T::MaximumReasonLength::get() as usize, - Error::::ReasonTooBig - ); + let bounded_description: BoundedVec<_, _> = + description.try_into().map_err(|()| Error::::ReasonTooBig)?; ensure!(value >= T::BountyValueMinimum::get(), Error::::InvalidValue); let index = Self::bounty_count(); // reserve deposit for new bounty let bond = T::BountyDepositBase::get() + - T::DataDepositPerByte::get() * (description.len() as u32).into(); + T::DataDepositPerByte::get() * (bounded_description.len() as u32).into(); T::Currency::reserve(&proposer, bond) .map_err(|_| Error::::InsufficientProposersBalance)?; @@ -751,9 +830,9 @@ impl Pallet { }; Bounties::::insert(index, &bounty); - BountyDescriptions::::insert(index, description); + BountyDescriptions::::insert(index, bounded_description); - Self::deposit_event(Event::::BountyProposed(index)); + Self::deposit_event(Event::::BountyProposed { index }); Ok(()) } @@ -787,7 +866,7 @@ impl pallet_treasury::SpendFunds for Pallet { bounty.value, )); - Self::deposit_event(Event::::BountyBecameActive(index)); + Self::deposit_event(Event::::BountyBecameActive { index }); false } else { *missed_any = true; @@ -804,3 +883,14 @@ impl pallet_treasury::SpendFunds for Pallet { *total_weight += ::WeightInfo::spend_funds(bounties_len); } } + +// Default impl for when ChildBounties is not being used in the runtime. +impl ChildBountyManager for () { + fn child_bounties_count(_bounty_id: BountyIndex) -> BountyIndex { + Default::default() + } + + fn children_curator_fees(_bounty_id: BountyIndex) -> Balance { + Zero::zero() + } +} diff --git a/frame/bounties/src/migrations/mod.rs b/frame/bounties/src/migrations/mod.rs index 26d07a0cd5ac..235d0f1c7cf1 100644 --- a/frame/bounties/src/migrations/mod.rs +++ b/frame/bounties/src/migrations/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/bounties/src/migrations/v4.rs b/frame/bounties/src/migrations/v4.rs index a1ca0e47680b..8f5f3ebe55bf 100644 --- a/frame/bounties/src/migrations/v4.rs +++ b/frame/bounties/src/migrations/v4.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/bounties/src/tests.rs b/frame/bounties/src/tests.rs index 96c09581fdd1..9a84bd687abc 100644 --- a/frame/bounties/src/tests.rs +++ b/frame/bounties/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,8 +24,11 @@ use crate as pallet_bounties; use std::cell::RefCell; use frame_support::{ - assert_noop, assert_ok, pallet_prelude::GenesisBuild, parameter_types, traits::OnInitialize, - weights::Weight, PalletId, + assert_noop, assert_ok, + pallet_prelude::GenesisBuild, + parameter_types, + traits::{ConstU32, ConstU64, OnInitialize}, + PalletId, }; use sp_core::H256; @@ -54,12 +57,11 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; - pub const MaximumBlockWeight: Weight = 1024; - pub const MaximumBlockLength: u32 = 2 * 1024; pub const AvailableBlockRatio: Perbill = Perbill::one(); } +type Balance = u64; + impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; type BlockWeights = (); @@ -75,7 +77,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -84,18 +86,17 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } -parameter_types! { - pub const ExistentialDeposit: u64 = 1; -} + impl pallet_balances::Config for Test { type MaxLocks = (); type MaxReserves = (); type ReserveIdentifier = [u8; 8]; - type Balance = u64; + type Balance = Balance; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } @@ -104,13 +105,10 @@ thread_local! { } parameter_types! { pub const ProposalBond: Permill = Permill::from_percent(5); - pub const ProposalBondMinimum: u64 = 1; - pub const SpendPeriod: u64 = 2; pub const Burn: Permill = Permill::from_percent(50); - pub const DataDepositPerByte: u64 = 1; pub const TreasuryPalletId: PalletId = PalletId(*b"py/trsry"); - pub const MaxApprovals: u32 = 100; } + // impl pallet_treasury::Config for Test { impl pallet_treasury::Config for Test { type PalletId = TreasuryPalletId; @@ -120,32 +118,37 @@ impl pallet_treasury::Config for Test { type Event = Event; type OnSlash = (); type ProposalBond = ProposalBond; - type ProposalBondMinimum = ProposalBondMinimum; - type SpendPeriod = SpendPeriod; + type ProposalBondMinimum = ConstU64<1>; + type ProposalBondMaximum = (); + type SpendPeriod = ConstU64<2>; type Burn = Burn; type BurnDestination = (); // Just gets burned. type WeightInfo = (); type SpendFunds = Bounties; - type MaxApprovals = MaxApprovals; + type MaxApprovals = ConstU32<100>; } + parameter_types! { - pub const BountyDepositBase: u64 = 80; - pub const BountyDepositPayoutDelay: u64 = 3; - pub const BountyUpdatePeriod: u32 = 20; - pub const BountyCuratorDeposit: Permill = Permill::from_percent(50); - pub const BountyValueMinimum: u64 = 1; - pub const MaximumReasonLength: u32 = 16384; + // This will be 50% of the bounty fee. + pub const CuratorDepositMultiplier: Permill = Permill::from_percent(50); + pub const CuratorDepositMax: Balance = 1_000; + pub const CuratorDepositMin: Balance = 3; + } + impl Config for Test { type Event = Event; - type BountyDepositBase = BountyDepositBase; - type BountyDepositPayoutDelay = BountyDepositPayoutDelay; - type BountyUpdatePeriod = BountyUpdatePeriod; - type BountyCuratorDeposit = BountyCuratorDeposit; - type BountyValueMinimum = BountyValueMinimum; - type DataDepositPerByte = DataDepositPerByte; - type MaximumReasonLength = MaximumReasonLength; + type BountyDepositBase = ConstU64<80>; + type BountyDepositPayoutDelay = ConstU64<3>; + type BountyUpdatePeriod = ConstU64<20>; + type CuratorDepositMultiplier = CuratorDepositMultiplier; + type CuratorDepositMax = CuratorDepositMax; + type CuratorDepositMin = CuratorDepositMin; + type BountyValueMinimum = ConstU64<1>; + type DataDepositPerByte = ConstU64<1>; + type MaximumReasonLength = ConstU32<16384>; type WeightInfo = (); + type ChildBountyManager = (); } type TreasuryError = pallet_treasury::Error; @@ -398,7 +401,7 @@ fn propose_bounty_works() { assert_ok!(Bounties::propose_bounty(Origin::signed(0), 10, b"1234567890".to_vec())); - assert_eq!(last_event(), BountiesEvent::BountyProposed(0)); + assert_eq!(last_event(), BountiesEvent::BountyProposed { index: 0 }); let deposit: u64 = 85 + 5; assert_eq!(Balances::reserved_balance(0), deposit); @@ -460,7 +463,7 @@ fn close_bounty_works() { let deposit: u64 = 80 + 5; - assert_eq!(last_event(), BountiesEvent::BountyRejected(0, deposit)); + assert_eq!(last_event(), BountiesEvent::BountyRejected { index: 0, bond: deposit }); assert_eq!(Balances::reserved_balance(0), 0); assert_eq!(Balances::free_balance(0), 100 - deposit); @@ -550,13 +553,14 @@ fn assign_curator_works() { Error::::InvalidFee ); - assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 4)); + let fee = 4; + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, fee)); assert_eq!( Bounties::bounties(0).unwrap(), Bounty { proposer: 0, - fee: 4, + fee, curator_deposit: 0, value: 50, bond: 85, @@ -574,20 +578,22 @@ fn assign_curator_works() { assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + let expected_deposit = Bounties::calculate_curator_deposit(&fee); + assert_eq!( Bounties::bounties(0).unwrap(), Bounty { proposer: 0, - fee: 4, - curator_deposit: 2, + fee, + curator_deposit: expected_deposit, value: 50, bond: 85, status: BountyStatus::Active { curator: 4, update_due: 22 }, } ); - assert_eq!(Balances::free_balance(&4), 8); - assert_eq!(Balances::reserved_balance(&4), 2); + assert_eq!(Balances::free_balance(&4), 10 - expected_deposit); + assert_eq!(Balances::reserved_balance(&4), expected_deposit); }); } @@ -603,17 +609,17 @@ fn unassign_curator_works() { System::set_block_number(2); >::on_initialize(2); - assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 4)); + let fee = 4; + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, fee)); assert_noop!(Bounties::unassign_curator(Origin::signed(1), 0), BadOrigin); - assert_ok!(Bounties::unassign_curator(Origin::signed(4), 0)); assert_eq!( Bounties::bounties(0).unwrap(), Bounty { proposer: 0, - fee: 4, + fee, curator_deposit: 0, value: 50, bond: 85, @@ -621,19 +627,17 @@ fn unassign_curator_works() { } ); - assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 4)); - + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, fee)); Balances::make_free_balance_be(&4, 10); - assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); - + let expected_deposit = Bounties::calculate_curator_deposit(&fee); assert_ok!(Bounties::unassign_curator(Origin::root(), 0)); assert_eq!( Bounties::bounties(0).unwrap(), Bounty { proposer: 0, - fee: 4, + fee, curator_deposit: 0, value: 50, bond: 85, @@ -641,8 +645,8 @@ fn unassign_curator_works() { } ); - assert_eq!(Balances::free_balance(&4), 8); - assert_eq!(Balances::reserved_balance(&4), 0); // slashed 2 + assert_eq!(Balances::free_balance(&4), 10 - expected_deposit); + assert_eq!(Balances::reserved_balance(&4), 0); // slashed curator deposit }); } @@ -659,10 +663,12 @@ fn award_and_claim_bounty_works() { System::set_block_number(2); >::on_initialize(2); - assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 4)); + let fee = 4; + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, fee)); assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); - assert_eq!(Balances::free_balance(4), 8); // inital 10 - 2 deposit + let expected_deposit = Bounties::calculate_curator_deposit(&fee); + assert_eq!(Balances::free_balance(4), 10 - expected_deposit); assert_noop!( Bounties::award_bounty(Origin::signed(1), 0, 3), @@ -675,8 +681,8 @@ fn award_and_claim_bounty_works() { Bounties::bounties(0).unwrap(), Bounty { proposer: 0, - fee: 4, - curator_deposit: 2, + fee, + curator_deposit: expected_deposit, value: 50, bond: 85, status: BountyStatus::PendingPayout { curator: 4, beneficiary: 3, unlock_at: 5 }, @@ -692,7 +698,10 @@ fn award_and_claim_bounty_works() { assert_ok!(Bounties::claim_bounty(Origin::signed(1), 0)); - assert_eq!(last_event(), BountiesEvent::BountyClaimed(0, 56, 3)); + assert_eq!( + last_event(), + BountiesEvent::BountyClaimed { index: 0, payout: 56, beneficiary: 3 } + ); assert_eq!(Balances::free_balance(4), 14); // initial 10 + fee 4 @@ -731,7 +740,10 @@ fn claim_handles_high_fee() { assert_ok!(Bounties::claim_bounty(Origin::signed(1), 0)); - assert_eq!(last_event(), BountiesEvent::BountyClaimed(0, 0, 3)); + assert_eq!( + last_event(), + BountiesEvent::BountyClaimed { index: 0, payout: 0, beneficiary: 3 } + ); assert_eq!(Balances::free_balance(4), 70); // 30 + 50 - 10 assert_eq!(Balances::free_balance(3), 0); @@ -808,7 +820,7 @@ fn award_and_cancel() { assert_ok!(Bounties::unassign_curator(Origin::root(), 0)); assert_ok!(Bounties::close_bounty(Origin::root(), 0)); - assert_eq!(last_event(), BountiesEvent::BountyCanceled(0)); + assert_eq!(last_event(), BountiesEvent::BountyCanceled { index: 0 }); assert_eq!(Balances::free_balance(Bounties::bounty_account_id(0)), 0); @@ -996,3 +1008,117 @@ fn genesis_funding_works() { assert_eq!(Treasury::pot(), initial_funding - Balances::minimum_balance()); }); } + +#[test] +fn unassign_curator_self() { + new_test_ext().execute_with(|| { + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + System::set_block_number(2); + >::on_initialize(2); + + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 1, 10)); + assert_ok!(Bounties::accept_curator(Origin::signed(1), 0)); + + assert_eq!(Balances::free_balance(1), 93); + assert_eq!(Balances::reserved_balance(1), 5); + + System::set_block_number(8); + >::on_initialize(8); + + assert_ok!(Bounties::unassign_curator(Origin::signed(1), 0)); + + assert_eq!( + Bounties::bounties(0).unwrap(), + Bounty { + proposer: 0, + fee: 10, + curator_deposit: 0, + value: 50, + bond: 85, + status: BountyStatus::Funded, + } + ); + + assert_eq!(Balances::free_balance(1), 98); + assert_eq!(Balances::reserved_balance(1), 0); // not slashed + }); +} + +#[test] +fn accept_curator_handles_different_deposit_calculations() { + // This test will verify that a bounty with and without a fee results + // in a different curator deposit: one using the value, and one using the fee. + new_test_ext().execute_with(|| { + // Case 1: With a fee + let user = 1; + let bounty_index = 0; + let value = 88; + let fee = 42; + + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + Balances::make_free_balance_be(&user, 100); + assert_ok!(Bounties::propose_bounty(Origin::signed(0), value, b"12345".to_vec())); + assert_ok!(Bounties::approve_bounty(Origin::root(), bounty_index)); + + System::set_block_number(2); + >::on_initialize(2); + + assert_ok!(Bounties::propose_curator(Origin::root(), bounty_index, user, fee)); + assert_ok!(Bounties::accept_curator(Origin::signed(user), bounty_index)); + + let expected_deposit = CuratorDepositMultiplier::get() * fee; + assert_eq!(Balances::free_balance(&user), 100 - expected_deposit); + assert_eq!(Balances::reserved_balance(&user), expected_deposit); + + // Case 2: Lower bound + let user = 2; + let bounty_index = 1; + let value = 35; + let fee = 0; + + Balances::make_free_balance_be(&Treasury::account_id(), 101); + Balances::make_free_balance_be(&user, 100); + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), value, b"12345".to_vec())); + assert_ok!(Bounties::approve_bounty(Origin::root(), bounty_index)); + + System::set_block_number(3); + >::on_initialize(3); + + assert_ok!(Bounties::propose_curator(Origin::root(), bounty_index, user, fee)); + assert_ok!(Bounties::accept_curator(Origin::signed(user), bounty_index)); + + let expected_deposit = CuratorDepositMin::get(); + assert_eq!(Balances::free_balance(&user), 100 - expected_deposit); + assert_eq!(Balances::reserved_balance(&user), expected_deposit); + + // Case 3: Upper bound + let user = 3; + let bounty_index = 2; + let value = 1_000_000; + let fee = 50_000; + let starting_balance = fee * 2; + + Balances::make_free_balance_be(&Treasury::account_id(), value * 2); + Balances::make_free_balance_be(&user, starting_balance); + Balances::make_free_balance_be(&0, starting_balance); + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), value, b"12345".to_vec())); + assert_ok!(Bounties::approve_bounty(Origin::root(), bounty_index)); + + System::set_block_number(3); + >::on_initialize(3); + + assert_ok!(Bounties::propose_curator(Origin::root(), bounty_index, user, fee)); + assert_ok!(Bounties::accept_curator(Origin::signed(user), bounty_index)); + + let expected_deposit = CuratorDepositMax::get(); + assert_eq!(Balances::free_balance(&user), starting_balance - expected_deposit); + assert_eq!(Balances::reserved_balance(&user), expected_deposit); + }); +} diff --git a/frame/bounties/src/weights.rs b/frame/bounties/src/weights.rs index be9363642439..405e11acea5a 100644 --- a/frame/bounties/src/weights.rs +++ b/frame/bounties/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_bounties //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-30, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/bounties/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -61,87 +62,91 @@ pub trait WeightInfo { /// Weights for pallet_bounties using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { - // Storage: Treasury BountyCount (r:1 w:1) + // Storage: Bounties BountyCount (r:1 w:1) // Storage: System Account (r:1 w:1) - // Storage: Treasury BountyDescriptions (r:0 w:1) - // Storage: Treasury Bounties (r:0 w:1) + // Storage: Bounties BountyDescriptions (r:0 w:1) + // Storage: Bounties Bounties (r:0 w:1) fn propose_bounty(d: u32, ) -> Weight { - (44_482_000 as Weight) + (23_620_000 as Weight) // Standard Error: 0 - .saturating_add((1_000 as Weight).saturating_mul(d as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(d as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) - // Storage: Treasury BountyApprovals (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) + // Storage: Bounties BountyApprovals (r:1 w:1) fn approve_bounty() -> Weight { - (11_955_000 as Weight) + (6_655_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) fn propose_curator() -> Weight { - (9_771_000 as Weight) + (5_720_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) // Storage: System Account (r:1 w:1) fn unassign_curator() -> Weight { - (40_683_000 as Weight) + (26_958_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) // Storage: System Account (r:1 w:1) fn accept_curator() -> Weight { - (36_390_000 as Weight) + (21_457_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) + // Storage: ChildBounties ParentChildBounties (r:1 w:0) fn award_bounty() -> Weight { - (25_187_000 as Weight) - .saturating_add(T::DbWeight::get().reads(1 as Weight)) + (17_483_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) // Storage: System Account (r:3 w:3) - // Storage: Treasury BountyDescriptions (r:0 w:1) + // Storage: ChildBounties ChildrenCuratorFees (r:1 w:1) + // Storage: Bounties BountyDescriptions (r:0 w:1) fn claim_bounty() -> Weight { - (124_785_000 as Weight) - .saturating_add(T::DbWeight::get().reads(4 as Weight)) - .saturating_add(T::DbWeight::get().writes(5 as Weight)) + (61_763_000 as Weight) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) + // Storage: ChildBounties ParentChildBounties (r:1 w:0) // Storage: System Account (r:1 w:1) - // Storage: Treasury BountyDescriptions (r:0 w:1) + // Storage: Bounties BountyDescriptions (r:0 w:1) fn close_bounty_proposed() -> Weight { - (39_483_000 as Weight) - .saturating_add(T::DbWeight::get().reads(2 as Weight)) + (28_862_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) + // Storage: ChildBounties ParentChildBounties (r:1 w:0) // Storage: System Account (r:2 w:2) - // Storage: Treasury BountyDescriptions (r:0 w:1) + // Storage: Bounties BountyDescriptions (r:0 w:1) fn close_bounty_active() -> Weight { - (83_453_000 as Weight) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (45_129_000 as Weight) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) fn extend_bounty_expiry() -> Weight { - (24_151_000 as Weight) + (15_169_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } - // Storage: Treasury BountyApprovals (r:1 w:1) - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties BountyApprovals (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) // Storage: System Account (r:2 w:2) fn spend_funds(b: u32, ) -> Weight { (0 as Weight) - // Standard Error: 16_000 - .saturating_add((58_004_000 as Weight).saturating_mul(b as Weight)) + // Standard Error: 14_000 + .saturating_add((28_850_000 as Weight).saturating_mul(b as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(b as Weight))) .saturating_add(T::DbWeight::get().writes(1 as Weight)) @@ -151,87 +156,91 @@ impl WeightInfo for SubstrateWeight { // For backwards compatibility and tests impl WeightInfo for () { - // Storage: Treasury BountyCount (r:1 w:1) + // Storage: Bounties BountyCount (r:1 w:1) // Storage: System Account (r:1 w:1) - // Storage: Treasury BountyDescriptions (r:0 w:1) - // Storage: Treasury Bounties (r:0 w:1) + // Storage: Bounties BountyDescriptions (r:0 w:1) + // Storage: Bounties Bounties (r:0 w:1) fn propose_bounty(d: u32, ) -> Weight { - (44_482_000 as Weight) + (23_620_000 as Weight) // Standard Error: 0 - .saturating_add((1_000 as Weight).saturating_mul(d as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(d as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) - // Storage: Treasury BountyApprovals (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) + // Storage: Bounties BountyApprovals (r:1 w:1) fn approve_bounty() -> Weight { - (11_955_000 as Weight) + (6_655_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) fn propose_curator() -> Weight { - (9_771_000 as Weight) + (5_720_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) // Storage: System Account (r:1 w:1) fn unassign_curator() -> Weight { - (40_683_000 as Weight) + (26_958_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) // Storage: System Account (r:1 w:1) fn accept_curator() -> Weight { - (36_390_000 as Weight) + (21_457_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) + // Storage: ChildBounties ParentChildBounties (r:1 w:0) fn award_bounty() -> Weight { - (25_187_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + (17_483_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) // Storage: System Account (r:3 w:3) - // Storage: Treasury BountyDescriptions (r:0 w:1) + // Storage: ChildBounties ChildrenCuratorFees (r:1 w:1) + // Storage: Bounties BountyDescriptions (r:0 w:1) fn claim_bounty() -> Weight { - (124_785_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(4 as Weight)) - .saturating_add(RocksDbWeight::get().writes(5 as Weight)) + (61_763_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) + // Storage: ChildBounties ParentChildBounties (r:1 w:0) // Storage: System Account (r:1 w:1) - // Storage: Treasury BountyDescriptions (r:0 w:1) + // Storage: Bounties BountyDescriptions (r:0 w:1) fn close_bounty_proposed() -> Weight { - (39_483_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + (28_862_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) + // Storage: ChildBounties ParentChildBounties (r:1 w:0) // Storage: System Account (r:2 w:2) - // Storage: Treasury BountyDescriptions (r:0 w:1) + // Storage: Bounties BountyDescriptions (r:0 w:1) fn close_bounty_active() -> Weight { - (83_453_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (45_129_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) fn extend_bounty_expiry() -> Weight { - (24_151_000 as Weight) + (15_169_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } - // Storage: Treasury BountyApprovals (r:1 w:1) - // Storage: Treasury Bounties (r:1 w:1) + // Storage: Bounties BountyApprovals (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:1) // Storage: System Account (r:2 w:2) fn spend_funds(b: u32, ) -> Weight { (0 as Weight) - // Standard Error: 16_000 - .saturating_add((58_004_000 as Weight).saturating_mul(b as Weight)) + // Standard Error: 14_000 + .saturating_add((28_850_000 as Weight).saturating_mul(b as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(b as Weight))) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) diff --git a/frame/child-bounties/Cargo.toml b/frame/child-bounties/Cargo.toml new file mode 100644 index 000000000000..9d2e1031604e --- /dev/null +++ b/frame/child-bounties/Cargo.toml @@ -0,0 +1,54 @@ +[package] +name = "pallet-child-bounties" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet to manage child bounties" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ + "derive", +] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +pallet-treasury = { version = "4.0.0-dev", default-features = false, path = "../treasury" } +pallet-bounties = { version = "4.0.0-dev", default-features = false, path = "../bounties" } +sp-io = { version = "6.0.0", path = "../../primitives/io", default-features = false } +sp-core = { version = "6.0.0", path = "../../primitives/core", default-features = false } +frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } +log = { version = "0.4.14", default-features = false } + +[dev-dependencies] +pallet-balances = { version = "4.0.0-dev", path = "../balances" } + +[features] +default = ["std"] +std = [ + "codec/std", + "sp-core/std", + "sp-io/std", + "scale-info/std", + "sp-std/std", + "sp-runtime/std", + "frame-support/std", + "frame-system/std", + "pallet-treasury/std", + "pallet-bounties/std", + "log/std", +] +runtime-benchmarks = [ + "frame-benchmarking", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", +] +try-runtime = ["frame-support/try-runtime"] diff --git a/frame/child-bounties/README.md b/frame/child-bounties/README.md new file mode 100644 index 000000000000..e07996d54957 --- /dev/null +++ b/frame/child-bounties/README.md @@ -0,0 +1,21 @@ +# Child Bounties Pallet (pallet-child-bounties) + +## Child Bounty + +> NOTE: This pallet is tightly coupled with pallet-treasury and pallet-bounties. + +With child bounties, a large bounty proposal can be divided into smaller chunks, for parallel execution, and for efficient governance and tracking of spent funds. + +A child-bounty is a smaller piece of work, extracted from a parent bounty. A curator is assigned after the child-bounty is created by the parent bounty curator, to be delegated with the responsibility of assigning a payout address once the specified set of tasks is completed. + +## Interface + +### Dispatchable Functions + +- `add_child_bounty` - Add a child-bounty for a parent-bounty to for dividing the work in smaller tasks. +- `propose_curator` - Assign an account to a child-bounty as candidate curator. +- `accept_curator` - Accept a child-bounty assignment from the parent-bounty curator, setting a curator deposit. +- `award_child_bounty` - Close and pay out the specified amount for the completed work. +- `claim_child_bounty` - Claim a specific child-bounty amount from the payout address. +- `unassign_curator` - Unassign an accepted curator from a specific child-bounty. +- `close_child_bounty` - Cancel the child-bounty for a specific treasury amount and close the bounty. diff --git a/frame/child-bounties/src/benchmarking.rs b/frame/child-bounties/src/benchmarking.rs new file mode 100644 index 000000000000..d9edf14bbc9d --- /dev/null +++ b/frame/child-bounties/src/benchmarking.rs @@ -0,0 +1,308 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Child-bounties pallet benchmarking. + +#![cfg(feature = "runtime-benchmarks")] + +use super::*; + +use frame_benchmarking::{account, benchmarks, whitelisted_caller}; +use frame_system::RawOrigin; + +use crate::Pallet as ChildBounties; +use pallet_bounties::Pallet as Bounties; +use pallet_treasury::Pallet as Treasury; + +const SEED: u32 = 0; + +#[derive(Clone)] +struct BenchmarkChildBounty { + /// Bounty ID. + bounty_id: BountyIndex, + /// ChildBounty ID. + child_bounty_id: BountyIndex, + /// The account proposing it. + caller: T::AccountId, + /// The master curator account. + curator: T::AccountId, + /// The child-bounty curator account. + child_curator: T::AccountId, + /// The (total) amount that should be paid if the bounty is rewarded. + value: BalanceOf, + /// The curator fee. included in value. + fee: BalanceOf, + /// The (total) amount that should be paid if the child-bounty is rewarded. + child_bounty_value: BalanceOf, + /// The child-bounty curator fee. included in value. + child_bounty_fee: BalanceOf, + /// Bounty description. + reason: Vec, +} + +fn setup_bounty( + user: u32, + description: u32, +) -> (T::AccountId, T::AccountId, BalanceOf, BalanceOf, Vec) { + let caller = account("caller", user, SEED); + let value: BalanceOf = T::BountyValueMinimum::get().saturating_mul(100u32.into()); + let fee = value / 2u32.into(); + let deposit = T::BountyDepositBase::get() + + T::DataDepositPerByte::get() * T::MaximumReasonLength::get().into(); + let _ = T::Currency::make_free_balance_be(&caller, deposit); + let curator = account("curator", user, SEED); + let _ = T::Currency::make_free_balance_be(&curator, fee / 2u32.into()); + let reason = vec![0; description as usize]; + (caller, curator, fee, value, reason) +} + +fn setup_child_bounty(user: u32, description: u32) -> BenchmarkChildBounty { + let (caller, curator, fee, value, reason) = setup_bounty::(user, description); + let child_curator = account("child-curator", user, SEED); + let _ = T::Currency::make_free_balance_be(&child_curator, fee / 2u32.into()); + let child_bounty_value = (value - fee) / 4u32.into(); + let child_bounty_fee = child_bounty_value / 2u32.into(); + + BenchmarkChildBounty:: { + bounty_id: 0, + child_bounty_id: 0, + caller, + curator, + child_curator, + value, + fee, + child_bounty_value, + child_bounty_fee, + reason, + } +} + +fn activate_bounty( + user: u32, + description: u32, +) -> Result, &'static str> { + let mut child_bounty_setup = setup_child_bounty::(user, description); + let curator_lookup = T::Lookup::unlookup(child_bounty_setup.curator.clone()); + Bounties::::propose_bounty( + RawOrigin::Signed(child_bounty_setup.caller.clone()).into(), + child_bounty_setup.value, + child_bounty_setup.reason.clone(), + )?; + + child_bounty_setup.bounty_id = Bounties::::bounty_count() - 1; + + Bounties::::approve_bounty(RawOrigin::Root.into(), child_bounty_setup.bounty_id)?; + Treasury::::on_initialize(T::BlockNumber::zero()); + Bounties::::propose_curator( + RawOrigin::Root.into(), + child_bounty_setup.bounty_id, + curator_lookup.clone(), + child_bounty_setup.fee, + )?; + Bounties::::accept_curator( + RawOrigin::Signed(child_bounty_setup.curator.clone()).into(), + child_bounty_setup.bounty_id, + )?; + + Ok(child_bounty_setup) +} + +fn activate_child_bounty( + user: u32, + description: u32, +) -> Result, &'static str> { + let mut bounty_setup = activate_bounty::(user, description)?; + let child_curator_lookup = T::Lookup::unlookup(bounty_setup.child_curator.clone()); + + ChildBounties::::add_child_bounty( + RawOrigin::Signed(bounty_setup.curator.clone()).into(), + bounty_setup.bounty_id, + bounty_setup.child_bounty_value, + bounty_setup.reason.clone(), + )?; + + bounty_setup.child_bounty_id = ChildBountyCount::::get() - 1; + + ChildBounties::::propose_curator( + RawOrigin::Signed(bounty_setup.curator.clone()).into(), + bounty_setup.bounty_id, + bounty_setup.child_bounty_id, + child_curator_lookup.clone(), + bounty_setup.child_bounty_fee, + )?; + + ChildBounties::::accept_curator( + RawOrigin::Signed(bounty_setup.child_curator.clone()).into(), + bounty_setup.bounty_id, + bounty_setup.child_bounty_id, + )?; + + Ok(bounty_setup) +} + +fn setup_pot_account() { + let pot_account = Bounties::::account_id(); + let value = T::Currency::minimum_balance().saturating_mul(1_000_000_000u32.into()); + let _ = T::Currency::make_free_balance_be(&pot_account, value); +} + +fn assert_last_event(generic_event: ::Event) { + frame_system::Pallet::::assert_last_event(generic_event.into()); +} + +benchmarks! { + add_child_bounty { + let d in 0 .. T::MaximumReasonLength::get(); + setup_pot_account::(); + let bounty_setup = activate_bounty::(0, d)?; + }: _(RawOrigin::Signed(bounty_setup.curator), bounty_setup.bounty_id, + bounty_setup.child_bounty_value, bounty_setup.reason.clone()) + verify { + assert_last_event::(Event::Added { + index: bounty_setup.bounty_id, + child_index: bounty_setup.child_bounty_id, + }.into()) + } + + propose_curator { + setup_pot_account::(); + let bounty_setup = activate_bounty::(0, T::MaximumReasonLength::get())?; + let child_curator_lookup = T::Lookup::unlookup(bounty_setup.child_curator.clone()); + + ChildBounties::::add_child_bounty( + RawOrigin::Signed(bounty_setup.curator.clone()).into(), + bounty_setup.bounty_id, + bounty_setup.child_bounty_value, + bounty_setup.reason.clone(), + )?; + let child_bounty_id = ChildBountyCount::::get() - 1; + + }: _(RawOrigin::Signed(bounty_setup.curator), bounty_setup.bounty_id, + child_bounty_id, child_curator_lookup, bounty_setup.child_bounty_fee) + + accept_curator { + setup_pot_account::(); + let mut bounty_setup = activate_bounty::(0, T::MaximumReasonLength::get())?; + let child_curator_lookup = T::Lookup::unlookup(bounty_setup.child_curator.clone()); + + ChildBounties::::add_child_bounty( + RawOrigin::Signed(bounty_setup.curator.clone()).into(), + bounty_setup.bounty_id, + bounty_setup.child_bounty_value, + bounty_setup.reason.clone(), + )?; + bounty_setup.child_bounty_id = ChildBountyCount::::get() - 1; + + ChildBounties::::propose_curator( + RawOrigin::Signed(bounty_setup.curator.clone()).into(), + bounty_setup.bounty_id, + bounty_setup.child_bounty_id, + child_curator_lookup.clone(), + bounty_setup.child_bounty_fee, + )?; + }: _(RawOrigin::Signed(bounty_setup.child_curator), bounty_setup.bounty_id, + bounty_setup.child_bounty_id) + + // Worst case when curator is inactive and any sender un-assigns the curator. + unassign_curator { + setup_pot_account::(); + let bounty_setup = activate_child_bounty::(0, T::MaximumReasonLength::get())?; + Bounties::::on_initialize(T::BlockNumber::zero()); + frame_system::Pallet::::set_block_number(T::BountyUpdatePeriod::get() + 1u32.into()); + let caller = whitelisted_caller(); + }: _(RawOrigin::Signed(caller), bounty_setup.bounty_id, + bounty_setup.child_bounty_id) + + award_child_bounty { + setup_pot_account::(); + let bounty_setup = activate_child_bounty::(0, T::MaximumReasonLength::get())?; + let beneficiary_account: T::AccountId = account("beneficiary", 0, SEED); + let beneficiary = T::Lookup::unlookup(beneficiary_account.clone()); + }: _(RawOrigin::Signed(bounty_setup.child_curator), bounty_setup.bounty_id, + bounty_setup.child_bounty_id, beneficiary) + verify { + assert_last_event::(Event::Awarded { + index: bounty_setup.bounty_id, + child_index: bounty_setup.child_bounty_id, + beneficiary: beneficiary_account + }.into()) + } + + claim_child_bounty { + setup_pot_account::(); + let bounty_setup = activate_child_bounty::(0, T::MaximumReasonLength::get())?; + let beneficiary_account: T::AccountId = account("beneficiary", 0, SEED); + let beneficiary = T::Lookup::unlookup(beneficiary_account.clone()); + + ChildBounties::::award_child_bounty( + RawOrigin::Signed(bounty_setup.child_curator.clone()).into(), + bounty_setup.bounty_id, + bounty_setup.child_bounty_id, + beneficiary + )?; + + let beneficiary_account: T::AccountId = account("beneficiary", 0, SEED); + let beneficiary = T::Lookup::unlookup(beneficiary_account.clone()); + + frame_system::Pallet::::set_block_number(T::BountyDepositPayoutDelay::get()); + ensure!(T::Currency::free_balance(&beneficiary_account).is_zero(), + "Beneficiary already has balance."); + + }: _(RawOrigin::Signed(bounty_setup.curator), bounty_setup.bounty_id, + bounty_setup.child_bounty_id) + verify { + ensure!(!T::Currency::free_balance(&beneficiary_account).is_zero(), + "Beneficiary didn't get paid."); + } + + // Best case scenario. + close_child_bounty_added { + setup_pot_account::(); + let mut bounty_setup = activate_bounty::(0, T::MaximumReasonLength::get())?; + + ChildBounties::::add_child_bounty( + RawOrigin::Signed(bounty_setup.curator.clone()).into(), + bounty_setup.bounty_id, + bounty_setup.child_bounty_value, + bounty_setup.reason.clone(), + )?; + bounty_setup.child_bounty_id = ChildBountyCount::::get() - 1; + + }: close_child_bounty(RawOrigin::Root, bounty_setup.bounty_id, + bounty_setup.child_bounty_id) + verify { + assert_last_event::(Event::Canceled { + index: bounty_setup.bounty_id, + child_index: bounty_setup.child_bounty_id + }.into()) + } + + // Worst case scenario. + close_child_bounty_active { + setup_pot_account::(); + let bounty_setup = activate_child_bounty::(0, T::MaximumReasonLength::get())?; + Bounties::::on_initialize(T::BlockNumber::zero()); + }: close_child_bounty(RawOrigin::Root, bounty_setup.bounty_id, bounty_setup.child_bounty_id) + verify { + assert_last_event::(Event::Canceled { + index: bounty_setup.bounty_id, + child_index: bounty_setup.child_bounty_id, + }.into()) + } + + impl_benchmark_test_suite!(ChildBounties, crate::tests::new_test_ext(), crate::tests::Test) +} diff --git a/frame/child-bounties/src/lib.rs b/frame/child-bounties/src/lib.rs new file mode 100644 index 000000000000..2fea61c045cf --- /dev/null +++ b/frame/child-bounties/src/lib.rs @@ -0,0 +1,907 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Child Bounties Pallet ( pallet-child-bounties ) +//! +//! ## Child Bounty +//! +//! > NOTE: This pallet is tightly coupled with pallet-treasury and pallet-bounties. +//! +//! With child bounties, a large bounty proposal can be divided into smaller chunks, +//! for parallel execution, and for efficient governance and tracking of spent funds. +//! A child-bounty is a smaller piece of work, extracted from a parent bounty. +//! A curator is assigned after the child-bounty is created by the parent bounty curator, +//! to be delegated with the responsibility of assigning a payout address once the specified +//! set of tasks is completed. +//! +//! ## Interface +//! +//! ### Dispatchable Functions +//! +//! Child Bounty protocol: +//! - `add_child_bounty` - Add a child-bounty for a parent-bounty to for dividing the work in +//! smaller tasks. +//! - `propose_curator` - Assign an account to a child-bounty as candidate curator. +//! - `accept_curator` - Accept a child-bounty assignment from the parent-bounty curator, setting a +//! curator deposit. +//! - `award_child_bounty` - Close and pay out the specified amount for the completed work. +//! - `claim_child_bounty` - Claim a specific child-bounty amount from the payout address. +//! - `unassign_curator` - Unassign an accepted curator from a specific child-bounty. +//! - `close_child_bounty` - Cancel the child-bounty for a specific treasury amount and close the +//! bounty. + +// Most of the business logic in this pallet has been +// originally contributed by "https://github.com/shamb0", +// as part of the PR - https://github.com/paritytech/substrate/pull/7965. +// The code has been moved here and then refactored in order to +// extract child-bounties as a separate pallet. + +#![cfg_attr(not(feature = "std"), no_std)] + +mod benchmarking; +mod tests; +pub mod weights; + +use sp_std::prelude::*; + +use frame_support::traits::{ + Currency, + ExistenceRequirement::{AllowDeath, KeepAlive}, + Get, OnUnbalanced, ReservableCurrency, WithdrawReasons, +}; + +use sp_runtime::{ + traits::{AccountIdConversion, BadOrigin, CheckedSub, Saturating, StaticLookup, Zero}, + DispatchResult, RuntimeDebug, +}; + +use frame_support::pallet_prelude::*; +use frame_system::pallet_prelude::*; +use pallet_bounties::BountyStatus; +use scale_info::TypeInfo; +pub use weights::WeightInfo; + +pub use pallet::*; + +type BalanceOf = pallet_treasury::BalanceOf; +type BountiesError = pallet_bounties::Error; +type BountyIndex = pallet_bounties::BountyIndex; + +/// A child bounty proposal. +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub struct ChildBounty { + /// The parent of this child-bounty. + parent_bounty: BountyIndex, + /// The (total) amount that should be paid if this child-bounty is rewarded. + value: Balance, + /// The child bounty curator fee. + fee: Balance, + /// The deposit of child-bounty curator. + curator_deposit: Balance, + /// The status of this child-bounty. + status: ChildBountyStatus, +} + +/// The status of a child-bounty. +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub enum ChildBountyStatus { + /// The child-bounty is added and waiting for curator assignment. + Added, + /// A curator has been proposed by the parent-bounty curator. Waiting for + /// acceptance from the child-bounty curator. + CuratorProposed { + /// The assigned child-bounty curator of this bounty. + curator: AccountId, + }, + /// The child-bounty is active and waiting to be awarded. + Active { + /// The curator of this child-bounty. + curator: AccountId, + }, + /// The child-bounty is awarded and waiting to released after a delay. + PendingPayout { + /// The curator of this child-bounty. + curator: AccountId, + /// The beneficiary of the child-bounty. + beneficiary: AccountId, + /// When the child-bounty can be claimed. + unlock_at: BlockNumber, + }, +} + +#[frame_support::pallet] +pub mod pallet { + + use super::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: + frame_system::Config + pallet_treasury::Config + pallet_bounties::Config + { + /// Maximum number of child-bounties that can be added to a parent bounty. + #[pallet::constant] + type MaxActiveChildBountyCount: Get; + + /// Minimum value for a child-bounty. + #[pallet::constant] + type ChildBountyValueMinimum: Get>; + + /// The overarching event type. + type Event: From> + IsType<::Event>; + + /// Weight information for extrinsics in this pallet. + type WeightInfo: WeightInfo; + } + + #[pallet::error] + pub enum Error { + /// The parent bounty is not in active state. + ParentBountyNotActive, + /// The bounty balance is not enough to add new child-bounty. + InsufficientBountyBalance, + /// Number of child-bounties exceeds limit `MaxActiveChildBountyCount`. + TooManyChildBounties, + } + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// A child-bounty is added. + Added { index: BountyIndex, child_index: BountyIndex }, + /// A child-bounty is awarded to a beneficiary. + Awarded { index: BountyIndex, child_index: BountyIndex, beneficiary: T::AccountId }, + /// A child-bounty is claimed by beneficiary. + Claimed { + index: BountyIndex, + child_index: BountyIndex, + payout: BalanceOf, + beneficiary: T::AccountId, + }, + /// A child-bounty is cancelled. + Canceled { index: BountyIndex, child_index: BountyIndex }, + } + + /// Number of total child bounties. + #[pallet::storage] + #[pallet::getter(fn child_bounty_count)] + pub type ChildBountyCount = StorageValue<_, BountyIndex, ValueQuery>; + + /// Number of child-bounties per parent bounty. + /// Map of parent bounty index to number of child bounties. + #[pallet::storage] + #[pallet::getter(fn parent_child_bounties)] + pub type ParentChildBounties = + StorageMap<_, Twox64Concat, BountyIndex, u32, ValueQuery>; + + /// Child-bounties that have been added. + #[pallet::storage] + #[pallet::getter(fn child_bounties)] + pub type ChildBounties = StorageDoubleMap< + _, + Twox64Concat, + BountyIndex, + Twox64Concat, + BountyIndex, + ChildBounty, T::BlockNumber>, + >; + + /// The description of each child-bounty. + #[pallet::storage] + #[pallet::getter(fn child_bounty_descriptions)] + pub type ChildBountyDescriptions = + StorageMap<_, Twox64Concat, BountyIndex, BoundedVec>; + + /// The cumulative child-bounty curator fee for each parent bounty. + #[pallet::storage] + #[pallet::getter(fn children_curator_fees)] + pub type ChildrenCuratorFees = + StorageMap<_, Twox64Concat, BountyIndex, BalanceOf, ValueQuery>; + + #[pallet::call] + impl Pallet { + /// Add a new child-bounty. + /// + /// The dispatch origin for this call must be the curator of parent + /// bounty and the parent bounty must be in "active" state. + /// + /// Child-bounty gets added successfully & fund gets transferred from + /// parent bounty to child-bounty account, if parent bounty has enough + /// funds, else the call fails. + /// + /// Upper bound to maximum number of active child-bounties that can be + /// added are managed via runtime trait config + /// [`Config::MaxActiveChildBountyCount`]. + /// + /// If the call is success, the status of child-bounty is updated to + /// "Added". + /// + /// - `parent_bounty_id`: Index of parent bounty for which child-bounty is being added. + /// - `value`: Value for executing the proposal. + /// - `description`: Text description for the child-bounty. + #[pallet::weight(::WeightInfo::add_child_bounty(description.len() as u32))] + pub fn add_child_bounty( + origin: OriginFor, + #[pallet::compact] parent_bounty_id: BountyIndex, + #[pallet::compact] value: BalanceOf, + description: Vec, + ) -> DispatchResult { + let signer = ensure_signed(origin)?; + + // Verify the arguments. + let bounded_description = + description.try_into().map_err(|_| BountiesError::::ReasonTooBig)?; + ensure!(value >= T::ChildBountyValueMinimum::get(), BountiesError::::InvalidValue); + ensure!( + Self::parent_child_bounties(parent_bounty_id) <= + T::MaxActiveChildBountyCount::get() as u32, + Error::::TooManyChildBounties, + ); + + let (curator, _) = Self::ensure_bounty_active(parent_bounty_id)?; + ensure!(signer == curator, BountiesError::::RequireCurator); + + // Read parent bounty account info. + let parent_bounty_account = + pallet_bounties::Pallet::::bounty_account_id(parent_bounty_id); + + // Ensure parent bounty has enough balance after adding child-bounty. + let bounty_balance = T::Currency::free_balance(&parent_bounty_account); + let new_bounty_balance = bounty_balance + .checked_sub(&value) + .ok_or(Error::::InsufficientBountyBalance)?; + T::Currency::ensure_can_withdraw( + &parent_bounty_account, + value, + WithdrawReasons::TRANSFER, + new_bounty_balance, + )?; + + // Get child-bounty ID. + let child_bounty_id = Self::child_bounty_count(); + let child_bounty_account = Self::child_bounty_account_id(child_bounty_id); + + // Transfer funds from parent bounty to child-bounty. + T::Currency::transfer(&parent_bounty_account, &child_bounty_account, value, KeepAlive)?; + + // Increment the active child-bounty count. + >::mutate(parent_bounty_id, |count| count.saturating_inc()); + >::put(child_bounty_id.saturating_add(1)); + + // Create child-bounty instance. + Self::create_child_bounty( + parent_bounty_id, + child_bounty_id, + value, + bounded_description, + ); + Ok(()) + } + + /// Propose curator for funded child-bounty. + /// + /// The dispatch origin for this call must be curator of parent bounty. + /// + /// Parent bounty must be in active state, for this child-bounty call to + /// work. + /// + /// Child-bounty must be in "Added" state, for processing the call. And + /// state of child-bounty is moved to "CuratorProposed" on successful + /// call completion. + /// + /// - `parent_bounty_id`: Index of parent bounty. + /// - `child_bounty_id`: Index of child bounty. + /// - `curator`: Address of child-bounty curator. + /// - `fee`: payment fee to child-bounty curator for execution. + #[pallet::weight(::WeightInfo::propose_curator())] + pub fn propose_curator( + origin: OriginFor, + #[pallet::compact] parent_bounty_id: BountyIndex, + #[pallet::compact] child_bounty_id: BountyIndex, + curator: ::Source, + #[pallet::compact] fee: BalanceOf, + ) -> DispatchResult { + let signer = ensure_signed(origin)?; + let child_bounty_curator = T::Lookup::lookup(curator)?; + + let (curator, _) = Self::ensure_bounty_active(parent_bounty_id)?; + ensure!(signer == curator, BountiesError::::RequireCurator); + + // Mutate the child-bounty instance. + ChildBounties::::try_mutate_exists( + parent_bounty_id, + child_bounty_id, + |maybe_child_bounty| -> DispatchResult { + let mut child_bounty = + maybe_child_bounty.as_mut().ok_or(BountiesError::::InvalidIndex)?; + + // Ensure child-bounty is in expected state. + ensure!( + child_bounty.status == ChildBountyStatus::Added, + BountiesError::::UnexpectedStatus, + ); + + // Ensure child-bounty curator fee is less than child-bounty value. + ensure!(fee < child_bounty.value, BountiesError::::InvalidFee); + + // Add child-bounty curator fee to the cumulative sum. To be + // subtracted from the parent bounty curator when claiming + // bounty. + ChildrenCuratorFees::::mutate(parent_bounty_id, |value| { + *value = value.saturating_add(fee) + }); + + // Update the child-bounty curator fee. + child_bounty.fee = fee; + + // Update the child-bounty state. + child_bounty.status = + ChildBountyStatus::CuratorProposed { curator: child_bounty_curator }; + + Ok(()) + }, + ) + } + + /// Accept the curator role for the child-bounty. + /// + /// The dispatch origin for this call must be the curator of this + /// child-bounty. + /// + /// A deposit will be reserved from the curator and refund upon + /// successful payout or cancellation. + /// + /// Fee for curator is deducted from curator fee of parent bounty. + /// + /// Parent bounty must be in active state, for this child-bounty call to + /// work. + /// + /// Child-bounty must be in "CuratorProposed" state, for processing the + /// call. And state of child-bounty is moved to "Active" on successful + /// call completion. + /// + /// - `parent_bounty_id`: Index of parent bounty. + /// - `child_bounty_id`: Index of child bounty. + #[pallet::weight(::WeightInfo::accept_curator())] + pub fn accept_curator( + origin: OriginFor, + #[pallet::compact] parent_bounty_id: BountyIndex, + #[pallet::compact] child_bounty_id: BountyIndex, + ) -> DispatchResult { + let signer = ensure_signed(origin)?; + + let (parent_curator, _) = Self::ensure_bounty_active(parent_bounty_id)?; + // Mutate child-bounty. + ChildBounties::::try_mutate_exists( + parent_bounty_id, + child_bounty_id, + |maybe_child_bounty| -> DispatchResult { + let mut child_bounty = + maybe_child_bounty.as_mut().ok_or(BountiesError::::InvalidIndex)?; + + // Ensure child-bounty is in expected state. + if let ChildBountyStatus::CuratorProposed { ref curator } = child_bounty.status + { + ensure!(signer == *curator, BountiesError::::RequireCurator); + + // Reserve child-bounty curator deposit. + let deposit = Self::calculate_curator_deposit( + &parent_curator, + curator, + &child_bounty.fee, + ); + + T::Currency::reserve(curator, deposit)?; + child_bounty.curator_deposit = deposit; + + child_bounty.status = + ChildBountyStatus::Active { curator: curator.clone() }; + Ok(()) + } else { + Err(BountiesError::::UnexpectedStatus.into()) + } + }, + ) + } + + /// Unassign curator from a child-bounty. + /// + /// The dispatch origin for this call can be either `RejectOrigin`, or + /// the curator of the parent bounty, or any signed origin. + /// + /// For the origin other than T::RejectOrigin and the child-bounty + /// curator, parent-bounty must be in active state, for this call to + /// work. We allow child-bounty curator and T::RejectOrigin to execute + /// this call irrespective of the parent-bounty state. + /// + /// If this function is called by the `RejectOrigin` or the + /// parent-bounty curator, we assume that the child-bounty curator is + /// malicious or inactive. As a result, child-bounty curator deposit is + /// slashed. + /// + /// If the origin is the child-bounty curator, we take this as a sign + /// that they are unable to do their job, and are willingly giving up. + /// We could slash the deposit, but for now we allow them to unreserve + /// their deposit and exit without issue. (We may want to change this if + /// it is abused.) + /// + /// Finally, the origin can be anyone iff the child-bounty curator is + /// "inactive". Expiry update due of parent bounty is used to estimate + /// inactive state of child-bounty curator. + /// + /// This allows anyone in the community to call out that a child-bounty + /// curator is not doing their due diligence, and we should pick a new + /// one. In this case the child-bounty curator deposit is slashed. + /// + /// State of child-bounty is moved to Added state on successful call + /// completion. + /// + /// - `parent_bounty_id`: Index of parent bounty. + /// - `child_bounty_id`: Index of child bounty. + #[pallet::weight(::WeightInfo::unassign_curator())] + pub fn unassign_curator( + origin: OriginFor, + #[pallet::compact] parent_bounty_id: BountyIndex, + #[pallet::compact] child_bounty_id: BountyIndex, + ) -> DispatchResult { + let maybe_sender = ensure_signed(origin.clone()) + .map(Some) + .or_else(|_| T::RejectOrigin::ensure_origin(origin).map(|_| None))?; + + ChildBounties::::try_mutate_exists( + parent_bounty_id, + child_bounty_id, + |maybe_child_bounty| -> DispatchResult { + let mut child_bounty = + maybe_child_bounty.as_mut().ok_or(BountiesError::::InvalidIndex)?; + + let slash_curator = |curator: &T::AccountId, + curator_deposit: &mut BalanceOf| { + let imbalance = T::Currency::slash_reserved(curator, *curator_deposit).0; + T::OnSlash::on_unbalanced(imbalance); + *curator_deposit = Zero::zero(); + }; + + match child_bounty.status { + ChildBountyStatus::Added => { + // No curator to unassign at this point. + return Err(BountiesError::::UnexpectedStatus.into()) + }, + ChildBountyStatus::CuratorProposed { ref curator } => { + // A child-bounty curator has been proposed, but not accepted yet. + // Either `RejectOrigin`, parent-bounty curator or the proposed + // child-bounty curator can unassign the child-bounty curator. + ensure!( + maybe_sender.map_or(true, |sender| { + sender == *curator || + Self::ensure_bounty_active(parent_bounty_id) + .map_or(false, |(parent_curator, _)| { + sender == parent_curator + }) + }), + BadOrigin + ); + // Continue to change bounty status below. + }, + ChildBountyStatus::Active { ref curator } => { + // The child-bounty is active. + match maybe_sender { + // If the `RejectOrigin` is calling this function, slash the curator + // deposit. + None => { + slash_curator(curator, &mut child_bounty.curator_deposit); + // Continue to change child-bounty status below. + }, + Some(sender) if sender == *curator => { + // This is the child-bounty curator, willingly giving up their + // role. Give back their deposit. + T::Currency::unreserve(&curator, child_bounty.curator_deposit); + // Reset curator deposit. + child_bounty.curator_deposit = Zero::zero(); + // Continue to change bounty status below. + }, + Some(sender) => { + let (parent_curator, update_due) = + Self::ensure_bounty_active(parent_bounty_id)?; + if sender == parent_curator || + update_due < frame_system::Pallet::::block_number() + { + // Slash the child-bounty curator if + // + the call is made by the parent-bounty curator. + // + or the curator is inactive. + slash_curator(curator, &mut child_bounty.curator_deposit); + // Continue to change bounty status below. + } else { + // Curator has more time to give an update. + return Err(BountiesError::::Premature.into()) + } + }, + } + }, + ChildBountyStatus::PendingPayout { ref curator, .. } => { + let (parent_curator, _) = Self::ensure_bounty_active(parent_bounty_id)?; + ensure!( + maybe_sender.map_or(true, |sender| parent_curator == sender), + BadOrigin, + ); + slash_curator(curator, &mut child_bounty.curator_deposit); + // Continue to change child-bounty status below. + }, + }; + // Move the child-bounty state to Added. + child_bounty.status = ChildBountyStatus::Added; + Ok(()) + }, + ) + } + + /// Award child-bounty to a beneficiary. + /// + /// The beneficiary will be able to claim the funds after a delay. + /// + /// The dispatch origin for this call must be the master curator or + /// curator of this child-bounty. + /// + /// Parent bounty must be in active state, for this child-bounty call to + /// work. + /// + /// Child-bounty must be in active state, for processing the call. And + /// state of child-bounty is moved to "PendingPayout" on successful call + /// completion. + /// + /// - `parent_bounty_id`: Index of parent bounty. + /// - `child_bounty_id`: Index of child bounty. + /// - `beneficiary`: Beneficiary account. + #[pallet::weight(::WeightInfo::award_child_bounty())] + pub fn award_child_bounty( + origin: OriginFor, + #[pallet::compact] parent_bounty_id: BountyIndex, + #[pallet::compact] child_bounty_id: BountyIndex, + beneficiary: ::Source, + ) -> DispatchResult { + let signer = ensure_signed(origin)?; + let beneficiary = T::Lookup::lookup(beneficiary)?; + + // Ensure parent bounty exists, and is active. + let (parent_curator, _) = Self::ensure_bounty_active(parent_bounty_id)?; + + ChildBounties::::try_mutate_exists( + parent_bounty_id, + child_bounty_id, + |maybe_child_bounty| -> DispatchResult { + let mut child_bounty = + maybe_child_bounty.as_mut().ok_or(BountiesError::::InvalidIndex)?; + + // Ensure child-bounty is in active state. + if let ChildBountyStatus::Active { ref curator } = child_bounty.status { + ensure!( + signer == *curator || signer == parent_curator, + BountiesError::::RequireCurator, + ); + // Move the child-bounty state to pending payout. + child_bounty.status = ChildBountyStatus::PendingPayout { + curator: signer, + beneficiary: beneficiary.clone(), + unlock_at: frame_system::Pallet::::block_number() + + T::BountyDepositPayoutDelay::get(), + }; + Ok(()) + } else { + Err(BountiesError::::UnexpectedStatus.into()) + } + }, + )?; + + // Trigger the event Awarded. + Self::deposit_event(Event::::Awarded { + index: parent_bounty_id, + child_index: child_bounty_id, + beneficiary, + }); + + Ok(()) + } + + /// Claim the payout from an awarded child-bounty after payout delay. + /// + /// The dispatch origin for this call may be any signed origin. + /// + /// Call works independent of parent bounty state, No need for parent + /// bounty to be in active state. + /// + /// The Beneficiary is paid out with agreed bounty value. Curator fee is + /// paid & curator deposit is unreserved. + /// + /// Child-bounty must be in "PendingPayout" state, for processing the + /// call. And instance of child-bounty is removed from the state on + /// successful call completion. + /// + /// - `parent_bounty_id`: Index of parent bounty. + /// - `child_bounty_id`: Index of child bounty. + #[pallet::weight(::WeightInfo::claim_child_bounty())] + pub fn claim_child_bounty( + origin: OriginFor, + #[pallet::compact] parent_bounty_id: BountyIndex, + #[pallet::compact] child_bounty_id: BountyIndex, + ) -> DispatchResult { + let _ = ensure_signed(origin)?; + + // Ensure child-bounty is in expected state. + ChildBounties::::try_mutate_exists( + parent_bounty_id, + child_bounty_id, + |maybe_child_bounty| -> DispatchResult { + let child_bounty = + maybe_child_bounty.as_mut().ok_or(BountiesError::::InvalidIndex)?; + + if let ChildBountyStatus::PendingPayout { + ref curator, + ref beneficiary, + ref unlock_at, + } = child_bounty.status + { + // Ensure block number is elapsed for processing the + // claim. + ensure!( + frame_system::Pallet::::block_number() >= *unlock_at, + BountiesError::::Premature, + ); + + // Make curator fee payment. + let child_bounty_account = Self::child_bounty_account_id(child_bounty_id); + let balance = T::Currency::free_balance(&child_bounty_account); + let curator_fee = child_bounty.fee.min(balance); + let payout = balance.saturating_sub(curator_fee); + + // Unreserve the curator deposit. Should not fail + // because the deposit is always reserved when curator is + // assigned. + let _ = T::Currency::unreserve(&curator, child_bounty.curator_deposit); + + // Make payout to child-bounty curator. + // Should not fail because curator fee is always less than bounty value. + let fee_transfer_result = T::Currency::transfer( + &child_bounty_account, + &curator, + curator_fee, + AllowDeath, + ); + debug_assert!(fee_transfer_result.is_ok()); + + // Make payout to beneficiary. + // Should not fail. + let payout_transfer_result = T::Currency::transfer( + &child_bounty_account, + beneficiary, + payout, + AllowDeath, + ); + debug_assert!(payout_transfer_result.is_ok()); + + // Trigger the Claimed event. + Self::deposit_event(Event::::Claimed { + index: parent_bounty_id, + child_index: child_bounty_id, + payout, + beneficiary: beneficiary.clone(), + }); + + // Update the active child-bounty tracking count. + >::mutate(parent_bounty_id, |count| { + count.saturating_dec() + }); + + // Remove the child-bounty description. + >::remove(child_bounty_id); + + // Remove the child-bounty instance from the state. + *maybe_child_bounty = None; + + Ok(()) + } else { + Err(BountiesError::::UnexpectedStatus.into()) + } + }, + ) + } + + /// Cancel a proposed or active child-bounty. Child-bounty account funds + /// are transferred to parent bounty account. The child-bounty curator + /// deposit may be unreserved if possible. + /// + /// The dispatch origin for this call must be either parent curator or + /// `T::RejectOrigin`. + /// + /// If the state of child-bounty is `Active`, curator deposit is + /// unreserved. + /// + /// If the state of child-bounty is `PendingPayout`, call fails & + /// returns `PendingPayout` error. + /// + /// For the origin other than T::RejectOrigin, parent bounty must be in + /// active state, for this child-bounty call to work. For origin + /// T::RejectOrigin execution is forced. + /// + /// Instance of child-bounty is removed from the state on successful + /// call completion. + /// + /// - `parent_bounty_id`: Index of parent bounty. + /// - `child_bounty_id`: Index of child bounty. + #[pallet::weight(::WeightInfo::close_child_bounty_added() + .max(::WeightInfo::close_child_bounty_active()))] + pub fn close_child_bounty( + origin: OriginFor, + #[pallet::compact] parent_bounty_id: BountyIndex, + #[pallet::compact] child_bounty_id: BountyIndex, + ) -> DispatchResult { + let maybe_sender = ensure_signed(origin.clone()) + .map(Some) + .or_else(|_| T::RejectOrigin::ensure_origin(origin).map(|_| None))?; + + // Ensure parent bounty exist, get parent curator. + let (parent_curator, _) = Self::ensure_bounty_active(parent_bounty_id)?; + + ensure!(maybe_sender.map_or(true, |sender| parent_curator == sender), BadOrigin); + + Self::impl_close_child_bounty(parent_bounty_id, child_bounty_id)?; + Ok(()) + } + } +} + +impl Pallet { + // This function will calculate the deposit of a curator. + fn calculate_curator_deposit( + parent_curator: &T::AccountId, + child_curator: &T::AccountId, + bounty_fee: &BalanceOf, + ) -> BalanceOf { + if parent_curator == child_curator { + return Zero::zero() + } + + // We just use the same logic from the parent bounties pallet. + pallet_bounties::Pallet::::calculate_curator_deposit(bounty_fee) + } + + /// The account ID of a child-bounty account. + pub fn child_bounty_account_id(id: BountyIndex) -> T::AccountId { + // This function is taken from the parent (bounties) pallet, but the + // prefix is changed to have different AccountId when the index of + // parent and child is same. + T::PalletId::get().into_sub_account(("cb", id)) + } + + fn create_child_bounty( + parent_bounty_id: BountyIndex, + child_bounty_id: BountyIndex, + child_bounty_value: BalanceOf, + description: BoundedVec, + ) { + let child_bounty = ChildBounty { + parent_bounty: parent_bounty_id, + value: child_bounty_value, + fee: 0u32.into(), + curator_deposit: 0u32.into(), + status: ChildBountyStatus::Added, + }; + ChildBounties::::insert(parent_bounty_id, child_bounty_id, &child_bounty); + ChildBountyDescriptions::::insert(child_bounty_id, description); + Self::deposit_event(Event::Added { index: parent_bounty_id, child_index: child_bounty_id }); + } + + fn ensure_bounty_active( + bounty_id: BountyIndex, + ) -> Result<(T::AccountId, T::BlockNumber), DispatchError> { + let parent_bounty = pallet_bounties::Pallet::::bounties(bounty_id) + .ok_or(BountiesError::::InvalidIndex)?; + if let BountyStatus::Active { curator, update_due } = parent_bounty.get_status() { + Ok((curator, update_due)) + } else { + Err(Error::::ParentBountyNotActive.into()) + } + } + + fn impl_close_child_bounty( + parent_bounty_id: BountyIndex, + child_bounty_id: BountyIndex, + ) -> DispatchResult { + ChildBounties::::try_mutate_exists( + parent_bounty_id, + child_bounty_id, + |maybe_child_bounty| -> DispatchResult { + let child_bounty = + maybe_child_bounty.as_mut().ok_or(BountiesError::::InvalidIndex)?; + + match &child_bounty.status { + ChildBountyStatus::Added | ChildBountyStatus::CuratorProposed { .. } => { + // Nothing extra to do besides the removal of the child-bounty below. + }, + ChildBountyStatus::Active { curator } => { + // Cancelled by master curator or RejectOrigin, + // refund deposit of the working child-bounty curator. + let _ = T::Currency::unreserve(curator, child_bounty.curator_deposit); + // Then execute removal of the child-bounty below. + }, + ChildBountyStatus::PendingPayout { .. } => { + // Child-bounty is already in pending payout. If parent + // curator or RejectOrigin wants to close this + // child-bounty, it should mean the child-bounty curator + // was acting maliciously. So first unassign the + // child-bounty curator, slashing their deposit. + return Err(BountiesError::::PendingPayout.into()) + }, + } + + // Revert the curator fee back to parent-bounty curator & + // reduce the active child-bounty count. + ChildrenCuratorFees::::mutate(parent_bounty_id, |value| { + *value = value.saturating_sub(child_bounty.fee) + }); + >::mutate(parent_bounty_id, |count| { + *count = count.saturating_sub(1) + }); + + // Transfer fund from child-bounty to parent bounty. + let parent_bounty_account = + pallet_bounties::Pallet::::bounty_account_id(parent_bounty_id); + let child_bounty_account = Self::child_bounty_account_id(child_bounty_id); + let balance = T::Currency::free_balance(&child_bounty_account); + let transfer_result = T::Currency::transfer( + &child_bounty_account, + &parent_bounty_account, + balance, + AllowDeath, + ); // Should not fail; child bounty account gets this balance during creation. + debug_assert!(transfer_result.is_ok()); + + // Remove the child-bounty description. + >::remove(child_bounty_id); + + *maybe_child_bounty = None; + + Self::deposit_event(Event::::Canceled { + index: parent_bounty_id, + child_index: child_bounty_id, + }); + Ok(()) + }, + ) + } +} + +// Implement ChildBountyManager to connect with the bounties pallet. This is +// where we pass the active child-bounties and child curator fees to the parent +// bounty. +impl pallet_bounties::ChildBountyManager> for Pallet { + fn child_bounties_count( + bounty_id: pallet_bounties::BountyIndex, + ) -> pallet_bounties::BountyIndex { + Self::parent_child_bounties(bounty_id) + } + + fn children_curator_fees(bounty_id: pallet_bounties::BountyIndex) -> BalanceOf { + // This is asked for when the parent bounty is being claimed. No use of + // keeping it in state after that. Hence removing. + let children_fee_total = Self::children_curator_fees(bounty_id); + >::remove(bounty_id); + children_fee_total + } +} diff --git a/frame/child-bounties/src/tests.rs b/frame/child-bounties/src/tests.rs new file mode 100644 index 000000000000..61545561a26c --- /dev/null +++ b/frame/child-bounties/src/tests.rs @@ -0,0 +1,1409 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Child-bounties pallet tests. + +#![cfg(test)] + +use super::*; +use crate as pallet_child_bounties; +use std::cell::RefCell; + +use frame_support::{ + assert_noop, assert_ok, + pallet_prelude::GenesisBuild, + parameter_types, + traits::{ConstU32, ConstU64, OnInitialize}, + weights::Weight, + PalletId, +}; + +use sp_core::H256; +use sp_runtime::{ + testing::Header, + traits::{BadOrigin, BlakeTwo256, IdentityLookup}, + Perbill, Permill, +}; + +use super::Event as ChildBountiesEvent; + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; +type BountiesError = pallet_bounties::Error; + +frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, + Bounties: pallet_bounties::{Pallet, Call, Storage, Event}, + Treasury: pallet_treasury::{Pallet, Call, Storage, Config, Event}, + ChildBounties: pallet_child_bounties::{Pallet, Call, Storage, Event}, + } +); + +parameter_types! { + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockLength: u32 = 2 * 1024; + pub const AvailableBlockRatio: Perbill = Perbill::one(); +} + +type Balance = u64; + +impl frame_system::Config for Test { + type BaseCallFilter = frame_support::traits::Everything; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Call = Call; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u128; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = ConstU64<250>; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = ConstU32<16>; +} + +impl pallet_balances::Config for Test { + type MaxLocks = (); + type MaxReserves = (); + type ReserveIdentifier = [u8; 8]; + type Balance = Balance; + type Event = Event; + type DustRemoval = (); + type ExistentialDeposit = ConstU64<1>; + type AccountStore = System; + type WeightInfo = (); +} +thread_local! { + static TEN_TO_FOURTEEN: RefCell> = RefCell::new(vec![10,11,12,13,14]); +} +parameter_types! { + pub const ProposalBond: Permill = Permill::from_percent(5); + pub const Burn: Permill = Permill::from_percent(50); + pub const TreasuryPalletId: PalletId = PalletId(*b"py/trsry"); +} + +impl pallet_treasury::Config for Test { + type PalletId = TreasuryPalletId; + type Currency = pallet_balances::Pallet; + type ApproveOrigin = frame_system::EnsureRoot; + type RejectOrigin = frame_system::EnsureRoot; + type Event = Event; + type OnSlash = (); + type ProposalBond = ProposalBond; + type ProposalBondMinimum = ConstU64<1>; + type ProposalBondMaximum = (); + type SpendPeriod = ConstU64<2>; + type Burn = Burn; + type BurnDestination = (); + type WeightInfo = (); + type SpendFunds = Bounties; + type MaxApprovals = ConstU32<100>; +} +parameter_types! { + // This will be 50% of the bounty fee. + pub const CuratorDepositMultiplier: Permill = Permill::from_percent(50); + pub const CuratorDepositMax: Balance = 1_000; + pub const CuratorDepositMin: Balance = 3; + +} +impl pallet_bounties::Config for Test { + type Event = Event; + type BountyDepositBase = ConstU64<80>; + type BountyDepositPayoutDelay = ConstU64<3>; + type BountyUpdatePeriod = ConstU64<10>; + type CuratorDepositMultiplier = CuratorDepositMultiplier; + type CuratorDepositMax = CuratorDepositMax; + type CuratorDepositMin = CuratorDepositMin; + type BountyValueMinimum = ConstU64<5>; + type DataDepositPerByte = ConstU64<1>; + type MaximumReasonLength = ConstU32<300>; + type WeightInfo = (); + type ChildBountyManager = ChildBounties; +} +impl pallet_child_bounties::Config for Test { + type Event = Event; + type MaxActiveChildBountyCount = ConstU32<2>; + type ChildBountyValueMinimum = ConstU64<1>; + type WeightInfo = (); +} + +pub fn new_test_ext() -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + pallet_balances::GenesisConfig:: { + // Total issuance will be 200 with treasury account initialized at ED. + balances: vec![(0, 100), (1, 98), (2, 1)], + } + .assimilate_storage(&mut t) + .unwrap(); + GenesisBuild::::assimilate_storage(&pallet_treasury::GenesisConfig, &mut t).unwrap(); + t.into() +} + +fn last_event() -> ChildBountiesEvent { + System::events() + .into_iter() + .map(|r| r.event) + .filter_map(|e| if let Event::ChildBounties(inner) = e { Some(inner) } else { None }) + .last() + .unwrap() +} + +#[test] +fn genesis_config_works() { + new_test_ext().execute_with(|| { + assert_eq!(Treasury::pot(), 0); + assert_eq!(Treasury::proposal_count(), 0); + }); +} + +#[test] +fn minting_works() { + new_test_ext().execute_with(|| { + // Check that accumulate works when we have Some value in Dummy already. + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_eq!(Treasury::pot(), 100); + }); +} + +#[test] +fn add_child_bounty() { + new_test_ext().execute_with(|| { + // TestProcedure. + // 1, Create bounty & move to active state with enough bounty fund & parent curator. + // 2, Parent curator adds child-bounty child-bounty-1, test for error like RequireCurator + // ,InsufficientProposersBalance, InsufficientBountyBalance with invalid arguments. + // 3, Parent curator adds child-bounty child-bounty-1, moves to "Approved" state & + // test for the event Added. + // 4, Test for DB state of `Bounties` & `ChildBounties`. + // 5, Observe fund transaction moment between Bounty, Child-bounty, + // Curator, child-bounty curator & beneficiary. + + // Make the parent bounty. + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + System::set_block_number(2); + >::on_initialize(2); + + let fee = 8; + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, fee)); + + Balances::make_free_balance_be(&4, 10); + + assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + + // This verifies that the accept curator logic took a deposit. + let expected_deposit = CuratorDepositMultiplier::get() * fee; + assert_eq!(Balances::reserved_balance(&4), expected_deposit); + assert_eq!(Balances::free_balance(&4), 10 - expected_deposit); + + // Add child-bounty. + // Acc-4 is the parent curator. + // Call from invalid origin & check for error "RequireCurator". + assert_noop!( + ChildBounties::add_child_bounty(Origin::signed(0), 0, 10, b"12345-p1".to_vec()), + BountiesError::RequireCurator, + ); + + // Update the parent curator balance. + Balances::make_free_balance_be(&4, 101); + + // parent curator fee is reserved on parent bounty account. + assert_eq!(Balances::free_balance(Bounties::bounty_account_id(0)), 50); + assert_eq!(Balances::reserved_balance(Bounties::bounty_account_id(0)), 0); + + assert_noop!( + ChildBounties::add_child_bounty(Origin::signed(4), 0, 50, b"12345-p1".to_vec()), + pallet_balances::Error::::KeepAlive, + ); + + assert_noop!( + ChildBounties::add_child_bounty(Origin::signed(4), 0, 100, b"12345-p1".to_vec()), + Error::::InsufficientBountyBalance, + ); + + // Add child-bounty with valid value, which can be funded by parent bounty. + assert_ok!(ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec())); + + // Check for the event child-bounty added. + assert_eq!(last_event(), ChildBountiesEvent::Added { index: 0, child_index: 0 }); + + assert_eq!(Balances::free_balance(4), 101); + assert_eq!(Balances::reserved_balance(4), expected_deposit); + + // DB check. + // Check the child-bounty status. + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee: 0, + curator_deposit: 0, + status: ChildBountyStatus::Added, + } + ); + + // Check the child-bounty count. + assert_eq!(ChildBounties::parent_child_bounties(0), 1); + + // Check the child-bounty description status. + assert_eq!(ChildBounties::child_bounty_descriptions(0).unwrap(), b"12345-p1".to_vec(),); + }); +} + +#[test] +fn child_bounty_assign_curator() { + new_test_ext().execute_with(|| { + // TestProcedure + // 1, Create bounty & move to active state with enough bounty fund & parent curator. + // 2, Parent curator adds child-bounty child-bounty-1, moves to "Active" state. + // 3, Test for DB state of `ChildBounties`. + + // Make the parent bounty. + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + Balances::make_free_balance_be(&4, 101); + Balances::make_free_balance_be(&8, 101); + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + System::set_block_number(2); + >::on_initialize(2); + + let fee = 4; + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, fee)); + assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + + // Bounty account status before adding child-bounty. + assert_eq!(Balances::free_balance(Bounties::bounty_account_id(0)), 50); + assert_eq!(Balances::reserved_balance(Bounties::bounty_account_id(0)), 0); + + // Check the balance of parent curator. + // Curator deposit is reserved for parent curator on parent bounty. + let expected_deposit = Bounties::calculate_curator_deposit(&fee); + assert_eq!(Balances::free_balance(4), 101 - expected_deposit); + assert_eq!(Balances::reserved_balance(4), expected_deposit); + + // Add child-bounty. + // Acc-4 is the parent curator & make sure enough deposit. + assert_ok!(ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec())); + + assert_eq!(last_event(), ChildBountiesEvent::Added { index: 0, child_index: 0 }); + + // Bounty account status after adding child-bounty. + assert_eq!(Balances::free_balance(Bounties::bounty_account_id(0)), 40); + assert_eq!(Balances::reserved_balance(Bounties::bounty_account_id(0)), 0); + + // Child-bounty account status. + assert_eq!(Balances::free_balance(ChildBounties::child_bounty_account_id(0)), 10); + assert_eq!(Balances::reserved_balance(ChildBounties::child_bounty_account_id(0)), 0); + + let fee = 6u64; + assert_ok!(ChildBounties::propose_curator(Origin::signed(4), 0, 0, 8, fee)); + + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee, + curator_deposit: 0, + status: ChildBountyStatus::CuratorProposed { curator: 8 }, + } + ); + + // Check the balance of parent curator. + assert_eq!(Balances::free_balance(4), 101 - expected_deposit); + assert_eq!(Balances::reserved_balance(4), expected_deposit); + + assert_noop!( + ChildBounties::accept_curator(Origin::signed(3), 0, 0), + BountiesError::RequireCurator, + ); + + assert_ok!(ChildBounties::accept_curator(Origin::signed(8), 0, 0)); + + let expected_child_deposit = CuratorDepositMultiplier::get() * fee; + + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee, + curator_deposit: expected_child_deposit, + status: ChildBountyStatus::Active { curator: 8 }, + } + ); + + // Deposit for child-bounty curator deposit is reserved. + assert_eq!(Balances::free_balance(8), 101 - expected_child_deposit); + assert_eq!(Balances::reserved_balance(8), expected_child_deposit); + + // Bounty account status at exit. + assert_eq!(Balances::free_balance(Bounties::bounty_account_id(0)), 40); + assert_eq!(Balances::reserved_balance(Bounties::bounty_account_id(0)), 0); + + // Child-bounty account status at exit. + assert_eq!(Balances::free_balance(ChildBounties::child_bounty_account_id(0)), 10); + assert_eq!(Balances::reserved_balance(ChildBounties::child_bounty_account_id(0)), 0); + + // Treasury account status at exit. + assert_eq!(Balances::free_balance(Treasury::account_id()), 26); + assert_eq!(Balances::reserved_balance(Treasury::account_id()), 0); + }); +} + +#[test] +fn award_claim_child_bounty() { + new_test_ext().execute_with(|| { + // Make the parent bounty. + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_eq!(Balances::free_balance(Treasury::account_id()), 101); + assert_eq!(Balances::reserved_balance(Treasury::account_id()), 0); + + // Bounty curator initial balance. + Balances::make_free_balance_be(&4, 101); // Parent-bounty curator. + Balances::make_free_balance_be(&8, 101); // Child-bounty curator. + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + System::set_block_number(2); + >::on_initialize(2); + + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 6)); + assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + + // Child-bounty. + assert_ok!(ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec())); + + assert_eq!(last_event(), ChildBountiesEvent::Added { index: 0, child_index: 0 }); + + // Propose and accept curator for child-bounty. + let fee = 8; + assert_ok!(ChildBounties::propose_curator(Origin::signed(4), 0, 0, 8, fee)); + assert_ok!(ChildBounties::accept_curator(Origin::signed(8), 0, 0)); + + // Award child-bounty. + // Test for non child-bounty curator. + assert_noop!( + ChildBounties::award_child_bounty(Origin::signed(3), 0, 0, 7), + BountiesError::RequireCurator, + ); + + assert_ok!(ChildBounties::award_child_bounty(Origin::signed(8), 0, 0, 7)); + + let expected_deposit = CuratorDepositMultiplier::get() * fee; + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee, + curator_deposit: expected_deposit, + status: ChildBountyStatus::PendingPayout { + curator: 8, + beneficiary: 7, + unlock_at: 5 + }, + } + ); + + // Claim child-bounty. + // Test for Premature condition. + assert_noop!( + ChildBounties::claim_child_bounty(Origin::signed(7), 0, 0), + BountiesError::Premature + ); + + System::set_block_number(9); + + assert_ok!(ChildBounties::claim_child_bounty(Origin::signed(7), 0, 0)); + + // Ensure child-bounty curator is paid with curator fee & deposit refund. + assert_eq!(Balances::free_balance(8), 101 + fee); + assert_eq!(Balances::reserved_balance(8), 0); + + // Ensure executor is paid with beneficiary amount. + assert_eq!(Balances::free_balance(7), 10 - fee); + assert_eq!(Balances::reserved_balance(7), 0); + + // Child-bounty account status. + assert_eq!(Balances::free_balance(ChildBounties::child_bounty_account_id(0)), 0); + assert_eq!(Balances::reserved_balance(ChildBounties::child_bounty_account_id(0)), 0); + + // Check the child-bounty count. + assert_eq!(ChildBounties::parent_child_bounties(0), 0); + }); +} + +#[test] +fn close_child_bounty_added() { + new_test_ext().execute_with(|| { + // Make the parent bounty. + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_eq!(Balances::free_balance(Treasury::account_id()), 101); + assert_eq!(Balances::reserved_balance(Treasury::account_id()), 0); + + // Bounty curator initial balance. + Balances::make_free_balance_be(&4, 101); // Parent-bounty curator. + Balances::make_free_balance_be(&8, 101); // Child-bounty curator. + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + System::set_block_number(2); + >::on_initialize(2); + + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 6)); + + assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + + // Child-bounty. + assert_ok!(ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec())); + + assert_eq!(last_event(), ChildBountiesEvent::Added { index: 0, child_index: 0 }); + + System::set_block_number(4); + + // Close child-bounty. + // Wrong origin. + assert_noop!(ChildBounties::close_child_bounty(Origin::signed(7), 0, 0), BadOrigin); + assert_noop!(ChildBounties::close_child_bounty(Origin::signed(8), 0, 0), BadOrigin); + + // Correct origin - parent curator. + assert_ok!(ChildBounties::close_child_bounty(Origin::signed(4), 0, 0)); + + // Check the child-bounty count. + assert_eq!(ChildBounties::parent_child_bounties(0), 0); + + // Parent-bounty account status. + assert_eq!(Balances::free_balance(Bounties::bounty_account_id(0)), 50); + assert_eq!(Balances::reserved_balance(Bounties::bounty_account_id(0)), 0); + + // Child-bounty account status. + assert_eq!(Balances::free_balance(ChildBounties::child_bounty_account_id(0)), 0); + assert_eq!(Balances::reserved_balance(ChildBounties::child_bounty_account_id(0)), 0); + }); +} + +#[test] +fn close_child_bounty_active() { + new_test_ext().execute_with(|| { + // Make the parent bounty. + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_eq!(Balances::free_balance(Treasury::account_id()), 101); + assert_eq!(Balances::reserved_balance(Treasury::account_id()), 0); + + // Bounty curator initial balance. + Balances::make_free_balance_be(&4, 101); // Parent-bounty curator. + Balances::make_free_balance_be(&8, 101); // Child-bounty curator. + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + System::set_block_number(2); + >::on_initialize(2); + + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 6)); + + assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + + // Child-bounty. + assert_ok!(ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec())); + + assert_eq!(last_event(), ChildBountiesEvent::Added { index: 0, child_index: 0 }); + + // Propose and accept curator for child-bounty. + assert_ok!(ChildBounties::propose_curator(Origin::signed(4), 0, 0, 8, 2)); + assert_ok!(ChildBounties::accept_curator(Origin::signed(8), 0, 0)); + + // Close child-bounty in active state. + assert_ok!(ChildBounties::close_child_bounty(Origin::signed(4), 0, 0)); + + // Check the child-bounty count. + assert_eq!(ChildBounties::parent_child_bounties(0), 0); + + // Ensure child-bounty curator balance is unreserved. + assert_eq!(Balances::free_balance(8), 101); + assert_eq!(Balances::reserved_balance(8), 0); + + // Parent-bounty account status. + assert_eq!(Balances::free_balance(Bounties::bounty_account_id(0)), 50); + assert_eq!(Balances::reserved_balance(Bounties::bounty_account_id(0)), 0); + + // Child-bounty account status. + assert_eq!(Balances::free_balance(ChildBounties::child_bounty_account_id(0)), 0); + assert_eq!(Balances::reserved_balance(ChildBounties::child_bounty_account_id(0)), 0); + }); +} + +#[test] +fn close_child_bounty_pending() { + new_test_ext().execute_with(|| { + // Make the parent bounty. + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_eq!(Balances::free_balance(Treasury::account_id()), 101); + assert_eq!(Balances::reserved_balance(Treasury::account_id()), 0); + + // Bounty curator initial balance. + Balances::make_free_balance_be(&4, 101); // Parent-bounty curator. + Balances::make_free_balance_be(&8, 101); // Child-bounty curator. + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + System::set_block_number(2); + >::on_initialize(2); + + let parent_fee = 6; + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, parent_fee)); + assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + + // Child-bounty. + assert_ok!(ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec())); + + assert_eq!(last_event(), ChildBountiesEvent::Added { index: 0, child_index: 0 }); + + // Propose and accept curator for child-bounty. + let child_fee = 4; + assert_ok!(ChildBounties::propose_curator(Origin::signed(4), 0, 0, 8, child_fee)); + assert_ok!(ChildBounties::accept_curator(Origin::signed(8), 0, 0)); + let expected_child_deposit = CuratorDepositMin::get(); + + assert_ok!(ChildBounties::award_child_bounty(Origin::signed(8), 0, 0, 7)); + + // Close child-bounty in pending_payout state. + assert_noop!( + ChildBounties::close_child_bounty(Origin::signed(4), 0, 0), + BountiesError::PendingPayout + ); + + // Check the child-bounty count. + assert_eq!(ChildBounties::parent_child_bounties(0), 1); + + // Ensure no changes in child-bounty curator balance. + assert_eq!(Balances::reserved_balance(8), expected_child_deposit); + assert_eq!(Balances::free_balance(8), 101 - expected_child_deposit); + + // Child-bounty account status. + assert_eq!(Balances::free_balance(ChildBounties::child_bounty_account_id(0)), 10); + assert_eq!(Balances::reserved_balance(ChildBounties::child_bounty_account_id(0)), 0); + }); +} + +#[test] +fn child_bounty_added_unassign_curator() { + new_test_ext().execute_with(|| { + // Make the parent bounty. + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_eq!(Balances::free_balance(Treasury::account_id()), 101); + assert_eq!(Balances::reserved_balance(Treasury::account_id()), 0); + + // Bounty curator initial balance. + Balances::make_free_balance_be(&4, 101); // Parent-bounty curator. + Balances::make_free_balance_be(&8, 101); // Child-bounty curator. + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + System::set_block_number(2); + >::on_initialize(2); + + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 6)); + + assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + + // Child-bounty. + assert_ok!(ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec())); + + assert_eq!(last_event(), ChildBountiesEvent::Added { index: 0, child_index: 0 }); + + // Unassign curator in added state. + assert_noop!( + ChildBounties::unassign_curator(Origin::signed(4), 0, 0), + BountiesError::UnexpectedStatus + ); + }); +} + +#[test] +fn child_bounty_curator_proposed_unassign_curator() { + new_test_ext().execute_with(|| { + // Make the parent bounty. + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_eq!(Balances::free_balance(Treasury::account_id()), 101); + assert_eq!(Balances::reserved_balance(Treasury::account_id()), 0); + + // Bounty curator initial balance. + Balances::make_free_balance_be(&4, 101); // Parent-bounty curator. + Balances::make_free_balance_be(&8, 101); // Child-bounty curator. + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + System::set_block_number(2); + >::on_initialize(2); + + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 6)); + + assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + + // Child-bounty. + assert_ok!(ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec())); + + assert_eq!(last_event(), ChildBountiesEvent::Added { index: 0, child_index: 0 }); + + // Propose curator for child-bounty. + assert_ok!(ChildBounties::propose_curator(Origin::signed(4), 0, 0, 8, 2)); + + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee: 2, + curator_deposit: 0, + status: ChildBountyStatus::CuratorProposed { curator: 8 }, + } + ); + + // Random account cannot unassign the curator when in proposed state. + assert_noop!(ChildBounties::unassign_curator(Origin::signed(99), 0, 0), BadOrigin); + + // Unassign curator. + assert_ok!(ChildBounties::unassign_curator(Origin::signed(4), 0, 0)); + + // Verify updated child-bounty status. + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee: 2, + curator_deposit: 0, + status: ChildBountyStatus::Added, + } + ); + }); +} + +#[test] +fn child_bounty_active_unassign_curator() { + // Covers all scenarios with all origin types. + // Step 1: Setup bounty, child bounty. + // Step 2: Assign, accept curator for child bounty. Unassign from reject origin. Should slash. + // Step 3: Assign, accept another curator for child bounty. Unassign from parent-bounty curator. + // Should slash. Step 4: Assign, accept another curator for child bounty. Unassign from + // child-bounty curator. Should NOT slash. Step 5: Assign, accept another curator for child + // bounty. Unassign from random account. Should slash. + new_test_ext().execute_with(|| { + // Make the parent bounty. + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_eq!(Balances::free_balance(Treasury::account_id()), 101); + assert_eq!(Balances::reserved_balance(Treasury::account_id()), 0); + + // Bounty curator initial balance. + Balances::make_free_balance_be(&4, 101); // Parent-bounty curator. + Balances::make_free_balance_be(&6, 101); // Child-bounty curator 1. + Balances::make_free_balance_be(&7, 101); // Child-bounty curator 2. + Balances::make_free_balance_be(&8, 101); // Child-bounty curator 3. + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + System::set_block_number(2); + >::on_initialize(2); + + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 6)); + assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + + // Create Child-bounty. + assert_ok!(ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec())); + assert_eq!(last_event(), ChildBountiesEvent::Added { index: 0, child_index: 0 }); + + System::set_block_number(3); + >::on_initialize(3); + + // Propose and accept curator for child-bounty. + let fee = 6; + assert_ok!(ChildBounties::propose_curator(Origin::signed(4), 0, 0, 8, fee)); + assert_ok!(ChildBounties::accept_curator(Origin::signed(8), 0, 0)); + let expected_child_deposit = CuratorDepositMultiplier::get() * fee; + + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee, + curator_deposit: expected_child_deposit, + status: ChildBountyStatus::Active { curator: 8 }, + } + ); + + System::set_block_number(4); + >::on_initialize(4); + + // Unassign curator - from reject origin. + assert_ok!(ChildBounties::unassign_curator(Origin::root(), 0, 0)); + + // Verify updated child-bounty status. + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee, + curator_deposit: 0, + status: ChildBountyStatus::Added, + } + ); + + // Ensure child-bounty curator was slashed. + assert_eq!(Balances::free_balance(8), 101 - expected_child_deposit); + assert_eq!(Balances::reserved_balance(8), 0); // slashed + + // Propose and accept curator for child-bounty again. + let fee = 2; + assert_ok!(ChildBounties::propose_curator(Origin::signed(4), 0, 0, 7, fee)); + assert_ok!(ChildBounties::accept_curator(Origin::signed(7), 0, 0)); + let expected_child_deposit = CuratorDepositMin::get(); + + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee, + curator_deposit: expected_child_deposit, + status: ChildBountyStatus::Active { curator: 7 }, + } + ); + + System::set_block_number(5); + >::on_initialize(5); + + // Unassign curator again - from parent curator. + assert_ok!(ChildBounties::unassign_curator(Origin::signed(4), 0, 0)); + + // Verify updated child-bounty status. + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee: 2, + curator_deposit: 0, + status: ChildBountyStatus::Added, + } + ); + + // Ensure child-bounty curator was slashed. + assert_eq!(Balances::free_balance(7), 101 - expected_child_deposit); + assert_eq!(Balances::reserved_balance(7), 0); // slashed + + // Propose and accept curator for child-bounty again. + assert_ok!(ChildBounties::propose_curator(Origin::signed(4), 0, 0, 6, 2)); + assert_ok!(ChildBounties::accept_curator(Origin::signed(6), 0, 0)); + + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee, + curator_deposit: expected_child_deposit, + status: ChildBountyStatus::Active { curator: 6 }, + } + ); + + System::set_block_number(6); + >::on_initialize(6); + + // Unassign curator again - from child-bounty curator. + assert_ok!(ChildBounties::unassign_curator(Origin::signed(6), 0, 0)); + + // Verify updated child-bounty status. + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee: 2, + curator_deposit: 0, + status: ChildBountyStatus::Added, + } + ); + + // Ensure child-bounty curator was **not** slashed. + assert_eq!(Balances::free_balance(6), 101); // not slashed + assert_eq!(Balances::reserved_balance(6), 0); + + // Propose and accept curator for child-bounty one last time. + let fee = 2; + assert_ok!(ChildBounties::propose_curator(Origin::signed(4), 0, 0, 6, fee)); + assert_ok!(ChildBounties::accept_curator(Origin::signed(6), 0, 0)); + let expected_child_deposit = CuratorDepositMin::get(); + + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee, + curator_deposit: expected_child_deposit, + status: ChildBountyStatus::Active { curator: 6 }, + } + ); + + System::set_block_number(7); + >::on_initialize(7); + + // Unassign curator again - from non curator; non reject origin; some random guy. + // Bounty update period is not yet complete. + assert_noop!( + ChildBounties::unassign_curator(Origin::signed(3), 0, 0), + BountiesError::Premature + ); + + System::set_block_number(20); + >::on_initialize(20); + + // Unassign child curator from random account after inactivity. + assert_ok!(ChildBounties::unassign_curator(Origin::signed(3), 0, 0)); + + // Verify updated child-bounty status. + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee: 2, + curator_deposit: 0, + status: ChildBountyStatus::Added, + } + ); + + // Ensure child-bounty curator was slashed. + assert_eq!(Balances::free_balance(6), 101 - expected_child_deposit); // slashed + assert_eq!(Balances::reserved_balance(6), 0); + }); +} + +#[test] +fn parent_bounty_inactive_unassign_curator_child_bounty() { + // Unassign curator when parent bounty in not in active state. + // This can happen when the curator of parent bounty has been unassigned. + new_test_ext().execute_with(|| { + // Make the parent bounty. + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_eq!(Balances::free_balance(Treasury::account_id()), 101); + assert_eq!(Balances::reserved_balance(Treasury::account_id()), 0); + + // Bounty curator initial balance. + Balances::make_free_balance_be(&4, 101); // Parent-bounty curator 1. + Balances::make_free_balance_be(&5, 101); // Parent-bounty curator 2. + Balances::make_free_balance_be(&6, 101); // Child-bounty curator 1. + Balances::make_free_balance_be(&7, 101); // Child-bounty curator 2. + Balances::make_free_balance_be(&8, 101); // Child-bounty curator 3. + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + System::set_block_number(2); + >::on_initialize(2); + + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 6)); + assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + + // Create Child-bounty. + assert_ok!(ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec())); + assert_eq!(last_event(), ChildBountiesEvent::Added { index: 0, child_index: 0 }); + + System::set_block_number(3); + >::on_initialize(3); + + // Propose and accept curator for child-bounty. + let fee = 8; + assert_ok!(ChildBounties::propose_curator(Origin::signed(4), 0, 0, 8, fee)); + assert_ok!(ChildBounties::accept_curator(Origin::signed(8), 0, 0)); + let expected_child_deposit = CuratorDepositMultiplier::get() * fee; + + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee, + curator_deposit: expected_child_deposit, + status: ChildBountyStatus::Active { curator: 8 }, + } + ); + + System::set_block_number(4); + >::on_initialize(4); + + // Unassign parent bounty curator. + assert_ok!(Bounties::unassign_curator(Origin::root(), 0)); + + System::set_block_number(5); + >::on_initialize(5); + + // Try unassign child-bounty curator - from non curator; non reject + // origin; some random guy. Bounty update period is not yet complete. + assert_noop!( + ChildBounties::unassign_curator(Origin::signed(3), 0, 0), + Error::::ParentBountyNotActive + ); + + // Unassign curator - from reject origin. + assert_ok!(ChildBounties::unassign_curator(Origin::root(), 0, 0)); + + // Verify updated child-bounty status. + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee, + curator_deposit: 0, + status: ChildBountyStatus::Added, + } + ); + + // Ensure child-bounty curator was slashed. + assert_eq!(Balances::free_balance(8), 101 - expected_child_deposit); + assert_eq!(Balances::reserved_balance(8), 0); // slashed + + System::set_block_number(6); + >::on_initialize(6); + + // Propose and accept curator for parent-bounty again. + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 5, 6)); + assert_ok!(Bounties::accept_curator(Origin::signed(5), 0)); + + System::set_block_number(7); + >::on_initialize(7); + + // Propose and accept curator for child-bounty again. + let fee = 2; + assert_ok!(ChildBounties::propose_curator(Origin::signed(5), 0, 0, 7, fee)); + assert_ok!(ChildBounties::accept_curator(Origin::signed(7), 0, 0)); + let expected_deposit = CuratorDepositMin::get(); + + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee, + curator_deposit: expected_deposit, + status: ChildBountyStatus::Active { curator: 7 }, + } + ); + + System::set_block_number(8); + >::on_initialize(8); + + assert_noop!( + ChildBounties::unassign_curator(Origin::signed(3), 0, 0), + BountiesError::Premature + ); + + // Unassign parent bounty curator again. + assert_ok!(Bounties::unassign_curator(Origin::signed(5), 0)); + + System::set_block_number(9); + >::on_initialize(9); + + // Unassign curator again - from parent curator. + assert_ok!(ChildBounties::unassign_curator(Origin::signed(7), 0, 0)); + + // Verify updated child-bounty status. + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee: 2, + curator_deposit: 0, + status: ChildBountyStatus::Added, + } + ); + + // Ensure child-bounty curator was not slashed. + assert_eq!(Balances::free_balance(7), 101); + assert_eq!(Balances::reserved_balance(7), 0); // slashed + }); +} + +#[test] +fn close_parent_with_child_bounty() { + new_test_ext().execute_with(|| { + // Make the parent bounty. + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_eq!(Balances::free_balance(Treasury::account_id()), 101); + assert_eq!(Balances::reserved_balance(Treasury::account_id()), 0); + + // Bounty curator initial balance. + Balances::make_free_balance_be(&4, 101); // Parent-bounty curator. + Balances::make_free_balance_be(&8, 101); // Child-bounty curator. + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + // Try add child-bounty. + // Should fail, parent bounty not active yet. + assert_noop!( + ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec()), + Error::::ParentBountyNotActive + ); + + System::set_block_number(2); + >::on_initialize(2); + + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 6)); + assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + + // Child-bounty. + assert_ok!(ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec())); + assert_eq!(last_event(), ChildBountiesEvent::Added { index: 0, child_index: 0 }); + + System::set_block_number(4); + >::on_initialize(4); + + // Try close parent-bounty. + // Child bounty active, can't close parent. + assert_noop!( + Bounties::close_bounty(Origin::root(), 0), + BountiesError::HasActiveChildBounty + ); + + System::set_block_number(2); + + // Close child-bounty. + assert_ok!(ChildBounties::close_child_bounty(Origin::root(), 0, 0)); + + // Check the child-bounty count. + assert_eq!(ChildBounties::parent_child_bounties(0), 0); + + // Try close parent-bounty again. + // Should pass this time. + assert_ok!(Bounties::close_bounty(Origin::root(), 0)); + }); +} + +#[test] +fn children_curator_fee_calculation_test() { + // Tests the calculation of subtracting child-bounty curator fee + // from parent bounty fee when claiming bounties. + new_test_ext().execute_with(|| { + // Make the parent bounty. + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), 101); + assert_eq!(Balances::free_balance(Treasury::account_id()), 101); + assert_eq!(Balances::reserved_balance(Treasury::account_id()), 0); + + // Bounty curator initial balance. + Balances::make_free_balance_be(&4, 101); // Parent-bounty curator. + Balances::make_free_balance_be(&8, 101); // Child-bounty curator. + + assert_ok!(Bounties::propose_bounty(Origin::signed(0), 50, b"12345".to_vec())); + assert_ok!(Bounties::approve_bounty(Origin::root(), 0)); + + System::set_block_number(2); + >::on_initialize(2); + + assert_ok!(Bounties::propose_curator(Origin::root(), 0, 4, 6)); + assert_ok!(Bounties::accept_curator(Origin::signed(4), 0)); + + // Child-bounty. + assert_ok!(ChildBounties::add_child_bounty(Origin::signed(4), 0, 10, b"12345-p1".to_vec())); + assert_eq!(last_event(), ChildBountiesEvent::Added { index: 0, child_index: 0 }); + + System::set_block_number(4); + >::on_initialize(4); + + let fee = 6; + + // Propose curator for child-bounty. + assert_ok!(ChildBounties::propose_curator(Origin::signed(4), 0, 0, 8, fee)); + // Check curator fee added to the sum. + assert_eq!(ChildBounties::children_curator_fees(0), fee); + // Accept curator for child-bounty. + assert_ok!(ChildBounties::accept_curator(Origin::signed(8), 0, 0)); + // Award child-bounty. + assert_ok!(ChildBounties::award_child_bounty(Origin::signed(8), 0, 0, 7)); + + let expected_child_deposit = CuratorDepositMultiplier::get() * fee; + + assert_eq!( + ChildBounties::child_bounties(0, 0).unwrap(), + ChildBounty { + parent_bounty: 0, + value: 10, + fee, + curator_deposit: expected_child_deposit, + status: ChildBountyStatus::PendingPayout { + curator: 8, + beneficiary: 7, + unlock_at: 7 + }, + } + ); + + System::set_block_number(9); + + // Claim child-bounty. + assert_ok!(ChildBounties::claim_child_bounty(Origin::signed(7), 0, 0)); + + // Check the child-bounty count. + assert_eq!(ChildBounties::parent_child_bounties(0), 0); + + // Award the parent bounty. + assert_ok!(Bounties::award_bounty(Origin::signed(4), 0, 9)); + + System::set_block_number(15); + + // Claim the parent bounty. + assert_ok!(Bounties::claim_bounty(Origin::signed(9), 0)); + + // Ensure parent-bounty curator received correctly reduced fee. + assert_eq!(Balances::free_balance(4), 101 + 6 - fee); // 101 + 6 - 2 + assert_eq!(Balances::reserved_balance(4), 0); + + // Verify parent-bounty beneficiary balance. + assert_eq!(Balances::free_balance(9), 34); + assert_eq!(Balances::reserved_balance(9), 0); + }); +} + +#[test] +fn accept_curator_handles_different_deposit_calculations() { + // This test will verify that a bounty with and without a fee results + // in a different curator deposit, and if the child curator matches the parent curator. + new_test_ext().execute_with(|| { + // Setup a parent bounty. + let parent_curator = 0; + let parent_index = 0; + let parent_value = 1_000_000; + let parent_fee = 10_000; + + System::set_block_number(1); + Balances::make_free_balance_be(&Treasury::account_id(), parent_value * 3); + Balances::make_free_balance_be(&parent_curator, parent_fee * 100); + assert_ok!(Bounties::propose_bounty( + Origin::signed(parent_curator), + parent_value, + b"12345".to_vec() + )); + assert_ok!(Bounties::approve_bounty(Origin::root(), parent_index)); + + System::set_block_number(2); + >::on_initialize(2); + + assert_ok!(Bounties::propose_curator( + Origin::root(), + parent_index, + parent_curator, + parent_fee + )); + assert_ok!(Bounties::accept_curator(Origin::signed(parent_curator), parent_index)); + + // Now we can start creating some child bounties. + // Case 1: Parent and child curator are not the same. + + let child_index = 0; + let child_curator = 1; + let child_value = 1_000; + let child_fee = 100; + let starting_balance = 100 * child_fee + child_value; + + Balances::make_free_balance_be(&child_curator, starting_balance); + assert_ok!(ChildBounties::add_child_bounty( + Origin::signed(parent_curator), + parent_index, + child_value, + b"12345-p1".to_vec() + )); + System::set_block_number(3); + >::on_initialize(3); + assert_ok!(ChildBounties::propose_curator( + Origin::signed(parent_curator), + parent_index, + child_index, + child_curator, + child_fee + )); + assert_ok!(ChildBounties::accept_curator( + Origin::signed(child_curator), + parent_index, + child_index + )); + + let expected_deposit = CuratorDepositMultiplier::get() * child_fee; + assert_eq!(Balances::free_balance(child_curator), starting_balance - expected_deposit); + assert_eq!(Balances::reserved_balance(child_curator), expected_deposit); + + // Case 2: Parent and child curator are the same. + + let child_index = 1; + let child_curator = parent_curator; // The same as parent bounty curator + let child_value = 1_000; + let child_fee = 10; + + let free_before = Balances::free_balance(&parent_curator); + let reserved_before = Balances::reserved_balance(&parent_curator); + + assert_ok!(ChildBounties::add_child_bounty( + Origin::signed(parent_curator), + parent_index, + child_value, + b"12345-p1".to_vec() + )); + System::set_block_number(4); + >::on_initialize(4); + assert_ok!(ChildBounties::propose_curator( + Origin::signed(parent_curator), + parent_index, + child_index, + child_curator, + child_fee + )); + assert_ok!(ChildBounties::accept_curator( + Origin::signed(child_curator), + parent_index, + child_index + )); + + // No expected deposit + assert_eq!(Balances::free_balance(child_curator), free_before); + assert_eq!(Balances::reserved_balance(child_curator), reserved_before); + + // Case 3: Upper Limit + + let child_index = 2; + let child_curator = 2; + let child_value = 10_000; + let child_fee = 5_000; + + Balances::make_free_balance_be(&child_curator, starting_balance); + assert_ok!(ChildBounties::add_child_bounty( + Origin::signed(parent_curator), + parent_index, + child_value, + b"12345-p1".to_vec() + )); + System::set_block_number(5); + >::on_initialize(5); + assert_ok!(ChildBounties::propose_curator( + Origin::signed(parent_curator), + parent_index, + child_index, + child_curator, + child_fee + )); + assert_ok!(ChildBounties::accept_curator( + Origin::signed(child_curator), + parent_index, + child_index + )); + + let expected_deposit = CuratorDepositMax::get(); + assert_eq!(Balances::free_balance(child_curator), starting_balance - expected_deposit); + assert_eq!(Balances::reserved_balance(child_curator), expected_deposit); + + // There is a max number of child bounties at a time. + assert_ok!(ChildBounties::impl_close_child_bounty(parent_index, child_index)); + + // Case 4: Lower Limit + + let child_index = 3; + let child_curator = 3; + let child_value = 10_000; + let child_fee = 0; + + Balances::make_free_balance_be(&child_curator, starting_balance); + assert_ok!(ChildBounties::add_child_bounty( + Origin::signed(parent_curator), + parent_index, + child_value, + b"12345-p1".to_vec() + )); + System::set_block_number(5); + >::on_initialize(5); + assert_ok!(ChildBounties::propose_curator( + Origin::signed(parent_curator), + parent_index, + child_index, + child_curator, + child_fee + )); + assert_ok!(ChildBounties::accept_curator( + Origin::signed(child_curator), + parent_index, + child_index + )); + + let expected_deposit = CuratorDepositMin::get(); + assert_eq!(Balances::free_balance(child_curator), starting_balance - expected_deposit); + assert_eq!(Balances::reserved_balance(child_curator), expected_deposit); + }); +} diff --git a/frame/child-bounties/src/weights.rs b/frame/child-bounties/src/weights.rs new file mode 100644 index 000000000000..002388810b8a --- /dev/null +++ b/frame/child-bounties/src/weights.rs @@ -0,0 +1,216 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for pallet_child_bounties +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2022-01-30, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 + +// Executed Command: +// ./target/production/substrate +// benchmark +// --chain=dev +// --steps=50 +// --repeat=20 +// --pallet=pallet_child_bounties +// --extrinsic=* +// --execution=wasm +// --wasm-execution=compiled +// --heap-pages=4096 +// --output=./frame/child-bounties/src/weights.rs +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use sp_std::marker::PhantomData; + +/// Weight functions needed for pallet_child_bounties. +pub trait WeightInfo { + fn add_child_bounty(d: u32, ) -> Weight; + fn propose_curator() -> Weight; + fn accept_curator() -> Weight; + fn unassign_curator() -> Weight; + fn award_child_bounty() -> Weight; + fn claim_child_bounty() -> Weight; + fn close_child_bounty_added() -> Weight; + fn close_child_bounty_active() -> Weight; +} + +/// Weights for pallet_child_bounties using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + // Storage: ChildBounties ParentChildBounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:0) + // Storage: System Account (r:2 w:2) + // Storage: ChildBounties ChildBountyCount (r:1 w:1) + // Storage: ChildBounties ChildBountyDescriptions (r:0 w:1) + // Storage: ChildBounties ChildBounties (r:0 w:1) + fn add_child_bounty(d: u32, ) -> Weight { + (44_503_000 as Weight) + // Standard Error: 0 + .saturating_add((3_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) + } + // Storage: Bounties Bounties (r:1 w:0) + // Storage: ChildBounties ChildBounties (r:1 w:1) + // Storage: ChildBounties ChildrenCuratorFees (r:1 w:1) + fn propose_curator() -> Weight { + (11_661_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Bounties Bounties (r:1 w:0) + // Storage: ChildBounties ChildBounties (r:1 w:1) + // Storage: System Account (r:1 w:1) + fn accept_curator() -> Weight { + (24_162_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: ChildBounties ChildBounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:0) + // Storage: System Account (r:1 w:1) + fn unassign_curator() -> Weight { + (27_924_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Bounties Bounties (r:1 w:0) + // Storage: ChildBounties ChildBounties (r:1 w:1) + fn award_child_bounty() -> Weight { + (19_332_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: ChildBounties ChildBounties (r:1 w:1) + // Storage: System Account (r:3 w:3) + // Storage: ChildBounties ParentChildBounties (r:1 w:1) + // Storage: ChildBounties ChildBountyDescriptions (r:0 w:1) + fn claim_child_bounty() -> Weight { + (60_363_000 as Weight) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) + } + // Storage: Bounties Bounties (r:1 w:0) + // Storage: ChildBounties ChildBounties (r:1 w:1) + // Storage: ChildBounties ChildrenCuratorFees (r:1 w:1) + // Storage: ChildBounties ParentChildBounties (r:1 w:1) + // Storage: System Account (r:2 w:2) + // Storage: ChildBounties ChildBountyDescriptions (r:0 w:1) + fn close_child_bounty_added() -> Weight { + (42_415_000 as Weight) + .saturating_add(T::DbWeight::get().reads(6 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) + } + // Storage: Bounties Bounties (r:1 w:0) + // Storage: ChildBounties ChildBounties (r:1 w:1) + // Storage: System Account (r:3 w:3) + // Storage: ChildBounties ChildrenCuratorFees (r:1 w:1) + // Storage: ChildBounties ParentChildBounties (r:1 w:1) + // Storage: ChildBounties ChildBountyDescriptions (r:0 w:1) + fn close_child_bounty_active() -> Weight { + (52_743_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(7 as Weight)) + } +} + +// For backwards compatibility and tests +impl WeightInfo for () { + // Storage: ChildBounties ParentChildBounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:0) + // Storage: System Account (r:2 w:2) + // Storage: ChildBounties ChildBountyCount (r:1 w:1) + // Storage: ChildBounties ChildBountyDescriptions (r:0 w:1) + // Storage: ChildBounties ChildBounties (r:0 w:1) + fn add_child_bounty(d: u32, ) -> Weight { + (44_503_000 as Weight) + // Standard Error: 0 + .saturating_add((3_000 as Weight).saturating_mul(d as Weight)) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + } + // Storage: Bounties Bounties (r:1 w:0) + // Storage: ChildBounties ChildBounties (r:1 w:1) + // Storage: ChildBounties ChildrenCuratorFees (r:1 w:1) + fn propose_curator() -> Weight { + (11_661_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Bounties Bounties (r:1 w:0) + // Storage: ChildBounties ChildBounties (r:1 w:1) + // Storage: System Account (r:1 w:1) + fn accept_curator() -> Weight { + (24_162_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: ChildBounties ChildBounties (r:1 w:1) + // Storage: Bounties Bounties (r:1 w:0) + // Storage: System Account (r:1 w:1) + fn unassign_curator() -> Weight { + (27_924_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Bounties Bounties (r:1 w:0) + // Storage: ChildBounties ChildBounties (r:1 w:1) + fn award_child_bounty() -> Weight { + (19_332_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: ChildBounties ChildBounties (r:1 w:1) + // Storage: System Account (r:3 w:3) + // Storage: ChildBounties ParentChildBounties (r:1 w:1) + // Storage: ChildBounties ChildBountyDescriptions (r:0 w:1) + fn claim_child_bounty() -> Weight { + (60_363_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + } + // Storage: Bounties Bounties (r:1 w:0) + // Storage: ChildBounties ChildBounties (r:1 w:1) + // Storage: ChildBounties ChildrenCuratorFees (r:1 w:1) + // Storage: ChildBounties ParentChildBounties (r:1 w:1) + // Storage: System Account (r:2 w:2) + // Storage: ChildBounties ChildBountyDescriptions (r:0 w:1) + fn close_child_bounty_added() -> Weight { + (42_415_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(6 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + } + // Storage: Bounties Bounties (r:1 w:0) + // Storage: ChildBounties ChildBounties (r:1 w:1) + // Storage: System Account (r:3 w:3) + // Storage: ChildBounties ChildrenCuratorFees (r:1 w:1) + // Storage: ChildBounties ParentChildBounties (r:1 w:1) + // Storage: ChildBounties ChildBountyDescriptions (r:0 w:1) + fn close_child_bounty_active() -> Weight { + (52_743_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(7 as Weight)) + } +} diff --git a/frame/collective/Cargo.toml b/frame/collective/Cargo.toml index e88f28d41773..c9793e155c2a 100644 --- a/frame/collective/Cargo.toml +++ b/frame/collective/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-collective" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Collective system: Members of a set of account IDs can make their collective feelings known through dispatched calls from one of two specialized origins." readme = "README.md" @@ -13,14 +13,14 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } log = { version = "0.4.14", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } @@ -41,7 +41,7 @@ std = [ "frame-system/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", diff --git a/frame/collective/src/benchmarking.rs b/frame/collective/src/benchmarking.rs index c7e695babf27..d5d0fc5f263e 100644 --- a/frame/collective/src/benchmarking.rs +++ b/frame/collective/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,9 +23,7 @@ use crate::Pallet as Collective; use sp_runtime::traits::Bounded; use sp_std::mem::size_of; -use frame_benchmarking::{ - account, benchmarks_instance_pallet, impl_benchmark_test_suite, whitelisted_caller, -}; +use frame_benchmarking::{account, benchmarks_instance_pallet, whitelisted_caller}; use frame_system::{Call as SystemCall, Pallet as System, RawOrigin as SystemOrigin}; const SEED: u32 = 0; @@ -45,9 +43,9 @@ benchmarks_instance_pallet! { // Set old members. // We compute the difference of old and new members, so it should influence timing. let mut old_members = vec![]; - let mut last_old_member = T::AccountId::default(); + let mut last_old_member = account::("old member", 0, SEED); for i in 0 .. m { - last_old_member = account("old member", i, SEED); + last_old_member = account::("old member", i, SEED); old_members.push(last_old_member.clone()); } let old_members_count = old_members.len() as u32; @@ -93,9 +91,9 @@ benchmarks_instance_pallet! { // Construct `new_members`. // It should influence timing since it will sort this vector. let mut new_members = vec![]; - let mut last_member = T::AccountId::default(); + let mut last_member = account::("member", 0, SEED); for i in 0 .. n { - last_member = account("member", i, SEED); + last_member = account::("member", i, SEED); new_members.push(last_member.clone()); } @@ -114,7 +112,7 @@ benchmarks_instance_pallet! { // Construct `members`. let mut members = vec![]; for i in 0 .. m - 1 { - let member = account("member", i, SEED); + let member = account::("member", i, SEED); members.push(member); } @@ -130,7 +128,7 @@ benchmarks_instance_pallet! { let proposal_hash = T::Hashing::hash_of(&proposal); // Note that execution fails due to mis-matched origin assert_last_event::( - Event::MemberExecuted(proposal_hash, Err(DispatchError::BadOrigin)).into() + Event::MemberExecuted { proposal_hash, result: Err(DispatchError::BadOrigin) }.into() ); } @@ -144,7 +142,7 @@ benchmarks_instance_pallet! { // Construct `members`. let mut members = vec![]; for i in 0 .. m - 1 { - let member = account("member", i, SEED); + let member = account::("member", i, SEED); members.push(member); } @@ -161,7 +159,7 @@ benchmarks_instance_pallet! { let proposal_hash = T::Hashing::hash_of(&proposal); // Note that execution fails due to mis-matched origin assert_last_event::( - Event::Executed(proposal_hash, Err(DispatchError::BadOrigin)).into() + Event::Executed { proposal_hash, result: Err(DispatchError::BadOrigin) }.into() ); } @@ -176,7 +174,7 @@ benchmarks_instance_pallet! { // Construct `members`. let mut members = vec![]; for i in 0 .. m - 1 { - let member = account("member", i, SEED); + let member = account::("member", i, SEED); members.push(member); } let caller: T::AccountId = whitelisted_caller(); @@ -205,7 +203,7 @@ benchmarks_instance_pallet! { // New proposal is recorded assert_eq!(Collective::::proposals().len(), p as usize); let proposal_hash = T::Hashing::hash_of(&proposal); - assert_last_event::(Event::Proposed(caller, p - 1, proposal_hash, threshold).into()); + assert_last_event::(Event::Proposed { account: caller, proposal_index: p - 1, proposal_hash, threshold }.into()); } vote { @@ -218,13 +216,13 @@ benchmarks_instance_pallet! { // Construct `members`. let mut members = vec![]; - let proposer: T::AccountId = account("proposer", 0, SEED); + let proposer: T::AccountId = account::("proposer", 0, SEED); members.push(proposer.clone()); for i in 1 .. m - 1 { - let member = account("member", i, SEED); + let member = account::("member", i, SEED); members.push(member); } - let voter: T::AccountId = account("voter", 0, SEED); + let voter: T::AccountId = account::("voter", 0, SEED); members.push(voter.clone()); Collective::::set_members(SystemOrigin::Root.into(), members.clone(), None, T::MaxMembers::get())?; @@ -293,13 +291,13 @@ benchmarks_instance_pallet! { // Construct `members`. let mut members = vec![]; - let proposer: T::AccountId = account("proposer", 0, SEED); + let proposer = account::("proposer", 0, SEED); members.push(proposer.clone()); for i in 1 .. m - 1 { - let member = account("member", i, SEED); + let member = account::("member", i, SEED); members.push(member); } - let voter: T::AccountId = account("voter", 0, SEED); + let voter = account::("voter", 0, SEED); members.push(voter.clone()); Collective::::set_members(SystemOrigin::Root.into(), members.clone(), None, T::MaxMembers::get())?; @@ -361,7 +359,7 @@ benchmarks_instance_pallet! { verify { // The last proposal is removed. assert_eq!(Collective::::proposals().len(), (p - 1) as usize); - assert_last_event::(Event::Disapproved(last_hash).into()); + assert_last_event::(Event::Disapproved { proposal_hash: last_hash }.into()); } close_early_approved { @@ -375,7 +373,7 @@ benchmarks_instance_pallet! { // Construct `members`. let mut members = vec![]; for i in 0 .. m - 1 { - let member = account("member", i, SEED); + let member = account::("member", i, SEED); members.push(member); } let caller: T::AccountId = whitelisted_caller(); @@ -442,7 +440,7 @@ benchmarks_instance_pallet! { verify { // The last proposal is removed. assert_eq!(Collective::::proposals().len(), (p - 1) as usize); - assert_last_event::(Event::Executed(last_hash, Err(DispatchError::BadOrigin)).into()); + assert_last_event::(Event::Executed { proposal_hash: last_hash, result: Err(DispatchError::BadOrigin) }.into()); } close_disapproved { @@ -456,7 +454,7 @@ benchmarks_instance_pallet! { // Construct `members`. let mut members = vec![]; for i in 0 .. m - 1 { - let member = account("member", i, SEED); + let member = account::("member", i, SEED); members.push(member); } let caller: T::AccountId = whitelisted_caller(); @@ -516,7 +514,7 @@ benchmarks_instance_pallet! { }: close(SystemOrigin::Signed(caller), last_hash, index, Weight::max_value(), bytes_in_storage) verify { assert_eq!(Collective::::proposals().len(), (p - 1) as usize); - assert_last_event::(Event::Disapproved(last_hash).into()); + assert_last_event::(Event::Disapproved { proposal_hash: last_hash }.into()); } close_approved { @@ -530,7 +528,7 @@ benchmarks_instance_pallet! { // Construct `members`. let mut members = vec![]; for i in 0 .. m - 1 { - let member = account("member", i, SEED); + let member = account::("member", i, SEED); members.push(member); } let caller: T::AccountId = whitelisted_caller(); @@ -588,7 +586,7 @@ benchmarks_instance_pallet! { }: close(SystemOrigin::Signed(caller), last_hash, p - 1, Weight::max_value(), bytes_in_storage) verify { assert_eq!(Collective::::proposals().len(), (p - 1) as usize); - assert_last_event::(Event::Executed(last_hash, Err(DispatchError::BadOrigin)).into()); + assert_last_event::(Event::Executed { proposal_hash: last_hash, result: Err(DispatchError::BadOrigin) }.into()); } disapprove_proposal { @@ -601,10 +599,10 @@ benchmarks_instance_pallet! { // Construct `members`. let mut members = vec![]; for i in 0 .. m - 1 { - let member = account("member", i, SEED); + let member = account::("member", i, SEED); members.push(member); } - let caller: T::AccountId = account("caller", 0, SEED); + let caller = account::("caller", 0, SEED); members.push(caller.clone()); Collective::::set_members( SystemOrigin::Root.into(), @@ -636,8 +634,8 @@ benchmarks_instance_pallet! { }: _(SystemOrigin::Root, last_hash) verify { assert_eq!(Collective::::proposals().len(), (p - 1) as usize); - assert_last_event::(Event::Disapproved(last_hash).into()); + assert_last_event::(Event::Disapproved { proposal_hash: last_hash }.into()); } -} -impl_benchmark_test_suite!(Collective, crate::tests::new_test_ext(), crate::tests::Test); + impl_benchmark_test_suite!(Collective, crate::tests::new_test_ext(), crate::tests::Test); +} diff --git a/frame/collective/src/lib.rs b/frame/collective/src/lib.rs index 89d4c8a150c3..e876343ec33d 100644 --- a/frame/collective/src/lib.rs +++ b/frame/collective/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -43,7 +43,6 @@ #![recursion_limit = "128"] use scale_info::TypeInfo; -use sp_core::u32_trait::Value as U32; use sp_io::storage; use sp_runtime::{traits::Hash, RuntimeDebug}; use sp_std::{marker::PhantomData, prelude::*, result}; @@ -172,6 +171,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] #[pallet::storage_version(STORAGE_VERSION)] + #[pallet::without_storage_info] pub struct Pallet(PhantomData<(T, I)>); #[pallet::config] @@ -279,27 +279,31 @@ pub mod pallet { pub enum Event, I: 'static = ()> { /// A motion (given hash) has been proposed (by given account) with a threshold (given /// `MemberCount`). - /// \[account, proposal_index, proposal_hash, threshold\] - Proposed(T::AccountId, ProposalIndex, T::Hash, MemberCount), + Proposed { + account: T::AccountId, + proposal_index: ProposalIndex, + proposal_hash: T::Hash, + threshold: MemberCount, + }, /// A motion (given hash) has been voted on by given account, leaving /// a tally (yes votes and no votes given respectively as `MemberCount`). - /// \[account, proposal_hash, voted, yes, no\] - Voted(T::AccountId, T::Hash, bool, MemberCount, MemberCount), + Voted { + account: T::AccountId, + proposal_hash: T::Hash, + voted: bool, + yes: MemberCount, + no: MemberCount, + }, /// A motion was approved by the required threshold. - /// \[proposal_hash\] - Approved(T::Hash), + Approved { proposal_hash: T::Hash }, /// A motion was not approved by the required threshold. - /// \[proposal_hash\] - Disapproved(T::Hash), + Disapproved { proposal_hash: T::Hash }, /// A motion was executed; result will be `Ok` if it returned without error. - /// \[proposal_hash, result\] - Executed(T::Hash, DispatchResult), + Executed { proposal_hash: T::Hash, result: DispatchResult }, /// A single member did some action; result will be `Ok` if it returned without error. - /// \[proposal_hash, result\] - MemberExecuted(T::Hash, DispatchResult), + MemberExecuted { proposal_hash: T::Hash, result: DispatchResult }, /// A proposal was closed because its threshold was reached or after its duration was up. - /// \[proposal_hash, yes, no\] - Closed(T::Hash, MemberCount, MemberCount), + Closed { proposal_hash: T::Hash, yes: MemberCount, no: MemberCount }, } /// Old name generated by `decl_event`. @@ -345,6 +349,13 @@ pub mod pallet { /// NOTE: Does not enforce the expected `MaxMembers` limit on the amount of members, but /// the weight estimations rely on it to estimate dispatchable weight. /// + /// # WARNING: + /// + /// The `pallet-collective` can also be managed by logic outside of the pallet through the + /// implementation of the trait [`ChangeMembers`]. + /// Any call to `set_members` must be careful that the member set doesn't get out of sync + /// with other logic managing the member set. + /// /// # /// ## Weight /// - `O(MP + N)` where: @@ -435,10 +446,10 @@ pub mod pallet { let proposal_hash = T::Hashing::hash_of(&proposal); let result = proposal.dispatch(RawOrigin::Member(who).into()); - Self::deposit_event(Event::MemberExecuted( + Self::deposit_event(Event::MemberExecuted { proposal_hash, - result.map(|_| ()).map_err(|e| e.error), - )); + result: result.map(|_| ()).map_err(|e| e.error), + }); Ok(get_result_weight(result) .map(|w| { @@ -514,10 +525,10 @@ pub mod pallet { if threshold < 2 { let seats = Self::members().len() as MemberCount; let result = proposal.dispatch(RawOrigin::Members(1, seats).into()); - Self::deposit_event(Event::Executed( + Self::deposit_event(Event::Executed { proposal_hash, - result.map(|_| ()).map_err(|e| e.error), - )); + result: result.map(|_| ()).map_err(|e| e.error), + }); Ok(get_result_weight(result) .map(|w| { @@ -545,7 +556,12 @@ pub mod pallet { }; >::insert(proposal_hash, votes); - Self::deposit_event(Event::Proposed(who, index, proposal_hash, threshold)); + Self::deposit_event(Event::Proposed { + account: who, + proposal_index: index, + proposal_hash, + threshold, + }); Ok(Some(T::WeightInfo::propose_proposed( proposal_len as u32, // B @@ -613,7 +629,13 @@ pub mod pallet { let yes_votes = voting.ayes.len() as MemberCount; let no_votes = voting.nays.len() as MemberCount; - Self::deposit_event(Event::Voted(who, proposal, approve, yes_votes, no_votes)); + Self::deposit_event(Event::Voted { + account: who, + proposal_hash: proposal, + voted: approve, + yes: yes_votes, + no: no_votes, + }); Voting::::insert(&proposal, voting); @@ -694,7 +716,7 @@ pub mod pallet { length_bound, proposal_weight_bound, )?; - Self::deposit_event(Event::Closed(proposal_hash, yes_votes, no_votes)); + Self::deposit_event(Event::Closed { proposal_hash, yes: yes_votes, no: no_votes }); let (proposal_weight, proposal_count) = Self::do_approve_proposal(seats, yes_votes, proposal_hash, proposal); return Ok(( @@ -706,7 +728,7 @@ pub mod pallet { ) .into()) } else if disapproved { - Self::deposit_event(Event::Closed(proposal_hash, yes_votes, no_votes)); + Self::deposit_event(Event::Closed { proposal_hash, yes: yes_votes, no: no_votes }); let proposal_count = Self::do_disapprove_proposal(proposal_hash); return Ok(( Some(T::WeightInfo::close_early_disapproved(seats, proposal_count)), @@ -739,7 +761,7 @@ pub mod pallet { length_bound, proposal_weight_bound, )?; - Self::deposit_event(Event::Closed(proposal_hash, yes_votes, no_votes)); + Self::deposit_event(Event::Closed { proposal_hash, yes: yes_votes, no: no_votes }); let (proposal_weight, proposal_count) = Self::do_approve_proposal(seats, yes_votes, proposal_hash, proposal); Ok(( @@ -751,7 +773,7 @@ pub mod pallet { ) .into()) } else { - Self::deposit_event(Event::Closed(proposal_hash, yes_votes, no_votes)); + Self::deposit_event(Event::Closed { proposal_hash, yes: yes_votes, no: no_votes }); let proposal_count = Self::do_disapprove_proposal(proposal_hash); Ok((Some(T::WeightInfo::close_disapproved(seats, proposal_count)), Pays::No).into()) } @@ -841,15 +863,15 @@ impl, I: 'static> Pallet { proposal_hash: T::Hash, proposal: >::Proposal, ) -> (Weight, u32) { - Self::deposit_event(Event::Approved(proposal_hash)); + Self::deposit_event(Event::Approved { proposal_hash }); let dispatch_weight = proposal.get_dispatch_info().weight; let origin = RawOrigin::Members(yes_votes, seats).into(); let result = proposal.dispatch(origin); - Self::deposit_event(Event::Executed( + Self::deposit_event(Event::Executed { proposal_hash, - result.map(|_| ()).map_err(|e| e.error), - )); + result: result.map(|_| ()).map_err(|e| e.error), + }); // default to the dispatch info weight for safety let proposal_weight = get_result_weight(result).unwrap_or(dispatch_weight); // P1 @@ -859,7 +881,7 @@ impl, I: 'static> Pallet { fn do_disapprove_proposal(proposal_hash: T::Hash) -> u32 { // disapproved - Self::deposit_event(Event::Disapproved(proposal_hash)); + Self::deposit_event(Event::Disapproved { proposal_hash }); Self::remove_proposal(proposal_hash) } @@ -967,8 +989,8 @@ where pub struct EnsureMember(PhantomData<(AccountId, I)>); impl< O: Into, O>> + From>, - AccountId: Default, I, + AccountId: Decode, > EnsureOrigin for EnsureMember { type Success = AccountId; @@ -981,47 +1003,50 @@ impl< #[cfg(feature = "runtime-benchmarks")] fn successful_origin() -> O { - O::from(RawOrigin::Member(Default::default())) + let zero_account_id = + AccountId::decode(&mut sp_runtime::traits::TrailingZeroInput::zeroes()) + .expect("infinite length input; no invalid inputs for type; qed"); + O::from(RawOrigin::Member(zero_account_id)) } } -pub struct EnsureMembers(PhantomData<(N, AccountId, I)>); +pub struct EnsureMembers(PhantomData<(AccountId, I)>); impl< O: Into, O>> + From>, - N: U32, AccountId, I, - > EnsureOrigin for EnsureMembers + const N: u32, + > EnsureOrigin for EnsureMembers { type Success = (MemberCount, MemberCount); fn try_origin(o: O) -> Result { o.into().and_then(|o| match o { - RawOrigin::Members(n, m) if n >= N::VALUE => Ok((n, m)), + RawOrigin::Members(n, m) if n >= N => Ok((n, m)), r => Err(O::from(r)), }) } #[cfg(feature = "runtime-benchmarks")] fn successful_origin() -> O { - O::from(RawOrigin::Members(N::VALUE, N::VALUE)) + O::from(RawOrigin::Members(N, N)) } } -pub struct EnsureProportionMoreThan( - PhantomData<(N, D, AccountId, I)>, +pub struct EnsureProportionMoreThan( + PhantomData<(AccountId, I)>, ); impl< O: Into, O>> + From>, - N: U32, - D: U32, AccountId, I, - > EnsureOrigin for EnsureProportionMoreThan + const N: u32, + const D: u32, + > EnsureOrigin for EnsureProportionMoreThan { type Success = (); fn try_origin(o: O) -> Result { o.into().and_then(|o| match o { - RawOrigin::Members(n, m) if n * D::VALUE > N::VALUE * m => Ok(()), + RawOrigin::Members(n, m) if n * D > N * m => Ok(()), r => Err(O::from(r)), }) } @@ -1032,21 +1057,21 @@ impl< } } -pub struct EnsureProportionAtLeast( - PhantomData<(N, D, AccountId, I)>, +pub struct EnsureProportionAtLeast( + PhantomData<(AccountId, I)>, ); impl< O: Into, O>> + From>, - N: U32, - D: U32, AccountId, I, - > EnsureOrigin for EnsureProportionAtLeast + const N: u32, + const D: u32, + > EnsureOrigin for EnsureProportionAtLeast { type Success = (); fn try_origin(o: O) -> Result { o.into().and_then(|o| match o { - RawOrigin::Members(n, m) if n * D::VALUE >= N::VALUE * m => Ok(()), + RawOrigin::Members(n, m) if n * D >= N * m => Ok(()), r => Err(O::from(r)), }) } diff --git a/frame/collective/src/migrations/mod.rs b/frame/collective/src/migrations/mod.rs index 26d07a0cd5ac..235d0f1c7cf1 100644 --- a/frame/collective/src/migrations/mod.rs +++ b/frame/collective/src/migrations/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/collective/src/migrations/v4.rs b/frame/collective/src/migrations/v4.rs index 68284ba4df91..4e6cd0558413 100644 --- a/frame/collective/src/migrations/v4.rs +++ b/frame/collective/src/migrations/v4.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/collective/src/tests.rs b/frame/collective/src/tests.rs index b8feb64867cf..a8abfb0c5235 100644 --- a/frame/collective/src/tests.rs +++ b/frame/collective/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,13 +18,13 @@ use super::{Event as CollectiveEvent, *}; use crate as pallet_collective; use frame_support::{ - assert_noop, assert_ok, parameter_types, traits::GenesisBuild, weights::Pays, Hashable, + assert_noop, assert_ok, parameter_types, + traits::{ConstU32, ConstU64, GenesisBuild}, + weights::Pays, + Hashable, }; use frame_system::{EventRecord, Phase}; -use sp_core::{ - u32_trait::{_3, _4}, - H256, -}; +use sp_core::H256; use sp_runtime::{ testing::Header, traits::{BlakeTwo256, IdentityLookup}, @@ -52,9 +52,8 @@ mod mock_democracy { pub use pallet::*; #[frame_support::pallet] pub mod pallet { - use frame_support::{pallet_prelude::*, traits::EnsureOrigin}; + use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; - use sp_runtime::DispatchResult; #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] @@ -84,11 +83,11 @@ mod mock_democracy { } } +pub type MaxMembers = ConstU32<100>; + parameter_types! { - pub const BlockHashCount: u64 = 250; pub const MotionDuration: u64 = 3; pub const MaxProposals: u32 = 100; - pub const MaxMembers: u32 = 100; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -107,7 +106,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -116,12 +115,13 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl Config for Test { type Origin = Origin; type Proposal = Call; type Event = Event; - type MotionDuration = MotionDuration; + type MotionDuration = ConstU64<3>; type MaxProposals = MaxProposals; type MaxMembers = MaxMembers; type DefaultVote = PrimeDefaultVote; @@ -131,7 +131,7 @@ impl Config for Test { type Origin = Origin; type Proposal = Call; type Event = Event; - type MotionDuration = MotionDuration; + type MotionDuration = ConstU64<3>; type MaxProposals = MaxProposals; type MaxMembers = MaxMembers; type DefaultVote = MoreThanMajorityThenPrimeDefaultVote; @@ -139,13 +139,13 @@ impl Config for Test { } impl mock_democracy::Config for Test { type Event = Event; - type ExternalMajorityOrigin = EnsureProportionAtLeast<_3, _4, u64, Instance1>; + type ExternalMajorityOrigin = EnsureProportionAtLeast; } impl Config for Test { type Origin = Origin; type Proposal = Call; type Event = Event; - type MotionDuration = MotionDuration; + type MotionDuration = ConstU64<3>; type MaxProposals = MaxProposals; type MaxMembers = MaxMembers; type DefaultVote = PrimeDefaultVote; @@ -172,7 +172,7 @@ pub fn new_test_ext() -> sp_io::TestExternalities { } fn make_proposal(value: u64) -> Call { - Call::System(frame_system::Call::remark { remark: value.encode() }) + Call::System(frame_system::Call::remark_with_event { remark: value.to_be_bytes().to_vec() }) } fn record(event: Event) -> EventRecord { @@ -216,11 +216,32 @@ fn close_works() { assert_eq!( System::events(), vec![ - record(Event::Collective(CollectiveEvent::Proposed(1, 0, hash, 3))), - record(Event::Collective(CollectiveEvent::Voted(1, hash, true, 1, 0))), - record(Event::Collective(CollectiveEvent::Voted(2, hash, true, 2, 0))), - record(Event::Collective(CollectiveEvent::Closed(hash, 2, 1))), - record(Event::Collective(CollectiveEvent::Disapproved(hash))) + record(Event::Collective(CollectiveEvent::Proposed { + account: 1, + proposal_index: 0, + proposal_hash: hash, + threshold: 3 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 1, + proposal_hash: hash, + voted: true, + yes: 1, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 2, + proposal_hash: hash, + voted: true, + yes: 2, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Closed { + proposal_hash: hash, + yes: 2, + no: 1 + })), + record(Event::Collective(CollectiveEvent::Disapproved { proposal_hash: hash })) ] ); }); @@ -315,11 +336,32 @@ fn close_with_prime_works() { assert_eq!( System::events(), vec![ - record(Event::Collective(CollectiveEvent::Proposed(1, 0, hash, 3))), - record(Event::Collective(CollectiveEvent::Voted(1, hash, true, 1, 0))), - record(Event::Collective(CollectiveEvent::Voted(2, hash, true, 2, 0))), - record(Event::Collective(CollectiveEvent::Closed(hash, 2, 1))), - record(Event::Collective(CollectiveEvent::Disapproved(hash))) + record(Event::Collective(CollectiveEvent::Proposed { + account: 1, + proposal_index: 0, + proposal_hash: hash, + threshold: 3 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 1, + proposal_hash: hash, + voted: true, + yes: 1, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 2, + proposal_hash: hash, + voted: true, + yes: 2, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Closed { + proposal_hash: hash, + yes: 2, + no: 1 + })), + record(Event::Collective(CollectiveEvent::Disapproved { proposal_hash: hash })) ] ); }); @@ -354,15 +396,36 @@ fn close_with_voting_prime_works() { assert_eq!( System::events(), vec![ - record(Event::Collective(CollectiveEvent::Proposed(1, 0, hash, 3))), - record(Event::Collective(CollectiveEvent::Voted(1, hash, true, 1, 0))), - record(Event::Collective(CollectiveEvent::Voted(2, hash, true, 2, 0))), - record(Event::Collective(CollectiveEvent::Closed(hash, 3, 0))), - record(Event::Collective(CollectiveEvent::Approved(hash))), - record(Event::Collective(CollectiveEvent::Executed( - hash, - Err(DispatchError::BadOrigin) - ))) + record(Event::Collective(CollectiveEvent::Proposed { + account: 1, + proposal_index: 0, + proposal_hash: hash, + threshold: 3 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 1, + proposal_hash: hash, + voted: true, + yes: 1, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 2, + proposal_hash: hash, + voted: true, + yes: 2, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Closed { + proposal_hash: hash, + yes: 3, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Approved { proposal_hash: hash })), + record(Event::Collective(CollectiveEvent::Executed { + proposal_hash: hash, + result: Err(DispatchError::BadOrigin) + })) ] ); }); @@ -404,16 +467,45 @@ fn close_with_no_prime_but_majority_works() { assert_eq!( System::events(), vec![ - record(Event::CollectiveMajority(CollectiveEvent::Proposed(1, 0, hash, 5))), - record(Event::CollectiveMajority(CollectiveEvent::Voted(1, hash, true, 1, 0))), - record(Event::CollectiveMajority(CollectiveEvent::Voted(2, hash, true, 2, 0))), - record(Event::CollectiveMajority(CollectiveEvent::Voted(3, hash, true, 3, 0))), - record(Event::CollectiveMajority(CollectiveEvent::Closed(hash, 5, 0))), - record(Event::CollectiveMajority(CollectiveEvent::Approved(hash))), - record(Event::CollectiveMajority(CollectiveEvent::Executed( - hash, - Err(DispatchError::BadOrigin) - ))) + record(Event::CollectiveMajority(CollectiveEvent::Proposed { + account: 1, + proposal_index: 0, + proposal_hash: hash, + threshold: 5 + })), + record(Event::CollectiveMajority(CollectiveEvent::Voted { + account: 1, + proposal_hash: hash, + voted: true, + yes: 1, + no: 0 + })), + record(Event::CollectiveMajority(CollectiveEvent::Voted { + account: 2, + proposal_hash: hash, + voted: true, + yes: 2, + no: 0 + })), + record(Event::CollectiveMajority(CollectiveEvent::Voted { + account: 3, + proposal_hash: hash, + voted: true, + yes: 3, + no: 0 + })), + record(Event::CollectiveMajority(CollectiveEvent::Closed { + proposal_hash: hash, + yes: 5, + no: 0 + })), + record(Event::CollectiveMajority(CollectiveEvent::Approved { + proposal_hash: hash + })), + record(Event::CollectiveMajority(CollectiveEvent::Executed { + proposal_hash: hash, + result: Err(DispatchError::BadOrigin) + })) ] ); }); @@ -537,7 +629,12 @@ fn propose_works() { assert_eq!( System::events(), - vec![record(Event::Collective(CollectiveEvent::Proposed(1, 0, hash, 3)))] + vec![record(Event::Collective(CollectiveEvent::Proposed { + account: 1, + proposal_index: 0, + proposal_hash: hash, + threshold: 3 + }))] ); }); } @@ -696,9 +793,26 @@ fn motions_vote_after_works() { assert_eq!( System::events(), vec![ - record(Event::Collective(CollectiveEvent::Proposed(1, 0, hash, 2))), - record(Event::Collective(CollectiveEvent::Voted(1, hash, true, 1, 0))), - record(Event::Collective(CollectiveEvent::Voted(1, hash, false, 0, 1))), + record(Event::Collective(CollectiveEvent::Proposed { + account: 1, + proposal_index: 0, + proposal_hash: hash, + threshold: 2 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 1, + proposal_hash: hash, + voted: true, + yes: 1, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 1, + proposal_hash: hash, + voted: false, + yes: 0, + no: 1 + })), ] ); }); @@ -812,15 +926,36 @@ fn motions_approval_with_enough_votes_and_lower_voting_threshold_works() { assert_eq!( System::events(), vec![ - record(Event::Collective(CollectiveEvent::Proposed(1, 0, hash, 2))), - record(Event::Collective(CollectiveEvent::Voted(1, hash, true, 1, 0))), - record(Event::Collective(CollectiveEvent::Voted(2, hash, true, 2, 0))), - record(Event::Collective(CollectiveEvent::Closed(hash, 2, 0))), - record(Event::Collective(CollectiveEvent::Approved(hash))), - record(Event::Collective(CollectiveEvent::Executed( - hash, - Err(DispatchError::BadOrigin) - ))), + record(Event::Collective(CollectiveEvent::Proposed { + account: 1, + proposal_index: 0, + proposal_hash: hash, + threshold: 2 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 1, + proposal_hash: hash, + voted: true, + yes: 1, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 2, + proposal_hash: hash, + voted: true, + yes: 2, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Closed { + proposal_hash: hash, + yes: 2, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Approved { proposal_hash: hash })), + record(Event::Collective(CollectiveEvent::Executed { + proposal_hash: hash, + result: Err(DispatchError::BadOrigin) + })), ] ); @@ -840,14 +975,44 @@ fn motions_approval_with_enough_votes_and_lower_voting_threshold_works() { assert_eq!( System::events(), vec![ - record(Event::Collective(CollectiveEvent::Proposed(1, 1, hash, 2))), - record(Event::Collective(CollectiveEvent::Voted(1, hash, true, 1, 0))), - record(Event::Collective(CollectiveEvent::Voted(2, hash, true, 2, 0))), - record(Event::Collective(CollectiveEvent::Voted(3, hash, true, 3, 0))), - record(Event::Collective(CollectiveEvent::Closed(hash, 3, 0))), - record(Event::Collective(CollectiveEvent::Approved(hash))), + record(Event::Collective(CollectiveEvent::Proposed { + account: 1, + proposal_index: 1, + proposal_hash: hash, + threshold: 2 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 1, + proposal_hash: hash, + voted: true, + yes: 1, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 2, + proposal_hash: hash, + voted: true, + yes: 2, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 3, + proposal_hash: hash, + voted: true, + yes: 3, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Closed { + proposal_hash: hash, + yes: 3, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Approved { proposal_hash: hash })), record(Event::Democracy(mock_democracy::pallet::Event::::ExternalProposed)), - record(Event::Collective(CollectiveEvent::Executed(hash, Ok(())))), + record(Event::Collective(CollectiveEvent::Executed { + proposal_hash: hash, + result: Ok(()) + })), ] ); }); @@ -873,11 +1038,32 @@ fn motions_disapproval_works() { assert_eq!( System::events(), vec![ - record(Event::Collective(CollectiveEvent::Proposed(1, 0, hash, 3))), - record(Event::Collective(CollectiveEvent::Voted(1, hash, true, 1, 0))), - record(Event::Collective(CollectiveEvent::Voted(2, hash, false, 1, 1))), - record(Event::Collective(CollectiveEvent::Closed(hash, 1, 1))), - record(Event::Collective(CollectiveEvent::Disapproved(hash))), + record(Event::Collective(CollectiveEvent::Proposed { + account: 1, + proposal_index: 0, + proposal_hash: hash, + threshold: 3 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 1, + proposal_hash: hash, + voted: true, + yes: 1, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 2, + proposal_hash: hash, + voted: false, + yes: 1, + no: 1 + })), + record(Event::Collective(CollectiveEvent::Closed { + proposal_hash: hash, + yes: 1, + no: 1 + })), + record(Event::Collective(CollectiveEvent::Disapproved { proposal_hash: hash })), ] ); }); @@ -903,15 +1089,36 @@ fn motions_approval_works() { assert_eq!( System::events(), vec![ - record(Event::Collective(CollectiveEvent::Proposed(1, 0, hash, 2))), - record(Event::Collective(CollectiveEvent::Voted(1, hash, true, 1, 0))), - record(Event::Collective(CollectiveEvent::Voted(2, hash, true, 2, 0))), - record(Event::Collective(CollectiveEvent::Closed(hash, 2, 0))), - record(Event::Collective(CollectiveEvent::Approved(hash))), - record(Event::Collective(CollectiveEvent::Executed( - hash, - Err(DispatchError::BadOrigin) - ))), + record(Event::Collective(CollectiveEvent::Proposed { + account: 1, + proposal_index: 0, + proposal_hash: hash, + threshold: 2 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 1, + proposal_hash: hash, + voted: true, + yes: 1, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 2, + proposal_hash: hash, + voted: true, + yes: 2, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Closed { + proposal_hash: hash, + yes: 2, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Approved { proposal_hash: hash })), + record(Event::Collective(CollectiveEvent::Executed { + proposal_hash: hash, + result: Err(DispatchError::BadOrigin) + })), ] ); }); @@ -932,7 +1139,12 @@ fn motion_with_no_votes_closes_with_disapproval() { )); assert_eq!( System::events()[0], - record(Event::Collective(CollectiveEvent::Proposed(1, 0, hash, 3))) + record(Event::Collective(CollectiveEvent::Proposed { + account: 1, + proposal_index: 0, + proposal_hash: hash, + threshold: 3 + })) ); // Closing the motion too early is not possible because it has neither @@ -951,11 +1163,15 @@ fn motion_with_no_votes_closes_with_disapproval() { // Events show that the close ended in a disapproval. assert_eq!( System::events()[1], - record(Event::Collective(CollectiveEvent::Closed(hash, 0, 3))) + record(Event::Collective(CollectiveEvent::Closed { + proposal_hash: hash, + yes: 0, + no: 3 + })) ); assert_eq!( System::events()[2], - record(Event::Collective(CollectiveEvent::Disapproved(hash))) + record(Event::Collective(CollectiveEvent::Disapproved { proposal_hash: hash })) ); }) } @@ -1015,10 +1231,27 @@ fn disapprove_proposal_works() { assert_eq!( System::events(), vec![ - record(Event::Collective(CollectiveEvent::Proposed(1, 0, hash, 2))), - record(Event::Collective(CollectiveEvent::Voted(1, hash, true, 1, 0))), - record(Event::Collective(CollectiveEvent::Voted(2, hash, true, 2, 0))), - record(Event::Collective(CollectiveEvent::Disapproved(hash))), + record(Event::Collective(CollectiveEvent::Proposed { + account: 1, + proposal_index: 0, + proposal_hash: hash, + threshold: 2 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 1, + proposal_hash: hash, + voted: true, + yes: 1, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Voted { + account: 2, + proposal_hash: hash, + voted: true, + yes: 2, + no: 0 + })), + record(Event::Collective(CollectiveEvent::Disapproved { proposal_hash: hash })), ] ); }) diff --git a/frame/collective/src/weights.rs b/frame/collective/src/weights.rs index 40ac9eabdd6e..1280ced89eee 100644 --- a/frame/collective/src/weights.rs +++ b/frame/collective/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_collective //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-30, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/collective/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -60,132 +61,132 @@ pub trait WeightInfo { /// Weights for pallet_collective using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { - // Storage: Instance1Collective Members (r:1 w:1) - // Storage: Instance1Collective Proposals (r:1 w:0) - // Storage: Instance1Collective Voting (r:100 w:100) - // Storage: Instance1Collective Prime (r:0 w:1) + // Storage: Council Members (r:1 w:1) + // Storage: Council Proposals (r:1 w:0) + // Storage: Council Voting (r:100 w:100) + // Storage: Council Prime (r:0 w:1) fn set_members(m: u32, n: u32, p: u32, ) -> Weight { (0 as Weight) - // Standard Error: 4_000 - .saturating_add((14_084_000 as Weight).saturating_mul(m as Weight)) - // Standard Error: 4_000 - .saturating_add((161_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 4_000 - .saturating_add((19_201_000 as Weight).saturating_mul(p as Weight)) + // Standard Error: 10_000 + .saturating_add((14_493_000 as Weight).saturating_mul(m as Weight)) + // Standard Error: 10_000 + .saturating_add((23_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 10_000 + .saturating_add((16_909_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(p as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(p as Weight))) } - // Storage: Instance1Collective Members (r:1 w:0) + // Storage: Council Members (r:1 w:0) fn execute(b: u32, m: u32, ) -> Weight { - (22_748_000 as Weight) + (12_790_000 as Weight) // Standard Error: 0 - .saturating_add((3_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(b as Weight)) // Standard Error: 0 - .saturating_add((92_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((73_000 as Weight).saturating_mul(m as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) } - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective ProposalOf (r:1 w:0) + // Storage: Council Members (r:1 w:0) + // Storage: Council ProposalOf (r:1 w:0) fn propose_execute(b: u32, m: u32, ) -> Weight { - (27_465_000 as Weight) + (15_087_000 as Weight) // Standard Error: 0 - .saturating_add((3_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(b as Weight)) // Standard Error: 0 - .saturating_add((178_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((135_000 as Weight).saturating_mul(m as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) } - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective ProposalOf (r:1 w:1) - // Storage: Instance1Collective Proposals (r:1 w:1) - // Storage: Instance1Collective ProposalCount (r:1 w:1) - // Storage: Instance1Collective Voting (r:0 w:1) + // Storage: Council Members (r:1 w:0) + // Storage: Council ProposalOf (r:1 w:1) + // Storage: Council Proposals (r:1 w:1) + // Storage: Council ProposalCount (r:1 w:1) + // Storage: Council Voting (r:0 w:1) fn propose_proposed(b: u32, m: u32, p: u32, ) -> Weight { - (39_869_000 as Weight) + (18_269_000 as Weight) // Standard Error: 0 .saturating_add((8_000 as Weight).saturating_mul(b as Weight)) // Standard Error: 1_000 - .saturating_add((107_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((77_000 as Weight).saturating_mul(m as Weight)) // Standard Error: 1_000 - .saturating_add((406_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((203_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective Voting (r:1 w:1) + // Storage: Council Members (r:1 w:0) + // Storage: Council Voting (r:1 w:1) fn vote(m: u32, ) -> Weight { - (37_387_000 as Weight) + (26_624_000 as Weight) // Standard Error: 2_000 - .saturating_add((223_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((161_000 as Weight).saturating_mul(m as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } - // Storage: Instance1Collective Voting (r:1 w:1) - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective Proposals (r:1 w:1) - // Storage: Instance1Collective ProposalOf (r:0 w:1) + // Storage: Council Voting (r:1 w:1) + // Storage: Council Members (r:1 w:0) + // Storage: Council Proposals (r:1 w:1) + // Storage: Council ProposalOf (r:0 w:1) fn close_early_disapproved(m: u32, p: u32, ) -> Weight { - (45_670_000 as Weight) + (26_527_000 as Weight) // Standard Error: 1_000 - .saturating_add((170_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((127_000 as Weight).saturating_mul(m as Weight)) // Standard Error: 1_000 - .saturating_add((358_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((155_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Collective Voting (r:1 w:1) - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective ProposalOf (r:1 w:1) - // Storage: Instance1Collective Proposals (r:1 w:1) + // Storage: Council Voting (r:1 w:1) + // Storage: Council Members (r:1 w:0) + // Storage: Council ProposalOf (r:1 w:1) + // Storage: Council Proposals (r:1 w:1) fn close_early_approved(b: u32, m: u32, p: u32, ) -> Weight { - (52_529_000 as Weight) + (26_352_000 as Weight) // Standard Error: 0 - .saturating_add((7_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((6_000 as Weight).saturating_mul(b as Weight)) // Standard Error: 1_000 - .saturating_add((206_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((154_000 as Weight).saturating_mul(m as Weight)) // Standard Error: 1_000 - .saturating_add((412_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((203_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Collective Voting (r:1 w:1) - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective Prime (r:1 w:0) - // Storage: Instance1Collective Proposals (r:1 w:1) - // Storage: Instance1Collective ProposalOf (r:0 w:1) + // Storage: Council Voting (r:1 w:1) + // Storage: Council Members (r:1 w:0) + // Storage: Council Prime (r:1 w:0) + // Storage: Council Proposals (r:1 w:1) + // Storage: Council ProposalOf (r:0 w:1) fn close_disapproved(m: u32, p: u32, ) -> Weight { - (50_427_000 as Weight) + (28_638_000 as Weight) // Standard Error: 1_000 - .saturating_add((170_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((133_000 as Weight).saturating_mul(m as Weight)) // Standard Error: 1_000 - .saturating_add((354_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((162_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Collective Voting (r:1 w:1) - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective Prime (r:1 w:0) - // Storage: Instance1Collective ProposalOf (r:1 w:1) - // Storage: Instance1Collective Proposals (r:1 w:1) + // Storage: Council Voting (r:1 w:1) + // Storage: Council Members (r:1 w:0) + // Storage: Council Prime (r:1 w:0) + // Storage: Council ProposalOf (r:1 w:1) + // Storage: Council Proposals (r:1 w:1) fn close_approved(b: u32, m: u32, p: u32, ) -> Weight { - (57_031_000 as Weight) + (29_946_000 as Weight) // Standard Error: 0 - .saturating_add((7_000 as Weight).saturating_mul(b as Weight)) - // Standard Error: 1_000 - .saturating_add((208_000 as Weight).saturating_mul(m as Weight)) - // Standard Error: 1_000 - .saturating_add((408_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((5_000 as Weight).saturating_mul(b as Weight)) + // Standard Error: 2_000 + .saturating_add((151_000 as Weight).saturating_mul(m as Weight)) + // Standard Error: 2_000 + .saturating_add((201_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Collective Proposals (r:1 w:1) - // Storage: Instance1Collective Voting (r:0 w:1) - // Storage: Instance1Collective ProposalOf (r:0 w:1) + // Storage: Council Proposals (r:1 w:1) + // Storage: Council Voting (r:0 w:1) + // Storage: Council ProposalOf (r:0 w:1) fn disapprove_proposal(p: u32, ) -> Weight { - (27_458_000 as Weight) + (15_778_000 as Weight) // Standard Error: 1_000 - .saturating_add((402_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((206_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -193,132 +194,132 @@ impl WeightInfo for SubstrateWeight { // For backwards compatibility and tests impl WeightInfo for () { - // Storage: Instance1Collective Members (r:1 w:1) - // Storage: Instance1Collective Proposals (r:1 w:0) - // Storage: Instance1Collective Voting (r:100 w:100) - // Storage: Instance1Collective Prime (r:0 w:1) + // Storage: Council Members (r:1 w:1) + // Storage: Council Proposals (r:1 w:0) + // Storage: Council Voting (r:100 w:100) + // Storage: Council Prime (r:0 w:1) fn set_members(m: u32, n: u32, p: u32, ) -> Weight { (0 as Weight) - // Standard Error: 4_000 - .saturating_add((14_084_000 as Weight).saturating_mul(m as Weight)) - // Standard Error: 4_000 - .saturating_add((161_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 4_000 - .saturating_add((19_201_000 as Weight).saturating_mul(p as Weight)) + // Standard Error: 10_000 + .saturating_add((14_493_000 as Weight).saturating_mul(m as Weight)) + // Standard Error: 10_000 + .saturating_add((23_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 10_000 + .saturating_add((16_909_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(p as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(p as Weight))) } - // Storage: Instance1Collective Members (r:1 w:0) + // Storage: Council Members (r:1 w:0) fn execute(b: u32, m: u32, ) -> Weight { - (22_748_000 as Weight) + (12_790_000 as Weight) // Standard Error: 0 - .saturating_add((3_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(b as Weight)) // Standard Error: 0 - .saturating_add((92_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((73_000 as Weight).saturating_mul(m as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) } - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective ProposalOf (r:1 w:0) + // Storage: Council Members (r:1 w:0) + // Storage: Council ProposalOf (r:1 w:0) fn propose_execute(b: u32, m: u32, ) -> Weight { - (27_465_000 as Weight) + (15_087_000 as Weight) // Standard Error: 0 - .saturating_add((3_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(b as Weight)) // Standard Error: 0 - .saturating_add((178_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((135_000 as Weight).saturating_mul(m as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) } - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective ProposalOf (r:1 w:1) - // Storage: Instance1Collective Proposals (r:1 w:1) - // Storage: Instance1Collective ProposalCount (r:1 w:1) - // Storage: Instance1Collective Voting (r:0 w:1) + // Storage: Council Members (r:1 w:0) + // Storage: Council ProposalOf (r:1 w:1) + // Storage: Council Proposals (r:1 w:1) + // Storage: Council ProposalCount (r:1 w:1) + // Storage: Council Voting (r:0 w:1) fn propose_proposed(b: u32, m: u32, p: u32, ) -> Weight { - (39_869_000 as Weight) + (18_269_000 as Weight) // Standard Error: 0 .saturating_add((8_000 as Weight).saturating_mul(b as Weight)) // Standard Error: 1_000 - .saturating_add((107_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((77_000 as Weight).saturating_mul(m as Weight)) // Standard Error: 1_000 - .saturating_add((406_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((203_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective Voting (r:1 w:1) + // Storage: Council Members (r:1 w:0) + // Storage: Council Voting (r:1 w:1) fn vote(m: u32, ) -> Weight { - (37_387_000 as Weight) + (26_624_000 as Weight) // Standard Error: 2_000 - .saturating_add((223_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((161_000 as Weight).saturating_mul(m as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } - // Storage: Instance1Collective Voting (r:1 w:1) - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective Proposals (r:1 w:1) - // Storage: Instance1Collective ProposalOf (r:0 w:1) + // Storage: Council Voting (r:1 w:1) + // Storage: Council Members (r:1 w:0) + // Storage: Council Proposals (r:1 w:1) + // Storage: Council ProposalOf (r:0 w:1) fn close_early_disapproved(m: u32, p: u32, ) -> Weight { - (45_670_000 as Weight) + (26_527_000 as Weight) // Standard Error: 1_000 - .saturating_add((170_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((127_000 as Weight).saturating_mul(m as Weight)) // Standard Error: 1_000 - .saturating_add((358_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((155_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Collective Voting (r:1 w:1) - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective ProposalOf (r:1 w:1) - // Storage: Instance1Collective Proposals (r:1 w:1) + // Storage: Council Voting (r:1 w:1) + // Storage: Council Members (r:1 w:0) + // Storage: Council ProposalOf (r:1 w:1) + // Storage: Council Proposals (r:1 w:1) fn close_early_approved(b: u32, m: u32, p: u32, ) -> Weight { - (52_529_000 as Weight) + (26_352_000 as Weight) // Standard Error: 0 - .saturating_add((7_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((6_000 as Weight).saturating_mul(b as Weight)) // Standard Error: 1_000 - .saturating_add((206_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((154_000 as Weight).saturating_mul(m as Weight)) // Standard Error: 1_000 - .saturating_add((412_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((203_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Collective Voting (r:1 w:1) - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective Prime (r:1 w:0) - // Storage: Instance1Collective Proposals (r:1 w:1) - // Storage: Instance1Collective ProposalOf (r:0 w:1) + // Storage: Council Voting (r:1 w:1) + // Storage: Council Members (r:1 w:0) + // Storage: Council Prime (r:1 w:0) + // Storage: Council Proposals (r:1 w:1) + // Storage: Council ProposalOf (r:0 w:1) fn close_disapproved(m: u32, p: u32, ) -> Weight { - (50_427_000 as Weight) + (28_638_000 as Weight) // Standard Error: 1_000 - .saturating_add((170_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((133_000 as Weight).saturating_mul(m as Weight)) // Standard Error: 1_000 - .saturating_add((354_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((162_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Collective Voting (r:1 w:1) - // Storage: Instance1Collective Members (r:1 w:0) - // Storage: Instance1Collective Prime (r:1 w:0) - // Storage: Instance1Collective ProposalOf (r:1 w:1) - // Storage: Instance1Collective Proposals (r:1 w:1) + // Storage: Council Voting (r:1 w:1) + // Storage: Council Members (r:1 w:0) + // Storage: Council Prime (r:1 w:0) + // Storage: Council ProposalOf (r:1 w:1) + // Storage: Council Proposals (r:1 w:1) fn close_approved(b: u32, m: u32, p: u32, ) -> Weight { - (57_031_000 as Weight) + (29_946_000 as Weight) // Standard Error: 0 - .saturating_add((7_000 as Weight).saturating_mul(b as Weight)) - // Standard Error: 1_000 - .saturating_add((208_000 as Weight).saturating_mul(m as Weight)) - // Standard Error: 1_000 - .saturating_add((408_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((5_000 as Weight).saturating_mul(b as Weight)) + // Standard Error: 2_000 + .saturating_add((151_000 as Weight).saturating_mul(m as Weight)) + // Standard Error: 2_000 + .saturating_add((201_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Collective Proposals (r:1 w:1) - // Storage: Instance1Collective Voting (r:0 w:1) - // Storage: Instance1Collective ProposalOf (r:0 w:1) + // Storage: Council Proposals (r:1 w:1) + // Storage: Council Voting (r:0 w:1) + // Storage: Council ProposalOf (r:0 w:1) fn disapprove_proposal(p: u32, ) -> Weight { - (27_458_000 as Weight) + (15_778_000 as Weight) // Standard Error: 1_000 - .saturating_add((402_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((206_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } diff --git a/frame/contracts/CHANGELOG.md b/frame/contracts/CHANGELOG.md index eaedd28bf3e4..ab3998e6dc4f 100644 --- a/frame/contracts/CHANGELOG.md +++ b/frame/contracts/CHANGELOG.md @@ -23,13 +23,9 @@ In other words: Upgrading this pallet will not break pre-existing contracts. - Allow contracts to dispatch calls into the runtime (**unstable**) [#9276](https://github.com/paritytech/substrate/pull/9276) -- New **unstable** version of `seal_call` that offers more features. +- New version of `seal_call` that offers more features. [#8909](https://github.com/paritytech/substrate/pull/8909) -- New **unstable** `seal_rent_params` and `seal_rent_status` contract callable function. -[#8231](https://github.com/paritytech/substrate/pull/8231) -[#8780](https://github.com/paritytech/substrate/pull/8780) - - New `instantiate` RPC that allows clients to dry-run contract instantiation. [#8451](https://github.com/paritytech/substrate/pull/8451) @@ -38,6 +34,10 @@ In other words: Upgrading this pallet will not break pre-existing contracts. ### Changed +- Replaced storage rent with automatic storage deposits +[#9669](https://github.com/paritytech/substrate/pull/9669) +[#10082](https://github.com/paritytech/substrate/pull/10082) + - Replaced `seal_println` with the `seal_debug_message` API which allows outputting debug messages to the console and RPC clients. [#8773](https://github.com/paritytech/substrate/pull/8773) diff --git a/frame/contracts/COMPLEXITY.md b/frame/contracts/COMPLEXITY.md deleted file mode 100644 index 1fc1932fe1b5..000000000000 --- a/frame/contracts/COMPLEXITY.md +++ /dev/null @@ -1,487 +0,0 @@ -# Complexity - -This analysis is on the computing and memory complexity of specific procedures. It provides a rough estimate of operations performed in general and especially focusing on DB reads and writes. It is also an attempt to estimate the memory consumption at its peak. - -The primary goal is to come up with decent pricing for functions that can be invoked by a user (via extrinsics) or by untrusted code that prevents DoS attacks. - -## Sandboxing - -It makes sense to describe the sandboxing module first because the smart-contract module is built upon it. - -### Memory - -#### set - -Copies data from the supervisor's memory to the guest's memory. - -**complexity**: It doesn't allocate, and the computational complexity is proportional to the number of bytes to copy. - -#### get - -Copies data from the guest's memory to the supervisor's memory. - -**complexity**: It doesn't allocate, and the computational complexity is proportional to the number of bytes to copy. - -## Instance - -### Instantiation - -Instantiation of a sandbox module consists of the following steps: - -1. Loading the wasm module in the in-memory representation, -2. Performing validation of the wasm code, -3. Setting up the environment which will be used to instantiate the module, -4. Performing the standard wasm instantiation process, which includes (but is not limited to): - 1. Allocating of memory requested by the instance, - 2. Copying static data from the module to newly allocated memory, - 3. Executing the `start` function. - -**Note** that the `start` function can be viewed as a normal function and can do anything that a normal function can do, including allocation of more memory or calling the host environment. The complexity of running the `start` function should be considered separately. - -In order to start the process of instantiation, the supervisor should provide the wasm module code being instantiated and the environment definition (a set of functions, memories (and maybe globals and tables in the future) available for import by the guest module) for that module. While the environment definition typically is of the constant size (unless mechanisms like dynamic linking are used), the size of wasm is not. - -Validation and instantiation in WebAssembly are designed to be able to be performed in linear time. The allocation and computational complexity of loading a wasm module depend on the underlying wasm VM being used. For example, for JIT compilers it can and probably will be non-linear because of compilation. However, for wasmi, it should be linear. We can try to use other VMs that are able to compile code with memory and time consumption proportional to the size of the code. - -Since the module itself requests memory, the amount of allocation depends on the module code itself. If untrusted code is being instantiated, it's up to the supervisor to limit the amount of memory available to allocate. - -**complexity**: The computational complexity is proportional to the size of wasm code. Memory complexity is proportional to the size of wasm code and the amount of memory requested by the module. - -### Preparation to invoke - -Invocation of an exported function in the sandboxed module consists of the following steps: - -1. Marshalling, copying and unmarshalling the arguments when passing them between the supervisor and executor, -2. Calling into the underlying VM, -3. Marshalling, copying and unmarshalling the result when passing it between the executor and supervisor, - -**Note** that the complexity of running the function code itself should be considered separately. - -The actual complexity of invocation depends on the underlying VM. Wasmi will reserve a relatively large chunk of memory for the stack before execution of the code, although it's of constant size. - -The size of the arguments and the return value depends on the exact function in question, but can be considered as constant. - -**complexity**: Memory and computational complexity can be considered as a constant. - -### Call from the guest to the supervisor - -The executor handles each call from the guest. The execution of it consists of the following steps: - -1. Marshalling, copying and unmarshalling the arguments when passing them between the guest and executor, -2. Calling into the supervisor, -3. Marshaling, copying and unmarshalling the result when passing it between the executor and guest. - -**Note** that the complexity of running the supervisor handler should be considered separately. - -Because calling into the supervisor requires invoking a wasm VM, the actual complexity of invocation depends on the actual VM used for the runtime/supervisor. Wasmi will reserve a relatively large chunk of memory for the stack before execution of the code, although it's of constant size. - -The size of the arguments and the return value depends on the exact function in question, but can be considered as a constant. - -**complexity**: Memory and computational complexity can be considered as a constant. - -## Transactional Storage - -The contracts module makes use of the nested storage transactions feature offered by -the underlying storage which allows efficient roll back of changes made by contracts. - -> ℹ️ The underlying storage has a overlay layer implemented as a `Map`. If the runtime reads a storage location and the -> respective key doesn't exist in the overlay, then the underlying storage performs a DB access, but the value won't be -> placed into the overlay. The overlay is only filled with writes. -> -> This means that the overlay can be abused in the following ways: -> -> - The overlay can be inflated by issuing a lot of writes to unique locations, -> - Deliberate cache misses can be induced by reading non-modified storage locations, - -It also worth noting that the performance degrades with more state stored in the trie. Due to this -there is not negligible chance that gas schedule will be updated for all operations that involve -storage access. - -## get_storage, get_code_hash, get_rent_allowance, get_balance, contract_exists - -Those query the underlying storage for the requested value. If the value was modified in the -current block they are served from the cache. Otherwise a database read is performed. - -**complexity**: The memory complexity is proportional to the size of the value. The computational complexity is proportional the size of the value; the cost is dominated by the DB read. - -## set_storage, set_balance, set_rent_allowance - -These function write to the underlying storage which caches those values and does not write -them to the database immediately. - -While these functions only modify the local cache, they trigger a database write later when -all changes that were not rolled back are written to storage. Moreover, if the balance of the -account is changed to be below `existential_deposit` then that account along with all its storage -will be removed, which requires time proportional to the number of storage entries that account has. -It should be ensured that pricing accounts for these facts. - -**complexity**: Each lookup has a logarithmical computing time to the number of already inserted entries. -No additional memory is required. - -## instantiate_contract - -Calls `contract_exists` and if it doesn't exist, do not modify the local `Map` similarly to `set_rent_allowance`. - -**complexity**: The computational complexity is proportional to the depth of the overlay cascade and the size of the value; the cost is dominated by the DB read though. No additional memory is required. - -## commit - -In this function, all values modified in the current transactions are committed to the parent -transaction. - -This will trigger `N` inserts into parent transaction (`O(log M)` complexity) or into the storage, where `N` is the size of the current transaction and `M` is the size of the parent transaction. Consider adjusting the price of modifying the -current transaction to account for this (since pricing for the count of entries in `commit` will make the price of commit way less predictable). No additional memory is required. - -Note that in case of storage modification we need to construct a key in the underlying storage. In order to do that we need: - -- perform `twox_128` hashing over a concatenation of some prefix literal and the `AccountId` of the storage owner. -- then perform `blake2_256` hashing of the storage key. -- concatenation of these hashes will constitute the key in the underlying storage. - -There is also a special case to think of: if the balance of some account goes below `existential_deposit`, then all storage entries of that account will be erased, which requires time proportional to the number of storage entries that account has. - -**complexity**: `N` inserts into a transaction or eventually into the storage (if committed). Every deleted account will induce removal of all its storage which is proportional to the number of storage entries that account has. - -## revert - -Consists of dropping (in the Rust sense) of the current transaction. - -**complexity**: Computing complexity is proportional to a number of changed entries in a overlay. No additional memory is required. - -## Executive - -### Transfer - -This function performs the following steps: - -1. Querying source and destination balances from the current transaction (see `get_balance`), -2. Querying `existential_deposit`. -3. Executing `ensure_account_liquid` hook. -4. Updating source and destination balance in the overlay (see `set_balance`). - -**Note** that the complexity of executing `ensure_account_liquid` hook should be considered separately. - -In the course of the execution this function can perform up to 2 DB reads to `get_balance` of source and destination accounts. It can also induce up to 2 DB writes via `set_balance` if flushed to the storage. - -Moreover, if the source balance goes below `existential_deposit` then the transfer is denied and -returns with an error. - -Assuming marshaled size of a balance value is of the constant size we can neglect its effect on the performance. - -**complexity**: up to 2 DB reads and up to 2 DB writes (if flushed to the storage) in the standard case. If removal of the source account takes place then it will additionally perform a DB write per one storage entry that the account has. Memorywise it can be assumed to be constant. - -### Initialization - -Before a call or instantiate can be performed the execution context must be initialized. - -For the first call or instantiation in the handling of an extrinsic, this involves two calls: - -1. `>::now()` -2. `>::block_number()` - -The complexity of initialization depends on the complexity of these functions. In the current -implementation they just involve a DB read. - -For subsequent calls and instantiations during contract execution, the initialization requires no -expensive operations. - -### Terminate - -This function performs the following steps: - -1. Check the calling contract is not already on the callstack by calling `is_live`. -2. `transfer` funds from caller to the beneficiary. -3. Flag the caller contract as deleted in the overlay. - -`is_live` does not do any database access nor does it allocate memory. It walks up the call -stack and therefore executes in linear time depending on size of the call stack. Because -the call stack is of a fixed maximum size we consider this operation as constant time. - -**complexity**: Database accesses as described in Transfer + Removal of the contract. Currently, -we are using child trie removal which is linear in the amount of stored keys. Upcoming changes -will make the account removal constant time. - -### Call - -This function receives input data for the contract execution. The execution consists of the following steps: - -1. Initialization of the execution context. -2. Checking rent payment. -3. Loading code from the DB. -4. Starting a new storage transaction. -5. `transfer`-ing funds between the caller and the destination account. -6. Executing the code of the destination account. -7. Committing or rolling back the storage transaction. - -**Note** that the complexity of executing the contract code should be considered separately. - -Checking for rent involves 2 unconditional DB reads: `ContractInfoOf` and `block_number` -and on top of that at most once per block: - -- DB read to `free_balance` and -- `rent_deposit_offset` and -- `rent_byte_price` and -- `Currency::minimum_balance` and -- `tombstone_deposit`. -- Calls to `ensure_can_withdraw`, `withdraw`, `make_free_balance_be` can perform arbitrary logic and should be considered separately, -- `child_storage_root` -- `kill_child_storage` -- mutation of `ContractInfoOf` - -Loading code most likely will trigger a DB read, since the code is immutable and therefore will not get into the cache (unless a suicide removes it, or it has been instantiated in the same call chain). - -Also, `transfer` can make up to 2 DB reads and up to 2 DB writes (if flushed to the storage) in the standard case. If removal of the source account takes place then it will additionally perform a DB write per one storage entry that the account has. - -Finally, the current storage transaction is closed. The complexity of this depends on the number of changes performed by the code. Thus, the pricing of storage modification should account for that. - -**complexity**: - -- Only for the first invocation of the contract: up to 5 DB reads and one DB write as well as logic executed by `ensure_can_withdraw`, `withdraw`, `make_free_balance_be`. -- On top of that for every invocation: Up to 5 DB reads. DB read of the code is of dynamic size. There can also be up to 2 DB writes (if flushed to the storage). Additionally, if the source account removal takes place a DB write will be performed per one storage entry that the account has. - -### Instantiate - -This function takes the code of the constructor and input data. Instantiation of a contract consists of the following steps: - -1. Initialization of the execution context. -2. Calling `DetermineContractAddress` hook to determine an address for the contract, -3. Starting a new storage transaction. -4. `transfer`-ing funds between self and the newly instantiated contract. -5. Executing the constructor code. This will yield the final code of the code. -6. Storing the code for the newly instantiated contract in the overlay. -7. Committing or rolling back the storage transaction. - -**Note** that the complexity of executing the constructor code should be considered separately. - -**Note** that the complexity of `DetermineContractAddress` hook should be considered separately as well. Most likely it will use some kind of hashing over the code of the constructor and input data. The default `SimpleAddressDeterminer` does precisely that. - -**Note** that the constructor returns code in the owned form and it's obtained via return facilities, which should have take fee for the return value. - -Also, `transfer` can make up to 2 DB reads and up to 2 DB writes (if flushed to the storage) in the standard case. If removal of the source account takes place then it will additionally perform a DB write per one storage entry that the account has. - -Storing the code in the overlay may induce another DB write (if flushed to the storage) with the size proportional to the size of the constructor code. - -Finally, the current storage transaction is closed.. The complexity of this depends on the number of changes performed by the constructor code. Thus, the pricing of storage modification should account for that. - -**complexity**: Up to 2 DB reads and induces up to 3 DB writes (if flushed to the storage), one of which is dependent on the size of the code. Additionally, if the source account removal takes place a DB write will be performed per one storage entry that the account has. - -## Contracts API - -Each API function invoked from a contract can involve some overhead. - -## Getter functions - -Those are simple getter functions which copy a requested value to contract memory. They -all have the following two arguments: - -- `output_ptr`: Pointer into contract memory where to copy the value. -- `output_len_ptr`: Pointer into contract memory where the size of the buffer is stored. The size of the copied value is also stored there. - -**complexity**: The size of the returned value is constant for a given runtime. Therefore we -consider its complexity constant even though some of them might involve at most one DB read. Some of those -functions call into other pallets of the runtime. The assumption here is that those functions are also -linear in regard to the size of the data that is returned and therefore considered constant for a -given runtime. - -This is the list of getters: - -- seal_caller -- seal_address -- seal_weight_to_fee -- seal_gas_left -- seal_balance -- seal_value_transferred -- seal_now -- seal_minimum_balance -- seal_tombstone_deposit -- seal_rent_allowance -- seal_block_number - -### seal_set_storage - -This function receives a `key` and `value` as arguments. It consists of the following steps: - -1. Reading the sandbox memory for `key` and `value` (see sandboxing memory get). -2. Setting the storage at the given `key` to the given `value` (see `set_storage`). - -**complexity**: Complexity is proportional to the size of the `value`. This function induces a DB write of size proportional to the `value` size (if flushed to the storage), so should be priced accordingly. - -### seal_clear_storage - -This function receives a `key` as argument. It consists of the following steps: - -1. Reading the sandbox memory for `key` (see sandboxing memory get). -2. Clearing the storage at the given `key` (see `set_storage`). - -**complexity**: Complexity is constant. This function induces a DB write to clear the storage entry -(upon being flushed to the storage) and should be priced accordingly. - -### seal_get_storage - -This function receives a `key` as an argument. It consists of the following steps: - -1. Reading the sandbox memory for `key` (see sandboxing memory get). -2. Reading the storage with the given key (see `get_storage`). It receives back the owned result buffer. -3. Writing the storage value to contract memory. - -Key is of a constant size. Therefore, the sandbox memory load can be considered to be of constant complexity. - -Unless the value is cached, a DB read will be performed. The size of the value is not known until the read is -performed. Moreover, the DB read has to be synchronous and no progress can be made until the value is fetched. - -**complexity**: The memory and computing complexity is proportional to the size of the fetched value. This function performs a DB read. - -### seal_transfer - -This function receives the following arguments: - -- `account` buffer of a marshaled `AccountId`, -- `value` buffer of a marshaled `Balance`, - -It consists of the following steps: - -1. Loading `account` buffer from the sandbox memory (see sandboxing memory get) and then decoding it. -2. Loading `value` buffer from the sandbox memory and then decoding it. -3. Invoking the executive function `transfer`. - -Loading of `account` and `value` buffers should be charged. This is because the sizes of buffers are specified by the calling code, even though marshaled representations are, essentially, of constant size. This can be fixed by assigning an upper bound for sizes of `AccountId` and `Balance`. - -### seal_call - -This function receives the following arguments: - -- `callee` buffer of a marshaled `AccountId`, -- `gas` limit which is plain u64, -- `value` buffer of a marshaled `Balance`, -- `input_data` an arbitrarily sized byte vector. -- `output_ptr` pointer to contract memory. - -It consists of the following steps: - -1. Loading `callee` buffer from the sandbox memory (see sandboxing memory get) and then decoding it. -2. Loading `value` buffer from the sandbox memory and then decoding it. -3. Loading `input_data` buffer from the sandbox memory. -4. Invoking the executive function `call`. -5. Writing output buffer to contract memory. - -Loading of `callee` and `value` buffers should be charged. This is because the sizes of buffers are specified by the calling code, even though marshaled representations are, essentially, of constant size. This can be fixed by assigning an upper bound for sizes of `AccountId` and `Balance`. - -Loading `input_data` should be charged in any case. - -**complexity**: All complexity comes from loading and writing buffers and executing `call` executive function. The former component is proportional to the sizes of `callee`, `value`, `input_data` and `output_ptr` buffers. The latter component completely depends on the complexity of `call` executive function, and also dominated by it. - -### seal_instantiate - -This function receives the following arguments: - -- `init_code`, a buffer which contains the code of the constructor. -- `gas` limit which is plain u64 -- `value` buffer of a marshaled `Balance` -- `input_data`. an arbitrarily sized byte vector. - -It consists of the following steps: - -1. Loading `init_code` buffer from the sandbox memory (see sandboxing memory get) and then decoding it. -2. Loading `value` buffer from the sandbox memory and then decoding it. -3. Loading `input_data` buffer from the sandbox memory. -4. Invoking `instantiate` executive function. - -Loading of `value` buffer should be charged. This is because the size of the buffer is specified by the calling code, even though marshaled representation is, essentially, of constant size. This can be fixed by assigning an upper bound for size for `Balance`. - -Loading `init_code` and `input_data` should be charged in any case. - -**complexity**: All complexity comes from loading buffers and executing `instantiate` executive function. The former component is proportional to the sizes of `init_code`, `value` and `input_data` buffers. The latter component completely depends on the complexity of `instantiate` executive function and also dominated by it. - -### seal_terminate - -This function receives the following arguments: - -- `beneficiary`, buffer of a marshaled `AccountId` - -It consists of the following steps: - -1. Loading `beneficiary` buffer from the sandbox memory (see sandboxing memory get) and then decoding it. - -Loading of the `beneficiary` buffer should be charged. This is because the sizes of buffers are specified by the calling code, even though marshaled representations are, essentially, of constant size. This can be fixed by assigning an upper bound for sizes of `AccountId`. - -**complexity**: All complexity comes from loading buffers and executing `terminate` executive function. The former component is proportional to the size of the `beneficiary` buffer. The latter component completely depends on the complexity of `terminate` executive function and also dominated by it. - -### seal_input - -This function receives a pointer to contract memory. It copies the input to the contract call to this location. - -**complexity**: The complextity is proportional to the size of the input buffer. - -### seal_return - -This function receives a `data` buffer and `flags` arguments. Execution of the function consists of the following steps: - -1. Loading `data` buffer from the sandbox memory (see sandboxing memory get). -2. Storing the `u32` flags value. -3. Trapping - -**complexity**: The complexity of this function is proportional to the size of the `data` buffer. - -### seal_deposit_event - -This function receives a `data` buffer as an argument. Execution of the function consists of the following steps: - -1. Loading `data` buffer from the sandbox memory (see sandboxing memory get), -2. Insert to nested context execution -3. Copies from nested to underlying contexts -4. Call system deposit event - -**complexity**: The complexity of this function is proportional to the size of the `data` buffer. - -### seal_set_rent_allowance - -This function receives the following argument: - -- `value` buffer of a marshaled `Balance`, - -It consists of the following steps: - -1. Loading `value` buffer from the sandbox memory and then decoding it. -2. Invoking `set_rent_allowance` AccountDB function. - -**complexity**: Complexity is proportional to the size of the `value`. This function induces a DB write of size proportional to the `value` size (if flushed to the storage), so should be priced accordingly. - -## Built-in hashing functions - -This paragraph concerns the following supported built-in hash functions: - -- `SHA2` with 256-bit width -- `KECCAK` with 256-bit width -- `BLAKE2` with 128-bit and 256-bit widths - -These functions compute a cryptographic hash on the given inputs and copy the -resulting hash directly back into the sandboxed Wasm contract output buffer. - -Execution of the function consists of the following steps: - -1. Load data stored in the input buffer into an intermediate buffer. -2. Compute the cryptographic hash `H` on the intermediate buffer. -3. Copy back the bytes of `H` into the contract side output buffer. - -**complexity**: Complexity is proportional to the size of the input buffer in bytes -as well as to the size of the output buffer in bytes. Also different cryptographic -algorithms have different inherent complexity so users must expect the above -mentioned crypto hashes to have varying gas costs. -The complexity of each cryptographic hash function highly depends on the underlying -implementation. - -### seal_ecdsa_recover - -This function receives the following arguments: - -- `signature` is 65 bytes buffer, -- `message_hash` is 32 bytes buffer, -- `output` is 33 bytes buffer to return compressed public key, - -It consists of the following steps: - -1. Loading `signature` buffer from the sandbox memory (see sandboxing memory get). -2. Loading `message_hash` buffer from the sandbox memory. -3. Invoking the executive function `secp256k1_ecdsa_recover_compressed`. -4. Copy the bytes of compressed public key into the contract side output buffer. - -**complexity**: Complexity is partially constant(it doesn't depend on input) but still depends on points of ECDSA and calculation. \ No newline at end of file diff --git a/frame/contracts/Cargo.toml b/frame/contracts/Cargo.toml index 80dc0b05e751..2e906b69f32f 100644 --- a/frame/contracts/Cargo.toml +++ b/frame/contracts/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-contracts" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for WASM contracts" readme = "README.md" @@ -14,13 +14,13 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] bitflags = "1.3" -codec = { package = "parity-scale-codec", version = "2.2.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", "max-encoded-len", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } log = { version = "0.4", default-features = false } -pwasm-utils = { version = "0.18.2", default-features = false } +wasm-instrument = { version = "0.1", default-features = false } serde = { version = "1", optional = true, features = ["derive"] } smallvec = { version = "1", default-features = false, features = [ "const_generics", @@ -28,26 +28,26 @@ smallvec = { version = "1", default-features = false, features = [ wasmi-validation = { version = "0.4", default-features = false } # Only used in benchmarking to generate random contract code -libsecp256k1 = { version = "0.3.5", optional = true, default-features = false, features = ["hmac"] } -rand = { version = "0.7.3", optional = true, default-features = false } -rand_pcg = { version = "0.2", optional = true } +rand = { version = "0.8", optional = true, default-features = false } +rand_pcg = { version = "0.3", optional = true } # Substrate Dependencies frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -pallet-contracts-primitives = { version = "4.0.0-dev", default-features = false, path = "common" } +pallet-contracts-primitives = { version = "6.0.0", default-features = false, path = "common" } pallet-contracts-proc-macro = { version = "4.0.0-dev", path = "proc-macro" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-sandbox = { version = "0.10.0-dev", default-features = false, path = "../../primitives/sandbox" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } [dev-dependencies] assert_matches = "1" +env_logger = "0.9" hex-literal = "0.3" -pretty_assertions = "0.7" +pretty_assertions = "1" wat = "1" # Substrate Dependencies @@ -55,6 +55,7 @@ pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } pallet-randomness-collective-flip = { version = "4.0.0-dev", path = "../randomness-collective-flip" } pallet-utility = { version = "4.0.0-dev", path = "../utility" } +sp-keystore = { version = "0.12.0", path = "../../primitives/keystore" } [features] default = ["std"] @@ -70,17 +71,15 @@ std = [ "frame-benchmarking/std", "frame-support/std", "frame-system/std", - "pwasm-utils/std", + "wasm-instrument/std", "wasmi-validation/std", "pallet-contracts-primitives/std", "pallet-contracts-proc-macro/full", "log/std", "rand/std", - "libsecp256k1/std", ] runtime-benchmarks = [ - "frame-benchmarking", - "libsecp256k1", + "frame-benchmarking/runtime-benchmarks", "rand", "rand_pcg", "unstable-interface", diff --git a/frame/contracts/README.md b/frame/contracts/README.md index f3a8d13f6e77..8a8e4918f2e4 100644 --- a/frame/contracts/README.md +++ b/frame/contracts/README.md @@ -34,7 +34,7 @@ reverted at the current call's contract level. For example, if contract A calls then all of B's calls are reverted. Assuming correct error handling by contract A, A's other calls and state changes still persist. -One gas is equivalent to one [weight](https://substrate.dev/docs/en/knowledgebase/learn-substrate/weight) +One gas is equivalent to one [weight](https://docs.substrate.io/v3/runtime/weights-and-fees) which is defined as one picosecond of execution time on the runtime's reference machine. ### Notable Scenarios @@ -49,6 +49,34 @@ fails, A can decide how to handle that failure, either proceeding or reverting A Those are documented in the [reference documentation](https://docs.rs/pallet-contracts/latest/pallet_contracts/#dispatchable-functions). +### Interface exposed to contracts + +Each contract is one WebAssembly module that looks like this: + +```wat +(module + ;; Invoked by pallet-contracts when a contract is instantiated. + ;; No arguments and empty return type. + (func (export "deploy")) + + ;; Invoked by pallet-contracts when a contract is called. + ;; No arguments and empty return type. + (func (export "call")) + + ;; If a contract uses memory it must be imported. Memory is optional. + ;; The maximum allowed memory size depends on the pallet-contracts configuration. + (import "env" "memory" (memory 1 1)) + + ;; This is one of many functions that can be imported and is implemented by pallet-contracts. + ;; This function is used to copy the result buffer and flags back to the caller. + (import "seal0" "seal_return" (func $seal_return (param i32 i32 i32))) +) +``` + +The documentation of all importable functions can be found +[here](https://github.com/paritytech/substrate/blob/master/frame/contracts/src/wasm/runtime.rs). +Look for the `define_env!` macro invocation. + ## Usage This module executes WebAssembly smart contracts. These can potentially be written in any language @@ -73,7 +101,7 @@ by block production. A good starting point for observing them on the console is command line in the root directory of the substrate repository: ```bash -cargo run --release -- --dev --tmp -lerror,runtime::contracts=debug +cargo run --release -- --dev -lerror,runtime::contracts=debug ``` This raises the log level of `runtime::contracts` to `debug` and all other targets diff --git a/frame/contracts/common/Cargo.toml b/frame/contracts/common/Cargo.toml index b441d88453ae..49d7973ab155 100644 --- a/frame/contracts/common/Cargo.toml +++ b/frame/contracts/common/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "pallet-contracts-primitives" -version = "4.0.0-dev" +version = "6.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "A crate that hosts a common definitions that are relevant for the pallet-contracts." readme = "README.md" @@ -14,14 +14,15 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] bitflags = "1.0" -codec = { package = "parity-scale-codec", version = "2", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } serde = { version = "1", features = ["derive"], optional = true } # Substrate Dependencies (This crate should not rely on frame) -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core", default-features = false } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../../primitives/core", default-features = false } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } +sp-rpc = { version = "6.0.0", path = "../../../primitives/rpc", optional = true } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } [features] default = ["std"] @@ -31,5 +32,6 @@ std = [ "sp-core/std", "sp-runtime/std", "sp-std/std", + "sp-rpc", "serde", ] diff --git a/frame/contracts/common/src/lib.rs b/frame/contracts/common/src/lib.rs index c57f728c26b6..49f91f676929 100644 --- a/frame/contracts/common/src/lib.rs +++ b/frame/contracts/common/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,19 +22,32 @@ use bitflags::bitflags; use codec::{Decode, Encode}; use sp_core::Bytes; -use sp_runtime::{DispatchError, RuntimeDebug}; +use sp_runtime::{ + traits::{Saturating, Zero}, + DispatchError, RuntimeDebug, +}; use sp_std::prelude::*; #[cfg(feature = "std")] use serde::{Deserialize, Serialize}; +#[cfg(feature = "std")] +use sp_rpc::number::NumberOrHex; + /// Result type of a `bare_call` or `bare_instantiate` call. /// /// It contains the execution result together with some auxiliary information. #[derive(Eq, PartialEq, Encode, Decode, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] -pub struct ContractResult { +#[cfg_attr( + feature = "std", + serde( + rename_all = "camelCase", + bound(serialize = "R: Serialize, Balance: Copy + Into"), + bound(deserialize = "R: Deserialize<'de>, Balance: TryFrom") + ) +)] +pub struct ContractResult { /// How much gas was consumed during execution. pub gas_consumed: u64, /// How much gas is required as gas limit in order to execute this call. @@ -45,7 +58,14 @@ pub struct ContractResult { /// /// This can only different from [`Self::gas_consumed`] when weight pre charging /// is used. Currently, only `seal_call_runtime` makes use of pre charging. + /// Additionally, any `seal_call` or `seal_instantiate` makes use of pre-charging + /// when a non-zero `gas_limit` argument is supplied. pub gas_required: u64, + /// How much balance was deposited and reserved during execution in order to pay for storage. + /// + /// The storage deposit is never actually charged from the caller in case of [`Self::result`] + /// is `Err`. This is because on error all storage changes are rolled back. + pub storage_deposit: StorageDeposit, /// An optional debug message. This message is only filled when explicitly requested /// by the code that calls into the contract. Otherwise it is empty. /// @@ -63,15 +83,20 @@ pub struct ContractResult { #[cfg_attr(feature = "std", serde(with = "as_string"))] pub debug_message: Vec, /// The execution result of the wasm code. - pub result: T, + pub result: R, } /// Result type of a `bare_call` call. -pub type ContractExecResult = ContractResult>; +pub type ContractExecResult = + ContractResult, Balance>; /// Result type of a `bare_instantiate` call. -pub type ContractInstantiateResult = - ContractResult, DispatchError>>; +pub type ContractInstantiateResult = + ContractResult, DispatchError>, Balance>; + +/// Result type of a `bare_code_upload` call. +pub type CodeUploadResult = + Result, DispatchError>; /// Result type of a `get_storage` call. pub type GetStorageResult = Result>, ContractAccessError>; @@ -106,9 +131,9 @@ pub struct ExecReturnValue { } impl ExecReturnValue { - /// We understand the absense of a revert flag as success. - pub fn is_success(&self) -> bool { - !self.flags.contains(ReturnFlags::REVERT) + /// The contract did revert all storage changes. + pub fn did_revert(&self) -> bool { + self.flags.contains(ReturnFlags::REVERT) } } @@ -123,6 +148,25 @@ pub struct InstantiateReturnValue { pub account_id: AccountId, } +/// The result of succesfully uploading a contract. +#[derive(PartialEq, Eq, Encode, Decode, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr( + feature = "std", + serde( + rename_all = "camelCase", + bound(serialize = "CodeHash: Serialize, Balance: Copy + Into"), + bound(deserialize = "CodeHash: Deserialize<'de>, Balance: TryFrom") + ) +)] +pub struct CodeUploadReturnValue { + /// The key under which the new code is stored. + pub code_hash: CodeHash, + /// The deposit that was reserved at the caller. Is zero when the code already existed. + #[cfg_attr(feature = "std", serde(with = "as_hex"))] + pub deposit: Balance, +} + /// Reference to an existing code hash or a new wasm module. #[derive(Eq, PartialEq, Encode, Decode, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] @@ -134,6 +178,122 @@ pub enum Code { Existing(Hash), } +impl>, Hash> From for Code { + fn from(from: T) -> Self { + Code::Upload(Bytes(from.into())) + } +} + +/// The amount of balance that was either charged or refunded in order to pay for storage. +#[derive(Eq, PartialEq, Ord, PartialOrd, Encode, Decode, RuntimeDebug, Clone)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +#[cfg_attr( + feature = "std", + serde( + rename_all = "camelCase", + bound(serialize = "Balance: Copy + Into"), + bound(deserialize = "Balance: TryFrom") + ) +)] +pub enum StorageDeposit { + /// The transaction reduced storage consumption. + /// + /// This means that the specified amount of balance was transferred from the involved + /// contracts to the call origin. + #[cfg_attr(feature = "std", serde(with = "as_hex"))] + Refund(Balance), + /// The transaction increased overall storage usage. + /// + /// This means that the specified amount of balance was transferred from the call origin + /// to the contracts involved. + #[cfg_attr(feature = "std", serde(with = "as_hex"))] + Charge(Balance), +} + +impl Default for StorageDeposit { + fn default() -> Self { + Self::Charge(Zero::zero()) + } +} + +impl StorageDeposit { + /// Returns how much balance is charged or `0` in case of a refund. + pub fn charge_or_zero(&self) -> Balance { + match self { + Self::Charge(amount) => *amount, + Self::Refund(_) => Zero::zero(), + } + } + + pub fn is_zero(&self) -> bool { + match self { + Self::Charge(amount) => amount.is_zero(), + Self::Refund(amount) => amount.is_zero(), + } + } +} + +impl StorageDeposit +where + Balance: Saturating + Ord + Copy, +{ + /// This is essentially a saturating signed add. + pub fn saturating_add(&self, rhs: &Self) -> Self { + use StorageDeposit::*; + match (self, rhs) { + (Charge(lhs), Charge(rhs)) => Charge(lhs.saturating_add(*rhs)), + (Refund(lhs), Refund(rhs)) => Refund(lhs.saturating_add(*rhs)), + (Charge(lhs), Refund(rhs)) => + if lhs >= rhs { + Charge(lhs.saturating_sub(*rhs)) + } else { + Refund(rhs.saturating_sub(*lhs)) + }, + (Refund(lhs), Charge(rhs)) => + if lhs > rhs { + Refund(lhs.saturating_sub(*rhs)) + } else { + Charge(rhs.saturating_sub(*lhs)) + }, + } + } + + /// This is essentially a saturating signed sub. + pub fn saturating_sub(&self, rhs: &Self) -> Self { + use StorageDeposit::*; + match (self, rhs) { + (Charge(lhs), Refund(rhs)) => Charge(lhs.saturating_add(*rhs)), + (Refund(lhs), Charge(rhs)) => Refund(lhs.saturating_add(*rhs)), + (Charge(lhs), Charge(rhs)) => + if lhs >= rhs { + Charge(lhs.saturating_sub(*rhs)) + } else { + Refund(rhs.saturating_sub(*lhs)) + }, + (Refund(lhs), Refund(rhs)) => + if lhs > rhs { + Refund(lhs.saturating_sub(*rhs)) + } else { + Charge(rhs.saturating_sub(*lhs)) + }, + } + } + + /// If the amount of deposit (this type) is constrained by a `limit` this calcuates how + /// much balance (if any) is still available from this limit. + /// + /// # Note + /// + /// In case of a refund the return value can be larger than `limit`. + pub fn available(&self, limit: &Balance) -> Balance { + use StorageDeposit::*; + match self { + Charge(amount) => limit.saturating_sub(*amount), + Refund(amount) => limit.saturating_add(*amount), + } + } +} + #[cfg(feature = "std")] mod as_string { use super::*; @@ -149,3 +309,26 @@ mod as_string { Ok(String::deserialize(deserializer)?.into_bytes()) } } + +#[cfg(feature = "std")] +mod as_hex { + use super::*; + use serde::{de::Error as _, Deserializer, Serializer}; + + pub fn serialize(balance: &Balance, serializer: S) -> Result + where + S: Serializer, + Balance: Copy + Into, + { + Into::::into(*balance).serialize(serializer) + } + + pub fn deserialize<'de, D, Balance>(deserializer: D) -> Result + where + D: Deserializer<'de>, + Balance: TryFrom, + { + Balance::try_from(NumberOrHex::deserialize(deserializer)?) + .map_err(|_| D::Error::custom("Cannot decode NumberOrHex to Balance")) + } +} diff --git a/frame/contracts/fixtures/delegate_call.wat b/frame/contracts/fixtures/delegate_call.wat new file mode 100644 index 000000000000..7fe422af4551 --- /dev/null +++ b/frame/contracts/fixtures/delegate_call.wat @@ -0,0 +1,111 @@ +(module + (import "seal0" "seal_input" (func $seal_input (param i32 i32))) + (import "seal0" "seal_get_storage" (func $seal_get_storage (param i32 i32 i32) (result i32))) + (import "seal0" "seal_set_storage" (func $seal_set_storage (param i32 i32 i32))) + (import "seal0" "seal_delegate_call" (func $seal_delegate_call (param i32 i32 i32 i32 i32 i32) (result i32))) + (import "env" "memory" (memory 3 3)) + + ;; [0, 32) storage key + (data (i32.const 0) "\01") + + ;; [32, 64) storage key + (data (i32.const 32) "\02") + + ;; [64, 96) buffer where input is copied + + ;; [96, 100) size of the input buffer + (data (i32.const 96) "\20") + + ;; [100, 104) size of buffer for seal_get_storage + (data (i32.const 100) "\20") + + ;; [104, 136) seal_get_storage buffer + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "call") + (local $exit_code i32) + + ;; Reading "callee" code_hash + (call $seal_input (i32.const 64) (i32.const 96)) + + ;; assert input size == 32 + (call $assert + (i32.eq + (i32.load (i32.const 96)) + (i32.const 32) + ) + ) + + ;; place a value in storage, the size of which is specified by the call input. + (call $seal_set_storage + (i32.const 0) ;; Pointer to storage key + (i32.const 32) ;; Pointer to initial value + (i32.load (i32.const 100)) ;; Size of value + ) + + (call $assert + (i32.eq + (call $seal_get_storage + (i32.const 0) ;; Pointer to storage key + (i32.const 104) ;; buffer where to copy result + (i32.const 100) ;; pointer to size of buffer + ) + (i32.const 0) ;; ReturnCode::Success + ) + ) + + (call $assert + (i32.eq + (i32.load (i32.const 104)) ;; value received from storage + (i32.load (i32.const 32)) ;; initial value + ) + ) + + ;; Call deployed library contract code. + (set_local $exit_code + (call $seal_delegate_call + (i32.const 0) ;; Set no call flags + (i32.const 64) ;; Pointer to "callee" code_hash. + (i32.const 0) ;; Input is ignored + (i32.const 0) ;; Length of the input + (i32.const 4294967295) ;; u32 max sentinel value: do not copy output + (i32.const 0) ;; Length is ignored in this case + ) + ) + + ;; Check for success exit status. + (call $assert + (i32.eq (get_local $exit_code) (i32.const 0)) ;; ReturnCode::Success + ) + + (call $assert + (i32.eq + (call $seal_get_storage + (i32.const 0) ;; Pointer to storage key + (i32.const 104) ;; buffer where to copy result + (i32.const 100) ;; pointer to size of buffer + ) + (i32.const 0) ;; ReturnCode::Success + ) + ) + + ;; Make sure that 'callee' code changed the value + (call $assert + (i32.eq + (i32.load (i32.const 104)) + (i32.const 1) + ) + ) + ) + + (func (export "deploy")) + +) diff --git a/frame/contracts/fixtures/delegate_call_lib.wat b/frame/contracts/fixtures/delegate_call_lib.wat new file mode 100644 index 000000000000..340b9699f875 --- /dev/null +++ b/frame/contracts/fixtures/delegate_call_lib.wat @@ -0,0 +1,79 @@ +(module + (import "seal0" "seal_set_storage" (func $seal_set_storage (param i32 i32 i32))) + (import "seal0" "seal_caller" (func $seal_caller (param i32 i32))) + (import "seal0" "seal_value_transferred" (func $seal_value_transferred (param i32 i32))) + (import "env" "memory" (memory 1 1)) + + ;; [0, 32) storage key + (data (i32.const 0) "\01") + + ;; [32, 64) buffer for transferred value + + ;; [64, 96) size of the buffer for transferred value + (data (i32.const 64) "\20") + + ;; [96, 128) buffer for the caller + + ;; [128, 160) size of the buffer for caller + (data (i32.const 128) "\20") + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "call") + ;; place a value in storage + (call $seal_set_storage + (i32.const 0) ;; Pointer to storage key + (i32.const 0) ;; Pointer to value + (i32.const 32) ;; Size of value + ) + + ;; This stores the value transferred in the buffer + (call $seal_value_transferred (i32.const 32) (i32.const 64)) + + ;; assert len == 8 + (call $assert + (i32.eq + (i32.load (i32.const 64)) + (i32.const 8) + ) + ) + + ;; assert that contents of the buffer is equal to the value + ;; passed to the `caller` contract: 1337 + (call $assert + (i64.eq + (i64.load (i32.const 32)) + (i64.const 1337) + ) + ) + + ;; fill the buffer with the caller. + (call $seal_caller (i32.const 96) (i32.const 128)) + + ;; assert len == 32 + (call $assert + (i32.eq + (i32.load (i32.const 128)) + (i32.const 32) + ) + ) + + ;; assert that the first 64 byte are the beginning of "ALICE", + ;; who is the caller of the `caller` contract + (call $assert + (i64.eq + (i64.load (i32.const 96)) + (i64.const 0x0101010101010101) + ) + ) + ) + + (func (export "deploy")) +) diff --git a/frame/contracts/fixtures/destroy_and_transfer.wat b/frame/contracts/fixtures/destroy_and_transfer.wat index aa13cd8b8107..255547955527 100644 --- a/frame/contracts/fixtures/destroy_and_transfer.wat +++ b/frame/contracts/fixtures/destroy_and_transfer.wat @@ -9,7 +9,7 @@ )) (import "env" "memory" (memory 1 1)) - ;; [0, 8) Endowment to send when creating contract. + ;; [0, 8) value to send when creating contract. (data (i32.const 0) "\00\00\01") ;; [8, 16) Value to send when calling contract. diff --git a/frame/contracts/fixtures/drain.wat b/frame/contracts/fixtures/drain.wat index 546026ac9598..94c651842266 100644 --- a/frame/contracts/fixtures/drain.wat +++ b/frame/contracts/fixtures/drain.wat @@ -33,7 +33,9 @@ ) ) - ;; Self-destruct by sending full balance to the 0 address. + ;; Try to self-destruct by sending full balance to the 0 address. + ;; All the *free* balance will be send away, which is a valid thing to do + ;; because the storage deposits will keep the account alive. (call $assert (i32.eq (call $seal_transfer @@ -42,7 +44,7 @@ (i32.const 0) ;; Pointer to the buffer with value to transfer (i32.const 8) ;; Length of the buffer with value to transfer ) - (i32.const 4) ;; ReturnCode::BelowSubsistenceThreshold + (i32.const 0) ;; ReturnCode::Success ) ) ) diff --git a/frame/contracts/fixtures/invalid_import.wat b/frame/contracts/fixtures/invalid_import.wat new file mode 100644 index 000000000000..011f1a40e76d --- /dev/null +++ b/frame/contracts/fixtures/invalid_import.wat @@ -0,0 +1,6 @@ +;; A valid contract which does nothing at all but imports an invalid function +(module + (import "invalid" "invalid_88_99" (func (param i32 i32 i32))) + (func (export "deploy")) + (func (export "call")) +) diff --git a/frame/contracts/fixtures/multi_store.wat b/frame/contracts/fixtures/multi_store.wat new file mode 100644 index 000000000000..2592baf61835 --- /dev/null +++ b/frame/contracts/fixtures/multi_store.wat @@ -0,0 +1,54 @@ +;; Does two stores to two seperate storage items +;; Expects (len0, len1) as input. +(module + (import "seal0" "seal_set_storage" (func $seal_set_storage (param i32 i32 i32))) + (import "seal0" "seal_input" (func $seal_input (param i32 i32))) + (import "env" "memory" (memory 16 16)) + + ;; [0, 32) storage key 0 + (data (i32.const 0) "\01") + + ;; [32, 64) storage key 1 + (data (i32.const 32) "\02") + + ;; [64, 72) buffer where input is copied (expected sizes of storage items) + + ;; [72, 76) size of the input buffer + (data (i32.const 72) "\08") + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "call") + (call $seal_input (i32.const 64) (i32.const 72)) + + ;; assert input size == 8 + (call $assert + (i32.eq + (i32.load (i32.const 72)) + (i32.const 8) + ) + ) + + ;; place a values in storage sizes are specified in the input buffer + ;; we don't care about the contents of the storage item + (call $seal_set_storage + (i32.const 0) ;; Pointer to storage key + (i32.const 0) ;; Pointer to value + (i32.load (i32.const 64)) ;; Size of value + ) + (call $seal_set_storage + (i32.const 32) ;; Pointer to storage key + (i32.const 0) ;; Pointer to value + (i32.load (i32.const 68)) ;; Size of value + ) + ) + + (func (export "deploy")) +) diff --git a/frame/contracts/fixtures/new_set_code_hash_contract.wat b/frame/contracts/fixtures/new_set_code_hash_contract.wat new file mode 100644 index 000000000000..86ab2737be48 --- /dev/null +++ b/frame/contracts/fixtures/new_set_code_hash_contract.wat @@ -0,0 +1,13 @@ +(module + (import "seal0" "seal_return" (func $seal_return (param i32 i32 i32))) + (import "env" "memory" (memory 1 1)) + + ;; [0, 32) return value + (data (i32.const 0) "\02") + + (func (export "deploy")) + + (func (export "call") + (call $seal_return (i32.const 0) (i32.const 0) (i32.const 4)) + ) +) diff --git a/frame/contracts/fixtures/set_code_hash.wat b/frame/contracts/fixtures/set_code_hash.wat new file mode 100644 index 000000000000..0a7b2e7cbedf --- /dev/null +++ b/frame/contracts/fixtures/set_code_hash.wat @@ -0,0 +1,43 @@ +(module + (import "seal0" "seal_input" (func $seal_input (param i32 i32))) + (import "seal0" "seal_return" (func $seal_return (param i32 i32 i32))) + (import "__unstable__" "seal_set_code_hash" (func $seal_set_code_hash (param i32) (result i32))) + + (import "env" "memory" (memory 1 1)) + + ;; [0, 32) here we store input + + ;; [32, 36) input size + (data (i32.const 32) "\20") + + ;; [36, 40) return value + (data (i32.const 36) "\01") + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "call") + (local $exit_code i32) + + (call $seal_input (i32.const 0) (i32.const 32)) + + (set_local $exit_code + (call $seal_set_code_hash (i32.const 0)) ;; Pointer to the input data. + ) + (call $assert + (i32.eq (get_local $exit_code) (i32.const 0)) ;; ReturnCode::Success + ) + + ;; we return 1 after setting new code_hash + ;; next `call` will NOT return this value, because contract code has been changed + (call $seal_return (i32.const 0) (i32.const 36) (i32.const 4)) + ) + + (func (export "deploy")) +) diff --git a/frame/contracts/fixtures/store.wat b/frame/contracts/fixtures/store.wat new file mode 100644 index 000000000000..9e090d31801f --- /dev/null +++ b/frame/contracts/fixtures/store.wat @@ -0,0 +1,45 @@ +;; Stores a value of the passed size. +(module + (import "seal0" "seal_set_storage" (func $seal_set_storage (param i32 i32 i32))) + (import "seal0" "seal_input" (func $seal_input (param i32 i32))) + (import "env" "memory" (memory 16 16)) + + ;; [0, 32) storage key + (data (i32.const 0) "\01") + + ;; [32, 36) buffer where input is copied (expected size of storage item) + + ;; [36, 40) size of the input buffer + (data (i32.const 36) "\04") + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "call") + (call $seal_input (i32.const 32) (i32.const 36)) + + ;; assert input size == 4 + (call $assert + (i32.eq + (i32.load (i32.const 36)) + (i32.const 4) + ) + ) + + ;; place a value in storage, the size of which is specified by the call input. + ;; we don't care about the contents of the storage item + (call $seal_set_storage + (i32.const 0) ;; Pointer to storage key + (i32.const 0) ;; Pointer to value + (i32.load (i32.const 32)) ;; Size of value + ) + ) + + (func (export "deploy")) +) diff --git a/frame/contracts/proc-macro/Cargo.toml b/frame/contracts/proc-macro/Cargo.toml index 605c69fe73e2..db3c62039777 100644 --- a/frame/contracts/proc-macro/Cargo.toml +++ b/frame/contracts/proc-macro/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-contracts-proc-macro" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Procedural macros used in pallet_contracts" diff --git a/frame/contracts/proc-macro/src/lib.rs b/frame/contracts/proc-macro/src/lib.rs index 302a0d01a93d..43b59debc4ad 100644 --- a/frame/contracts/proc-macro/src/lib.rs +++ b/frame/contracts/proc-macro/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/contracts/rpc/Cargo.toml b/frame/contracts/rpc/Cargo.toml index b73039ba7191..e506e78a2fbd 100644 --- a/frame/contracts/rpc/Cargo.toml +++ b/frame/contracts/rpc/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-contracts-rpc" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Node-specific RPC methods for interaction with contracts." readme = "README.md" @@ -13,20 +13,20 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2" } +codec = { package = "parity-scale-codec", version = "3.0.0" } jsonrpc-core = "18.0.0" jsonrpc-core-client = "18.0.0" jsonrpc-derive = "18.0.0" serde = { version = "1", features = ["derive"] } # Substrate Dependencies -pallet-contracts-primitives = { version = "4.0.0-dev", path = "../common" } +pallet-contracts-primitives = { version = "6.0.0", path = "../common" } pallet-contracts-rpc-runtime-api = { version = "4.0.0-dev", path = "./runtime-api" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-rpc = { version = "4.0.0-dev", path = "../../../primitives/rpc" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-rpc = { version = "6.0.0", path = "../../../primitives/rpc" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } [dev-dependencies] serde_json = "1" diff --git a/frame/contracts/rpc/runtime-api/Cargo.toml b/frame/contracts/rpc/runtime-api/Cargo.toml index e5f6d1ec7eb8..e50a41624e7e 100644 --- a/frame/contracts/rpc/runtime-api/Cargo.toml +++ b/frame/contracts/rpc/runtime-api/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-contracts-rpc-runtime-api" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Runtime API definition required by Contracts RPC extensions." readme = "README.md" @@ -13,14 +13,14 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } # Substrate Dependencies -pallet-contracts-primitives = { version = "4.0.0-dev", default-features = false, path = "../../common" } +pallet-contracts-primitives = { version = "6.0.0", default-features = false, path = "../../common" } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../../primitives/api" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../../../primitives/std" } [features] default = ["std"] diff --git a/frame/contracts/rpc/runtime-api/src/lib.rs b/frame/contracts/rpc/runtime-api/src/lib.rs index 20dfbe210e5c..59622a21a659 100644 --- a/frame/contracts/rpc/runtime-api/src/lib.rs +++ b/frame/contracts/rpc/runtime-api/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,7 +25,7 @@ use codec::Codec; use pallet_contracts_primitives::{ - Code, ContractExecResult, ContractInstantiateResult, GetStorageResult, + Code, CodeUploadResult, ContractExecResult, ContractInstantiateResult, GetStorageResult, }; use sp_std::vec::Vec; @@ -45,20 +45,32 @@ sp_api::decl_runtime_apis! { dest: AccountId, value: Balance, gas_limit: u64, + storage_deposit_limit: Option, input_data: Vec, - ) -> ContractExecResult; + ) -> ContractExecResult; /// Instantiate a new contract. /// /// See `pallet_contracts::Pallet::instantiate`. fn instantiate( origin: AccountId, - endowment: Balance, + value: Balance, gas_limit: u64, + storage_deposit_limit: Option, code: Code, data: Vec, salt: Vec, - ) -> ContractInstantiateResult; + ) -> ContractInstantiateResult; + + + /// Upload new code without instantiating a contract from it. + /// + /// See `pallet_contracts::Pallet::upload_code`. + fn upload_code( + origin: AccountId, + code: Vec, + storage_deposit_limit: Option, + ) -> CodeUploadResult; /// Query a given storage key in a given contract. /// diff --git a/frame/contracts/rpc/src/lib.rs b/frame/contracts/rpc/src/lib.rs index e0796af05654..e83e4e6249b9 100644 --- a/frame/contracts/rpc/src/lib.rs +++ b/frame/contracts/rpc/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,9 @@ use std::sync::Arc; use codec::Codec; use jsonrpc_core::{Error, ErrorCode, Result}; use jsonrpc_derive::rpc; -use pallet_contracts_primitives::{Code, ContractExecResult, ContractInstantiateResult}; +use pallet_contracts_primitives::{ + Code, CodeUploadResult, ContractExecResult, ContractInstantiateResult, +}; use serde::{Deserialize, Serialize}; use sp_api::ProvideRuntimeApi; use sp_blockchain::HeaderBackend; @@ -32,7 +34,6 @@ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Header as HeaderT}, }; -use std::convert::{TryFrom, TryInto}; pub use pallet_contracts_rpc_runtime_api::ContractsApi as ContractsRuntimeApi; @@ -79,6 +80,7 @@ pub struct CallRequest { dest: AccountId, value: NumberOrHex, gas_limit: NumberOrHex, + storage_deposit_limit: Option, input_data: Bytes, } @@ -88,32 +90,47 @@ pub struct CallRequest { #[serde(deny_unknown_fields)] pub struct InstantiateRequest { origin: AccountId, - endowment: NumberOrHex, + value: NumberOrHex, gas_limit: NumberOrHex, + storage_deposit_limit: Option, code: Code, data: Bytes, salt: Bytes, } +/// A struct that encodes RPC parameters required for a call to upload a new code. +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct CodeUploadRequest { + origin: AccountId, + code: Bytes, + storage_deposit_limit: Option, +} + /// Contracts RPC methods. #[rpc] -pub trait ContractsApi { +pub trait ContractsApi +where + Balance: Copy + TryFrom + Into, +{ /// Executes a call to a contract. /// /// This call is performed locally without submitting any transactions. Thus executing this /// won't change any state. Nonetheless, the calling state-changing contracts is still possible. /// - /// This method is useful for calling getter-like methods on contracts. + /// This method is useful for calling getter-like methods on contracts or to dry-run a + /// a contract call in order to determine the `gas_limit`. #[rpc(name = "contracts_call")] fn call( &self, call_request: CallRequest, at: Option, - ) -> Result; + ) -> Result>; /// Instantiate a new contract. /// - /// This call is performed locally without submitting any transactions. Thus the contract + /// This instantiate is performed locally without submitting any transactions. Thus the contract /// is not actually created. /// /// This method is useful for UIs to dry-run contract instantiations. @@ -122,7 +139,20 @@ pub trait ContractsApi { &self, instantiate_request: InstantiateRequest, at: Option, - ) -> Result>; + ) -> Result>; + + /// Upload new code without instantiating a contract from it. + /// + /// This upload is performed locally without submitting any transactions. Thus executing this + /// won't change any state. + /// + /// This method is useful for UIs to dry-run code upload. + #[rpc(name = "contracts_upload_code")] + fn upload_code( + &self, + upload_request: CodeUploadRequest, + at: Option, + ) -> Result>; /// Returns the value under a specified storage `key` in a contract given by `address` param, /// or `None` if it is not set. @@ -166,54 +196,88 @@ where Hash, >, AccountId: Codec, - Balance: Codec + TryFrom, + Balance: Codec + Copy + TryFrom + Into, Hash: Codec, { fn call( &self, call_request: CallRequest, at: Option<::Hash>, - ) -> Result { + ) -> Result> { let api = self.client.runtime_api(); let at = BlockId::hash(at.unwrap_or_else(|| // If the block hash is not supplied assume the best block. self.client.info().best_hash)); - let CallRequest { origin, dest, value, gas_limit, input_data } = call_request; + let CallRequest { origin, dest, value, gas_limit, storage_deposit_limit, input_data } = + call_request; let value: Balance = decode_hex(value, "balance")?; let gas_limit: Weight = decode_hex(gas_limit, "weight")?; + let storage_deposit_limit: Option = + storage_deposit_limit.map(|l| decode_hex(l, "balance")).transpose()?; limit_gas(gas_limit)?; - let exec_result = api - .call(&at, origin, dest, value, gas_limit, input_data.to_vec()) - .map_err(runtime_error_into_rpc_err)?; - - Ok(exec_result) + api.call(&at, origin, dest, value, gas_limit, storage_deposit_limit, input_data.to_vec()) + .map_err(runtime_error_into_rpc_err) } fn instantiate( &self, instantiate_request: InstantiateRequest, at: Option<::Hash>, - ) -> Result> { + ) -> Result> { let api = self.client.runtime_api(); let at = BlockId::hash(at.unwrap_or_else(|| // If the block hash is not supplied assume the best block. self.client.info().best_hash)); - let InstantiateRequest { origin, endowment, gas_limit, code, data, salt } = - instantiate_request; + let InstantiateRequest { + origin, + value, + gas_limit, + storage_deposit_limit, + code, + data, + salt, + } = instantiate_request; - let endowment: Balance = decode_hex(endowment, "balance")?; + let value: Balance = decode_hex(value, "balance")?; let gas_limit: Weight = decode_hex(gas_limit, "weight")?; + let storage_deposit_limit: Option = + storage_deposit_limit.map(|l| decode_hex(l, "balance")).transpose()?; limit_gas(gas_limit)?; - let exec_result = api - .instantiate(&at, origin, endowment, gas_limit, code, data.to_vec(), salt.to_vec()) - .map_err(runtime_error_into_rpc_err)?; + api.instantiate( + &at, + origin, + value, + gas_limit, + storage_deposit_limit, + code, + data.to_vec(), + salt.to_vec(), + ) + .map_err(runtime_error_into_rpc_err) + } + + fn upload_code( + &self, + upload_request: CodeUploadRequest, + at: Option<::Hash>, + ) -> Result> { + let api = self.client.runtime_api(); + let at = BlockId::hash(at.unwrap_or_else(|| + // If the block hash is not supplied assume the best block. + self.client.info().best_hash)); + + let CodeUploadRequest { origin, code, storage_deposit_limit } = upload_request; - Ok(exec_result) + let storage_deposit_limit: Option = + storage_deposit_limit.map(|l| decode_hex(l, "balance")).transpose()?; + + api.upload_code(&at, origin, code.to_vec(), storage_deposit_limit) + .map_err(runtime_error_into_rpc_err) } fn get_storage( @@ -238,11 +302,11 @@ where } /// Converts a runtime trap into an RPC error. -fn runtime_error_into_rpc_err(err: impl std::fmt::Debug) -> Error { +fn runtime_error_into_rpc_err(err: impl std::fmt::Display) -> Error { Error { code: ErrorCode::ServerError(RUNTIME_ERROR), message: "Runtime error".into(), - data: Some(format!("{:?}", err).into()), + data: Some(err.to_string().into()), } } @@ -288,12 +352,14 @@ mod tests { "dest": "5DRakbLVnjVrW6niwLfHGW24EeCEvDAFGEXrtaYS5M4ynoom", "value": "0x112210f4B16c1cb1", "gasLimit": 1000000000000, + "storageDepositLimit": 5000, "inputData": "0x8c97db39" } "#, ) .unwrap(); assert_eq!(req.gas_limit.into_u256(), U256::from(0xe8d4a51000u64)); + assert_eq!(req.storage_deposit_limit.map(|l| l.into_u256()), Some(5000.into())); assert_eq!(req.value.into_u256(), U256::from(1234567890987654321u128)); } @@ -304,7 +370,7 @@ mod tests { r#" { "origin": "5CiPPseXPECbkjWCa6MnjNokrgYjMqmKndv2rSnekmSK2DjL", - "endowment": "0x88", + "value": "0x88", "gasLimit": 42, "code": { "existing": "0x1122" }, "data": "0x4299", @@ -315,8 +381,9 @@ mod tests { .unwrap(); assert_eq!(req.origin, "5CiPPseXPECbkjWCa6MnjNokrgYjMqmKndv2rSnekmSK2DjL"); - assert_eq!(req.endowment.into_u256(), 0x88.into()); + assert_eq!(req.value.into_u256(), 0x88.into()); assert_eq!(req.gas_limit.into_u256(), 42.into()); + assert_eq!(req.storage_deposit_limit, None); assert_eq!(&*req.data, [0x42, 0x99].as_ref()); assert_eq!(&*req.salt, [0x99, 0x88].as_ref()); let code = match req.code { @@ -326,10 +393,28 @@ mod tests { assert_eq!(&code, "0x1122"); } + #[test] + fn code_upload_request_should_serialize_deserialize_properly() { + type Req = CodeUploadRequest; + let req: Req = serde_json::from_str( + r#" + { + "origin": "5CiPPseXPECbkjWCa6MnjNokrgYjMqmKndv2rSnekmSK2DjL", + "code": "0x8c97db39", + "storageDepositLimit": 5000 + } + "#, + ) + .unwrap(); + assert_eq!(req.origin, "5CiPPseXPECbkjWCa6MnjNokrgYjMqmKndv2rSnekmSK2DjL"); + assert_eq!(&*req.code, [0x8c, 0x97, 0xdb, 0x39].as_ref()); + assert_eq!(req.storage_deposit_limit.map(|l| l.into_u256()), Some(5000.into())); + } + #[test] fn call_result_should_serialize_deserialize_properly() { fn test(expected: &str) { - let res: ContractExecResult = serde_json::from_str(expected).unwrap(); + let res: ContractExecResult = serde_json::from_str(expected).unwrap(); let actual = serde_json::to_string(&res).unwrap(); assert_eq!(actual, trim(expected).as_str()); } @@ -337,6 +422,7 @@ mod tests { r#"{ "gasConsumed": 5000, "gasRequired": 8000, + "storageDeposit": {"charge": 42000}, "debugMessage": "HelloWorld", "result": { "Ok": { @@ -350,6 +436,7 @@ mod tests { r#"{ "gasConsumed": 3400, "gasRequired": 5200, + "storageDeposit": {"refund": 12000}, "debugMessage": "HelloWorld", "result": { "Err": "BadOrigin" @@ -361,7 +448,8 @@ mod tests { #[test] fn instantiate_result_should_serialize_deserialize_properly() { fn test(expected: &str) { - let res: ContractInstantiateResult = serde_json::from_str(expected).unwrap(); + let res: ContractInstantiateResult = + serde_json::from_str(expected).unwrap(); let actual = serde_json::to_string(&res).unwrap(); assert_eq!(actual, trim(expected).as_str()); } @@ -369,6 +457,7 @@ mod tests { r#"{ "gasConsumed": 5000, "gasRequired": 8000, + "storageDeposit": {"refund": 12000}, "debugMessage": "HelloWorld", "result": { "Ok": { @@ -385,6 +474,7 @@ mod tests { r#"{ "gasConsumed": 3400, "gasRequired": 5200, + "storageDeposit": {"charge": 0}, "debugMessage": "HelloWorld", "result": { "Err": "BadOrigin" @@ -392,4 +482,26 @@ mod tests { }"#, ); } + + #[test] + fn code_upload_result_should_serialize_deserialize_properly() { + fn test(expected: &str) { + let res: CodeUploadResult = serde_json::from_str(expected).unwrap(); + let actual = serde_json::to_string(&res).unwrap(); + assert_eq!(actual, trim(expected).as_str()); + } + test( + r#"{ + "Ok": { + "codeHash": 4711, + "deposit": 99 + } + }"#, + ); + test( + r#"{ + "Err": "BadOrigin" + }"#, + ); + } } diff --git a/frame/contracts/src/benchmarking/code.rs b/frame/contracts/src/benchmarking/code.rs index b24005ec5869..2544fd6b7f92 100644 --- a/frame/contracts/src/benchmarking/code.rs +++ b/frame/contracts/src/benchmarking/code.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,17 +26,28 @@ use crate::Config; use frame_support::traits::Get; -use pwasm_utils::parity_wasm::{ +use sp_core::crypto::UncheckedFrom; +use sp_runtime::traits::Hash; +use sp_sandbox::{ + default_executor::{EnvironmentDefinitionBuilder, Memory}, + SandboxEnvironmentBuilder, SandboxMemory, +}; +use sp_std::{borrow::ToOwned, prelude::*}; +use wasm_instrument::parity_wasm::{ builder, elements::{ self, BlockType, CustomSection, External, FuncBody, Instruction, Instructions, Module, Section, ValueType, }, }; -use sp_core::crypto::UncheckedFrom; -use sp_runtime::traits::Hash; -use sp_sandbox::{EnvironmentDefinitionBuilder, Memory}; -use sp_std::{borrow::ToOwned, convert::TryFrom, prelude::*}; + +/// The location where to put the genrated code. +pub enum Location { + /// Generate all code into the `call` exported function. + Call, + /// Generate all code into the `deploy` exported function. + Deploy, +} /// Pass to `create_code` in order to create a compiled `WasmModule`. /// @@ -305,7 +316,8 @@ where /// Creates a wasm module of `target_bytes` size. Used to benchmark the performance of /// `instantiate_with_code` for different sizes of wasm modules. The generated module maximizes /// instrumentation runtime by nesting blocks as deeply as possible given the byte budget. - pub fn sized(target_bytes: u32) -> Self { + /// `code_location`: Whether to place the code into `deploy` or `call`. + pub fn sized(target_bytes: u32, code_location: Location) -> Self { use self::elements::Instruction::{End, I32Const, If, Return}; // Base size of a contract is 63 bytes and each expansion adds 6 bytes. // We do one expansion less to account for the code section and function body @@ -314,23 +326,25 @@ where // because of the maximum code size that is enforced by `instantiate_with_code`. let expansions = (target_bytes.saturating_sub(63) / 6).saturating_sub(1); const EXPANSION: [Instruction; 4] = [I32Const(0), If(BlockType::NoResult), Return, End]; - ModuleDefinition { - call_body: Some(body::repeated(expansions, &EXPANSION)), - memory: Some(ImportedMemory::max::()), - ..Default::default() + let mut module = + ModuleDefinition { memory: Some(ImportedMemory::max::()), ..Default::default() }; + let body = Some(body::repeated(expansions, &EXPANSION)); + match code_location { + Location::Call => module.call_body = body, + Location::Deploy => module.deploy_body = body, } - .into() + module.into() } /// Creates a wasm module that calls the imported function named `getter_name` `repeat` /// times. The imported function is expected to have the "getter signature" of /// (out_ptr: u32, len_ptr: u32) -> (). - pub fn getter(getter_name: &'static str, repeat: u32) -> Self { + pub fn getter(module_name: &'static str, getter_name: &'static str, repeat: u32) -> Self { let pages = max_pages::(); ModuleDefinition { memory: Some(ImportedMemory::max::()), imported_functions: vec![ImportedFunction { - module: "seal0", + module: module_name, name: getter_name, params: vec![ValueType::I32, ValueType::I32], return_type: None, @@ -492,11 +506,11 @@ pub mod body { vec![Instruction::I32Const(current as i32)] }, DynInstr::RandomUnaligned(low, high) => { - let unaligned = rng.gen_range(*low, *high) | 1; + let unaligned = rng.gen_range(*low..*high) | 1; vec![Instruction::I32Const(unaligned as i32)] }, DynInstr::RandomI32(low, high) => { - vec![Instruction::I32Const(rng.gen_range(*low, *high))] + vec![Instruction::I32Const(rng.gen_range(*low..*high))] }, DynInstr::RandomI32Repeated(num) => (&mut rng) .sample_iter(Standard) @@ -509,19 +523,19 @@ pub mod body { .map(|val| Instruction::I64Const(val)) .collect(), DynInstr::RandomGetLocal(low, high) => { - vec![Instruction::GetLocal(rng.gen_range(*low, *high))] + vec![Instruction::GetLocal(rng.gen_range(*low..*high))] }, DynInstr::RandomSetLocal(low, high) => { - vec![Instruction::SetLocal(rng.gen_range(*low, *high))] + vec![Instruction::SetLocal(rng.gen_range(*low..*high))] }, DynInstr::RandomTeeLocal(low, high) => { - vec![Instruction::TeeLocal(rng.gen_range(*low, *high))] + vec![Instruction::TeeLocal(rng.gen_range(*low..*high))] }, DynInstr::RandomGetGlobal(low, high) => { - vec![Instruction::GetGlobal(rng.gen_range(*low, *high))] + vec![Instruction::GetGlobal(rng.gen_range(*low..*high))] }, DynInstr::RandomSetGlobal(low, high) => { - vec![Instruction::SetGlobal(rng.gen_range(*low, *high))] + vec![Instruction::SetGlobal(rng.gen_range(*low..*high))] }, }) .chain(sp_std::iter::once(Instruction::End)) @@ -548,10 +562,13 @@ where fn inject_gas_metering(module: Module) -> Module { let schedule = T::Schedule::get(); let gas_rules = schedule.rules(&module); - pwasm_utils::inject_gas_counter(module, &gas_rules, "seal0").unwrap() + wasm_instrument::gas_metering::inject(module, &gas_rules, "seal0").unwrap() } fn inject_stack_metering(module: Module) -> Module { - let height = T::Schedule::get().limits.stack_height; - pwasm_utils::stack_height::inject_limiter(module, height).unwrap() + if let Some(height) = T::Schedule::get().limits.stack_height { + wasm_instrument::inject_stack_limiter(module, height).unwrap() + } else { + module + } } diff --git a/frame/contracts/src/benchmarking/mod.rs b/frame/contracts/src/benchmarking/mod.rs index 981af218ea5a..de83f51a0152 100644 --- a/frame/contracts/src/benchmarking/mod.rs +++ b/frame/contracts/src/benchmarking/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,7 +25,7 @@ mod sandbox; use self::{ code::{ body::{self, DynInstr::*}, - DataSegment, ImportedFunction, ImportedMemory, ModuleDefinition, WasmModule, + DataSegment, ImportedFunction, ImportedMemory, Location, ModuleDefinition, WasmModule, }, sandbox::Sandbox, }; @@ -33,18 +33,19 @@ use crate::{ exec::{AccountIdOf, StorageKey}, schedule::{API_BENCHMARK_BATCH_SIZE, INSTR_BENCHMARK_BATCH_SIZE}, storage::Storage, + wasm::CallFlags, Pallet as Contracts, *, }; -use codec::Encode; -use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite, whitelisted_caller}; +use codec::{Encode, MaxEncodedLen}; +use frame_benchmarking::{account, benchmarks, whitelisted_caller}; use frame_support::weights::Weight; use frame_system::RawOrigin; -use pwasm_utils::parity_wasm::elements::{BlockType, BrTableData, Instruction, ValueType}; use sp_runtime::{ traits::{Bounded, Hash}, Perbill, }; -use sp_std::{convert::TryInto, default::Default, vec, vec::Vec}; +use sp_std::prelude::*; +use wasm_instrument::parity_wasm::elements::{BlockType, BrTableData, Instruction, ValueType}; /// How many batches we do per API benchmark. const API_BENCHMARK_BATCHES: u32 = 20; @@ -57,13 +58,14 @@ struct Contract { caller: T::AccountId, account_id: T::AccountId, addr: ::Source, - endowment: BalanceOf, + value: BalanceOf, } impl Contract where T: Config, T::AccountId: UncheckedFrom + AsRef<[u8]>, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { /// Create new contract and use a default account id as instantiator. fn new(module: WasmModule, data: Vec) -> Result, &'static str> { @@ -85,27 +87,24 @@ where module: WasmModule, data: Vec, ) -> Result, &'static str> { - let endowment = contract_funding::(); + let value = T::Currency::minimum_balance(); T::Currency::make_free_balance_be(&caller, caller_funding::()); let salt = vec![0xff]; let addr = Contracts::::contract_address(&caller, &module.hash, &salt); - Contracts::::store_code_raw(module.code)?; + Contracts::::store_code_raw(module.code, caller.clone())?; Contracts::::instantiate( RawOrigin::Signed(caller.clone()).into(), - endowment, - Weight::max_value(), + value, + Weight::MAX, + None, module.hash, data, salt, )?; - let result = Contract { - caller, - account_id: addr.clone(), - addr: T::Lookup::unlookup(addr), - endowment, - }; + let result = + Contract { caller, account_id: addr.clone(), addr: T::Lookup::unlookup(addr), value }; ContractInfoOf::::insert(&result.account_id, result.info()?); @@ -134,9 +133,9 @@ where /// Store the supplied storage items into this contracts storage. fn store(&self, items: &Vec<(StorageKey, Vec)>) -> Result<(), &'static str> { - let mut info = self.info()?; + let info = self.info()?; for item in items { - Storage::::write(&mut info, &item.0, Some(item.1.clone())) + Storage::::write(&info.trie_id, &item.0, Some(item.1.clone()), None, false) .map_err(|_| "Failed to write storage to restoration dest")?; } >::insert(&self.account_id, info.clone()); @@ -152,6 +151,25 @@ where fn info(&self) -> Result, &'static str> { Self::address_info(&self.account_id) } + + /// Set the balance of the contract to the supplied amount. + fn set_balance(&self, balance: BalanceOf) { + T::Currency::make_free_balance_be(&self.account_id, balance); + } + + /// Returns `true` iff all storage entries related to code storage exist. + fn code_exists(hash: &CodeHash) -> bool { + >::contains_key(hash) && + >::contains_key(&hash) && + >::contains_key(&hash) + } + + /// Returns `true` iff no storage entry related to code storage exist. + fn code_removed(hash: &CodeHash) -> bool { + !>::contains_key(hash) && + !>::contains_key(&hash) && + !>::contains_key(&hash) + } } /// The funding that each account that either calls or instantiates contracts is funded with. @@ -159,11 +177,6 @@ fn caller_funding() -> BalanceOf { BalanceOf::::max_value() / 2u32.into() } -/// The funding used for contracts. It is less than `caller_funding` in purpose. -fn contract_funding() -> BalanceOf { - caller_funding::().saturating_sub(T::Currency::minimum_balance() * 100u32.into()) -} - /// Load the specified contract file from disk by including it into the runtime. /// /// We need to load a different version of ink! contracts when the benchmark is run as @@ -186,11 +199,12 @@ benchmarks! { where_clause { where T::AccountId: UncheckedFrom, T::AccountId: AsRef<[u8]>, + as codec::HasCompact>::Type: Clone + Eq + PartialEq + sp_std::fmt::Debug + scale_info::TypeInfo + codec::Encode, } // The base weight without any actual work performed apart from the setup costs. on_initialize {}: { - Storage::::process_deletion_queue_batch(Weight::max_value()) + Storage::::process_deletion_queue_batch(Weight::MAX) } #[skip_meta] @@ -199,7 +213,7 @@ benchmarks! { let instance = Contract::::with_storage(WasmModule::dummy(), k, T::Schedule::get().limits.payload_len)?; Storage::::queue_trie_for_deletion(&instance.info()?)?; }: { - Storage::::process_deletion_queue_batch(Weight::max_value()) + Storage::::process_deletion_queue_batch(Weight::MAX) } on_initialize_per_queue_item { @@ -210,45 +224,45 @@ benchmarks! { ContractInfoOf::::remove(instance.account_id); } }: { - Storage::::process_deletion_queue_batch(Weight::max_value()) + Storage::::process_deletion_queue_batch(Weight::MAX) } // This benchmarks the additional weight that is charged when a contract is executed the // first time after a new schedule was deployed: For every new schedule a contract needs // to re-run the instrumentation once. - instrument { - let c in 0 .. T::Schedule::get().limits.code_len / 1024; - let WasmModule { code, hash, .. } = WasmModule::::sized(c * 1024); - Contracts::::store_code_raw(code)?; - let mut module = PrefabWasmModule::from_storage_noinstr(hash)?; + reinstrument { + let c in 0 .. T::Schedule::get().limits.code_len; + let WasmModule { code, hash, .. } = WasmModule::::sized(c, Location::Call); + Contracts::::store_code_raw(code, whitelisted_caller())?; let schedule = T::Schedule::get(); + let mut gas_meter = GasMeter::new(Weight::MAX); + let mut module = PrefabWasmModule::from_storage(hash, &schedule, &mut gas_meter)?; }: { Contracts::::reinstrument_module(&mut module, &schedule)?; } - // The weight of loading and decoding of a contract's code per kilobyte. - code_load { - let c in 0 .. T::Schedule::get().limits.code_len / 1024; - let WasmModule { code, hash, .. } = WasmModule::::dummy_with_bytes(c * 1024); - Contracts::::store_code_raw(code)?; - }: { - >::from_storage_noinstr(hash)?; - } - - // The weight of changing the refcount of a contract's code per kilobyte. - code_refcount { - let c in 0 .. T::Schedule::get().limits.code_len / 1024; - let WasmModule { code, hash, .. } = WasmModule::::dummy_with_bytes(c * 1024); - Contracts::::store_code_raw(code)?; - let mut gas_meter = GasMeter::new(Weight::max_value()); - }: { - >::add_user(hash, &mut gas_meter)?; - } + // This benchmarks the overhead of loading a code of size `c` byte from storage and into + // the sandbox. This does **not** include the actual execution for which the gas meter + // is responsible. This is achieved by generating all code to the `deploy` function + // which is in the wasm module but not executed on `call`. + // The results are supposed to be used as `call_with_code_kb(c) - call_with_code_kb(0)`. + call_with_code_per_byte { + let c in 0 .. T::Schedule::get().limits.code_len; + let instance = Contract::::with_caller( + whitelisted_caller(), WasmModule::sized(c, Location::Deploy), vec![], + )?; + let value = T::Currency::minimum_balance(); + let origin = RawOrigin::Signed(instance.caller.clone()); + let callee = instance.addr.clone(); + }: call(origin, callee, value, Weight::MAX, None, vec![]) // This constructs a contract that is maximal expensive to instrument. // It creates a maximum number of metering blocks per byte. // The size of the salt influences the runtime because is is hashed in order to - // determine the contract address. + // determine the contract address. All code is generated to the `call` function so that + // we don't benchmark the actual execution of this code but merely what it takes to load + // a code of that size into the sandbox. + // // `c`: Size of the code in kilobytes. // `s`: Size of the salt in kilobytes. // @@ -257,21 +271,27 @@ benchmarks! { // We cannot let `c` grow to the maximum code size because the code is not allowed // to be larger than the maximum size **after instrumentation**. instantiate_with_code { - let c in 0 .. Perbill::from_percent(50).mul_ceil(T::Schedule::get().limits.code_len / 1024); - let s in 0 .. code::max_pages::() * 64; - let salt = vec![42u8; (s * 1024) as usize]; - let endowment = contract_funding::() / 3u32.into(); + let c in 0 .. Perbill::from_percent(49).mul_ceil(T::Schedule::get().limits.code_len); + let s in 0 .. code::max_pages::() * 64 * 1024; + let salt = vec![42u8; s as usize]; + let value = T::Currency::minimum_balance(); let caller = whitelisted_caller(); T::Currency::make_free_balance_be(&caller, caller_funding::()); - let WasmModule { code, hash, .. } = WasmModule::::sized(c * 1024); + let WasmModule { code, hash, .. } = WasmModule::::sized(c, Location::Call); let origin = RawOrigin::Signed(caller.clone()); let addr = Contracts::::contract_address(&caller, &hash, &salt); - }: _(origin, endowment, Weight::max_value(), code, vec![], salt) + }: _(origin, value, Weight::MAX, None, code, vec![], salt) verify { - // endowment was removed from the caller - assert_eq!(T::Currency::free_balance(&caller), caller_funding::() - endowment); - // contract has the full endowment - assert_eq!(T::Currency::free_balance(&addr), endowment); + // the contract itself does not trigger any reserves + let deposit = T::Currency::reserved_balance(&addr); + // uploading the code reserves some balance in the callers account + let code_deposit = T::Currency::reserved_balance(&caller); + assert_eq!( + T::Currency::free_balance(&caller), + caller_funding::() - value - deposit - code_deposit, + ); + // contract has the full value + assert_eq!(T::Currency::free_balance(&addr), value); // instantiate should leave a contract Contract::::address_info(&addr)?; } @@ -279,21 +299,23 @@ benchmarks! { // Instantiate uses a dummy contract constructor to measure the overhead of the instantiate. // `s`: Size of the salt in kilobytes. instantiate { - let s in 0 .. code::max_pages::() * 64; - let salt = vec![42u8; (s * 1024) as usize]; - let endowment = contract_funding::() / 3u32.into(); + let s in 0 .. code::max_pages::() * 64 * 1024; + let salt = vec![42u8; s as usize]; + let value = T::Currency::minimum_balance(); let caller = whitelisted_caller(); T::Currency::make_free_balance_be(&caller, caller_funding::()); let WasmModule { code, hash, .. } = WasmModule::::dummy(); let origin = RawOrigin::Signed(caller.clone()); let addr = Contracts::::contract_address(&caller, &hash, &salt); - Contracts::::store_code_raw(code)?; - }: _(origin, endowment, Weight::max_value(), hash, vec![], salt) + Contracts::::store_code_raw(code, caller.clone())?; + }: _(origin, value, Weight::MAX, None, hash, vec![], salt) verify { - // endowment was removed from the caller - assert_eq!(T::Currency::free_balance(&caller), caller_funding::() - endowment); - // contract has the full endowment - assert_eq!(T::Currency::free_balance(&addr), endowment); + // the contract itself does not trigger any reserves + let deposit = T::Currency::reserved_balance(&addr); + // value was removed from the caller + assert_eq!(T::Currency::free_balance(&caller), caller_funding::() - value - deposit); + // contract has the full value + assert_eq!(T::Currency::free_balance(&addr), value); // instantiate should leave a contract Contract::::address_info(&addr)?; } @@ -302,7 +324,9 @@ benchmarks! { // The size of the data has no influence on the costs of this extrinsic as long as the contract // won't call `seal_input` in its constructor to copy the data to contract memory. // The dummy contract used here does not do this. The costs for the data copy is billed as - // part of `seal_input`. + // part of `seal_input`. The costs for invoking a contract of a specific size are not part + // of this benchmark because we cannot know the size of the contract when issuing a call + // transaction. See `invoke_per_code_kb` for this. call { let data = vec![42u8; 1024]; let instance = Contract::::with_caller( @@ -312,12 +336,14 @@ benchmarks! { let origin = RawOrigin::Signed(instance.caller.clone()); let callee = instance.addr.clone(); let before = T::Currency::free_balance(&instance.account_id); - }: _(origin, callee, value, Weight::max_value(), data) + }: _(origin, callee, value, Weight::MAX, None, data) verify { - // endowment and value transfered via call should be removed from the caller + // the contract itself does not trigger any reserves + let deposit = T::Currency::reserved_balance(&instance.account_id); + // value and value transfered via call should be removed from the caller assert_eq!( T::Currency::free_balance(&instance.caller), - caller_funding::() - instance.endowment - value, + caller_funding::() - instance.value - value - deposit, ); // contract should have received the value assert_eq!(T::Currency::free_balance(&instance.account_id), before + value); @@ -325,77 +351,219 @@ benchmarks! { instance.info()?; } + // This constructs a contract that is maximal expensive to instrument. + // It creates a maximum number of metering blocks per byte. + // `c`: Size of the code in kilobytes. + // + // # Note + // + // We cannot let `c` grow to the maximum code size because the code is not allowed + // to be larger than the maximum size **after instrumentation**. + upload_code { + let c in 0 .. Perbill::from_percent(50).mul_ceil(T::Schedule::get().limits.code_len); + let caller = whitelisted_caller(); + T::Currency::make_free_balance_be(&caller, caller_funding::()); + let WasmModule { code, hash, .. } = WasmModule::::sized(c, Location::Call); + let origin = RawOrigin::Signed(caller.clone()); + }: _(origin, code, None) + verify { + // uploading the code reserves some balance in the callers account + assert!(T::Currency::reserved_balance(&caller) > 0u32.into()); + assert!(>::code_exists(&hash)); + } + + // Removing code does not depend on the size of the contract because all the information + // needed to verify the removal claim (refcount, owner) is stored in a separate storage + // item (`OwnerInfoOf`). + remove_code { + let caller = whitelisted_caller(); + T::Currency::make_free_balance_be(&caller, caller_funding::()); + let WasmModule { code, hash, .. } = WasmModule::::dummy(); + let origin = RawOrigin::Signed(caller.clone()); + let uploaded = >::bare_upload_code(caller.clone(), code, None)?; + assert_eq!(uploaded.code_hash, hash); + assert_eq!(uploaded.deposit, T::Currency::reserved_balance(&caller)); + assert!(>::code_exists(&hash)); + }: _(origin, hash) + verify { + // removing the code should have unreserved the deposit + assert_eq!(T::Currency::reserved_balance(&caller), 0u32.into()); + assert!(>::code_removed(&hash)); + } + seal_caller { let r in 0 .. API_BENCHMARK_BATCHES; let instance = Contract::::new(WasmModule::getter( - "seal_caller", r * API_BENCHMARK_BATCH_SIZE + "seal0", "seal_caller", r * API_BENCHMARK_BATCH_SIZE ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) - seal_address { + seal_is_contract { + let r in 0 .. API_BENCHMARK_BATCHES; + let accounts = (0 .. r * API_BENCHMARK_BATCH_SIZE) + .map(|n| account::("account", n, 0)) + .collect::>(); + let account_len = accounts.get(0).map(|i| i.encode().len()).unwrap_or(0); + let accounts_bytes = accounts.iter().map(|a| a.encode()).flatten().collect::>(); + let code = WasmModule::::from(ModuleDefinition { + memory: Some(ImportedMemory::max::()), + imported_functions: vec![ImportedFunction { + module: "seal0", + name: "seal_is_contract", + params: vec![ValueType::I32], + return_type: Some(ValueType::I32), + }], + data_segments: vec![ + DataSegment { + offset: 0, + value: accounts_bytes + }, + ], + call_body: Some(body::repeated_dyn(r * API_BENCHMARK_BATCH_SIZE, vec![ + Counter(0, account_len as u32), // address_ptr + Regular(Instruction::Call(0)), + Regular(Instruction::Drop), + ])), + .. Default::default() + }); + let instance = Contract::::new(code, vec![])?; + let info = instance.info()?; + // every account would be a contract (worst case) + for acc in accounts.iter() { + >::insert(acc, info.clone()); + } + let origin = RawOrigin::Signed(instance.caller.clone()); + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) + + seal_code_hash { + let r in 0 .. API_BENCHMARK_BATCHES; + let accounts = (0 .. r * API_BENCHMARK_BATCH_SIZE) + .map(|n| account::("account", n, 0)) + .collect::>(); + let account_len = accounts.get(0).map(|i| i.encode().len()).unwrap_or(0); + let accounts_bytes = accounts.iter().map(|a| a.encode()).flatten().collect::>(); + let accounts_len = accounts_bytes.len(); + let pages = code::max_pages::(); + let code = WasmModule::::from(ModuleDefinition { + memory: Some(ImportedMemory::max::()), + imported_functions: vec![ImportedFunction { + module: "__unstable__", + name: "seal_code_hash", + params: vec![ValueType::I32, ValueType::I32, ValueType::I32], + return_type: Some(ValueType::I32), + }], + data_segments: vec![ + DataSegment { + offset: 0, + value: 32u32.to_le_bytes().to_vec(), // output length + }, + DataSegment { + offset: 36, + value: accounts_bytes, + }, + ], + call_body: Some(body::repeated_dyn(r * API_BENCHMARK_BATCH_SIZE, vec![ + Counter(36, account_len as u32), // address_ptr + Regular(Instruction::I32Const(4)), // ptr to output data + Regular(Instruction::I32Const(0)), // ptr to output length + Regular(Instruction::Call(0)), + Regular(Instruction::Drop), + ])), + .. Default::default() + }); + let instance = Contract::::new(code, vec![])?; + let info = instance.info()?; + // every account would be a contract (worst case) + for acc in accounts.iter() { + >::insert(acc, info.clone()); + } + let origin = RawOrigin::Signed(instance.caller.clone()); + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) + + seal_own_code_hash { let r in 0 .. API_BENCHMARK_BATCHES; let instance = Contract::::new(WasmModule::getter( - "seal_address", r * API_BENCHMARK_BATCH_SIZE + "__unstable__", "seal_own_code_hash", r * API_BENCHMARK_BATCH_SIZE ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) - seal_gas_left { + seal_caller_is_origin { + let r in 0 .. API_BENCHMARK_BATCHES; + let code = WasmModule::::from(ModuleDefinition { + memory: Some(ImportedMemory::max::()), + imported_functions: vec![ImportedFunction { + module: "seal0", + name: "seal_caller_is_origin", + params: vec![], + return_type: Some(ValueType::I32), + }], + call_body: Some(body::repeated(r * API_BENCHMARK_BATCH_SIZE, &[ + Instruction::Call(0), + Instruction::Drop, + ])), + .. Default::default() + }); + let instance = Contract::::new(code, vec![])?; + let origin = RawOrigin::Signed(instance.caller.clone()); + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) + + seal_address { let r in 0 .. API_BENCHMARK_BATCHES; let instance = Contract::::new(WasmModule::getter( - "seal_gas_left", r * API_BENCHMARK_BATCH_SIZE + "seal0", "seal_address", r * API_BENCHMARK_BATCH_SIZE ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) - seal_balance { + seal_gas_left { let r in 0 .. API_BENCHMARK_BATCHES; let instance = Contract::::new(WasmModule::getter( - "seal_balance", r * API_BENCHMARK_BATCH_SIZE + "seal0", "seal_gas_left", r * API_BENCHMARK_BATCH_SIZE ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) - seal_value_transferred { + seal_balance { let r in 0 .. API_BENCHMARK_BATCHES; let instance = Contract::::new(WasmModule::getter( - "seal_value_transferred", r * API_BENCHMARK_BATCH_SIZE + "seal0", "seal_balance", r * API_BENCHMARK_BATCH_SIZE ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) - seal_minimum_balance { + seal_value_transferred { let r in 0 .. API_BENCHMARK_BATCHES; let instance = Contract::::new(WasmModule::getter( - "seal_minimum_balance", r * API_BENCHMARK_BATCH_SIZE + "seal0", "seal_value_transferred", r * API_BENCHMARK_BATCH_SIZE ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) - seal_tombstone_deposit { + seal_minimum_balance { let r in 0 .. API_BENCHMARK_BATCHES; let instance = Contract::::new(WasmModule::getter( - "seal_tombstone_deposit", r * API_BENCHMARK_BATCH_SIZE + "seal0", "seal_minimum_balance", r * API_BENCHMARK_BATCH_SIZE ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) seal_block_number { let r in 0 .. API_BENCHMARK_BATCHES; let instance = Contract::::new(WasmModule::getter( - "seal_block_number", r * API_BENCHMARK_BATCH_SIZE + "seal0", "seal_block_number", r * API_BENCHMARK_BATCH_SIZE ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) seal_now { let r in 0 .. API_BENCHMARK_BATCHES; let instance = Contract::::new(WasmModule::getter( - "seal_now", r * API_BENCHMARK_BATCH_SIZE + "seal0", "seal_now", r * API_BENCHMARK_BATCH_SIZE ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) seal_weight_to_fee { let r in 0 .. API_BENCHMARK_BATCHES; @@ -422,7 +590,7 @@ benchmarks! { }); let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) seal_gas { let r in 0 .. API_BENCHMARK_BATCHES; @@ -442,7 +610,7 @@ benchmarks! { let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) seal_input { let r in 0 .. API_BENCHMARK_BATCHES; @@ -469,7 +637,7 @@ benchmarks! { }); let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) seal_input_per_kb { let n in 0 .. code::max_pages::() * 64; @@ -499,7 +667,7 @@ benchmarks! { let instance = Contract::::new(code, vec![])?; let data = vec![42u8; (n * 1024).min(buffer_size) as usize]; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), data) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, data) // We cannot call `seal_return` multiple times. Therefore our weight determination is not // as precise as with other APIs. Because this function can only be called once per @@ -524,7 +692,7 @@ benchmarks! { }); let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) seal_return_per_kb { let n in 0 .. code::max_pages::() * 64; @@ -547,7 +715,7 @@ benchmarks! { }); let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // The same argument as for `seal_return` is true here. seal_terminate { @@ -579,12 +747,13 @@ benchmarks! { let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); assert_eq!(T::Currency::total_balance(&beneficiary), 0u32.into()); - assert_eq!(T::Currency::total_balance(&instance.account_id), contract_funding::()); - }: call(origin, instance.addr.clone(), 0u32.into(), Weight::max_value(), vec![]) + assert_eq!(T::Currency::free_balance(&instance.account_id), T::Currency::minimum_balance()); + assert_ne!(T::Currency::reserved_balance(&instance.account_id), 0u32.into()); + }: call(origin, instance.addr.clone(), 0u32.into(), Weight::MAX, None, vec![]) verify { if r > 0 { assert_eq!(T::Currency::total_balance(&instance.account_id), 0u32.into()); - assert_eq!(T::Currency::total_balance(&beneficiary), contract_funding::()); + assert_eq!(T::Currency::total_balance(&beneficiary), T::Currency::minimum_balance()); } } @@ -621,7 +790,7 @@ benchmarks! { }); let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // Overhead of calling the function without any topic. // We benchmark for the worst case (largest event). @@ -646,7 +815,7 @@ benchmarks! { }); let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // Benchmark the overhead that topics generate. // `t`: Number of topics @@ -684,7 +853,7 @@ benchmarks! { }); let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // The size of the supplied message does not influence the weight because as it is never // processed during on-chain execution: It is only ever read during debugging which happens @@ -710,93 +879,364 @@ benchmarks! { }); let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) + + // Only the overhead of calling the function itself with minimal arguments. + // The contract is a bit more complex because I needs to use different keys in order + // to generate unique storage accesses. However, it is still dominated by the storage + // accesses. + #[skip_meta] + seal_set_storage { + let r in 0 .. API_BENCHMARK_BATCHES; + let keys = (0 .. r * API_BENCHMARK_BATCH_SIZE) + .map(|n| T::Hashing::hash_of(&n).as_ref().to_vec()) + .collect::>(); + let key_len = keys.get(0).map(|i| i.len() as u32).unwrap_or(0); + let key_bytes = keys.iter().flatten().cloned().collect::>(); + let code = WasmModule::::from(ModuleDefinition { + memory: Some(ImportedMemory::max::()), + imported_functions: vec![ImportedFunction { + module: "__unstable__", + name: "seal_set_storage", + params: vec![ValueType::I32, ValueType::I32, ValueType::I32], + return_type: Some(ValueType::I32), + }], + data_segments: vec![ + DataSegment { + offset: 0, + value: key_bytes, + }, + ], + call_body: Some(body::repeated_dyn(r * API_BENCHMARK_BATCH_SIZE, vec![ + Counter(0, key_len as u32), // key_ptr + Regular(Instruction::I32Const(0)), // value_ptr + Regular(Instruction::I32Const(0)), // value_len + Regular(Instruction::Call(0)), + Regular(Instruction::Drop), + ])), + .. Default::default() + }); + let instance = Contract::::new(code, vec![])?; + let info = instance.info()?; + for key in keys { + Storage::::write( + &info.trie_id, + key.as_slice().try_into().map_err(|e| "Key has wrong length")?, + Some(vec![]), + None, + false, + ) + .map_err(|_| "Failed to write to storage during setup.")?; + } + let origin = RawOrigin::Signed(instance.caller.clone()); + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) + + #[skip_meta] + seal_set_storage_per_new_kb { + let n in 0 .. T::Schedule::get().limits.payload_len / 1024; + let keys = (0 .. API_BENCHMARK_BATCH_SIZE) + .map(|n| T::Hashing::hash_of(&n).as_ref().to_vec()) + .collect::>(); + let key_len = keys.get(0).map(|i| i.len() as u32).unwrap_or(0); + let key_bytes = keys.iter().flatten().cloned().collect::>(); + let code = WasmModule::::from(ModuleDefinition { + memory: Some(ImportedMemory::max::()), + imported_functions: vec![ImportedFunction { + module: "__unstable__", + name: "seal_set_storage", + params: vec![ValueType::I32, ValueType::I32, ValueType::I32], + return_type: Some(ValueType::I32), + }], + data_segments: vec![ + DataSegment { + offset: 0, + value: key_bytes, + }, + ], + call_body: Some(body::repeated_dyn(API_BENCHMARK_BATCH_SIZE, vec![ + Counter(0, key_len as u32), // key_ptr + Regular(Instruction::I32Const(0)), // value_ptr + Regular(Instruction::I32Const((n * 1024) as i32)), // value_len + Regular(Instruction::Call(0)), + Regular(Instruction::Drop), + ])), + .. Default::default() + }); + let instance = Contract::::new(code, vec![])?; + let info = instance.info()?; + for key in keys { + Storage::::write( + &info.trie_id, + key.as_slice().try_into().map_err(|e| "Key has wrong length")?, + Some(vec![]), + None, + false, + ) + .map_err(|_| "Failed to write to storage during setup.")?; + } + let origin = RawOrigin::Signed(instance.caller.clone()); + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) + + #[skip_meta] + seal_set_storage_per_old_kb { + let n in 0 .. T::Schedule::get().limits.payload_len / 1024; + let keys = (0 .. API_BENCHMARK_BATCH_SIZE) + .map(|n| T::Hashing::hash_of(&n).as_ref().to_vec()) + .collect::>(); + let key_len = keys.get(0).map(|i| i.len() as u32).unwrap_or(0); + let key_bytes = keys.iter().flatten().cloned().collect::>(); + let code = WasmModule::::from(ModuleDefinition { + memory: Some(ImportedMemory::max::()), + imported_functions: vec![ImportedFunction { + module: "__unstable__", + name: "seal_set_storage", + params: vec![ValueType::I32, ValueType::I32, ValueType::I32], + return_type: Some(ValueType::I32), + }], + data_segments: vec![ + DataSegment { + offset: 0, + value: key_bytes, + }, + ], + call_body: Some(body::repeated_dyn(API_BENCHMARK_BATCH_SIZE, vec![ + Counter(0, key_len as u32), // key_ptr + Regular(Instruction::I32Const(0)), // value_ptr + Regular(Instruction::I32Const(0)), // value_len + Regular(Instruction::Call(0)), + Regular(Instruction::Drop), + ])), + .. Default::default() + }); + let instance = Contract::::new(code, vec![])?; + let info = instance.info()?; + for key in keys { + Storage::::write( + &info.trie_id, + key.as_slice().try_into().map_err(|e| "Key has wrong length")?, + Some(vec![42u8; (n * 1024) as usize]), + None, + false, + ) + .map_err(|_| "Failed to write to storage during setup.")?; + } + let origin = RawOrigin::Signed(instance.caller.clone()); + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) + + // Similar to seal_set_storage. However, we store all the keys that we are about to + // delete beforehand in order to prevent any optimizations that could occur when + // deleting a non existing key. + #[skip_meta] + seal_clear_storage { + let r in 0 .. API_BENCHMARK_BATCHES; + let keys = (0 .. r * API_BENCHMARK_BATCH_SIZE) + .map(|n| T::Hashing::hash_of(&n).as_ref().to_vec()) + .collect::>(); + let key_bytes = keys.iter().flatten().cloned().collect::>(); + let key_len = keys.get(0).map(|i| i.len() as u32).unwrap_or(0); + let code = WasmModule::::from(ModuleDefinition { + memory: Some(ImportedMemory::max::()), + imported_functions: vec![ImportedFunction { + module: "__unstable__", + name: "seal_clear_storage", + params: vec![ValueType::I32], + return_type: Some(ValueType::I32), + }], + data_segments: vec![ + DataSegment { + offset: 0, + value: key_bytes, + }, + ], + call_body: Some(body::repeated_dyn(r * API_BENCHMARK_BATCH_SIZE, vec![ + Counter(0, key_len as u32), + Regular(Instruction::Call(0)), + Regular(Instruction::Drop), + ])), + .. Default::default() + }); + let instance = Contract::::new(code, vec![])?; + let info = instance.info()?; + for key in keys { + Storage::::write( + &info.trie_id, + key.as_slice().try_into().map_err(|e| "Key has wrong length")?, + Some(vec![]), + None, + false, + ) + .map_err(|_| "Failed to write to storage during setup.")?; + } + >::insert(&instance.account_id, info.clone()); + let origin = RawOrigin::Signed(instance.caller.clone()); + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) + + #[skip_meta] + seal_clear_storage_per_kb { + let n in 0 .. T::Schedule::get().limits.payload_len / 1024; + let keys = (0 .. API_BENCHMARK_BATCH_SIZE) + .map(|n| T::Hashing::hash_of(&n).as_ref().to_vec()) + .collect::>(); + let key_len = keys.get(0).map(|i| i.len() as u32).unwrap_or(0); + let key_bytes = keys.iter().flatten().cloned().collect::>(); + let code = WasmModule::::from(ModuleDefinition { + memory: Some(ImportedMemory::max::()), + imported_functions: vec![ImportedFunction { + module: "__unstable__", + name: "seal_clear_storage", + params: vec![ValueType::I32], + return_type: Some(ValueType::I32), + }], + data_segments: vec![ + DataSegment { + offset: 0, + value: key_bytes, + }, + ], + call_body: Some(body::repeated_dyn(API_BENCHMARK_BATCH_SIZE, vec![ + Counter(0, key_len as u32), // key_ptr + Regular(Instruction::Call(0)), + Regular(Instruction::Drop), + ])), + .. Default::default() + }); + let instance = Contract::::new(code, vec![])?; + let info = instance.info()?; + for key in keys { + Storage::::write( + &info.trie_id, + key.as_slice().try_into().map_err(|e| "Key has wrong length")?, + Some(vec![42u8; (n * 1024) as usize]), + None, + false, + ) + .map_err(|_| "Failed to write to storage during setup.")?; + } + let origin = RawOrigin::Signed(instance.caller.clone()); + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) - // Only the overhead of calling the function itself with minimal arguments. - // The contract is a bit more complex because I needs to use different keys in order - // to generate unique storage accesses. However, it is still dominated by the storage - // accesses. + // We make sure that all storage accesses are to unique keys. #[skip_meta] - seal_set_storage { + seal_get_storage { let r in 0 .. API_BENCHMARK_BATCHES; let keys = (0 .. r * API_BENCHMARK_BATCH_SIZE) - .flat_map(|n| T::Hashing::hash_of(&n).as_ref().to_vec()) + .map(|n| T::Hashing::hash_of(&n).as_ref().to_vec()) .collect::>(); - let key_len = sp_std::mem::size_of::<::Output>(); + let key_len = keys.get(0).map(|i| i.len() as u32).unwrap_or(0); + let key_bytes = keys.iter().flatten().cloned().collect::>(); + let key_bytes_len = key_bytes.len(); let code = WasmModule::::from(ModuleDefinition { memory: Some(ImportedMemory::max::()), imported_functions: vec![ImportedFunction { module: "seal0", - name: "seal_set_storage", + name: "seal_get_storage", params: vec![ValueType::I32, ValueType::I32, ValueType::I32], - return_type: None, + return_type: Some(ValueType::I32), }], data_segments: vec![ DataSegment { offset: 0, - value: keys, + value: key_bytes, + }, + DataSegment { + offset: key_bytes_len as u32, + value: T::Schedule::get().limits.payload_len.to_le_bytes().into(), }, ], call_body: Some(body::repeated_dyn(r * API_BENCHMARK_BATCH_SIZE, vec![ Counter(0, key_len as u32), // key_ptr - Regular(Instruction::I32Const(0)), // value_ptr - Regular(Instruction::I32Const(0)), // value_len + Regular(Instruction::I32Const((key_bytes_len + 4) as i32)), // out_ptr + Regular(Instruction::I32Const(key_bytes_len as i32)), // out_len_ptr Regular(Instruction::Call(0)), + Regular(Instruction::Drop), ])), .. Default::default() }); let instance = Contract::::new(code, vec![])?; + let info = instance.info()?; + for key in keys { + Storage::::write( + &info.trie_id, + key.as_slice().try_into().map_err(|e| "Key has wrong length")?, + Some(vec![]), + None, + false, + ) + .map_err(|_| "Failed to write to storage during setup.")?; + } + >::insert(&instance.account_id, info.clone()); let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) - seal_set_storage_per_kb { + #[skip_meta] + seal_get_storage_per_kb { let n in 0 .. T::Schedule::get().limits.payload_len / 1024; - let key = T::Hashing::hash_of(&1u32).as_ref().to_vec(); - let key_len = key.len(); + let keys = (0 .. API_BENCHMARK_BATCH_SIZE) + .map(|n| T::Hashing::hash_of(&n).as_ref().to_vec()) + .collect::>(); + let key_len = keys.get(0).map(|i| i.len() as u32).unwrap_or(0); + let key_bytes = keys.iter().flatten().cloned().collect::>(); + let key_bytes_len = key_bytes.len(); let code = WasmModule::::from(ModuleDefinition { memory: Some(ImportedMemory::max::()), imported_functions: vec![ImportedFunction { module: "seal0", - name: "seal_set_storage", + name: "seal_get_storage", params: vec![ValueType::I32, ValueType::I32, ValueType::I32], - return_type: None, + return_type: Some(ValueType::I32), }], data_segments: vec![ DataSegment { offset: 0, - value: key, + value: key_bytes, + }, + DataSegment { + offset: key_bytes_len as u32, + value: T::Schedule::get().limits.payload_len.to_le_bytes().into(), }, ], - call_body: Some(body::repeated(API_BENCHMARK_BATCH_SIZE, &[ - Instruction::I32Const(0), // key_ptr - Instruction::I32Const(0), // value_ptr - Instruction::I32Const((n * 1024) as i32), // value_len - Instruction::Call(0), + call_body: Some(body::repeated_dyn(API_BENCHMARK_BATCH_SIZE, vec![ + Counter(0, key_len as u32), // key_ptr + Regular(Instruction::I32Const((key_bytes_len + 4) as i32)), // out_ptr + Regular(Instruction::I32Const(key_bytes_len as i32)), // out_len_ptr + Regular(Instruction::Call(0)), + Regular(Instruction::Drop), ])), .. Default::default() }); let instance = Contract::::new(code, vec![])?; + let info = instance.info()?; + for key in keys { + Storage::::write( + &info.trie_id, + key.as_slice().try_into().map_err(|e| "Key has wrong length")?, + Some(vec![42u8; (n * 1024) as usize]), + None, + false, + ) + .map_err(|_| "Failed to write to storage during setup.")?; + } + >::insert(&instance.account_id, info.clone()); let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) - // Similar to seal_set_storage. However, we store all the keys that we are about to - // delete beforehand in order to prevent any optimizations that could occur when - // deleting a non existing key. + // We make sure that all storage accesses are to unique keys. #[skip_meta] - seal_clear_storage { + seal_contains_storage { let r in 0 .. API_BENCHMARK_BATCHES; let keys = (0 .. r * API_BENCHMARK_BATCH_SIZE) .map(|n| T::Hashing::hash_of(&n).as_ref().to_vec()) .collect::>(); + let key_len = keys.get(0).map(|i| i.len() as u32).unwrap_or(0); let key_bytes = keys.iter().flatten().cloned().collect::>(); - let key_len = sp_std::mem::size_of::<::Output>(); + let key_bytes_len = key_bytes.len(); let code = WasmModule::::from(ModuleDefinition { memory: Some(ImportedMemory::max::()), imported_functions: vec![ImportedFunction { - module: "seal0", - name: "seal_clear_storage", + module: "__unstable__", + name: "seal_contains_storage", params: vec![ValueType::I32], - return_type: None, + return_type: Some(ValueType::I32), }], data_segments: vec![ DataSegment { @@ -805,40 +1245,87 @@ benchmarks! { }, ], call_body: Some(body::repeated_dyn(r * API_BENCHMARK_BATCH_SIZE, vec![ - Counter(0, key_len as u32), + Counter(0, key_len as u32), // key_ptr Regular(Instruction::Call(0)), + Regular(Instruction::Drop), ])), .. Default::default() }); let instance = Contract::::new(code, vec![])?; - let mut info = instance.info()?; + let info = instance.info()?; for key in keys { Storage::::write( - &mut info, + &info.trie_id, key.as_slice().try_into().map_err(|e| "Key has wrong length")?, - Some(vec![42; T::Schedule::get().limits.payload_len as usize]) + Some(vec![]), + None, + false, ) .map_err(|_| "Failed to write to storage during setup.")?; } >::insert(&instance.account_id, info.clone()); let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) - // We make sure that all storage accesses are to unique keys. #[skip_meta] - seal_get_storage { + seal_contains_storage_per_kb { + let n in 0 .. T::Schedule::get().limits.payload_len / 1024; + let keys = (0 .. API_BENCHMARK_BATCH_SIZE) + .map(|n| T::Hashing::hash_of(&n).as_ref().to_vec()) + .collect::>(); + let key_len = keys.get(0).map(|i| i.len() as u32).unwrap_or(0); + let key_bytes = keys.iter().flatten().cloned().collect::>(); + let code = WasmModule::::from(ModuleDefinition { + memory: Some(ImportedMemory::max::()), + imported_functions: vec![ImportedFunction { + module: "__unstable__", + name: "seal_contains_storage", + params: vec![ValueType::I32], + return_type: Some(ValueType::I32), + }], + data_segments: vec![ + DataSegment { + offset: 0, + value: key_bytes, + }, + ], + call_body: Some(body::repeated_dyn(API_BENCHMARK_BATCH_SIZE, vec![ + Counter(0, key_len as u32), // key_ptr + Regular(Instruction::Call(0)), + Regular(Instruction::Drop), + ])), + .. Default::default() + }); + let instance = Contract::::new(code, vec![])?; + let info = instance.info()?; + for key in keys { + Storage::::write( + &info.trie_id, + key.as_slice().try_into().map_err(|e| "Key has wrong length")?, + Some(vec![42u8; (n * 1024) as usize]), + None, + false, + ) + .map_err(|_| "Failed to write to storage during setup.")?; + } + >::insert(&instance.account_id, info.clone()); + let origin = RawOrigin::Signed(instance.caller.clone()); + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) + + #[skip_meta] + seal_take_storage { let r in 0 .. API_BENCHMARK_BATCHES; let keys = (0 .. r * API_BENCHMARK_BATCH_SIZE) .map(|n| T::Hashing::hash_of(&n).as_ref().to_vec()) .collect::>(); - let key_len = sp_std::mem::size_of::<::Output>(); + let key_len = keys.get(0).map(|i| i.len() as u32).unwrap_or(0); let key_bytes = keys.iter().flatten().cloned().collect::>(); let key_bytes_len = key_bytes.len(); let code = WasmModule::::from(ModuleDefinition { memory: Some(ImportedMemory::max::()), imported_functions: vec![ImportedFunction { - module: "seal0", - name: "seal_get_storage", + module: "__unstable__", + name: "seal_take_storage", params: vec![ValueType::I32, ValueType::I32, ValueType::I32], return_type: Some(ValueType::I32), }], @@ -847,6 +1334,10 @@ benchmarks! { offset: 0, value: key_bytes, }, + DataSegment { + offset: key_bytes_len as u32, + value: T::Schedule::get().limits.payload_len.to_le_bytes().into(), + }, ], call_body: Some(body::repeated_dyn(r * API_BENCHMARK_BATCH_SIZE, vec![ Counter(0, key_len as u32), // key_ptr @@ -858,62 +1349,72 @@ benchmarks! { .. Default::default() }); let instance = Contract::::new(code, vec![])?; - let mut info = instance.info()?; + let info = instance.info()?; for key in keys { Storage::::write( - &mut info, + &info.trie_id, key.as_slice().try_into().map_err(|e| "Key has wrong length")?, - Some(vec![]) + Some(vec![]), + None, + false, ) .map_err(|_| "Failed to write to storage during setup.")?; } >::insert(&instance.account_id, info.clone()); let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) - seal_get_storage_per_kb { + #[skip_meta] + seal_take_storage_per_kb { let n in 0 .. T::Schedule::get().limits.payload_len / 1024; - let key = T::Hashing::hash_of(&1u32).as_ref().to_vec(); - let key_len = key.len(); + let keys = (0 .. API_BENCHMARK_BATCH_SIZE) + .map(|n| T::Hashing::hash_of(&n).as_ref().to_vec()) + .collect::>(); + let key_len = keys.get(0).map(|i| i.len() as u32).unwrap_or(0); + let key_bytes = keys.iter().flatten().cloned().collect::>(); + let key_bytes_len = key_bytes.len(); let code = WasmModule::::from(ModuleDefinition { memory: Some(ImportedMemory::max::()), imported_functions: vec![ImportedFunction { - module: "seal0", - name: "seal_get_storage", + module: "__unstable__", + name: "seal_take_storage", params: vec![ValueType::I32, ValueType::I32, ValueType::I32], return_type: Some(ValueType::I32), }], data_segments: vec![ DataSegment { offset: 0, - value: key.clone(), + value: key_bytes, }, DataSegment { - offset: key_len as u32, + offset: key_bytes_len as u32, value: T::Schedule::get().limits.payload_len.to_le_bytes().into(), }, ], - call_body: Some(body::repeated(API_BENCHMARK_BATCH_SIZE, &[ - // call at key_ptr - Instruction::I32Const(0), // key_ptr - Instruction::I32Const((key_len + 4) as i32), // out_ptr - Instruction::I32Const(key_len as i32), // out_len_ptr - Instruction::Call(0), - Instruction::Drop, + call_body: Some(body::repeated_dyn(API_BENCHMARK_BATCH_SIZE, vec![ + Counter(0, key_len as u32), // key_ptr + Regular(Instruction::I32Const((key_bytes_len + 4) as i32)), // out_ptr + Regular(Instruction::I32Const(key_bytes_len as i32)), // out_len_ptr + Regular(Instruction::Call(0)), + Regular(Instruction::Drop), ])), .. Default::default() }); let instance = Contract::::new(code, vec![])?; - let mut info = instance.info()?; - Storage::::write( - &mut info, - key.as_slice().try_into().map_err(|e| "Key has wrong length")?, - Some(vec![42u8; (n * 1024) as usize]) - ) - .map_err(|_| "Failed to write to storage during setup.")?; + let info = instance.info()?; + for key in keys { + Storage::::write( + &info.trie_id, + key.as_slice().try_into().map_err(|e| "Key has wrong length")?, + Some(vec![42u8; (n * 1024) as usize]), + None, + false, + ) + .map_err(|_| "Failed to write to storage during setup.")?; + } >::insert(&instance.account_id, info.clone()); let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // We transfer to unique accounts. seal_transfer { @@ -923,7 +1424,7 @@ benchmarks! { .collect::>(); let account_len = accounts.get(0).map(|i| i.encode().len()).unwrap_or(0); let account_bytes = accounts.iter().flat_map(|x| x.encode()).collect(); - let value = Contracts::::subsistence_threshold(); + let value = T::Currency::minimum_balance(); assert!(value > 0u32.into()); let value_bytes = value.encode(); let value_len = value_bytes.len(); @@ -956,11 +1457,12 @@ benchmarks! { .. Default::default() }); let instance = Contract::::new(code, vec![])?; + instance.set_balance(value * (r * API_BENCHMARK_BATCH_SIZE + 1).into()); let origin = RawOrigin::Signed(instance.caller.clone()); for account in &accounts { assert_eq!(T::Currency::total_balance(account), 0u32.into()); } - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) verify { for account in &accounts { assert_eq!(T::Currency::total_balance(account), value); @@ -1015,7 +1517,7 @@ benchmarks! { Regular(Instruction::I32Const(value_len as i32)), // value_len Regular(Instruction::I32Const(0)), // input_data_ptr Regular(Instruction::I32Const(0)), // input_data_len - Regular(Instruction::I32Const(u32::max_value() as i32)), // output_ptr + Regular(Instruction::I32Const(SENTINEL as i32)), // output_ptr Regular(Instruction::I32Const(0)), // output_len_ptr Regular(Instruction::Call(0)), Regular(Instruction::Drop), @@ -1024,46 +1526,75 @@ benchmarks! { }); let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) - seal_call_per_transfer_input_output_kb { - let t in 0 .. 1; - let i in 0 .. code::max_pages::() * 64; - let o in 0 .. (code::max_pages::() - 1) * 64; - let callee_code = WasmModule::::from(ModuleDefinition { + seal_delegate_call { + let r in 0 .. API_BENCHMARK_BATCHES; + let hashes = (0..r * API_BENCHMARK_BATCH_SIZE) + .map(|i| { + let code = WasmModule::::dummy_with_bytes(i); + Contracts::::store_code_raw(code.code, whitelisted_caller())?; + Ok(code.hash) + }) + .collect::, &'static str>>()?; + let hash_len = hashes.get(0).map(|x| x.encode().len()).unwrap_or(0); + let hashes_bytes = hashes.iter().flat_map(|x| x.encode()).collect::>(); + let hashes_len = hashes_bytes.len(); + let hashes_offset = 0; + + let code = WasmModule::::from(ModuleDefinition { memory: Some(ImportedMemory::max::()), imported_functions: vec![ImportedFunction { module: "seal0", - name: "seal_return", + name: "seal_delegate_call", params: vec![ ValueType::I32, ValueType::I32, ValueType::I32, + ValueType::I32, + ValueType::I32, + ValueType::I32, ], - return_type: None, + return_type: Some(ValueType::I32), }], - call_body: Some(body::plain(vec![ - Instruction::I32Const(0), // flags - Instruction::I32Const(0), // data_ptr - Instruction::I32Const((o * 1024) as i32), // data_len - Instruction::Call(0), - Instruction::End, + data_segments: vec![ + DataSegment { + offset: hashes_offset as u32, + value: hashes_bytes, + }, + ], + call_body: Some(body::repeated_dyn(r * API_BENCHMARK_BATCH_SIZE, vec![ + Regular(Instruction::I32Const(0)), // flags + Counter(hashes_offset as u32, hash_len as u32), // code_hash_ptr + Regular(Instruction::I32Const(0)), // input_data_ptr + Regular(Instruction::I32Const(0)), // input_data_len + Regular(Instruction::I32Const(u32::max_value() as i32)), // output_ptr + Regular(Instruction::I32Const(0)), // output_len_ptr + Regular(Instruction::Call(0)), + Regular(Instruction::Drop), ])), .. Default::default() }); + let instance = Contract::::new(code, vec![])?; + let callee = instance.addr.clone(); + let origin = RawOrigin::Signed(instance.caller.clone()); + }: call(origin, callee, 0u32.into(), Weight::MAX, None, vec![]) + + seal_call_per_transfer_clone_kb { + let t in 0 .. 1; + let c in 0 .. code::max_pages::() * 64; let callees = (0..API_BENCHMARK_BATCH_SIZE) - .map(|i| Contract::with_index(i + 1, callee_code.clone(), vec![])) + .map(|i| Contract::with_index(i + 1, >::dummy(), vec![])) .collect::, _>>()?; let callee_len = callees.get(0).map(|i| i.account_id.encode().len()).unwrap_or(0); let callee_bytes = callees.iter().flat_map(|x| x.account_id.encode()).collect::>(); - let callees_len = callee_bytes.len(); let value: BalanceOf = t.into(); let value_bytes = value.encode(); let value_len = value_bytes.len(); let code = WasmModule::::from(ModuleDefinition { memory: Some(ImportedMemory::max::()), imported_functions: vec![ImportedFunction { - module: "seal0", + module: "seal1", name: "seal_call", params: vec![ ValueType::I32, @@ -1074,7 +1605,6 @@ benchmarks! { ValueType::I32, ValueType::I32, ValueType::I32, - ValueType::I32, ], return_type: Some(ValueType::I32), }], @@ -1087,21 +1617,16 @@ benchmarks! { offset: value_len as u32, value: callee_bytes, }, - DataSegment { - offset: (value_len + callees_len) as u32, - value: (o * 1024).to_le_bytes().into(), - }, ], call_body: Some(body::repeated_dyn(API_BENCHMARK_BATCH_SIZE, vec![ + Regular(Instruction::I32Const(CallFlags::CLONE_INPUT.bits() as i32)), // flags Counter(value_len as u32, callee_len as u32), // callee_ptr - Regular(Instruction::I32Const(callee_len as i32)), // callee_len Regular(Instruction::I64Const(0)), // gas Regular(Instruction::I32Const(0)), // value_ptr - Regular(Instruction::I32Const(value_len as i32)), // value_len Regular(Instruction::I32Const(0)), // input_data_ptr - Regular(Instruction::I32Const((i * 1024) as i32)), // input_data_len - Regular(Instruction::I32Const((value_len + callees_len + 4) as i32)), // output_ptr - Regular(Instruction::I32Const((value_len + callees_len) as i32)), // output_len_ptr + Regular(Instruction::I32Const(0)), // input_data_len + Regular(Instruction::I32Const(SENTINEL as i32)), // output_ptr + Regular(Instruction::I32Const(0)), // output_len_ptr Regular(Instruction::Call(0)), Regular(Instruction::Drop), ])), @@ -1109,9 +1634,10 @@ benchmarks! { }); let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + let bytes = vec![42; (c * 1024) as usize]; + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, bytes) - // We assume that every instantiate sends at least the subsistence amount. + // We assume that every instantiate sends at least the minimum balance. seal_instantiate { let r in 0 .. API_BENCHMARK_BATCHES; let hashes = (0..r * API_BENCHMARK_BATCH_SIZE) @@ -1127,18 +1653,18 @@ benchmarks! { ])), .. Default::default() }); - Contracts::::store_code_raw(code.code)?; + Contracts::::store_code_raw(code.code, whitelisted_caller())?; Ok(code.hash) }) .collect::, &'static str>>()?; let hash_len = hashes.get(0).map(|x| x.encode().len()).unwrap_or(0); let hashes_bytes = hashes.iter().flat_map(|x| x.encode()).collect::>(); let hashes_len = hashes_bytes.len(); - let value = contract_funding::() / (r * API_BENCHMARK_BATCH_SIZE + 2).into(); + let value = T::Currency::minimum_balance(); assert!(value > 0u32.into()); let value_bytes = value.encode(); let value_len = value_bytes.len(); - let addr_len = sp_std::mem::size_of::(); + let addr_len = T::AccountId::max_encoded_len(); // offsets where to place static data in contract memory let value_offset = 0; @@ -1192,7 +1718,7 @@ benchmarks! { Regular(Instruction::I32Const(0)), // input_data_len Regular(Instruction::I32Const(addr_offset as i32)), // address_ptr Regular(Instruction::I32Const(addr_len_offset as i32)), // address_len_ptr - Regular(Instruction::I32Const(u32::max_value() as i32)), // output_ptr + Regular(Instruction::I32Const(SENTINEL as i32)), // output_ptr Regular(Instruction::I32Const(0)), // output_len_ptr Regular(Instruction::I32Const(0)), // salt_ptr Regular(Instruction::I32Const(0)), // salt_ptr_len @@ -1202,6 +1728,7 @@ benchmarks! { .. Default::default() }); let instance = Contract::::new(code, vec![])?; + instance.set_balance(value * (r * API_BENCHMARK_BATCH_SIZE + 1).into()); let origin = RawOrigin::Signed(instance.caller.clone()); let callee = instance.addr.clone(); let addresses = hashes @@ -1216,7 +1743,7 @@ benchmarks! { return Err("Expected that contract does not exist at this point.".into()); } } - }: call(origin, callee, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, callee, 0u32.into(), Weight::MAX, None, vec![]) verify { for addr in &addresses { ContractInfoOf::::get(&addr) @@ -1224,52 +1751,28 @@ benchmarks! { } } - seal_instantiate_per_input_output_salt_kb { - let i in 0 .. (code::max_pages::() - 1) * 64; - let o in 0 .. (code::max_pages::() - 1) * 64; + seal_instantiate_per_transfer_salt_kb { + let t in 0 .. 1; let s in 0 .. (code::max_pages::() - 1) * 64; - let callee_code = WasmModule::::from(ModuleDefinition { - memory: Some(ImportedMemory::max::()), - imported_functions: vec![ImportedFunction { - module: "seal0", - name: "seal_return", - params: vec![ - ValueType::I32, - ValueType::I32, - ValueType::I32, - ], - return_type: None, - }], - deploy_body: Some(body::plain(vec![ - Instruction::I32Const(0), // flags - Instruction::I32Const(0), // data_ptr - Instruction::I32Const((o * 1024) as i32), // data_len - Instruction::Call(0), - Instruction::End, - ])), - .. Default::default() - }); + let callee_code = WasmModule::::dummy(); let hash = callee_code.hash.clone(); let hash_bytes = callee_code.hash.encode(); let hash_len = hash_bytes.len(); - Contracts::::store_code_raw(callee_code.code)?; - let inputs = (0..API_BENCHMARK_BATCH_SIZE).map(|x| x.encode()).collect::>(); - let input_len = inputs.get(0).map(|x| x.len()).unwrap_or(0); - let input_bytes = inputs.iter().cloned().flatten().collect::>(); - let inputs_len = input_bytes.len(); - let value = contract_funding::() / (API_BENCHMARK_BATCH_SIZE + 2).into(); - assert!(value > 0u32.into()); + Contracts::::store_code_raw(callee_code.code, whitelisted_caller())?; + let salts = (0..API_BENCHMARK_BATCH_SIZE).map(|x| x.encode()).collect::>(); + let salt_len = salts.get(0).map(|x| x.len()).unwrap_or(0); + let salt_bytes = salts.iter().cloned().flatten().collect::>(); + let salts_len = salt_bytes.len(); + let value: BalanceOf = t.into(); let value_bytes = value.encode(); let value_len = value_bytes.len(); - let addr_len = sp_std::mem::size_of::(); + let addr_len = T::AccountId::max_encoded_len(); // offsets where to place static data in contract memory - let input_offset = 0; - let value_offset = inputs_len; + let salt_offset = 0; + let value_offset = salts_len; let hash_offset = value_offset + value_len; let addr_len_offset = hash_offset + hash_len; - let output_len_offset = addr_len_offset + 4; - let output_offset = output_len_offset + 4; let code = WasmModule::::from(ModuleDefinition { memory: Some(ImportedMemory::max::()), @@ -1295,8 +1798,8 @@ benchmarks! { }], data_segments: vec![ DataSegment { - offset: input_offset as u32, - value: input_bytes, + offset: salt_offset as u32, + value: salt_bytes, }, DataSegment { offset: value_offset as u32, @@ -1310,10 +1813,6 @@ benchmarks! { offset: addr_len_offset as u32, value: (addr_len as u32).to_le_bytes().into(), }, - DataSegment { - offset: output_len_offset as u32, - value: (o * 1024).to_le_bytes().into(), - }, ], call_body: Some(body::repeated_dyn(API_BENCHMARK_BATCH_SIZE, vec![ Regular(Instruction::I32Const(hash_offset as i32)), // code_hash_ptr @@ -1321,14 +1820,14 @@ benchmarks! { Regular(Instruction::I64Const(0)), // gas Regular(Instruction::I32Const(value_offset as i32)), // value_ptr Regular(Instruction::I32Const(value_len as i32)), // value_len - Counter(input_offset as u32, input_len as u32), // input_data_ptr - Regular(Instruction::I32Const((i * 1024).max(input_len as u32) as i32)), // input_data_len + Regular(Instruction::I32Const(0)), // input_data_ptr + Regular(Instruction::I32Const(0)), // input_data_len Regular(Instruction::I32Const((addr_len_offset + addr_len) as i32)), // address_ptr Regular(Instruction::I32Const(addr_len_offset as i32)), // address_len_ptr - Regular(Instruction::I32Const(output_offset as i32)), // output_ptr - Regular(Instruction::I32Const(output_len_offset as i32)), // output_len_ptr - Counter(input_offset as u32, input_len as u32), // salt_ptr - Regular(Instruction::I32Const((s * 1024).max(input_len as u32) as i32)), // salt_len + Regular(Instruction::I32Const(SENTINEL as i32)), // output_ptr + Regular(Instruction::I32Const(0)), // output_len_ptr + Counter(salt_offset as u32, salt_len as u32), // salt_ptr + Regular(Instruction::I32Const((s * 1024).max(salt_len as u32) as i32)), // salt_len Regular(Instruction::Call(0)), Regular(Instruction::I32Eqz), Regular(Instruction::If(BlockType::NoResult)), @@ -1340,8 +1839,9 @@ benchmarks! { .. Default::default() }); let instance = Contract::::new(code, vec![])?; + instance.set_balance(value * (API_BENCHMARK_BATCH_SIZE + 1).into()); let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // Only the overhead of calling the function itself with minimal arguments. seal_hash_sha2_256 { @@ -1350,7 +1850,7 @@ benchmarks! { "seal_hash_sha2_256", r * API_BENCHMARK_BATCH_SIZE, 0, ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // `n`: Input to hash in kilobytes seal_hash_sha2_256_per_kb { @@ -1359,7 +1859,7 @@ benchmarks! { "seal_hash_sha2_256", API_BENCHMARK_BATCH_SIZE, n * 1024, ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // Only the overhead of calling the function itself with minimal arguments. seal_hash_keccak_256 { @@ -1368,7 +1868,7 @@ benchmarks! { "seal_hash_keccak_256", r * API_BENCHMARK_BATCH_SIZE, 0, ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // `n`: Input to hash in kilobytes seal_hash_keccak_256_per_kb { @@ -1377,7 +1877,7 @@ benchmarks! { "seal_hash_keccak_256", API_BENCHMARK_BATCH_SIZE, n * 1024, ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // Only the overhead of calling the function itself with minimal arguments. seal_hash_blake2_256 { @@ -1386,7 +1886,7 @@ benchmarks! { "seal_hash_blake2_256", r * API_BENCHMARK_BATCH_SIZE, 0, ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // `n`: Input to hash in kilobytes seal_hash_blake2_256_per_kb { @@ -1395,7 +1895,7 @@ benchmarks! { "seal_hash_blake2_256", API_BENCHMARK_BATCH_SIZE, n * 1024, ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // Only the overhead of calling the function itself with minimal arguments. seal_hash_blake2_128 { @@ -1404,7 +1904,7 @@ benchmarks! { "seal_hash_blake2_128", r * API_BENCHMARK_BATCH_SIZE, 0, ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // `n`: Input to hash in kilobytes seal_hash_blake2_128_per_kb { @@ -1413,26 +1913,20 @@ benchmarks! { "seal_hash_blake2_128", API_BENCHMARK_BATCH_SIZE, n * 1024, ), vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // Only calling the function itself with valid arguments. // It generates different private keys and signatures for the message "Hello world". seal_ecdsa_recover { let r in 0 .. API_BENCHMARK_BATCHES; - use rand::SeedableRng; - let mut rng = rand_pcg::Pcg32::seed_from_u64(123456); let message_hash = sp_io::hashing::blake2_256("Hello world".as_bytes()); + let key_type = sp_core::crypto::KeyTypeId(*b"code"); let signatures = (0..r * API_BENCHMARK_BATCH_SIZE) .map(|i| { - use secp256k1::{SecretKey, Message, sign}; - - let private_key = SecretKey::random(&mut rng); - let (signature, recovery_id) = sign(&Message::parse(&message_hash), &private_key); - let mut full_signature = [0; 65]; - full_signature[..64].copy_from_slice(&signature.serialize()); - full_signature[64] = recovery_id.serialize(); - full_signature + let pub_key = sp_io::crypto::ecdsa_generate(key_type, None); + let sig = sp_io::crypto::ecdsa_sign_prehashed(key_type, &pub_key, &message_hash).expect("Generates signature"); + AsRef::<[u8; 65]>::as_ref(&sig).to_vec() }) .collect::>(); let signatures = signatures.iter().flatten().cloned().collect::>(); @@ -1467,7 +1961,47 @@ benchmarks! { }); let instance = Contract::::new(code, vec![])?; let origin = RawOrigin::Signed(instance.caller.clone()); - }: call(origin, instance.addr, 0u32.into(), Weight::max_value(), vec![]) + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) + + seal_set_code_hash { + let r in 0 .. API_BENCHMARK_BATCHES; + let code_hashes = (0..r * API_BENCHMARK_BATCH_SIZE) + .map(|i| { + let new_code = WasmModule::::dummy_with_bytes(i); + Contracts::::store_code_raw(new_code.code, whitelisted_caller())?; + Ok(new_code.hash) + }) + .collect::, &'static str>>()?; + let code_hash_len = code_hashes.get(0).map(|x| x.encode().len()).unwrap_or(0); + let code_hashes_bytes = code_hashes.iter().flat_map(|x| x.encode()).collect::>(); + let code_hashes_len = code_hashes_bytes.len(); + + let code = WasmModule::::from(ModuleDefinition { + memory: Some(ImportedMemory::max::()), + imported_functions: vec![ImportedFunction { + module: "__unstable__", + name: "seal_set_code_hash", + params: vec![ + ValueType::I32, + ], + return_type: Some(ValueType::I32), + }], + data_segments: vec![ + DataSegment { + offset: 0, + value: code_hashes_bytes, + }, + ], + call_body: Some(body::repeated_dyn(r * API_BENCHMARK_BATCH_SIZE, vec![ + Counter(0, code_hash_len as u32), // code_hash_ptr + Regular(Instruction::Call(0)), + Regular(Instruction::Drop), + ])), + .. Default::default() + }); + let instance = Contract::::new(code, vec![])?; + let origin = RawOrigin::Signed(instance.caller.clone()); + }: call(origin, instance.addr, 0u32.into(), Weight::MAX, None, vec![]) // We make the assumption that pushing a constant and dropping a value takes roughly // the same amount of time. We follow that `t.load` and `drop` both have the weight @@ -1763,7 +2297,7 @@ benchmarks! { // w_local_get = w_bench - 1 * w_param instr_local_get { let r in 0 .. INSTR_BENCHMARK_BATCHES; - let max_locals = T::Schedule::get().limits.stack_height; + let max_locals = T::Schedule::get().limits.stack_height.unwrap_or(512); let mut call_body = body::repeated_dyn(r * INSTR_BENCHMARK_BATCH_SIZE, vec![ RandomGetLocal(0, max_locals), Regular(Instruction::Drop), @@ -1780,7 +2314,7 @@ benchmarks! { // w_local_set = w_bench - 1 * w_param instr_local_set { let r in 0 .. INSTR_BENCHMARK_BATCHES; - let max_locals = T::Schedule::get().limits.stack_height; + let max_locals = T::Schedule::get().limits.stack_height.unwrap_or(512); let mut call_body = body::repeated_dyn(r * INSTR_BENCHMARK_BATCH_SIZE, vec![ RandomI64Repeated(1), RandomSetLocal(0, max_locals), @@ -1797,7 +2331,7 @@ benchmarks! { // w_local_tee = w_bench - 2 * w_param instr_local_tee { let r in 0 .. INSTR_BENCHMARK_BATCHES; - let max_locals = T::Schedule::get().limits.stack_height; + let max_locals = T::Schedule::get().limits.stack_height.unwrap_or(512); let mut call_body = body::repeated_dyn(r * INSTR_BENCHMARK_BATCH_SIZE, vec![ RandomI64Repeated(1), RandomTeeLocal(0, max_locals), @@ -1860,7 +2394,7 @@ benchmarks! { } // w_memory_grow = w_bench - 2 * w_param - // We can only allow allocate as much memory as it is allowed in a a contract. + // We can only allow allocate as much memory as it is allowed in a contract. // Therefore the repeat count is limited by the maximum memory any contract can have. // Using a contract with more memory will skew the benchmark because the runtime of grow // depends on how much memory is already allocated. @@ -2221,7 +2755,7 @@ benchmarks! { // configured `Schedule` during benchmark development. // It can be outputed using the following command: // cargo run --manifest-path=bin/node/cli/Cargo.toml --release \ - // --features runtime-benchmarks -- benchmark --dev --execution=native \ + // --features runtime-benchmarks -- benchmark --extra --dev --execution=native \ // -p pallet_contracts -e print_schedule --no-median-slopes --no-min-squares #[extra] print_schedule { @@ -2274,6 +2808,7 @@ benchmarks! { instance.account_id, 0u32.into(), Weight::MAX, + None, data, false, ) @@ -2320,15 +2855,16 @@ benchmarks! { instance.account_id, 0u32.into(), Weight::MAX, + None, data, false, ) .result?; } -} -impl_benchmark_test_suite!( - Contracts, - crate::tests::ExtBuilder::default().build(), - crate::tests::Test, -); + impl_benchmark_test_suite!( + Contracts, + crate::tests::ExtBuilder::default().build(), + crate::tests::Test, + ) +} diff --git a/frame/contracts/src/benchmarking/sandbox.rs b/frame/contracts/src/benchmarking/sandbox.rs index 320ac90cce64..451d2fe43391 100644 --- a/frame/contracts/src/benchmarking/sandbox.rs +++ b/frame/contracts/src/benchmarking/sandbox.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,10 @@ /// ! environment that provides the seal interface as imported functions. use super::{code::WasmModule, Config}; use sp_core::crypto::UncheckedFrom; -use sp_sandbox::{EnvironmentDefinitionBuilder, Instance, Memory}; +use sp_sandbox::{ + default_executor::{EnvironmentDefinitionBuilder, Instance, Memory}, + SandboxEnvironmentBuilder, SandboxInstance, +}; /// Minimal execution environment without any exported functions. pub struct Sandbox { diff --git a/frame/contracts/src/chain_extension.rs b/frame/contracts/src/chain_extension.rs index 14080102933c..ed447719933b 100644 --- a/frame/contracts/src/chain_extension.rs +++ b/frame/contracts/src/chain_extension.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -329,7 +329,7 @@ where /// /// If the contract supplied buffer is smaller than the passed `buffer` an `Err` is returned. /// If `allow_skip` is set to true the contract is allowed to skip the copying of the buffer - /// by supplying the guard value of `u32::MAX` as `out_ptr`. The + /// by supplying the guard value of `pallet-contracts::SENTINEL` as `out_ptr`. The /// `weight_per_byte` is only charged when the write actually happens and is not skipped or /// failed due to a too small output buffer. pub fn write( diff --git a/frame/contracts/src/exec.rs b/frame/contracts/src/exec.rs index cc468466c292..7aa5c0b731fa 100644 --- a/frame/contracts/src/exec.rs +++ b/frame/contracts/src/exec.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,14 +16,15 @@ // limitations under the License. use crate::{ - gas::GasMeter, storage::Storage, AccountCounter, BalanceOf, CodeHash, Config, ContractInfo, - ContractInfoOf, Error, Event, Pallet as Contracts, Schedule, + gas::GasMeter, + storage::{self, Storage, WriteOutcome}, + BalanceOf, CodeHash, Config, ContractInfo, ContractInfoOf, Error, Event, Nonce, + Pallet as Contracts, Schedule, }; use frame_support::{ dispatch::{DispatchError, DispatchResult, DispatchResultWithPostInfo, Dispatchable}, - ensure, storage::{with_transaction, TransactionOutcome}, - traits::{Contains, Currency, ExistenceRequirement, Get, OriginTrait, Randomness, Time}, + traits::{Contains, Currency, ExistenceRequirement, OriginTrait, Randomness, Time}, weights::Weight, }; use frame_system::RawOrigin; @@ -31,14 +32,9 @@ use pallet_contracts_primitives::ExecReturnValue; use smallvec::{Array, SmallVec}; use sp_core::crypto::UncheckedFrom; use sp_io::crypto::secp256k1_ecdsa_recover_compressed; -use sp_runtime::traits::{Convert, Saturating}; +use sp_runtime::traits::Convert; use sp_std::{marker::PhantomData, mem, prelude::*}; -/// When fields are added to the [`ContractInfo`] that can change during execution this -/// variable needs to be set to true. This will also force changes to the -/// `in_memory_changes_not_discarded` test. -const CONTRACT_INFO_CAN_CHANGE: bool = false; - pub type AccountIdOf = ::AccountId; pub type MomentOf = <::Time as Time>::Moment; pub type SeedOf = ::Hash; @@ -95,10 +91,6 @@ pub trait Ext: sealing::Sealed { /// Call (possibly transferring some amount of funds) into the specified account. /// /// Returns the original code size of the called contract. - /// - /// # Return Value - /// - /// Result<(ExecReturnValue, CodeSize), (ExecError, CodeSize)> fn call( &mut self, gas_limit: Weight, @@ -108,15 +100,20 @@ pub trait Ext: sealing::Sealed { allows_reentry: bool, ) -> Result; + /// Execute code in the current frame. + /// + /// Returns the original code size of the called contract. + fn delegate_call( + &mut self, + code: CodeHash, + input_data: Vec, + ) -> Result; + /// Instantiate a contract from the given code. /// /// Returns the original code size of the called contract. /// The newly created account will be associated with `code`. `value` specifies the amount of - /// value transferred from this to the newly created account (also known as endowment). - /// - /// # Return Value - /// - /// Result<(AccountId, ExecReturnValue, CodeSize), (ExecError, CodeSize)> + /// value transferred from this to the newly created account. fn instantiate( &mut self, gas_limit: Weight, @@ -144,13 +141,41 @@ pub trait Ext: sealing::Sealed { /// was deleted. fn get_storage(&mut self, key: &StorageKey) -> Option>; + /// Returns `Some(len)` (in bytes) if a storage item exists at `key`. + /// + /// Returns `None` if the `key` wasn't previously set by `set_storage` or + /// was deleted. + fn get_storage_size(&mut self, key: &StorageKey) -> Option; + /// Sets the storage entry by the given key to the specified value. If `value` is `None` then /// the storage entry is deleted. - fn set_storage(&mut self, key: StorageKey, value: Option>) -> DispatchResult; + fn set_storage( + &mut self, + key: StorageKey, + value: Option>, + take_old: bool, + ) -> Result; /// Returns a reference to the account id of the caller. fn caller(&self) -> &AccountIdOf; + /// Check if a contract lives at the specified `address`. + fn is_contract(&self, address: &AccountIdOf) -> bool; + + /// Returns the code hash of the contract for the given `address`. + /// + /// Returns `None` if the `address` does not belong to a contract. + fn code_hash(&self, address: &AccountIdOf) -> Option>; + + /// Returns the code hash of the contract being executed. + fn own_code_hash(&mut self) -> &CodeHash; + + /// Check if the caller of the current contract is the origin of the whole call stack. + /// + /// This can be checked with `is_contract(self.caller())` as well. + /// However, this function does not require any storage lookup and therefore uses less weight. + fn caller_is_origin(&self) -> bool; + /// Returns a reference to the account id of the current contract. fn address(&self) -> &AccountIdOf; @@ -159,7 +184,7 @@ pub trait Ext: sealing::Sealed { /// The `value_transferred` is already added. fn balance(&self) -> BalanceOf; - /// Returns the value transferred along with this call or as endowment. + /// Returns the value transferred along with this call. fn value_transferred(&self) -> BalanceOf; /// Returns a reference to the timestamp of the current block @@ -168,9 +193,6 @@ pub trait Ext: sealing::Sealed { /// Returns the minimum balance that is required for creating an account. fn minimum_balance(&self) -> BalanceOf; - /// Returns the deposit required to instantiate a contract. - fn contract_deposit(&self) -> BalanceOf; - /// Returns a random number for the current block with the given subject. fn random(&self, subject: &[u8]) -> (SeedOf, BlockNumberOf); @@ -209,6 +231,13 @@ pub trait Ext: sealing::Sealed { /// Recovers ECDSA compressed public key based on signature and message hash. fn ecdsa_recover(&self, signature: &[u8; 65], message_hash: &[u8; 32]) -> Result<[u8; 33], ()>; + + /// Tests sometimes need to modify and inspect the contract info directly. + #[cfg(test)] + fn contract_info(&mut self) -> &mut ContractInfo; + + /// Sets new code hash for existing contract. + fn set_code_hash(&mut self, hash: CodeHash) -> Result<(), DispatchError>; } /// Describes the different functions that can be exported by an [`Executable`]. @@ -235,37 +264,17 @@ pub trait Executable: Sized { gas_meter: &mut GasMeter, ) -> Result; - /// Load the module from storage without re-instrumenting it. + /// Increment the refcount of a code in-storage by one. /// - /// A code module is re-instrumented on-load when it was originally instrumented with - /// an older schedule. This skips this step for cases where the code storage is - /// queried for purposes other than execution. + /// This is needed when the code is not set via instantiate but `seal_set_code_hash`. /// - /// # Note + /// # Errors /// - /// Does not charge from the gas meter. Do not call in contexts where this is important. - fn from_storage_noinstr(code_hash: CodeHash) -> Result; + /// [`Error::CodeNotFound`] is returned if the specified `code_hash` does not exist. + fn add_user(code_hash: CodeHash) -> Result<(), DispatchError>; - /// Increment the refcount by one. Fails if the code does not exist on-chain. - /// - /// Returns the size of the original code. - /// - /// # Note - /// - /// Charges weight proportional to the code size from the gas meter. - fn add_user(code_hash: CodeHash, gas_meter: &mut GasMeter) -> Result<(), DispatchError>; - - /// Decrement the refcount by one and remove the code when it drops to zero. - /// - /// Returns the size of the original code. - /// - /// # Note - /// - /// Charges weight proportional to the code size from the gas meter - fn remove_user( - code_hash: CodeHash, - gas_meter: &mut GasMeter, - ) -> Result<(), DispatchError>; + /// Decrement the refcount by one if the code exists. + fn remove_user(code_hash: CodeHash); /// Execute the specified exported function and return the result. /// @@ -288,12 +297,6 @@ pub trait Executable: Sized { /// Size of the instrumented code in bytes. fn code_len(&self) -> u32; - - /// Sum of instrumented and pristine code len. - fn aggregate_code_len(&self) -> u32; - - // The number of contracts using this executable. - fn refcount(&self) -> u32; } /// The complete call stack of a contract execution. @@ -314,13 +317,16 @@ pub struct Stack<'a, T: Config, E> { schedule: &'a Schedule, /// The gas meter where costs are charged to. gas_meter: &'a mut GasMeter, + /// The storage meter makes sure that the storage deposit limit is obeyed. + storage_meter: &'a mut storage::meter::Meter, /// The timestamp at the point of call stack instantiation. timestamp: MomentOf, /// The block number at the time of call stack instantiation. block_number: T::BlockNumber, - /// The account counter is cached here when accessed. It is written back when the call stack - /// finishes executing. - account_counter: Option, + /// The nonce is cached here when accessed. It is written back when the call stack + /// finishes executing. Please refer to [`Nonce`] to a description of + /// the nonce itself. + nonce: Option, /// The actual call stack. One entry per nested contract called/instantiated. /// This does **not** include the [`Self::first_frame`]. frames: SmallVec, @@ -354,9 +360,21 @@ pub struct Frame { /// Determines whether this is a call or instantiate frame. entry_point: ExportedFunction, /// The gas meter capped to the supplied gas limit. - nested_meter: GasMeter, + nested_gas: GasMeter, + /// The storage meter for the individual call. + nested_storage: storage::meter::NestedMeter, /// If `false` the contract enabled its defense against reentrance attacks. allows_reentry: bool, + /// The caller of the currently executing frame which was spawned by `delegate_call`. + delegate_caller: Option, +} + +/// Used in a delegate call frame arguments in order to override the executable and caller. +struct DelegatedCall { + /// The executable which is run instead of the contracts own `executable`. + executable: E, + /// The account id of the caller contract. + caller: T::AccountId, } /// Parameter passed in when creating a new `Frame`. @@ -368,12 +386,16 @@ enum FrameArgs<'a, T: Config, E> { dest: T::AccountId, /// If `None` the contract info needs to be reloaded from storage. cached_info: Option>, + /// This frame was created by `seal_delegate_call` and hence uses different code than + /// what is stored at [`Self::dest`]. Its caller ([`Frame::delegated_caller`]) is the + /// account which called the caller contract + delegated_call: Option>, }, Instantiate { /// The contract or signed origin which instantiates the new contract. sender: T::AccountId, - /// The seed that should be used to derive a new trie id for the contract. - trie_seed: u64, + /// The nonce that should be used to derive a new trie id for the contract. + nonce: u64, /// The executable whose `deploy` function is run. executable: E, /// A salt used in the contract address deriviation of the new contract. @@ -396,6 +418,26 @@ enum CachedContract { Terminated, } +impl CachedContract { + /// Return `Some(ContractInfo)` if the contract is in cached state. `None` otherwise. + fn into_contract(self) -> Option> { + if let CachedContract::Cached(contract) = self { + Some(contract) + } else { + None + } + } + + /// Return `Some(&mut ContractInfo)` if the contract is in cached state. `None` otherwise. + fn as_contract(&mut self) -> Option<&mut ContractInfo> { + if let CachedContract::Cached(contract) = self { + Some(contract) + } else { + None + } + } +} + impl Frame { /// Return the `contract_info` of the current contract. fn contract_info(&mut self) -> &mut ContractInfo { @@ -432,6 +474,26 @@ macro_rules! get_cached_or_panic_after_load { }}; } +/// Same as [`Stack::top_frame`]. +/// +/// We need this access as a macro because sometimes hiding the lifetimes behind +/// a function won't work out. +macro_rules! top_frame { + ($stack:expr) => { + $stack.frames.last().unwrap_or(&$stack.first_frame) + }; +} + +/// Same as [`Stack::top_frame_mut`]. +/// +/// We need this access as a macro because sometimes hiding the lifetimes behind +/// a function won't work out. +macro_rules! top_frame_mut { + ($stack:expr) => { + $stack.frames.last_mut().unwrap_or(&mut $stack.first_frame) + }; +} + impl CachedContract { /// Load the `contract_info` from storage if necessary. fn load(&mut self, account_id: &T::AccountId) { @@ -462,7 +524,7 @@ where T::AccountId: UncheckedFrom + AsRef<[u8]>, E: Executable, { - /// Create an run a new call stack by calling into `dest`. + /// Create and run a new call stack by calling into `dest`. /// /// # Note /// @@ -476,15 +538,17 @@ where origin: T::AccountId, dest: T::AccountId, gas_meter: &'a mut GasMeter, + storage_meter: &'a mut storage::meter::Meter, schedule: &'a Schedule, value: BalanceOf, input_data: Vec, debug_message: Option<&'a mut Vec>, ) -> Result { let (mut stack, executable) = Self::new( - FrameArgs::Call { dest, cached_info: None }, + FrameArgs::Call { dest, cached_info: None, delegated_call: None }, origin, gas_meter, + storage_meter, schedule, value, debug_message, @@ -506,6 +570,7 @@ where origin: T::AccountId, executable: E, gas_meter: &'a mut GasMeter, + storage_meter: &'a mut storage::meter::Meter, schedule: &'a Schedule, value: BalanceOf, input_data: Vec, @@ -515,12 +580,13 @@ where let (mut stack, executable) = Self::new( FrameArgs::Instantiate { sender: origin.clone(), - trie_seed: Self::initial_trie_seed(), + nonce: Self::initial_nonce(), executable, salt, }, origin, gas_meter, + storage_meter, schedule, value, debug_message, @@ -534,18 +600,21 @@ where args: FrameArgs, origin: T::AccountId, gas_meter: &'a mut GasMeter, + storage_meter: &'a mut storage::meter::Meter, schedule: &'a Schedule, value: BalanceOf, debug_message: Option<&'a mut Vec>, ) -> Result<(Self, E), ExecError> { - let (first_frame, executable) = Self::new_frame(args, value, gas_meter, 0, &schedule)?; + let (first_frame, executable, nonce) = + Self::new_frame(args, value, gas_meter, storage_meter, 0, &schedule)?; let stack = Self { origin, schedule, gas_meter, + storage_meter, timestamp: T::Time::now(), block_number: >::block_number(), - account_counter: None, + nonce, first_frame, frames: Default::default(), debug_message, @@ -559,48 +628,64 @@ where /// /// This does not take `self` because when constructing the first frame `self` is /// not initialized, yet. - fn new_frame( + fn new_frame( frame_args: FrameArgs, value_transferred: BalanceOf, gas_meter: &mut GasMeter, + storage_meter: &mut storage::meter::GenericMeter, gas_limit: Weight, schedule: &Schedule, - ) -> Result<(Frame, E), ExecError> { - let (account_id, contract_info, executable, entry_point) = match frame_args { - FrameArgs::Call { dest, cached_info } => { - let contract = if let Some(contract) = cached_info { - contract - } else { - >::get(&dest).ok_or(>::ContractNotFound)? - }; - - let executable = E::from_storage(contract.code_hash, schedule, gas_meter)?; - - (dest, contract, executable, ExportedFunction::Call) - }, - FrameArgs::Instantiate { sender, trie_seed, executable, salt } => { - let account_id = - >::contract_address(&sender, executable.code_hash(), &salt); - let trie_id = Storage::::generate_trie_id(&account_id, trie_seed); - let contract = Storage::::new_contract( - &account_id, - trie_id, - executable.code_hash().clone(), - )?; - (account_id, contract, executable, ExportedFunction::Constructor) - }, - }; + ) -> Result<(Frame, E, Option), ExecError> { + let (account_id, contract_info, executable, delegate_caller, entry_point, nonce) = + match frame_args { + FrameArgs::Call { dest, cached_info, delegated_call } => { + let contract = if let Some(contract) = cached_info { + contract + } else { + >::get(&dest).ok_or(>::ContractNotFound)? + }; + + let (executable, delegate_caller) = + if let Some(DelegatedCall { executable, caller }) = delegated_call { + (executable, Some(caller)) + } else { + (E::from_storage(contract.code_hash, schedule, gas_meter)?, None) + }; + + (dest, contract, executable, delegate_caller, ExportedFunction::Call, None) + }, + FrameArgs::Instantiate { sender, nonce, executable, salt } => { + let account_id = + >::contract_address(&sender, executable.code_hash(), &salt); + let trie_id = Storage::::generate_trie_id(&account_id, nonce); + let contract = Storage::::new_contract( + &account_id, + trie_id, + executable.code_hash().clone(), + )?; + ( + account_id, + contract, + executable, + None, + ExportedFunction::Constructor, + Some(nonce), + ) + }, + }; let frame = Frame { + delegate_caller, value_transferred, contract_info: CachedContract::Cached(contract_info), account_id, entry_point, - nested_meter: gas_meter.nested(gas_limit)?, + nested_gas: gas_meter.nested(gas_limit)?, + nested_storage: storage_meter.nested(), allows_reentry: true, }; - Ok((frame, executable)) + Ok((frame, executable, nonce)) } /// Create a subsequent nested frame. @@ -614,23 +699,28 @@ where return Err(Error::::MaxCallDepthReached.into()) } - if CONTRACT_INFO_CAN_CHANGE { - // We need to make sure that changes made to the contract info are not discarded. - // See the `in_memory_changes_not_discarded` test for more information. - // We do not store on instantiate because we do not allow to call into a contract - // from its own constructor. - let frame = self.top_frame(); - if let (CachedContract::Cached(contract), ExportedFunction::Call) = - (&frame.contract_info, frame.entry_point) - { - >::insert(frame.account_id.clone(), contract.clone()); - } + // We need to make sure that changes made to the contract info are not discarded. + // See the `in_memory_changes_not_discarded` test for more information. + // We do not store on instantiate because we do not allow to call into a contract + // from its own constructor. + let frame = self.top_frame(); + if let (CachedContract::Cached(contract), ExportedFunction::Call) = + (&frame.contract_info, frame.entry_point) + { + >::insert(frame.account_id.clone(), contract.clone()); } - let nested_meter = - &mut self.frames.last_mut().unwrap_or(&mut self.first_frame).nested_meter; - let (frame, executable) = - Self::new_frame(frame_args, value_transferred, nested_meter, gas_limit, self.schedule)?; + let frame = top_frame_mut!(self); + let nested_gas = &mut frame.nested_gas; + let nested_storage = &mut frame.nested_storage; + let (frame, executable, _) = Self::new_frame( + frame_args, + value_transferred, + nested_gas, + nested_storage, + gas_limit, + self.schedule, + )?; self.frames.push(frame); Ok(executable) } @@ -641,6 +731,17 @@ where fn run(&mut self, executable: E, input_data: Vec) -> Result { let entry_point = self.top_frame().entry_point; let do_transaction = || { + // We need to charge the storage deposit before the initial transfer so that + // it can create the account in case the initial transfer is < ed. + if entry_point == ExportedFunction::Constructor { + let top_frame = top_frame_mut!(self); + top_frame.nested_storage.charge_instantiate( + &self.origin, + &top_frame.account_id, + &mut top_frame.contract_info.get(&top_frame.account_id), + )?; + } + // Every call or instantiate also optionally transferres balance. self.initial_transfer()?; @@ -650,17 +751,22 @@ where .map_err(|e| ExecError { error: e.error, origin: ErrorOrigin::Callee })?; // Additional work needs to be performed in case of an instantiation. - if output.is_success() && entry_point == ExportedFunction::Constructor { - let frame = self.top_frame_mut(); - let account_id = frame.account_id.clone(); + if !output.did_revert() && entry_point == ExportedFunction::Constructor { + let frame = self.top_frame(); // It is not allowed to terminate a contract inside its constructor. - if let CachedContract::Terminated = frame.contract_info { + if matches!(frame.contract_info, CachedContract::Terminated) { return Err(Error::::TerminatedInConstructor.into()) } // Deposit an instantiation event. - deposit_event::(vec![], Event::Instantiated(self.caller().clone(), account_id)); + deposit_event::( + vec![], + Event::Instantiated { + deployer: self.caller().clone(), + contract: frame.account_id.clone(), + }, + ); } Ok(output) @@ -668,14 +774,27 @@ where // All changes performed by the contract are executed under a storage transaction. // This allows for roll back on error. Changes to the cached contract_info are - // comitted or rolled back when popping the frame. - let (success, output) = with_transaction(|| { - let output = do_transaction(); - match &output { - Ok(result) if result.is_success() => TransactionOutcome::Commit((true, output)), - _ => TransactionOutcome::Rollback((false, output)), - } - }); + // committed or rolled back when popping the frame. + // + // `with_transactional` may return an error caused by a limit in the + // transactional storage depth. + let transaction_outcome = + with_transaction(|| -> TransactionOutcome> { + let output = do_transaction(); + match &output { + Ok(result) if !result.did_revert() => + TransactionOutcome::Commit(Ok((true, output))), + _ => TransactionOutcome::Rollback(Ok((false, output))), + } + }); + + let (success, output) = match transaction_outcome { + // `with_transactional` executed successfully, and we have the expected output. + Ok((success, output)) => (success, output), + // `with_transactional` returned an error, and we propagate that error and note no state + // has changed. + Err(error) => (false, Err(error.into())), + }; self.pop_frame(success); output } @@ -685,9 +804,9 @@ where /// This is called after running the current frame. It commits cached values to storage /// and invalidates all stale references to it that might exist further down the call stack. fn pop_frame(&mut self, persist: bool) { - // Revert the account counter in case of a failed instantiation. + // Revert changes to the nonce in case of a failed instantiation. if !persist && self.top_frame().entry_point == ExportedFunction::Constructor { - self.account_counter.as_mut().map(|c| *c = c.wrapping_sub(1)); + self.nonce.as_mut().map(|c| *c = c.wrapping_sub(1)); } // Pop the current frame from the stack and return it in case it needs to interact @@ -695,15 +814,34 @@ where // A `None` means that we are returning from the `first_frame`. let frame = self.frames.pop(); - if let Some(frame) = frame { - let prev = self.top_frame_mut(); + // Both branches do essentially the same with the exception. The difference is that + // the else branch does consume the hardcoded `first_frame`. + if let Some(mut frame) = frame { let account_id = &frame.account_id; - prev.nested_meter.absorb_nested(frame.nested_meter); + let prev = top_frame_mut!(self); + + prev.nested_gas.absorb_nested(frame.nested_gas); + // Only gas counter changes are persisted in case of a failure. if !persist { return } - if let CachedContract::Cached(contract) = frame.contract_info { + + // Record the storage meter changes of the nested call into the parent meter. + // If the dropped frame's contract wasn't terminated we update the deposit counter + // in its contract info. The load is necessary to to pull it from storage in case + // it was invalidated. + frame.contract_info.load(account_id); + let mut contract = frame.contract_info.into_contract(); + prev.nested_storage.absorb( + frame.nested_storage, + &self.origin, + account_id, + contract.as_mut(), + ); + + // In case the contract wasn't terminated we need to persist changes made to it. + if let Some(contract) = contract { // optimization: Predecessor is the same contract. // We can just copy the contract into the predecessor without a storage write. // This is possible when there is no other contract in-between that could @@ -731,54 +869,35 @@ where core::str::from_utf8(msg).unwrap_or(""), ); } - // Write back to the root gas meter. - self.gas_meter.absorb_nested(mem::take(&mut self.first_frame.nested_meter)); - // Only gas counter changes are persisted in case of a failure. + self.gas_meter.absorb_nested(mem::take(&mut self.first_frame.nested_gas)); if !persist { return } - if let CachedContract::Cached(contract) = &self.first_frame.contract_info { - >::insert(&self.first_frame.account_id, contract.clone()); + let mut contract = self.first_frame.contract_info.as_contract(); + self.storage_meter.absorb( + mem::take(&mut self.first_frame.nested_storage), + &self.origin, + &self.first_frame.account_id, + contract.as_deref_mut(), + ); + if let Some(contract) = contract { + >::insert(&self.first_frame.account_id, contract); } - if let Some(counter) = self.account_counter { - >::set(counter); + if let Some(nonce) = self.nonce { + >::set(nonce); } } } /// Transfer some funds from `from` to `to`. - /// - /// We only allow allow for draining all funds of the sender if `allow_death` is - /// is specified as `true`. Otherwise, any transfer that would bring the sender below the - /// subsistence threshold (for contracts) or the existential deposit (for plain accounts) - /// results in an error. fn transfer( - sender_is_contract: bool, - allow_death: bool, + existence_requirement: ExistenceRequirement, from: &T::AccountId, to: &T::AccountId, value: BalanceOf, ) -> DispatchResult { - if value == 0u32.into() { - return Ok(()) - } - - let existence_requirement = match (allow_death, sender_is_contract) { - (true, _) => ExistenceRequirement::AllowDeath, - (false, true) => { - ensure!( - T::Currency::total_balance(from).saturating_sub(value) >= - Contracts::::subsistence_threshold(), - Error::::BelowSubsistenceThreshold, - ); - ExistenceRequirement::KeepAlive - }, - (false, false) => ExistenceRequirement::KeepAlive, - }; - T::Currency::transfer(from, to, value, existence_requirement) .map_err(|_| Error::::TransferFailed)?; - Ok(()) } @@ -786,31 +905,18 @@ where fn initial_transfer(&self) -> DispatchResult { let frame = self.top_frame(); let value = frame.value_transferred; - let subsistence_threshold = >::subsistence_threshold(); - // If the value transferred to a new contract is less than the subsistence threshold - // we can error out early. This avoids executing the constructor in cases where - // we already know that the contract has too little balance. - if frame.entry_point == ExportedFunction::Constructor && value < subsistence_threshold { - return Err(>::NewContractNotFunded.into()) - } - - Self::transfer(self.caller_is_origin(), false, self.caller(), &frame.account_id, value) - } - - /// Wether the caller is the initiator of the call stack. - fn caller_is_origin(&self) -> bool { - !self.frames.is_empty() + Self::transfer(ExistenceRequirement::KeepAlive, self.caller(), &frame.account_id, value) } /// Reference to the current (top) frame. fn top_frame(&self) -> &Frame { - self.frames.last().unwrap_or(&self.first_frame) + top_frame!(self) } /// Mutable reference to the current (top) frame. fn top_frame_mut(&mut self) -> &mut Frame { - self.frames.last_mut().unwrap_or(&mut self.first_frame) + top_frame_mut!(self) } /// Iterator over all frames. @@ -836,20 +942,20 @@ where !self.frames().any(|f| &f.account_id == id && !f.allows_reentry) } - /// Increments the cached account id and returns the value to be used for the trie_id. - fn next_trie_seed(&mut self) -> u64 { - let next = if let Some(current) = self.account_counter { - current + 1 + /// Increments and returns the next nonce. Pulls it from storage if it isn't in cache. + fn next_nonce(&mut self) -> u64 { + let next = if let Some(current) = self.nonce { + current.wrapping_add(1) } else { - Self::initial_trie_seed() + Self::initial_nonce() }; - self.account_counter = Some(next); + self.nonce = Some(next); next } - /// The account seed to be used to instantiate the account counter cache. - fn initial_trie_seed() -> u64 { - >::get().wrapping_add(1) + /// Pull the current nonce from storage. + fn initial_nonce() -> u64 { + >::get().wrapping_add(1) } } @@ -888,8 +994,11 @@ where CachedContract::Cached(contract) => Some(contract.clone()), _ => None, }); - let executable = - self.push_frame(FrameArgs::Call { dest: to, cached_info }, value, gas_limit)?; + let executable = self.push_frame( + FrameArgs::Call { dest: to, cached_info, delegated_call: None }, + value, + gas_limit, + )?; self.run(executable, input_data) }; @@ -902,24 +1011,46 @@ where result } + fn delegate_call( + &mut self, + code_hash: CodeHash, + input_data: Vec, + ) -> Result { + let executable = E::from_storage(code_hash, &self.schedule, self.gas_meter())?; + let top_frame = self.top_frame_mut(); + let contract_info = top_frame.contract_info().clone(); + let account_id = top_frame.account_id.clone(); + let value = top_frame.value_transferred.clone(); + let executable = self.push_frame( + FrameArgs::Call { + dest: account_id, + cached_info: Some(contract_info), + delegated_call: Some(DelegatedCall { executable, caller: self.caller().clone() }), + }, + value, + 0, + )?; + self.run(executable, input_data) + } + fn instantiate( &mut self, gas_limit: Weight, code_hash: CodeHash, - endowment: BalanceOf, + value: BalanceOf, input_data: Vec, salt: &[u8], ) -> Result<(AccountIdOf, ExecReturnValue), ExecError> { let executable = E::from_storage(code_hash, &self.schedule, self.gas_meter())?; - let trie_seed = self.next_trie_seed(); + let nonce = self.next_nonce(); let executable = self.push_frame( FrameArgs::Instantiate { sender: self.top_frame().account_id.clone(), - trie_seed, + nonce, executable, salt, }, - endowment, + value, gas_limit, )?; let account_id = self.top_frame().account_id.clone(); @@ -932,34 +1063,49 @@ where } let frame = self.top_frame_mut(); let info = frame.terminate(); + frame.nested_storage.terminate(&info); Storage::::queue_trie_for_deletion(&info)?; >::transfer( - true, - true, + ExistenceRequirement::AllowDeath, &frame.account_id, beneficiary, T::Currency::free_balance(&frame.account_id), )?; ContractInfoOf::::remove(&frame.account_id); - E::remove_user(info.code_hash, &mut frame.nested_meter)?; - Contracts::::deposit_event(Event::Terminated( - frame.account_id.clone(), - beneficiary.clone(), - )); + E::remove_user(info.code_hash); + Contracts::::deposit_event(Event::Terminated { + contract: frame.account_id.clone(), + beneficiary: beneficiary.clone(), + }); Ok(()) } fn transfer(&mut self, to: &T::AccountId, value: BalanceOf) -> DispatchResult { - Self::transfer(true, false, &self.top_frame().account_id, to, value) + Self::transfer(ExistenceRequirement::KeepAlive, &self.top_frame().account_id, to, value) } fn get_storage(&mut self, key: &StorageKey) -> Option> { Storage::::read(&self.top_frame_mut().contract_info().trie_id, key) } - fn set_storage(&mut self, key: StorageKey, value: Option>) -> DispatchResult { + fn get_storage_size(&mut self, key: &StorageKey) -> Option { + Storage::::size(&self.top_frame_mut().contract_info().trie_id, key) + } + + fn set_storage( + &mut self, + key: StorageKey, + value: Option>, + take_old: bool, + ) -> Result { let frame = self.top_frame_mut(); - Storage::::write(frame.contract_info(), &key, value) + Storage::::write( + &frame.contract_info.get(&frame.account_id).trie_id, + &key, + value, + Some(&mut frame.nested_storage), + take_old, + ) } fn address(&self) -> &T::AccountId { @@ -967,7 +1113,27 @@ where } fn caller(&self) -> &T::AccountId { - self.frames().nth(1).map(|f| &f.account_id).unwrap_or(&self.origin) + if let Some(caller) = &self.top_frame().delegate_caller { + &caller + } else { + self.frames().nth(1).map(|f| &f.account_id).unwrap_or(&self.origin) + } + } + + fn is_contract(&self, address: &T::AccountId) -> bool { + ContractInfoOf::::contains_key(&address) + } + + fn code_hash(&self, address: &T::AccountId) -> Option> { + >::get(&address).map(|contract| contract.code_hash) + } + + fn own_code_hash(&mut self) -> &CodeHash { + &self.top_frame_mut().contract_info().code_hash + } + + fn caller_is_origin(&self) -> bool { + self.caller() == &self.origin } fn balance(&self) -> BalanceOf { @@ -990,14 +1156,10 @@ where T::Currency::minimum_balance() } - fn contract_deposit(&self) -> BalanceOf { - T::ContractDeposit::get() - } - fn deposit_event(&mut self, topics: Vec, data: Vec) { deposit_event::( topics, - Event::ContractEmitted(self.top_frame().account_id.clone(), data), + Event::ContractEmitted { contract: self.top_frame().account_id.clone(), data }, ); } @@ -1006,7 +1168,7 @@ where } fn max_value_size(&self) -> u32 { - T::Schedule::get().limits.payload_len + self.schedule.limits.payload_len } fn get_weight_price(&self, weight: Weight) -> BalanceOf { @@ -1018,7 +1180,7 @@ where } fn gas_meter(&mut self) -> &mut GasMeter { - &mut self.top_frame_mut().nested_meter + &mut self.top_frame_mut().nested_gas } fn append_debug_buffer(&mut self, msg: &str) -> bool { @@ -1041,6 +1203,25 @@ where fn ecdsa_recover(&self, signature: &[u8; 65], message_hash: &[u8; 32]) -> Result<[u8; 33], ()> { secp256k1_ecdsa_recover_compressed(&signature, &message_hash).map_err(|_| ()) } + + #[cfg(test)] + fn contract_info(&mut self) -> &mut ContractInfo { + self.top_frame_mut().contract_info() + } + + fn set_code_hash(&mut self, hash: CodeHash) -> Result<(), DispatchError> { + E::add_user(hash)?; + let top_frame = self.top_frame_mut(); + let prev_hash = top_frame.contract_info().code_hash.clone(); + E::remove_user(prev_hash.clone()); + top_frame.contract_info().code_hash = hash; + Contracts::::deposit_event(Event::ContractCodeUpdated { + contract: top_frame.account_id.clone(), + new_code_hash: hash, + old_code_hash: prev_hash, + }); + Ok(()) + } } fn deposit_event(topics: Vec, event: Event) { @@ -1078,9 +1259,9 @@ mod tests { storage::Storage, tests::{ test_utils::{get_balance, place_contract, set_balance}, - Call, Event as MetaEvent, ExtBuilder, Test, TestFilter, ALICE, BOB, CHARLIE, + Call, Event as MetaEvent, ExtBuilder, Test, TestFilter, ALICE, BOB, CHARLIE, GAS_LIMIT, }, - Error, Weight, + Error, }; use assert_matches::assert_matches; use codec::{Decode, Encode}; @@ -1089,18 +1270,17 @@ mod tests { use pallet_contracts_primitives::ReturnFlags; use pretty_assertions::assert_eq; use sp_core::Bytes; - use sp_runtime::{ - traits::{BadOrigin, Hash}, - DispatchError, + use sp_runtime::{traits::Hash, DispatchError}; + use std::{ + cell::RefCell, + collections::hash_map::{Entry, HashMap}, + rc::Rc, }; - use std::{cell::RefCell, collections::HashMap, rc::Rc}; type System = frame_system::Pallet; type MockStack<'a> = Stack<'a, Test, MockExecutable>; - const GAS_LIMIT: Weight = 10_000_000_000; - thread_local! { static LOADER: RefCell = RefCell::new(MockLoader::default()); } @@ -1157,15 +1337,15 @@ mod tests { }) } - fn increment_refcount(code_hash: CodeHash) { + fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError> { LOADER.with(|loader| { let mut loader = loader.borrow_mut(); - loader - .map - .entry(code_hash) - .and_modify(|executable| executable.refcount += 1) - .or_insert_with(|| panic!("code_hash does not exist")); - }); + match loader.map.entry(code_hash) { + Entry::Vacant(_) => Err(>::CodeNotFound)?, + Entry::Occupied(mut entry) => entry.get_mut().refcount += 1, + } + Ok(()) + }) } fn decrement_refcount(code_hash: CodeHash) { @@ -1191,10 +1371,6 @@ mod tests { _schedule: &Schedule, _gas_meter: &mut GasMeter, ) -> Result { - Self::from_storage_noinstr(code_hash) - } - - fn from_storage_noinstr(code_hash: CodeHash) -> Result { LOADER.with(|loader| { loader .borrow_mut() @@ -1205,20 +1381,12 @@ mod tests { }) } - fn add_user( - code_hash: CodeHash, - _: &mut GasMeter, - ) -> Result<(), DispatchError> { - MockLoader::increment_refcount(code_hash); - Ok(()) + fn add_user(code_hash: CodeHash) -> Result<(), DispatchError> { + MockLoader::increment_refcount(code_hash) } - fn remove_user( - code_hash: CodeHash, - _: &mut GasMeter, - ) -> Result<(), DispatchError> { + fn remove_user(code_hash: CodeHash) { MockLoader::decrement_refcount(code_hash); - Ok(()) } fn execute>( @@ -1228,7 +1396,7 @@ mod tests { input_data: Vec, ) -> ExecResult { if let &Constructor = function { - MockLoader::increment_refcount(self.code_hash); + Self::add_user(self.code_hash).unwrap(); } if function == &self.func_type { (self.func)(MockCtx { ext, input_data }, &self) @@ -1244,14 +1412,6 @@ mod tests { fn code_len(&self) -> u32 { 0 } - - fn aggregate_code_len(&self) -> u32 { - 0 - } - - fn refcount(&self) -> u32 { - self.refcount as u32 - } } fn exec_success() -> ExecResult { @@ -1278,9 +1438,19 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let schedule = ::Schedule::get(); place_contract(&BOB, exec_ch); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), value).unwrap(); assert_matches!( - MockStack::run_call(ALICE, BOB, &mut gas_meter, &schedule, value, vec![], None,), + MockStack::run_call( + ALICE, + BOB, + &mut gas_meter, + &mut storage_meter, + &schedule, + value, + vec![], + None, + ), Ok(_) ); }); @@ -1299,7 +1469,7 @@ mod tests { set_balance(&origin, 100); set_balance(&dest, 0); - MockStack::transfer(true, false, &origin, &dest, 55).unwrap(); + MockStack::transfer(ExistenceRequirement::KeepAlive, &origin, &dest, 55).unwrap(); assert_eq!(get_balance(&origin), 45); assert_eq!(get_balance(&dest), 55); @@ -1322,11 +1492,13 @@ mod tests { place_contract(&dest, return_ch); set_balance(&origin, 100); let balance = get_balance(&dest); + let mut storage_meter = storage::meter::Meter::new(&origin, Some(0), 55).unwrap(); let output = MockStack::run_call( origin.clone(), dest.clone(), &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, 55, vec![], @@ -1334,7 +1506,7 @@ mod tests { ) .unwrap(); - assert!(!output.is_success()); + assert!(output.did_revert()); assert_eq!(get_balance(&origin), 100); assert_eq!(get_balance(&dest), balance); }); @@ -1350,7 +1522,7 @@ mod tests { ExtBuilder::default().build().execute_with(|| { set_balance(&origin, 0); - let result = MockStack::transfer(false, false, &origin, &dest, 100); + let result = MockStack::transfer(ExistenceRequirement::KeepAlive, &origin, &dest, 100); assert_eq!(result, Err(Error::::TransferFailed.into())); assert_eq!(get_balance(&origin), 0); @@ -1370,12 +1542,14 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let schedule = ::Schedule::get(); + let mut storage_meter = storage::meter::Meter::new(&origin, Some(0), 0).unwrap(); place_contract(&BOB, return_ch); let result = MockStack::run_call( origin, dest, &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, 0, vec![], @@ -1383,7 +1557,7 @@ mod tests { ); let output = result.unwrap(); - assert!(output.is_success()); + assert!(!output.did_revert()); assert_eq!(output.data, Bytes(vec![1, 2, 3, 4])); }); } @@ -1401,11 +1575,13 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let schedule = ::Schedule::get(); place_contract(&BOB, return_ch); + let mut storage_meter = storage::meter::Meter::new(&origin, Some(0), 0).unwrap(); let result = MockStack::run_call( origin, dest, &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, 0, vec![], @@ -1413,7 +1589,7 @@ mod tests { ); let output = result.unwrap(); - assert!(!output.is_success()); + assert!(output.did_revert()); assert_eq!(output.data, Bytes(vec![1, 2, 3, 4])); }); } @@ -1429,11 +1605,13 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let schedule = ::Schedule::get(); place_contract(&BOB, input_data_ch); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); let result = MockStack::run_call( ALICE, BOB, &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, 0, vec![1, 2, 3, 4], @@ -1453,19 +1631,21 @@ mod tests { // This one tests passing the input data into a contract via instantiate. ExtBuilder::default().build().execute_with(|| { let schedule = ::Schedule::get(); - let subsistence = Contracts::::subsistence_threshold(); + let min_balance = ::Currency::minimum_balance(); let mut gas_meter = GasMeter::::new(GAS_LIMIT); let executable = MockExecutable::from_storage(input_data_ch, &schedule, &mut gas_meter).unwrap(); - - set_balance(&ALICE, subsistence * 10); + set_balance(&ALICE, min_balance * 1000); + let mut storage_meter = + storage::meter::Meter::new(&ALICE, Some(min_balance * 100), min_balance).unwrap(); let result = MockStack::run_instantiate( ALICE, executable, &mut gas_meter, + &mut storage_meter, &schedule, - subsistence * 3, + min_balance, vec![1, 2, 3, 4], &[], None, @@ -1506,11 +1686,13 @@ mod tests { let schedule = ::Schedule::get(); set_balance(&BOB, 1); place_contract(&BOB, recurse_ch); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), value).unwrap(); let result = MockStack::run_call( ALICE, BOB, &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, value, vec![], @@ -1551,11 +1733,13 @@ mod tests { let schedule = ::Schedule::get(); place_contract(&dest, bob_ch); place_contract(&CHARLIE, charlie_ch); + let mut storage_meter = storage::meter::Meter::new(&origin, Some(0), 0).unwrap(); let result = MockStack::run_call( origin.clone(), dest.clone(), &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, 0, vec![], @@ -1569,6 +1753,126 @@ mod tests { WITNESSED_CALLER_CHARLIE.with(|caller| assert_eq!(*caller.borrow(), Some(dest))); } + #[test] + fn is_contract_returns_proper_values() { + let bob_ch = MockLoader::insert(Call, |ctx, _| { + // Verify that BOB is a contract + assert!(ctx.ext.is_contract(&BOB)); + // Verify that ALICE is not a contract + assert!(!ctx.ext.is_contract(&ALICE)); + exec_success() + }); + + ExtBuilder::default().build().execute_with(|| { + let schedule = ::Schedule::get(); + place_contract(&BOB, bob_ch); + + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); + let result = MockStack::run_call( + ALICE, + BOB, + &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, + &schedule, + 0, + vec![], + None, + ); + assert_matches!(result, Ok(_)); + }); + } + + #[test] + fn code_hash_returns_proper_values() { + let code_bob = MockLoader::insert(Call, |ctx, _| { + // ALICE is not a contract and hence she does not have a code_hash + assert!(ctx.ext.code_hash(&ALICE).is_none()); + // BOB is a contract and hence he has a code_hash + assert!(ctx.ext.code_hash(&BOB).is_some()); + exec_success() + }); + + ExtBuilder::default().build().execute_with(|| { + let schedule = ::Schedule::get(); + place_contract(&BOB, code_bob); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); + // ALICE (not contract) -> BOB (contract) + let result = MockStack::run_call( + ALICE, + BOB, + &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, + &schedule, + 0, + vec![0], + None, + ); + assert_matches!(result, Ok(_)); + }); + } + + #[test] + fn own_code_hash_returns_proper_values() { + let bob_ch = MockLoader::insert(Call, |ctx, _| { + let code_hash = ctx.ext.code_hash(&BOB).unwrap(); + assert_eq!(*ctx.ext.own_code_hash(), code_hash); + exec_success() + }); + + ExtBuilder::default().build().execute_with(|| { + let schedule = ::Schedule::get(); + place_contract(&BOB, bob_ch); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); + // ALICE (not contract) -> BOB (contract) + let result = MockStack::run_call( + ALICE, + BOB, + &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, + &schedule, + 0, + vec![0], + None, + ); + assert_matches!(result, Ok(_)); + }); + } + + #[test] + fn caller_is_origin_returns_proper_values() { + let code_charlie = MockLoader::insert(Call, |ctx, _| { + // BOB is not the origin of the stack call + assert!(!ctx.ext.caller_is_origin()); + exec_success() + }); + + let code_bob = MockLoader::insert(Call, |ctx, _| { + // ALICE is the origin of the call stack + assert!(ctx.ext.caller_is_origin()); + // BOB calls CHARLIE + ctx.ext.call(0, CHARLIE, 0, vec![], true) + }); + + ExtBuilder::default().build().execute_with(|| { + let schedule = ::Schedule::get(); + place_contract(&BOB, code_bob); + place_contract(&CHARLIE, code_charlie); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); + // ALICE -> BOB (caller is origin) -> CHARLIE (caller is not origin) + let result = MockStack::run_call( + ALICE, + BOB, + &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, + &schedule, + 0, + vec![0], + None, + ); + assert_matches!(result, Ok(_)); + }); + } + #[test] fn address_returns_proper_values() { let bob_ch = MockLoader::insert(Call, |ctx, _| { @@ -1588,11 +1892,13 @@ mod tests { let schedule = ::Schedule::get(); place_contract(&BOB, bob_ch); place_contract(&CHARLIE, charlie_ch); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); let result = MockStack::run_call( ALICE, BOB, &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, 0, vec![], @@ -1612,14 +1918,16 @@ mod tests { let mut gas_meter = GasMeter::::new(GAS_LIMIT); let executable = MockExecutable::from_storage(dummy_ch, &schedule, &mut gas_meter).unwrap(); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); assert_matches!( MockStack::run_instantiate( ALICE, executable, &mut gas_meter, + &mut storage_meter, &schedule, - 0, // <- zero endowment + 0, // <- zero value vec![], &[], None, @@ -1637,18 +1945,22 @@ mod tests { ExtBuilder::default().existential_deposit(15).build().execute_with(|| { let schedule = ::Schedule::get(); + let min_balance = ::Currency::minimum_balance(); let mut gas_meter = GasMeter::::new(GAS_LIMIT); let executable = MockExecutable::from_storage(dummy_ch, &schedule, &mut gas_meter).unwrap(); - set_balance(&ALICE, 1000); + set_balance(&ALICE, min_balance * 1000); + let mut storage_meter = + storage::meter::Meter::new(&ALICE, Some(min_balance * 100), min_balance).unwrap(); let instantiated_contract_address = assert_matches!( MockStack::run_instantiate( ALICE, executable, &mut gas_meter, + &mut storage_meter, &schedule, - 100, + min_balance, vec![], &[], None, @@ -1662,7 +1974,10 @@ mod tests { Storage::::code_hash(&instantiated_contract_address).unwrap(), dummy_ch ); - assert_eq!(&events(), &[Event::Instantiated(ALICE, instantiated_contract_address)]); + assert_eq!( + &events(), + &[Event::Instantiated { deployer: ALICE, contract: instantiated_contract_address }] + ); }); } @@ -1674,18 +1989,22 @@ mod tests { ExtBuilder::default().existential_deposit(15).build().execute_with(|| { let schedule = ::Schedule::get(); + let min_balance = ::Currency::minimum_balance(); let mut gas_meter = GasMeter::::new(GAS_LIMIT); let executable = MockExecutable::from_storage(dummy_ch, &schedule, &mut gas_meter).unwrap(); - set_balance(&ALICE, 1000); + set_balance(&ALICE, min_balance * 1000); + let mut storage_meter = + storage::meter::Meter::new(&ALICE, Some(min_balance * 100), min_balance).unwrap(); let instantiated_contract_address = assert_matches!( MockStack::run_instantiate( ALICE, executable, &mut gas_meter, + &mut storage_meter, &schedule, - 100, + min_balance, vec![], &[], None, @@ -1713,7 +2032,7 @@ mod tests { .instantiate( 0, dummy_ch, - Contracts::::subsistence_threshold() * 3, + ::Currency::minimum_balance(), vec![], &[48, 49, 50], ) @@ -1726,16 +2045,21 @@ mod tests { ExtBuilder::default().existential_deposit(15).build().execute_with(|| { let schedule = ::Schedule::get(); - set_balance(&ALICE, Contracts::::subsistence_threshold() * 100); + let min_balance = ::Currency::minimum_balance(); + set_balance(&ALICE, min_balance * 100); place_contract(&BOB, instantiator_ch); + let mut storage_meter = + storage::meter::Meter::new(&ALICE, Some(min_balance * 10), min_balance * 10) + .unwrap(); assert_matches!( MockStack::run_call( ALICE, BOB, &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, - 20, + min_balance * 10, vec![], None, ), @@ -1751,7 +2075,10 @@ mod tests { Storage::::code_hash(&instantiated_contract_address).unwrap(), dummy_ch ); - assert_eq!(&events(), &[Event::Instantiated(BOB, instantiated_contract_address)]); + assert_eq!( + &events(), + &[Event::Instantiated { deployer: BOB, contract: instantiated_contract_address }] + ); }); } @@ -1766,7 +2093,7 @@ mod tests { ctx.ext.instantiate( 0, dummy_ch, - Contracts::::subsistence_threshold(), + ::Currency::minimum_balance(), vec![], &[], ), @@ -1785,14 +2112,16 @@ mod tests { set_balance(&ALICE, 1000); set_balance(&BOB, 100); place_contract(&BOB, instantiator_ch); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(100), 0).unwrap(); assert_matches!( MockStack::run_call( ALICE, BOB, &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, - 20, + 0, vec![], None, ), @@ -1818,12 +2147,14 @@ mod tests { let executable = MockExecutable::from_storage(terminate_ch, &schedule, &mut gas_meter).unwrap(); set_balance(&ALICE, 1000); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(100), 100).unwrap(); assert_eq!( MockStack::run_instantiate( ALICE, executable, &mut gas_meter, + &mut storage_meter, &schedule, 100, vec![], @@ -1839,10 +2170,6 @@ mod tests { #[test] fn in_memory_changes_not_discarded() { - // Remove this assert and fill out the "DO" stubs once fields are added to the - // contract info that can be modified during exection. - assert!(!CONTRACT_INFO_CAN_CHANGE); - // Call stack: BOB -> CHARLIE (trap) -> BOB' (success) // This tests verfies some edge case of the contract info cache: // We change some value in our contract info before calling into a contract @@ -1853,9 +2180,11 @@ mod tests { // are made before calling into CHARLIE are not discarded. let code_bob = MockLoader::insert(Call, |ctx, _| { if ctx.input_data[0] == 0 { - // DO: modify medata (ContractInfo) of own contract through ctx.ext functions + let info = ctx.ext.contract_info(); + assert_eq!(info.storage_deposit, 0); + info.storage_deposit = 42; assert_eq!(ctx.ext.call(0, CHARLIE, 0, vec![], true), exec_trapped()); - // DO: check that the value is not discarded (query via ctx.ext) + assert_eq!(ctx.ext.contract_info().storage_deposit, 42); } exec_success() }); @@ -1869,11 +2198,13 @@ mod tests { let schedule = ::Schedule::get(); place_contract(&BOB, code_bob); place_contract(&CHARLIE, code_charlie); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); let result = MockStack::run_call( ALICE, BOB, &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, 0, vec![0], @@ -1896,18 +2227,20 @@ mod tests { // This one tests passing the input data into a contract via instantiate. ExtBuilder::default().build().execute_with(|| { let schedule = ::Schedule::get(); - let subsistence = Contracts::::subsistence_threshold(); + let min_balance = ::Currency::minimum_balance(); let mut gas_meter = GasMeter::::new(GAS_LIMIT); let executable = MockExecutable::from_storage(code, &schedule, &mut gas_meter).unwrap(); - - set_balance(&ALICE, subsistence * 10); + set_balance(&ALICE, min_balance * 1000); + let mut storage_meter = + storage::meter::Meter::new(&ALICE, Some(min_balance * 100), min_balance).unwrap(); let result = MockStack::run_instantiate( ALICE, executable, &mut gas_meter, + &mut storage_meter, &schedule, - subsistence * 3, + min_balance, vec![], &[], None, @@ -1927,15 +2260,17 @@ mod tests { let mut debug_buffer = Vec::new(); ExtBuilder::default().build().execute_with(|| { - let subsistence = Contracts::::subsistence_threshold(); + let min_balance = ::Currency::minimum_balance(); let schedule = ::Schedule::get(); let mut gas_meter = GasMeter::::new(GAS_LIMIT); - set_balance(&ALICE, subsistence * 10); + set_balance(&ALICE, min_balance * 10); place_contract(&BOB, code_hash); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); MockStack::run_call( ALICE, BOB, &mut gas_meter, + &mut storage_meter, &schedule, 0, vec![], @@ -1958,15 +2293,17 @@ mod tests { let mut debug_buffer = Vec::new(); ExtBuilder::default().build().execute_with(|| { - let subsistence = Contracts::::subsistence_threshold(); + let min_balance = ::Currency::minimum_balance(); let schedule = ::Schedule::get(); let mut gas_meter = GasMeter::::new(GAS_LIMIT); - set_balance(&ALICE, subsistence * 10); + set_balance(&ALICE, min_balance * 10); place_contract(&BOB, code_hash); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); let result = MockStack::run_call( ALICE, BOB, &mut gas_meter, + &mut storage_meter, &schedule, 0, vec![], @@ -1992,12 +2329,14 @@ mod tests { let schedule = ::Schedule::get(); place_contract(&BOB, code_bob); place_contract(&CHARLIE, code_charlie); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); // Calling another contract should succeed assert_ok!(MockStack::run_call( ALICE, BOB, &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, 0, CHARLIE.encode(), @@ -2010,6 +2349,7 @@ mod tests { ALICE, BOB, &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, 0, BOB.encode(), @@ -2039,6 +2379,7 @@ mod tests { let schedule = ::Schedule::get(); place_contract(&BOB, code_bob); place_contract(&CHARLIE, code_charlie); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); // BOB -> CHARLIE -> BOB fails as BOB denies reentry. assert_err!( @@ -2046,6 +2387,7 @@ mod tests { ALICE, BOB, &mut GasMeter::::new(GAS_LIMIT), + &mut storage_meter, &schedule, 0, vec![0], @@ -2068,20 +2410,34 @@ mod tests { }); ExtBuilder::default().build().execute_with(|| { - let subsistence = Contracts::::subsistence_threshold(); + let min_balance = ::Currency::minimum_balance(); let schedule = ::Schedule::get(); let mut gas_meter = GasMeter::::new(GAS_LIMIT); - set_balance(&ALICE, subsistence * 10); + set_balance(&ALICE, min_balance * 10); place_contract(&BOB, code_hash); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); System::reset_events(); - MockStack::run_call(ALICE, BOB, &mut gas_meter, &schedule, 0, vec![], None).unwrap(); + MockStack::run_call( + ALICE, + BOB, + &mut gas_meter, + &mut storage_meter, + &schedule, + 0, + vec![], + None, + ) + .unwrap(); let remark_hash = ::Hashing::hash(b"Hello World"); assert_eq!( System::events(), vec![EventRecord { phase: Phase::Initialization, - event: MetaEvent::System(frame_system::Event::Remarked(BOB, remark_hash)), + event: MetaEvent::System(frame_system::Event::Remarked { + sender: BOB, + hash: remark_hash + }), topics: vec![], },] ); @@ -2103,7 +2459,10 @@ mod tests { let forbidden_call = Call::Balances(BalanceCall::transfer { dest: CHARLIE, value: 22 }); // simple cases: direct call - assert_err!(ctx.ext.call_runtime(forbidden_call.clone()), BadOrigin); + assert_err!( + ctx.ext.call_runtime(forbidden_call.clone()), + frame_system::Error::::CallFiltered + ); // as part of a patch: return is OK (but it interrupted the batch) assert_ok!(ctx.ext.call_runtime(Call::Utility(UtilCall::batch { @@ -2122,13 +2481,24 @@ mod tests { }); ExtBuilder::default().build().execute_with(|| { - let subsistence = Contracts::::subsistence_threshold(); + let min_balance = ::Currency::minimum_balance(); let schedule = ::Schedule::get(); let mut gas_meter = GasMeter::::new(GAS_LIMIT); - set_balance(&ALICE, subsistence * 10); + set_balance(&ALICE, min_balance * 10); place_contract(&BOB, code_hash); + let mut storage_meter = storage::meter::Meter::new(&ALICE, Some(0), 0).unwrap(); System::reset_events(); - MockStack::run_call(ALICE, BOB, &mut gas_meter, &schedule, 0, vec![], None).unwrap(); + MockStack::run_call( + ALICE, + BOB, + &mut gas_meter, + &mut storage_meter, + &schedule, + 0, + vec![], + None, + ) + .unwrap(); let remark_hash = ::Hashing::hash(b"Hello"); assert_eq!( @@ -2136,7 +2506,10 @@ mod tests { vec![ EventRecord { phase: Phase::Initialization, - event: MetaEvent::System(frame_system::Event::Remarked(BOB, remark_hash)), + event: MetaEvent::System(frame_system::Event::Remarked { + sender: BOB, + hash: remark_hash + }), topics: vec![], }, EventRecord { @@ -2146,14 +2519,203 @@ mod tests { }, EventRecord { phase: Phase::Initialization, - event: MetaEvent::Utility(pallet_utility::Event::BatchInterrupted( - 1, - BadOrigin.into() - ),), + event: MetaEvent::Utility(pallet_utility::Event::BatchInterrupted { + index: 1, + error: frame_system::Error::::CallFiltered.into() + },), topics: vec![], }, ] ); }); } + + #[test] + fn nonce() { + let fail_code = MockLoader::insert(Constructor, |_, _| exec_trapped()); + let success_code = MockLoader::insert(Constructor, |_, _| exec_success()); + let succ_fail_code = MockLoader::insert(Constructor, move |ctx, _| { + ctx.ext + .instantiate(0, fail_code, ctx.ext.minimum_balance() * 100, vec![], &[]) + .ok(); + exec_success() + }); + let succ_succ_code = MockLoader::insert(Constructor, move |ctx, _| { + let (account_id, _) = ctx + .ext + .instantiate(0, success_code, ctx.ext.minimum_balance() * 100, vec![], &[]) + .unwrap(); + + // a plain call should not influence the account counter + ctx.ext.call(0, account_id, 0, vec![], false).unwrap(); + + exec_success() + }); + + ExtBuilder::default().build().execute_with(|| { + let schedule = ::Schedule::get(); + let min_balance = ::Currency::minimum_balance(); + let mut gas_meter = GasMeter::::new(GAS_LIMIT); + let fail_executable = + MockExecutable::from_storage(fail_code, &schedule, &mut gas_meter).unwrap(); + let success_executable = + MockExecutable::from_storage(success_code, &schedule, &mut gas_meter).unwrap(); + let succ_fail_executable = + MockExecutable::from_storage(succ_fail_code, &schedule, &mut gas_meter).unwrap(); + let succ_succ_executable = + MockExecutable::from_storage(succ_succ_code, &schedule, &mut gas_meter).unwrap(); + set_balance(&ALICE, min_balance * 1000); + let mut storage_meter = + storage::meter::Meter::new(&ALICE, Some(min_balance * 500), min_balance * 100) + .unwrap(); + + MockStack::run_instantiate( + ALICE, + fail_executable, + &mut gas_meter, + &mut storage_meter, + &schedule, + min_balance * 100, + vec![], + &[], + None, + ) + .ok(); + assert_eq!(>::get(), 0); + + assert_ok!(MockStack::run_instantiate( + ALICE, + success_executable, + &mut gas_meter, + &mut storage_meter, + &schedule, + min_balance * 100, + vec![], + &[], + None, + )); + assert_eq!(>::get(), 1); + + assert_ok!(MockStack::run_instantiate( + ALICE, + succ_fail_executable, + &mut gas_meter, + &mut storage_meter, + &schedule, + min_balance * 200, + vec![], + &[], + None, + )); + assert_eq!(>::get(), 2); + + assert_ok!(MockStack::run_instantiate( + ALICE, + succ_succ_executable, + &mut gas_meter, + &mut storage_meter, + &schedule, + min_balance * 200, + vec![], + &[], + None, + )); + assert_eq!(>::get(), 4); + }); + } + + #[test] + fn set_storage_works() { + let code_hash = MockLoader::insert(Call, |ctx, _| { + // Write + assert_eq!( + ctx.ext.set_storage([1; 32], Some(vec![1, 2, 3]), false), + Ok(WriteOutcome::New) + ); + assert_eq!( + ctx.ext.set_storage([2; 32], Some(vec![4, 5, 6]), true), + Ok(WriteOutcome::New) + ); + assert_eq!(ctx.ext.set_storage([3; 32], None, false), Ok(WriteOutcome::New)); + assert_eq!(ctx.ext.set_storage([4; 32], None, true), Ok(WriteOutcome::New)); + assert_eq!(ctx.ext.set_storage([5; 32], Some(vec![]), false), Ok(WriteOutcome::New)); + assert_eq!(ctx.ext.set_storage([6; 32], Some(vec![]), true), Ok(WriteOutcome::New)); + + // Overwrite + assert_eq!( + ctx.ext.set_storage([1; 32], Some(vec![42]), false), + Ok(WriteOutcome::Overwritten(3)) + ); + assert_eq!( + ctx.ext.set_storage([2; 32], Some(vec![48]), true), + Ok(WriteOutcome::Taken(vec![4, 5, 6])) + ); + assert_eq!(ctx.ext.set_storage([3; 32], None, false), Ok(WriteOutcome::New)); + assert_eq!(ctx.ext.set_storage([4; 32], None, true), Ok(WriteOutcome::New)); + assert_eq!( + ctx.ext.set_storage([5; 32], Some(vec![]), false), + Ok(WriteOutcome::Overwritten(0)) + ); + assert_eq!( + ctx.ext.set_storage([6; 32], Some(vec![]), true), + Ok(WriteOutcome::Taken(vec![])) + ); + + exec_success() + }); + + ExtBuilder::default().build().execute_with(|| { + let min_balance = ::Currency::minimum_balance(); + let schedule = ::Schedule::get(); + let mut gas_meter = GasMeter::::new(GAS_LIMIT); + set_balance(&ALICE, min_balance * 1000); + place_contract(&BOB, code_hash); + let mut storage_meter = storage::meter::Meter::new(&ALICE, None, 0).unwrap(); + assert_ok!(MockStack::run_call( + ALICE, + BOB, + &mut gas_meter, + &mut storage_meter, + &schedule, + 0, + vec![], + None, + )); + }); + } + + #[test] + fn get_storage_size_works() { + let code_hash = MockLoader::insert(Call, |ctx, _| { + assert_eq!( + ctx.ext.set_storage([1; 32], Some(vec![1, 2, 3]), false), + Ok(WriteOutcome::New) + ); + assert_eq!(ctx.ext.set_storage([2; 32], Some(vec![]), false), Ok(WriteOutcome::New)); + assert_eq!(ctx.ext.get_storage_size(&[1; 32]), Some(3)); + assert_eq!(ctx.ext.get_storage_size(&[2; 32]), Some(0)); + assert_eq!(ctx.ext.get_storage_size(&[3; 32]), None); + + exec_success() + }); + + ExtBuilder::default().build().execute_with(|| { + let min_balance = ::Currency::minimum_balance(); + let schedule = ::Schedule::get(); + let mut gas_meter = GasMeter::::new(GAS_LIMIT); + set_balance(&ALICE, min_balance * 1000); + place_contract(&BOB, code_hash); + let mut storage_meter = storage::meter::Meter::new(&ALICE, None, 0).unwrap(); + assert_ok!(MockStack::run_call( + ALICE, + BOB, + &mut gas_meter, + &mut storage_meter, + &schedule, + 0, + vec![], + None, + )); + }); + } } diff --git a/frame/contracts/src/gas.rs b/frame/contracts/src/gas.rs index 38d18c1e24c1..cdf0c1407c6b 100644 --- a/frame/contracts/src/gas.rs +++ b/frame/contracts/src/gas.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/contracts/src/lib.rs b/frame/contracts/src/lib.rs index 0d7e4cbf5647..4edf43a67215 100644 --- a/frame/contracts/src/lib.rs +++ b/frame/contracts/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -87,12 +87,12 @@ mod gas; mod benchmarking; mod exec; -mod migration; mod schedule; mod storage; mod wasm; pub mod chain_extension; +pub mod migration; pub mod weights; #[cfg(test)] @@ -106,24 +106,27 @@ pub use crate::{ use crate::{ exec::{AccountIdOf, ExecError, Executable, Stack as ExecStack}, gas::GasMeter, - storage::{ContractInfo, DeletedContract, Storage}, - wasm::PrefabWasmModule, + storage::{meter::Meter as StorageMeter, ContractInfo, DeletedContract, Storage}, + wasm::{OwnerInfo, PrefabWasmModule}, weights::WeightInfo, }; +use codec::{Encode, HasCompact}; use frame_support::{ dispatch::Dispatchable, ensure, - traits::{Contains, Currency, Get, Randomness, StorageVersion, Time}, - weights::{GetDispatchInfo, PostDispatchInfo, Weight}, + traits::{Contains, Currency, Get, Randomness, ReservableCurrency, StorageVersion, Time}, + weights::{GetDispatchInfo, Pays, PostDispatchInfo, Weight}, }; use frame_system::Pallet as System; use pallet_contracts_primitives::{ - Code, ContractAccessError, ContractExecResult, ContractInstantiateResult, ExecReturnValue, - GetStorageResult, InstantiateReturnValue, + Code, CodeUploadResult, CodeUploadReturnValue, ContractAccessError, ContractExecResult, + ContractInstantiateResult, ExecReturnValue, GetStorageResult, InstantiateReturnValue, + StorageDeposit, }; +use scale_info::TypeInfo; use sp_core::{crypto::UncheckedFrom, Bytes}; use sp_runtime::traits::{Convert, Hash, Saturating, StaticLookup}; -use sp_std::prelude::*; +use sp_std::{fmt::Debug, prelude::*}; type CodeHash = ::Hash; type TrieId = Vec; @@ -131,7 +134,64 @@ type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; /// The current storage version. -const STORAGE_VERSION: StorageVersion = StorageVersion::new(5); +const STORAGE_VERSION: StorageVersion = StorageVersion::new(7); + +/// Used as a sentinel value when reading and writing contract memory. +/// +/// It is usually used to signal `None` to a contract when only a primitive is allowed +/// and we don't want to go through encoding a full Rust type. Using `u32::Max` is a safe +/// sentinel because contracts are never allowed to use such a large amount of resources +/// that this value makes sense for a memory location or length. +const SENTINEL: u32 = u32::MAX; + +/// Provides the contract address generation method. +/// +/// See [`DefaultAddressGenerator`] for the default implementation. +pub trait AddressGenerator { + /// Generate the address of a contract based on the given instantiate parameters. + /// + /// # Note for implementors + /// 1. Make sure that there are no collisions, different inputs never lead to the same output. + /// 2. Make sure that the same inputs lead to the same output. + /// 3. Changing the implementation through a runtime upgrade without a proper storage migration + /// would lead to catastrophic misbehavior. + fn generate_address( + deploying_address: &T::AccountId, + code_hash: &CodeHash, + salt: &[u8], + ) -> T::AccountId; +} + +/// Default address generator. +/// +/// This is the default address generator used by contract instantiation. Its result +/// is only dependend on its inputs. It can therefore be used to reliably predict the +/// address of a contract. This is akin to the formular of eth's CREATE2 opcode. There +/// is no CREATE equivalent because CREATE2 is strictly more powerful. +/// +/// Formula: `hash(deploying_address ++ code_hash ++ salt)` +pub struct DefaultAddressGenerator; + +impl AddressGenerator for DefaultAddressGenerator +where + T: frame_system::Config, + T::AccountId: UncheckedFrom + AsRef<[u8]>, +{ + fn generate_address( + deploying_address: &T::AccountId, + code_hash: &CodeHash, + salt: &[u8], + ) -> T::AccountId { + let buf: Vec<_> = deploying_address + .as_ref() + .iter() + .chain(code_hash.as_ref()) + .chain(salt) + .cloned() + .collect(); + UncheckedFrom::unchecked_from(T::Hashing::hash(&buf)) + } +} #[frame_support::pallet] pub mod pallet { @@ -148,7 +208,7 @@ pub mod pallet { type Randomness: Randomness; /// The currency in which fees are paid and contract balances are held. - type Currency: Currency; + type Currency: ReservableCurrency; /// The overarching event type. type Event: From> + IsType<::Event>; @@ -165,12 +225,6 @@ pub mod pallet { /// This is applied in **addition** to [`frame_system::Config::BaseCallFilter`]. /// It is recommended to treat this as a whitelist. /// - /// # Subsistence Threshold - /// - /// The runtime **must** make sure that any allowed dispatchable makes sure that the - /// `total_balance` of the contract stays above [`Pallet::subsistence_threshold()`]. - /// Otherwise users could clutter the storage with contracts. - /// /// # Stability /// /// The runtime **must** make sure that all dispatchables that are callable by @@ -201,13 +255,6 @@ pub mod pallet { #[pallet::constant] type Schedule: Get>; - /// The deposit that must be placed into the contract's account to instantiate it. - /// This is in **addition** to the [`pallet_balances::Pallet::ExistenialDeposit`]. - /// The minimum balance for a contract's account can be queried using - /// [`Pallet::subsistence_threshold`]. - #[pallet::constant] - type ContractDeposit: Get>; - /// The type of the call stack determines the maximum nesting depth of contract calls. /// /// The allowed depth is `CallStack::size() + 1`. @@ -215,17 +262,56 @@ pub mod pallet { /// In other words only the origin called "root contract" is allowed to execute then. type CallStack: smallvec::Array>; - /// The maximum number of tries that can be queued for deletion. + /// The maximum number of contracts that can be pending for deletion. + /// + /// When a contract is deleted by calling `seal_terminate` it becomes inaccessible + /// immediately, but the deletion of the storage items it has accumulated is performed + /// later. The contract is put into the deletion queue. This defines how many + /// contracts can be queued up at the same time. If that limit is reached `seal_terminate` + /// will fail. The action must be retried in a later block in that case. + /// + /// The reasons for limiting the queue depth are: + /// + /// 1. The queue is in storage in order to be persistent between blocks. We want to limit + /// the amount of storage that can be consumed. + /// 2. The queue is stored in a vector and needs to be decoded as a whole when reading + /// it at the end of each block. Longer queues take more weight to decode and hence + /// limit the amount of items that can be deleted per block. #[pallet::constant] type DeletionQueueDepth: Get; /// The maximum amount of weight that can be consumed per block for lazy trie removal. + /// + /// The amount of weight that is dedicated per block to work on the deletion queue. Larger + /// values allow more trie keys to be deleted in each block but reduce the amount of + /// weight that is left for transactions. See [`Self::DeletionQueueDepth`] for more + /// information about the deletion queue. #[pallet::constant] type DeletionWeightLimit: Get; + + /// The amount of balance a caller has to pay for each byte of storage. + /// + /// # Note + /// + /// Changing this value for an existing chain might need a storage migration. + #[pallet::constant] + type DepositPerByte: Get>; + + /// The amount of balance a caller has to pay for each storage item. + /// + /// # Note + /// + /// Changing this value for an existing chain might need a storage migration. + #[pallet::constant] + type DepositPerItem: Get>; + + /// The address generator used to generate the addresses of contracts. + type AddressGenerator: AddressGenerator; } #[pallet::pallet] #[pallet::storage_version(STORAGE_VERSION)] + #[pallet::without_storage_info] pub struct Pallet(PhantomData); #[pallet::hooks] @@ -244,10 +330,6 @@ pub mod pallet { Storage::::process_deletion_queue_batch(weight_limit) .saturating_add(T::WeightInfo::on_initialize()) } - - fn on_runtime_upgrade() -> Weight { - migration::migrate::() - } } #[pallet::call] @@ -255,9 +337,19 @@ pub mod pallet { where T::AccountId: UncheckedFrom, T::AccountId: AsRef<[u8]>, + as HasCompact>::Type: Clone + Eq + PartialEq + Debug + TypeInfo + Encode, { /// Makes a call to an account, optionally transferring some balance. /// + /// # Parameters + /// + /// * `dest`: Address of the contract to call. + /// * `value`: The balance to transfer from the `origin` to `dest`. + /// * `gas_limit`: The gas limit enforced when executing the constructor. + /// * `storage_deposit_limit`: The maximum amount of balance that can be charged from the + /// caller to pay for the storage consumed. + /// * `data`: The input data to pass to the contract. + /// /// * If the account is a smart-contract account, the associated code will be /// executed and any value will be transferred. /// * If the account is a regular account, any value will be transferred. @@ -269,23 +361,41 @@ pub mod pallet { dest: ::Source, #[pallet::compact] value: BalanceOf, #[pallet::compact] gas_limit: Weight, + storage_deposit_limit: Option< as codec::HasCompact>::Type>, data: Vec, ) -> DispatchResultWithPostInfo { let origin = ensure_signed(origin)?; let dest = T::Lookup::lookup(dest)?; - let output = Self::internal_call(origin, dest, value, gas_limit, data, None); + let mut output = Self::internal_call( + origin, + dest, + value, + gas_limit, + storage_deposit_limit.map(Into::into), + data, + None, + ); + if let Ok(retval) = &output.result { + if retval.did_revert() { + output.result = Err(>::ContractReverted.into()); + } + } output.gas_meter.into_dispatch_result(output.result, T::WeightInfo::call()) } /// Instantiates a new contract from the supplied `code` optionally transferring /// some balance. /// - /// This is the only function that can deploy new code to the chain. + /// This dispatchable has the same effect as calling [`Self::upload_code`] + + /// [`Self::instantiate`]. Bundling them together provides efficiency gains. Please + /// also check the documentation of [`Self::upload_code`]. /// /// # Parameters /// - /// * `endowment`: The balance to transfer from the `origin` to the newly created contract. + /// * `value`: The balance to transfer from the `origin` to the newly created contract. /// * `gas_limit`: The gas limit enforced when executing the constructor. + /// * `storage_deposit_limit`: The maximum amount of balance that can be charged/reserved + /// from the caller to pay for the storage consumed. /// * `code`: The contract code to deploy in raw bytes. /// * `data`: The input data to pass to the contract constructor. /// * `salt`: Used for the address derivation. See [`Pallet::contract_address`]. @@ -297,19 +407,17 @@ pub mod pallet { /// - If the `code_hash` already exists on the chain the underlying `code` will be shared. /// - The destination address is computed based on the sender, code_hash and the salt. /// - The smart-contract account is created at the computed address. - /// - The `endowment` is transferred to the new account. + /// - The `value` is transferred to the new account. /// - The `deploy` function is executed in the context of the newly-created account. #[pallet::weight( - T::WeightInfo::instantiate_with_code( - code.len() as u32 / 1024, - salt.len() as u32 / 1024, - ) + T::WeightInfo::instantiate_with_code(code.len() as u32, salt.len() as u32) .saturating_add(*gas_limit) )] pub fn instantiate_with_code( origin: OriginFor, - #[pallet::compact] endowment: BalanceOf, + #[pallet::compact] value: BalanceOf, #[pallet::compact] gas_limit: Weight, + storage_deposit_limit: Option< as codec::HasCompact>::Type>, code: Vec, data: Vec, salt: Vec, @@ -317,18 +425,24 @@ pub mod pallet { let origin = ensure_signed(origin)?; let code_len = code.len() as u32; let salt_len = salt.len() as u32; - let output = Self::internal_instantiate( + let mut output = Self::internal_instantiate( origin, - endowment, + value, gas_limit, + storage_deposit_limit.map(Into::into), Code::Upload(Bytes(code)), data, salt, None, ); + if let Ok(retval) = &output.result { + if retval.1.did_revert() { + output.result = Err(>::ContractReverted.into()); + } + } output.gas_meter.into_dispatch_result( output.result.map(|(_address, result)| result), - T::WeightInfo::instantiate_with_code(code_len / 1024, salt_len / 1024), + T::WeightInfo::instantiate_with_code(code_len, salt_len), ) } @@ -338,100 +452,139 @@ pub mod pallet { /// code deployment step. Instead, the `code_hash` of an on-chain deployed wasm binary /// must be supplied. #[pallet::weight( - T::WeightInfo::instantiate(salt.len() as u32 / 1024).saturating_add(*gas_limit) + T::WeightInfo::instantiate(salt.len() as u32).saturating_add(*gas_limit) )] pub fn instantiate( origin: OriginFor, - #[pallet::compact] endowment: BalanceOf, + #[pallet::compact] value: BalanceOf, #[pallet::compact] gas_limit: Weight, + storage_deposit_limit: Option< as codec::HasCompact>::Type>, code_hash: CodeHash, data: Vec, salt: Vec, ) -> DispatchResultWithPostInfo { let origin = ensure_signed(origin)?; let salt_len = salt.len() as u32; - let output = Self::internal_instantiate( + let mut output = Self::internal_instantiate( origin, - endowment, + value, gas_limit, + storage_deposit_limit.map(Into::into), Code::Existing(code_hash), data, salt, None, ); + if let Ok(retval) = &output.result { + if retval.1.did_revert() { + output.result = Err(>::ContractReverted.into()); + } + } output.gas_meter.into_dispatch_result( output.result.map(|(_address, output)| output), - T::WeightInfo::instantiate(salt_len / 1024), + T::WeightInfo::instantiate(salt_len), ) } + + /// Upload new `code` without instantiating a contract from it. + /// + /// If the code does not already exist a deposit is reserved from the caller + /// and unreserved only when [`Self::remove_code`] is called. The size of the reserve + /// depends on the instrumented size of the the supplied `code`. + /// + /// If the code already exists in storage it will still return `Ok` and upgrades + /// the in storage version to the current + /// [`InstructionWeights::version`](InstructionWeights). + /// + /// # Note + /// + /// Anyone can instantiate a contract from any uploaded code and thus prevent its removal. + /// To avoid this situation a constructor could employ access control so that it can + /// only be instantiated by permissioned entities. The same is true when uploading + /// through [`Self::instantiate_with_code`]. + #[pallet::weight(T::WeightInfo::upload_code(code.len() as u32))] + pub fn upload_code( + origin: OriginFor, + code: Vec, + storage_deposit_limit: Option< as codec::HasCompact>::Type>, + ) -> DispatchResult { + let origin = ensure_signed(origin)?; + Self::bare_upload_code(origin, code, storage_deposit_limit.map(Into::into)).map(|_| ()) + } + + /// Remove the code stored under `code_hash` and refund the deposit to its owner. + /// + /// A code can only be removed by its original uploader (its owner) and only if it is + /// not used by any contract. + #[pallet::weight(T::WeightInfo::remove_code())] + pub fn remove_code( + origin: OriginFor, + code_hash: CodeHash, + ) -> DispatchResultWithPostInfo { + let origin = ensure_signed(origin)?; + >::remove(&origin, code_hash)?; + // we waive the fee because removing unused code is beneficial + Ok(Pays::No.into()) + } } #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// Contract deployed by address at the specified address. \[deployer, contract\] - Instantiated(T::AccountId, T::AccountId), + /// Contract deployed by address at the specified address. + Instantiated { deployer: T::AccountId, contract: T::AccountId }, /// Contract has been removed. - /// \[contract, beneficiary\] - /// - /// # Params - /// - /// - `contract`: The contract that was terminated. - /// - `beneficiary`: The account that received the contracts remaining balance. /// /// # Note /// /// The only way for a contract to be removed and emitting this event is by calling /// `seal_terminate`. - Terminated(T::AccountId, T::AccountId), + Terminated { + /// The contract that was terminated. + contract: T::AccountId, + /// The account that received the contracts remaining balance + beneficiary: T::AccountId, + }, - /// Code with the specified hash has been stored. \[code_hash\] - CodeStored(T::Hash), - - /// Triggered when the current schedule is updated. - /// \[version\] - /// - /// # Params - /// - /// - `version`: The version of the newly set schedule. - ScheduleUpdated(u32), + /// Code with the specified hash has been stored. + CodeStored { code_hash: T::Hash }, /// A custom event emitted by the contract. - /// \[contract, data\] - /// - /// # Params - /// - /// - `contract`: The contract that emitted the event. - /// - `data`: Data supplied by the contract. Metadata generated during contract compilation - /// is needed to decode it. - ContractEmitted(T::AccountId, Vec), + ContractEmitted { + /// The contract that emitted the event. + contract: T::AccountId, + /// Data supplied by the contract. Metadata generated during contract compilation + /// is needed to decode it. + data: Vec, + }, /// A code with the specified hash was removed. - /// \[code_hash\] - /// - /// This happens when the last contract that uses this code hash was removed. - CodeRemoved(T::Hash), + CodeRemoved { code_hash: T::Hash }, + + /// A contract's code was updated. + ContractCodeUpdated { + /// The contract that has been updated. + contract: T::AccountId, + /// New code hash that was set for the contract. + new_code_hash: T::Hash, + /// Previous code hash of the contract. + old_code_hash: T::Hash, + }, } #[pallet::error] pub enum Error { /// A new schedule must have a greater version than the current one. InvalidScheduleVersion, + /// Invalid combination of flags supplied to `seal_call` or `seal_delegate_call`. + InvalidCallFlags, /// The executed contract exhausted its gas limit. OutOfGas, /// The output buffer supplied to a contract API call was too small. OutputBufferTooSmall, - /// Performing the requested transfer would have brought the contract below - /// the subsistence threshold. No transfer is allowed to do this. Use `seal_terminate` - /// to recover a deposit. - BelowSubsistenceThreshold, - /// The newly created contract is below the subsistence threshold after executing - /// its contructor. No contracts are allowed to exist below that threshold. - NewContractNotFunded, - /// Performing the requested transfer failed for a reason originating in the - /// chosen currency implementation of the runtime. Most probably the balance is - /// too low or locks are placed on it. + /// Performing the requested transfer failed. Probably because there isn't enough + /// free balance in the sender's account. TransferFailed, /// Performing a call was denied because the calling depth reached the limit /// of what is specified in the schedule. @@ -472,11 +625,6 @@ pub mod pallet { /// The queue is filled by deleting contracts and emptied by a fixed amount each block. /// Trying again during another block is the only way to resolve this issue. DeletionQueueFull, - /// A storage modification exhausted the 32bit type that holds the storage size. - /// - /// This can either happen when the accumulated storage in bytes is too large or - /// when number of storage items is too large. - StorageExhausted, /// A contract with the same AccountId already exists. DuplicateContract, /// A contract self destructed in its constructor. @@ -487,6 +635,21 @@ pub mod pallet { DebugMessageInvalidUTF8, /// A call tried to invoke a contract that is flagged as non-reentrant. ReentranceDenied, + /// Origin doesn't have enough balance to pay the required storage deposits. + StorageDepositNotEnoughFunds, + /// More storage was created than allowed by the storage deposit limit. + StorageDepositLimitExhausted, + /// Code removal was denied because the code is still in use by at least one contract. + CodeInUse, + /// The contract ran to completion but decided to revert its storage changes. + /// Please note that this error is only returned from extrinsics. When called directly + /// or via RPC an `Ok` will be returned. In this case the caller needs to inspect the flags + /// to determine whether a reversion has taken place. + ContractReverted, + /// The contract's code was found to be invalid during validation or instrumentation. + /// A more detailed error can be found on the node console if debug messages are enabled + /// or in the debug buffer which is returned to RPC clients. + CodeRejected, } /// A mapping from an original code hash to the original code, untouched by instrumentation. @@ -498,9 +661,34 @@ pub mod pallet { pub(crate) type CodeStorage = StorageMap<_, Identity, CodeHash, PrefabWasmModule>; - /// The subtrie counter. + /// A mapping between an original code hash and its owner information. + #[pallet::storage] + pub(crate) type OwnerInfoOf = StorageMap<_, Identity, CodeHash, OwnerInfo>; + + /// This is a **monotonic** counter incremented on contract instantiation. + /// + /// This is used in order to generate unique trie ids for contracts. + /// The trie id of a new contract is calculated from hash(account_id, nonce). + /// The nonce is required because otherwise the following sequence would lead to + /// a possible collision of storage: + /// + /// 1. Create a new contract. + /// 2. Terminate the contract. + /// 3. Immediately recreate the contract with the same account_id. + /// + /// This is bad because the contents of a trie are deleted lazily and there might be + /// storage of the old instantiation still in it when the new contract is created. Please + /// note that we can't replace the counter by the block number because the sequence above + /// can happen in the same block. We also can't keep the account counter in memory only + /// because storage is the only way to communicate across different extrinsics in the + /// same block. + /// + /// # Note + /// + /// Do not use it to determine the number of contracts. It won't be decremented if + /// a contract is destroyed. #[pallet::storage] - pub(crate) type AccountCounter = StorageValue<_, u64, ValueQuery>; + pub(crate) type Nonce = StorageValue<_, u64, ValueQuery>; /// The code associated with a given account. /// @@ -527,6 +715,8 @@ type InternalInstantiateOutput = InternalOutput, ExecRetur struct InternalOutput { /// The gas meter that was used to execute the call. gas_meter: GasMeter, + /// The storage deposit used by the call. + storage_deposit: StorageDeposit>, /// The result of the call. result: Result, } @@ -552,16 +742,25 @@ where dest: T::AccountId, value: BalanceOf, gas_limit: Weight, + storage_deposit_limit: Option>, data: Vec, debug: bool, - ) -> ContractExecResult { + ) -> ContractExecResult> { let mut debug_message = if debug { Some(Vec::new()) } else { None }; - let output = - Self::internal_call(origin, dest, value, gas_limit, data, debug_message.as_mut()); + let output = Self::internal_call( + origin, + dest, + value, + gas_limit, + storage_deposit_limit, + data, + debug_message.as_mut(), + ); ContractExecResult { result: output.result.map_err(|r| r.error), gas_consumed: output.gas_meter.gas_consumed(), gas_required: output.gas_meter.gas_required(), + storage_deposit: output.storage_deposit, debug_message: debug_message.unwrap_or_default(), } } @@ -573,7 +772,6 @@ where /// /// It returns the execution result, account id and the amount of used weight. /// - /// /// # Note /// /// `debug` should only ever be set to `true` when executing as an RPC because @@ -581,18 +779,20 @@ where /// If set to `true` it returns additional human readable debugging information. pub fn bare_instantiate( origin: T::AccountId, - endowment: BalanceOf, + value: BalanceOf, gas_limit: Weight, + storage_deposit_limit: Option>, code: Code>, data: Vec, salt: Vec, debug: bool, - ) -> ContractInstantiateResult { + ) -> ContractInstantiateResult> { let mut debug_message = if debug { Some(Vec::new()) } else { None }; let output = Self::internal_instantiate( origin, - endowment, + value, gas_limit, + storage_deposit_limit, code, data, salt, @@ -605,10 +805,32 @@ where .map_err(|e| e.error), gas_consumed: output.gas_meter.gas_consumed(), gas_required: output.gas_meter.gas_required(), + storage_deposit: output.storage_deposit, debug_message: debug_message.unwrap_or_default(), } } + /// Upload new code without instantiating a contract from it. + /// + /// This function is similar to [`Self::upload_code`], but doesn't perform any address lookups + /// and better suitable for calling directly from Rust. + pub fn bare_upload_code( + origin: T::AccountId, + code: Vec, + storage_deposit_limit: Option>, + ) -> CodeUploadResult, BalanceOf> { + let schedule = T::Schedule::get(); + let module = PrefabWasmModule::from_code(code, &schedule, origin) + .map_err(|_| >::CodeRejected)?; + let deposit = module.open_deposit(); + if let Some(storage_deposit_limit) = storage_deposit_limit { + ensure!(storage_deposit_limit >= deposit, >::StorageDepositLimitExhausted); + } + let result = CodeUploadReturnValue { code_hash: *module.code_hash(), deposit }; + module.store()?; + Ok(result) + } + /// Query storage of a specified contract under a specified key. pub fn get_storage(address: T::AccountId, key: [u8; 32]) -> GetStorageResult { let contract_info = @@ -618,58 +840,26 @@ where Ok(maybe_value) } - /// Determine the address of a contract, + /// Determine the address of a contract. /// - /// This is the address generation function used by contract instantiation. Its result - /// is only dependend on its inputs. It can therefore be used to reliably predict the - /// address of a contract. This is akin to the formular of eth's CREATE2 opcode. There - /// is no CREATE equivalent because CREATE2 is strictly more powerful. - /// - /// Formula: `hash(deploying_address ++ code_hash ++ salt)` + /// This is the address generation function used by contract instantiation. See + /// [`DefaultAddressGenerator`] for the default implementation. pub fn contract_address( deploying_address: &T::AccountId, code_hash: &CodeHash, salt: &[u8], ) -> T::AccountId { - let buf: Vec<_> = deploying_address - .as_ref() - .iter() - .chain(code_hash.as_ref()) - .chain(salt) - .cloned() - .collect(); - UncheckedFrom::unchecked_from(T::Hashing::hash(&buf)) - } - - /// Subsistence threshold is the extension of the minimum balance (aka existential deposit) - /// by the contract deposit. It is the minimum balance any contract must hold. - /// - /// Any contract initiated balance transfer mechanism cannot make the balance lower - /// than the subsistence threshold. The only way to recover the balance is to remove - /// contract using `seal_terminate`. - pub fn subsistence_threshold() -> BalanceOf { - T::Currency::minimum_balance().saturating_add(T::ContractDeposit::get()) - } - - /// The in-memory size in bytes of the data structure associated with each contract. - /// - /// The data structure is also put into storage for each contract. The in-storage size - /// is never larger than the in-memory representation and usually smaller due to compact - /// encoding and lack of padding. - /// - /// # Note - /// - /// This returns the in-memory size because the in-storage size (SCALE encoded) cannot - /// be efficiently determined. Treat this as an upper bound of the in-storage size. - pub fn contract_info_size() -> u32 { - sp_std::mem::size_of::>() as u32 + T::AddressGenerator::generate_address(deploying_address, code_hash, salt) } /// Store code for benchmarks which does not check nor instrument the code. #[cfg(feature = "runtime-benchmarks")] - fn store_code_raw(code: Vec) -> frame_support::dispatch::DispatchResult { + fn store_code_raw( + code: Vec, + owner: T::AccountId, + ) -> frame_support::dispatch::DispatchResult { let schedule = T::Schedule::get(); - PrefabWasmModule::store_code_unchecked(code, &schedule)?; + PrefabWasmModule::store_code_unchecked(code, &schedule, owner)?; Ok(()) } @@ -679,7 +869,7 @@ where module: &mut PrefabWasmModule, schedule: &Schedule, ) -> frame_support::dispatch::DispatchResult { - self::wasm::reinstrument(module, schedule) + self::wasm::reinstrument(module, schedule).map(|_| ()) } /// Internal function that does the actual call. @@ -690,21 +880,32 @@ where dest: T::AccountId, value: BalanceOf, gas_limit: Weight, + storage_deposit_limit: Option>, data: Vec, debug_message: Option<&mut Vec>, ) -> InternalCallOutput { let mut gas_meter = GasMeter::new(gas_limit); + let mut storage_meter = match StorageMeter::new(&origin, storage_deposit_limit, value) { + Ok(meter) => meter, + Err(err) => + return InternalCallOutput { + result: Err(err.into()), + gas_meter, + storage_deposit: Default::default(), + }, + }; let schedule = T::Schedule::get(); let result = ExecStack::>::run_call( origin, dest, &mut gas_meter, + &mut storage_meter, &schedule, value, data, debug_message, ); - InternalCallOutput { gas_meter, result } + InternalCallOutput { result, gas_meter, storage_deposit: storage_meter.into_deposit() } } /// Internal function that does the actual instantiation. @@ -712,43 +913,65 @@ where /// Called by dispatchables and public functions. fn internal_instantiate( origin: T::AccountId, - endowment: BalanceOf, + value: BalanceOf, gas_limit: Weight, + storage_deposit_limit: Option>, code: Code>, data: Vec, salt: Vec, - debug_message: Option<&mut Vec>, + mut debug_message: Option<&mut Vec>, ) -> InternalInstantiateOutput { + let mut storage_deposit = Default::default(); let mut gas_meter = GasMeter::new(gas_limit); - let schedule = T::Schedule::get(); let try_exec = || { - let executable = match code { + let schedule = T::Schedule::get(); + let (extra_deposit, executable) = match code { Code::Upload(Bytes(binary)) => { ensure!( binary.len() as u32 <= schedule.limits.code_len, >::CodeTooLarge ); - let executable = PrefabWasmModule::from_code(binary, &schedule)?; + let executable = PrefabWasmModule::from_code(binary, &schedule, origin.clone()) + .map_err(|msg| { + debug_message.as_mut().map(|buffer| buffer.extend(msg.as_bytes())); + >::CodeRejected + })?; ensure!( executable.code_len() <= schedule.limits.code_len, >::CodeTooLarge ); - executable + // The open deposit will be charged during execution when the + // uploaded module does not already exist. This deposit is not part of the + // storage meter because it is not transfered to the contract but + // reserved on the uploading account. + (executable.open_deposit(), executable) }, - Code::Existing(hash) => + Code::Existing(hash) => ( + Default::default(), PrefabWasmModule::from_storage(hash, &schedule, &mut gas_meter)?, + ), }; - ExecStack::>::run_instantiate( + let mut storage_meter = StorageMeter::new( + &origin, + storage_deposit_limit, + value.saturating_add(extra_deposit), + )?; + let result = ExecStack::>::run_instantiate( origin, executable, &mut gas_meter, + &mut storage_meter, &schedule, - endowment, + value, data, &salt, debug_message, - ) + ); + storage_deposit = storage_meter + .into_deposit() + .saturating_add(&StorageDeposit::Charge(extra_deposit)); + result }; - InternalInstantiateOutput { result: try_exec(), gas_meter } + InternalInstantiateOutput { result: try_exec(), gas_meter, storage_deposit } } } diff --git a/frame/contracts/src/migration.rs b/frame/contracts/src/migration.rs index b7fa9575e23b..035e3b4409cf 100644 --- a/frame/contracts/src/migration.rs +++ b/frame/contracts/src/migration.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,13 +15,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{Config, Pallet, Weight}; +use crate::{BalanceOf, CodeHash, Config, Pallet, TrieId, Weight}; +use codec::{Decode, Encode}; use frame_support::{ + codec, generate_storage_alias, + pallet_prelude::*, storage::migration, traits::{Get, PalletInfoAccess}, + Identity, Twox64Concat, }; -use sp_std::prelude::*; +use sp_std::{marker::PhantomData, prelude::*}; +/// Wrapper for all migrations of this pallet, based on `StorageVersion`. pub fn migrate() -> Weight { use frame_support::traits::StorageVersion; @@ -38,6 +43,16 @@ pub fn migrate() -> Weight { StorageVersion::new(5).put::>(); } + if version < 6 { + weight = weight.saturating_add(v6::migrate::()); + StorageVersion::new(6).put::>(); + } + + if version < 7 { + weight = weight.saturating_add(v7::migrate::()); + StorageVersion::new(7).put::>(); + } + weight } @@ -54,11 +69,6 @@ mod v4 { /// V5: State rent is removed which obsoletes some fields in `ContractInfo`. mod v5 { use super::*; - use crate::{ - BalanceOf, CodeHash, ContractInfo, ContractInfoOf, DeletedContract, DeletionQueue, TrieId, - }; - use codec::Decode; - use sp_std::marker::PhantomData; type AliveContractInfo = RawAliveContractInfo, BalanceOf, ::BlockNumber>; @@ -95,6 +105,30 @@ mod v5 { trie_id: TrieId, } + pub type ContractInfo = RawContractInfo>; + + #[derive(Encode, Decode)] + pub struct RawContractInfo { + pub trie_id: TrieId, + pub code_hash: CodeHash, + pub _reserved: Option<()>, + } + + #[derive(Encode, Decode)] + struct DeletedContract { + trie_id: TrieId, + } + + generate_storage_alias!( + Contracts, + ContractInfoOf => Map<(Twox64Concat, T::AccountId), ContractInfo> + ); + + generate_storage_alias!( + Contracts, + DeletionQueue => Value> + ); + pub fn migrate() -> Weight { let mut weight: Weight = 0; @@ -110,7 +144,7 @@ mod v5 { } }); - >::translate(|old: Option>| { + DeletionQueue::translate(|old: Option>| { weight = weight.saturating_add(T::DbWeight::get().reads_writes(1, 1)); old.map(|old| old.into_iter().map(|o| DeletedContract { trie_id: o.trie_id }).collect()) }) @@ -119,3 +153,123 @@ mod v5 { weight } } + +/// V6: Added storage deposits +mod v6 { + use super::*; + + #[derive(Encode, Decode)] + struct OldPrefabWasmModule { + #[codec(compact)] + instruction_weights_version: u32, + #[codec(compact)] + initial: u32, + #[codec(compact)] + maximum: u32, + #[codec(compact)] + refcount: u64, + _reserved: Option<()>, + code: Vec, + original_code_len: u32, + } + + #[derive(Encode, Decode)] + struct PrefabWasmModule { + #[codec(compact)] + instruction_weights_version: u32, + #[codec(compact)] + initial: u32, + #[codec(compact)] + maximum: u32, + code: Vec, + } + + use v5::ContractInfo as OldContractInfo; + + #[derive(Encode, Decode)] + pub struct RawContractInfo { + trie_id: TrieId, + code_hash: CodeHash, + storage_deposit: Balance, + } + + #[derive(Encode, Decode)] + pub struct OwnerInfo { + owner: T::AccountId, + #[codec(compact)] + deposit: BalanceOf, + #[codec(compact)] + refcount: u64, + } + + type ContractInfo = RawContractInfo, BalanceOf>; + + generate_storage_alias!( + Contracts, + ContractInfoOf => Map<(Twox64Concat, T::AccountId), ContractInfo> + ); + + generate_storage_alias!( + Contracts, + CodeStorage => Map<(Identity, CodeHash), PrefabWasmModule> + ); + + generate_storage_alias!( + Contracts, + OwnerInfoOf => Map<(Identity, CodeHash), OwnerInfo> + ); + + pub fn migrate() -> Weight { + let mut weight: Weight = 0; + + >::translate(|_key, old: OldContractInfo| { + weight = weight.saturating_add(T::DbWeight::get().reads_writes(1, 1)); + Some(ContractInfo:: { + trie_id: old.trie_id, + code_hash: old.code_hash, + storage_deposit: Default::default(), + }) + }); + + let nobody = T::AccountId::decode(&mut sp_runtime::traits::TrailingZeroInput::zeroes()) + .expect("Infinite input; no dead input space; qed"); + + >::translate(|key, old: OldPrefabWasmModule| { + weight = weight.saturating_add(T::DbWeight::get().reads_writes(1, 2)); + >::insert( + key, + OwnerInfo { + refcount: old.refcount, + owner: nobody.clone(), + deposit: Default::default(), + }, + ); + Some(PrefabWasmModule { + instruction_weights_version: old.instruction_weights_version, + initial: old.initial, + maximum: old.maximum, + code: old.code, + }) + }); + + weight + } +} + +/// Rename `AccountCounter` to `Nonce`. +mod v7 { + use super::*; + + pub fn migrate() -> Weight { + generate_storage_alias!( + Contracts, + AccountCounter => Value + ); + generate_storage_alias!( + Contracts, + Nonce => Value + ); + Nonce::set(AccountCounter::take()); + T::DbWeight::get().reads_writes(1, 2) + } +} diff --git a/frame/contracts/src/schedule.rs b/frame/contracts/src/schedule.rs index c14165b4c6ae..b0c58d721d57 100644 --- a/frame/contracts/src/schedule.rs +++ b/frame/contracts/src/schedule.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,12 +23,12 @@ use crate::{weights::WeightInfo, Config}; use codec::{Decode, Encode}; use frame_support::{weights::Weight, DefaultNoBound}; use pallet_contracts_proc_macro::{ScheduleDebug, WeightDebug}; -use pwasm_utils::{parity_wasm::elements, rules}; use scale_info::TypeInfo; #[cfg(feature = "std")] use serde::{Deserialize, Serialize}; use sp_runtime::RuntimeDebug; use sp_std::{marker::PhantomData, vec::Vec}; +use wasm_instrument::{gas_metering, parity_wasm::elements}; /// How many API calls are executed in a single batch. The reason for increasing the amount /// of API calls in batches (per benchmark component increase) is so that the linear regression @@ -104,7 +104,13 @@ pub struct Limits { /// See to find out /// how the stack frame cost is calculated. Each element can be of one of the /// wasm value types. This means the maximum size per element is 64bit. - pub stack_height: u32, + /// + /// # Note + /// + /// It is safe to disable (pass `None`) the `stack_height` when the execution engine + /// is part of the runtime and hence there can be no indeterminism between different + /// client resident execution engines. + pub stack_height: Option, /// Maximum number of globals a module is allowed to declare. /// @@ -256,6 +262,18 @@ pub struct HostFnWeights { /// Weight of calling `seal_caller`. pub caller: Weight, + /// Weight of calling `seal_is_contract`. + pub is_contract: Weight, + + /// Weight of calling `seal_code_hash`. + pub code_hash: Weight, + + /// Weight of calling `seal_own_code_hash`. + pub own_code_hash: Weight, + + /// Weight of calling `seal_caller_is_origin`. + pub caller_is_origin: Weight, + /// Weight of calling `seal_address`. pub address: Weight, @@ -271,9 +289,6 @@ pub struct HostFnWeights { /// Weight of calling `seal_minimum_balance`. pub minimum_balance: Weight, - /// Weight of calling `seal_contract_deposit`. - pub contract_deposit: Weight, - /// Weight of calling `seal_block_number`. pub block_number: Weight, @@ -319,41 +334,59 @@ pub struct HostFnWeights { /// Weight of calling `seal_set_storage`. pub set_storage: Weight, - /// Weight per byte of an item stored with `seal_set_storage`. - pub set_storage_per_byte: Weight, + /// Weight per written byten of an item stored with `seal_set_storage`. + pub set_storage_per_new_byte: Weight, + + /// Weight per overwritten byte of an item stored with `seal_set_storage`. + pub set_storage_per_old_byte: Weight, + + /// Weight of calling `seal_set_code_hash`. + pub set_code_hash: Weight, /// Weight of calling `seal_clear_storage`. pub clear_storage: Weight, + /// Weight of calling `seal_clear_storage` per byte of the stored item. + pub clear_storage_per_byte: Weight, + + /// Weight of calling `seal_contains_storage`. + pub contains_storage: Weight, + + /// Weight of calling `seal_contains_storage` per byte of the stored item. + pub contains_storage_per_byte: Weight, + /// Weight of calling `seal_get_storage`. pub get_storage: Weight, /// Weight per byte of an item received via `seal_get_storage`. pub get_storage_per_byte: Weight, + /// Weight of calling `seal_take_storage`. + pub take_storage: Weight, + + /// Weight per byte of an item received via `seal_take_storage`. + pub take_storage_per_byte: Weight, + /// Weight of calling `seal_transfer`. pub transfer: Weight, /// Weight of calling `seal_call`. pub call: Weight, + /// Weight of calling `seal_delegate_call`. + pub delegate_call: Weight, + /// Weight surcharge that is claimed if `seal_call` does a balance transfer. pub call_transfer_surcharge: Weight, - /// Weight per input byte supplied to `seal_call`. - pub call_per_input_byte: Weight, - - /// Weight per output byte received through `seal_call`. - pub call_per_output_byte: Weight, + /// Weight per byte that is cloned by supplying the `CLONE_INPUT` flag. + pub call_per_cloned_byte: Weight, /// Weight of calling `seal_instantiate`. pub instantiate: Weight, - /// Weight per input byte supplied to `seal_instantiate`. - pub instantiate_per_input_byte: Weight, - - /// Weight per output byte received through `seal_instantiate`. - pub instantiate_per_output_byte: Weight, + /// Weight surcharge that is claimed if `seal_instantiate` does a balance transfer. + pub instantiate_transfer_surcharge: Weight, /// Weight per salt byte supplied to `seal_instantiate`. pub instantiate_per_salt_byte: Weight, @@ -475,8 +508,8 @@ impl Default for Limits { fn default() -> Self { Self { event_topics: 4, - // 512 * sizeof(i64) will give us a 4k stack. - stack_height: 512, + // No stack limit required because we use a runtime resident execution engine. + stack_height: None, globals: 256, parameters: 128, memory_pages: 16, @@ -556,12 +589,15 @@ impl Default for HostFnWeights { fn default() -> Self { Self { caller: cost_batched!(seal_caller), + is_contract: cost_batched!(seal_is_contract), + code_hash: cost_batched!(seal_code_hash), + own_code_hash: cost_batched!(seal_own_code_hash), + caller_is_origin: cost_batched!(seal_caller_is_origin), address: cost_batched!(seal_address), gas_left: cost_batched!(seal_gas_left), balance: cost_batched!(seal_balance), value_transferred: cost_batched!(seal_value_transferred), minimum_balance: cost_batched!(seal_minimum_balance), - contract_deposit: cost_batched!(seal_tombstone_deposit), block_number: cost_batched!(seal_block_number), now: cost_batched!(seal_now), weight_to_fee: cost_batched!(seal_weight_to_fee), @@ -581,46 +617,30 @@ impl Default for HostFnWeights { ), debug_message: cost_batched!(seal_debug_message), set_storage: cost_batched!(seal_set_storage), - set_storage_per_byte: cost_byte_batched!(seal_set_storage_per_kb), + set_code_hash: cost_batched!(seal_set_code_hash), + set_storage_per_new_byte: cost_byte_batched!(seal_set_storage_per_new_kb), + set_storage_per_old_byte: cost_byte_batched!(seal_set_storage_per_old_kb), clear_storage: cost_batched!(seal_clear_storage), + clear_storage_per_byte: cost_byte_batched!(seal_clear_storage_per_kb), + contains_storage: cost_batched!(seal_contains_storage), + contains_storage_per_byte: cost_byte_batched!(seal_contains_storage_per_kb), get_storage: cost_batched!(seal_get_storage), get_storage_per_byte: cost_byte_batched!(seal_get_storage_per_kb), + take_storage: cost_batched!(seal_take_storage), + take_storage_per_byte: cost_byte_batched!(seal_take_storage_per_kb), transfer: cost_batched!(seal_transfer), call: cost_batched!(seal_call), - call_transfer_surcharge: cost_batched_args!( - seal_call_per_transfer_input_output_kb, - 1, - 0, - 0 - ), - call_per_input_byte: cost_byte_batched_args!( - seal_call_per_transfer_input_output_kb, - 0, - 1, - 0 - ), - call_per_output_byte: cost_byte_batched_args!( - seal_call_per_transfer_input_output_kb, - 0, - 0, - 1 - ), + delegate_call: cost_batched!(seal_delegate_call), + call_transfer_surcharge: cost_batched_args!(seal_call_per_transfer_clone_kb, 1, 0), + call_per_cloned_byte: cost_batched_args!(seal_call_per_transfer_clone_kb, 0, 1), instantiate: cost_batched!(seal_instantiate), - instantiate_per_input_byte: cost_byte_batched_args!( - seal_instantiate_per_input_output_salt_kb, - 1, - 0, - 0 - ), - instantiate_per_output_byte: cost_byte_batched_args!( - seal_instantiate_per_input_output_salt_kb, - 0, + instantiate_transfer_surcharge: cost_byte_batched_args!( + seal_instantiate_per_transfer_salt_kb, 1, 0 ), instantiate_per_salt_byte: cost_byte_batched_args!( - seal_instantiate_per_input_output_salt_kb, - 0, + seal_instantiate_per_transfer_salt_kb, 0, 1 ), @@ -644,7 +664,7 @@ struct ScheduleRules<'a, T: Config> { } impl Schedule { - pub(crate) fn rules(&self, module: &elements::Module) -> impl rules::Rules + '_ { + pub(crate) fn rules(&self, module: &elements::Module) -> impl gas_metering::Rules + '_ { ScheduleRules { schedule: &self, params: module @@ -660,7 +680,7 @@ impl Schedule { } } -impl<'a, T: Config> rules::Rules for ScheduleRules<'a, T> { +impl<'a, T: Config> gas_metering::Rules for ScheduleRules<'a, T> { fn instruction_cost(&self, instruction: &elements::Instruction) -> Option { use self::elements::Instruction::*; let w = &self.schedule.instruction_weights; @@ -744,10 +764,10 @@ impl<'a, T: Config> rules::Rules for ScheduleRules<'a, T> { Some(weight) } - fn memory_grow_cost(&self) -> Option { + fn memory_grow_cost(&self) -> gas_metering::MemoryGrowCost { // We benchmarked the memory.grow instruction with the maximum allowed pages. // The cost for growing is therefore already included in the instruction cost. - None + gas_metering::MemoryGrowCost::Free } } diff --git a/frame/contracts/src/storage.rs b/frame/contracts/src/storage.rs index 41db0796717e..17022e942766 100644 --- a/frame/contracts/src/storage.rs +++ b/frame/contracts/src/storage.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,10 +17,12 @@ //! This module contains routines for accessing and altering a contract related state. +pub mod meter; + use crate::{ exec::{AccountIdOf, StorageKey}, weights::WeightInfo, - CodeHash, Config, ContractInfoOf, DeletionQueue, Error, TrieId, + BalanceOf, CodeHash, Config, ContractInfoOf, DeletionQueue, Error, TrieId, SENTINEL, }; use codec::{Decode, Encode}; use frame_support::{ @@ -32,24 +34,27 @@ use frame_support::{ use scale_info::TypeInfo; use sp_core::crypto::UncheckedFrom; use sp_io::hashing::blake2_256; -use sp_runtime::{traits::Hash, RuntimeDebug}; +use sp_runtime::{ + traits::{Hash, Zero}, + RuntimeDebug, +}; use sp_std::{marker::PhantomData, prelude::*}; -pub type ContractInfo = RawContractInfo>; +pub type ContractInfo = RawContractInfo, BalanceOf>; /// Information for managing an account and its sub trie abstraction. /// This is the required info to cache for an account. #[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo)] -pub struct RawContractInfo { +pub struct RawContractInfo { /// Unique ID for the subtree encoded as a bytes vector. pub trie_id: TrieId, /// The code associated with a given account. pub code_hash: CodeHash, - /// This field is reserved for future evolution of format. - pub _reserved: Option<()>, + /// The amount of balance that is currently deposited to pay for consumed storage. + pub storage_deposit: Balance, } -impl RawContractInfo { +impl RawContractInfo { /// Associated child trie unique id is built from the hash part of the trie id. #[cfg(test)] pub fn child_trie_info(&self) -> ChildInfo { @@ -67,6 +72,48 @@ pub struct DeletedContract { pub(crate) trie_id: TrieId, } +/// Information about what happended to the pre-existing value when calling [`Storage::write`]. +#[cfg_attr(test, derive(Debug, PartialEq))] +pub enum WriteOutcome { + /// No value existed at the specified key. + New, + /// A value of the returned length was overwritten. + Overwritten(u32), + /// The returned value was taken out of storage before being overwritten. + /// + /// This is only returned when specifically requested because it causes additional work + /// depending on the size of the pre-existing value. When not requested [`Self::Overwritten`] + /// is returned instead. + Taken(Vec), +} + +impl WriteOutcome { + /// Extracts the size of the overwritten value or `0` if there + /// was no value in storage. + pub fn old_len(&self) -> u32 { + match self { + Self::New => 0, + Self::Overwritten(len) => *len, + Self::Taken(value) => value.len() as u32, + } + } + + /// Extracts the size of the overwritten value or `SENTINEL` if there + /// was no value in storage. + /// + /// # Note + /// + /// We cannot use `0` as sentinel value because there could be a zero sized + /// storage entry which is different from a non existing one. + pub fn old_len_with_sentinel(&self) -> u32 { + match self { + Self::New => SENTINEL, + Self::Overwritten(len) => *len, + Self::Taken(value) => value.len() as u32, + } + } +} + pub struct Storage(PhantomData); impl Storage @@ -79,30 +126,72 @@ where /// The read is performed from the `trie_id` only. The `address` is not necessary. If the /// contract doesn't store under the given `key` `None` is returned. pub fn read(trie_id: &TrieId, key: &StorageKey) -> Option> { - child::get_raw(&child_trie_info(&trie_id), &blake2_256(key)) + child::get_raw(&child_trie_info(trie_id), &blake2_256(key)) + } + + /// Returns `Some(len)` (in bytes) if a storage item exists at `key`. + /// + /// Returns `None` if the `key` wasn't previously set by `set_storage` or + /// was deleted. + pub fn size(trie_id: &TrieId, key: &StorageKey) -> Option { + child::len(&child_trie_info(trie_id), &blake2_256(key)) } /// Update a storage entry into a contract's kv storage. /// - /// If the `opt_new_value` is `None` then the kv pair is removed. + /// If the `new_value` is `None` then the kv pair is removed. If `take` is true + /// a [`WriteOutcome::Taken`] is returned instead of a [`WriteOutcome::Overwritten`]. /// - /// This function also updates the bookkeeping info such as: number of total non-empty pairs a - /// contract owns, the last block the storage was written to, etc. That's why, in contrast to - /// `read`, this function also requires the `account` ID. + /// This function also records how much storage was created or removed if a `storage_meter` + /// is supplied. It should only be absent for testing or benchmarking code. pub fn write( - new_info: &mut ContractInfo, + trie_id: &TrieId, key: &StorageKey, - opt_new_value: Option>, - ) -> DispatchResult { + new_value: Option>, + storage_meter: Option<&mut meter::NestedMeter>, + take: bool, + ) -> Result { let hashed_key = blake2_256(key); - let child_trie_info = &child_trie_info(&new_info.trie_id); + let child_trie_info = &child_trie_info(trie_id); + let (old_len, old_value) = if take { + let val = child::get_raw(&child_trie_info, &hashed_key); + (val.as_ref().map(|v| v.len() as u32), val) + } else { + (child::len(&child_trie_info, &hashed_key), None) + }; + + if let Some(storage_meter) = storage_meter { + let mut diff = meter::Diff::default(); + match (old_len, new_value.as_ref().map(|v| v.len() as u32)) { + (Some(old_len), Some(new_len)) => + if new_len > old_len { + diff.bytes_added = new_len - old_len; + } else { + diff.bytes_removed = old_len - new_len; + }, + (None, Some(new_len)) => { + diff.bytes_added = new_len; + diff.items_added = 1; + }, + (Some(old_len), None) => { + diff.bytes_removed = old_len; + diff.items_removed = 1; + }, + (None, None) => (), + } + storage_meter.charge(&diff)?; + } - match opt_new_value { - Some(new_value) => child::put_raw(&child_trie_info, &hashed_key, &new_value[..]), + match &new_value { + Some(new_value) => child::put_raw(&child_trie_info, &hashed_key, new_value), None => child::kill(&child_trie_info, &hashed_key), } - Ok(()) + Ok(match (old_len, old_value) { + (None, _) => WriteOutcome::New, + (Some(old_len), None) => WriteOutcome::Overwritten(old_len), + (Some(_), Some(old_value)) => WriteOutcome::Taken(old_value), + }) } /// Creates a new contract descriptor in the storage with the given code hash at the given @@ -118,7 +207,8 @@ where return Err(Error::::DuplicateContract.into()) } - let contract = ContractInfo:: { code_hash: ch, trie_id, _reserved: None }; + let contract = + ContractInfo:: { code_hash: ch, trie_id, storage_deposit: >::zero() }; Ok(contract) } @@ -178,15 +268,17 @@ where let mut queue = >::get(); - if let (Some(trie), true) = (queue.get(0), remaining_key_budget > 0) { + while !queue.is_empty() && remaining_key_budget > 0 { + // Cannot panic due to loop condition + let trie = &mut queue[0]; let outcome = child::kill_storage(&child_trie_info(&trie.trie_id), Some(remaining_key_budget)); let keys_removed = match outcome { - // This should not happen as our budget was large enough to remove all keys. + // This happens when our budget wasn't large enough to remove all keys. KillStorageResult::SomeRemaining(count) => count, KillStorageResult::AllRemoved(count) => { // We do not care to preserve order. The contract is deleted already and - // noone waits for the trie to be deleted. + // no one waits for the trie to be deleted. queue.swap_remove(0); count }, @@ -198,10 +290,9 @@ where weight_limit.saturating_sub(weight_per_key.saturating_mul(remaining_key_budget as Weight)) } - /// This generator uses inner counter for account id and applies the hash over `AccountId + - /// accountid_counter`. - pub fn generate_trie_id(account_id: &AccountIdOf, seed: u64) -> TrieId { - let buf: Vec<_> = account_id.as_ref().iter().chain(&seed.to_le_bytes()).cloned().collect(); + /// Generates a unique trie id by returning `hash(account_id ++ nonce)`. + pub fn generate_trie_id(account_id: &AccountIdOf, nonce: u64) -> TrieId { + let buf: Vec<_> = account_id.as_ref().iter().chain(&nonce.to_le_bytes()).cloned().collect(); T::Hashing::hash(&buf).as_ref().into() } diff --git a/frame/contracts/src/storage/meter.rs b/frame/contracts/src/storage/meter.rs new file mode 100644 index 000000000000..28dc3356f714 --- /dev/null +++ b/frame/contracts/src/storage/meter.rs @@ -0,0 +1,723 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contains functions to meter the storage deposit. + +use crate::{storage::ContractInfo, BalanceOf, Config, Error}; +use codec::Encode; +use frame_support::{ + dispatch::DispatchError, + traits::{tokens::BalanceStatus, Currency, ExistenceRequirement, Get, ReservableCurrency}, + DefaultNoBound, +}; +use pallet_contracts_primitives::StorageDeposit as Deposit; +use sp_core::crypto::UncheckedFrom; +use sp_runtime::traits::{Saturating, Zero}; +use sp_std::marker::PhantomData; + +/// Deposit that uses the native currency's balance type. +pub type DepositOf = Deposit>; + +/// A production root storage meter that actually charges from its origin. +pub type Meter = RawMeter; + +/// A poduction nested storage meter that actually charges from its origin. +pub type NestedMeter = RawMeter; + +/// A poduction storage meter that actually charges from its origin. +/// +/// This can be used where we want to be generic over the state (Root vs. Nested). +pub type GenericMeter = RawMeter; + +/// A trait that allows to decouple the metering from the charging of balance. +/// +/// This mostly exists for testing so that the charging can be mocked. +pub trait Ext { + /// This checks whether `origin` is able to afford the storage deposit limit. + /// + /// It is necessary to do this check beforehand so that the charge won't fail later on. + /// + /// `origin`: The origin of the call stack from which is responsible for putting down a deposit. + /// `limit`: The limit with which the meter was constructed. + /// `min_leftover`: How much `free_balance` in addition to the ed should be left inside the + /// `origin` account. + /// + /// Returns the limit that should be used by the meter. If origin can't afford the `limit` + /// it returns `Err`. + fn check_limit( + origin: &T::AccountId, + limit: Option>, + min_leftover: BalanceOf, + ) -> Result, DispatchError>; + /// This is called to inform the implementer that some balance should be charged due to + /// some interaction of the `origin` with a `contract`. + /// + /// The balance transfer can either flow from `origin` to `contract` or the other way + /// around depending on whether `amount` constitutes a `Charge` or a `Refund`. + /// It is guaranteed that that this succeeds because no more balance than returned by + /// `check_limit` is ever charged. This is why this function is infallible. + /// `terminated` designates whether the `contract` was terminated. + fn charge( + origin: &T::AccountId, + contract: &T::AccountId, + amount: &DepositOf, + terminated: bool, + ); +} + +/// This [`Ext`] is used for actual on-chain execution when balance needs to be charged. +/// +/// It uses [`ReservableCurrency`] in order to do accomplish the reserves. +pub enum ReservingExt {} + +/// Used to implement a type state pattern for the meter. +/// +/// It is sealed and cannot be implemented outside of this module. +pub trait State: private::Sealed {} + +/// State parameter that constitutes a meter that is in its root state. +pub enum Root {} + +/// State parameter that constitutes a meter that is in its nested state. +pub enum Nested {} + +impl State for Root {} +impl State for Nested {} + +/// A type that allows the metering of consumed or freed storage of a single contract call stack. +#[derive(DefaultNoBound)] +pub struct RawMeter, S: State> { + /// The limit of how much balance this meter is allowed to consume. + limit: BalanceOf, + /// The amount of balance that was used in this meter and all of its already absorbed children. + total_deposit: DepositOf, + /// The amount of balance that was used in this meter alone. + own_deposit: DepositOf, + /// Only when a contract was terminated we allow it to drop below the minimum balance. + terminated: bool, + /// Type parameters are only used in impls. + _phantom: PhantomData<(E, S)>, +} + +/// This type is used to describe a storage change when charging from the meter. +#[derive(Default)] +pub struct Diff { + /// How many bytes were added to storage. + pub bytes_added: u32, + /// How many bytes were removed from storage. + pub bytes_removed: u32, + /// How many storage items were added to storage. + pub items_added: u32, + /// How many storage items were removed from storage. + pub items_removed: u32, + /// If set to true the derived deposit will always a `Charge` larger than the + /// the existential deposit. + pub require_ed: bool, +} + +impl Diff { + /// Calculate how much of a charge or refund results from applying the diff. + pub fn to_deposit(&self) -> DepositOf { + let mut deposit = Deposit::default(); + let per_byte = T::DepositPerByte::get(); + let per_item = T::DepositPerItem::get(); + + if self.bytes_added > self.bytes_removed { + deposit = deposit.saturating_add(&Deposit::Charge( + per_byte.saturating_mul((self.bytes_added - self.bytes_removed).into()), + )); + } else if self.bytes_removed > self.bytes_added { + deposit = deposit.saturating_add(&Deposit::Refund( + per_byte.saturating_mul((self.bytes_removed - self.bytes_added).into()), + )); + } + + if self.items_added > self.items_removed { + deposit = deposit.saturating_add(&Deposit::Charge( + per_item.saturating_mul((self.items_added - self.items_removed).into()), + )); + } else if self.items_removed > self.items_added { + deposit = deposit.saturating_add(&Deposit::Refund( + per_item.saturating_mul((self.items_removed - self.items_added).into()), + )); + } + + if self.require_ed { + deposit = deposit.max(Deposit::Charge(T::Currency::minimum_balance())) + } + + deposit + } +} + +/// Functions that apply to all states. +impl RawMeter +where + T: Config, + T::AccountId: UncheckedFrom + AsRef<[u8]>, + E: Ext, + S: State, +{ + /// Create a new child that has its `limit` set to whatever is remaining of it. + /// + /// This is called whenever a new subcall is initiated in order to track the storage + /// usage for this sub call separately. This is necessary because we want to exchange balance + /// with the current contract we are interacting with. + pub fn nested(&self) -> RawMeter { + RawMeter { limit: self.available(), ..Default::default() } + } + + /// Absorb a child that was spawned to handle a sub call. + /// + /// This should be called whenever a sub call comes to its end and it is **not** reverted. + /// This does the actual balance transfer from/to `origin` and `contract` based on the overall + /// storage consumption of the call. It also updates the supplied contract info. + /// + /// In case a contract reverted the child meter should just be dropped in order to revert + /// any changes it recorded. + /// + /// # Parameters + /// + /// `absorbed`: The child storage meter that should be absorbed. + /// `origin`: The origin that spawned the original root meter. + /// `contract`: The contract that this sub call belongs to. + /// `info`: The info of the contract in question. `None` if the contract was terminated. + pub fn absorb( + &mut self, + mut absorbed: RawMeter, + origin: &T::AccountId, + contract: &T::AccountId, + info: Option<&mut ContractInfo>, + ) { + // Absorbing from an existing (non terminated) contract. + if let Some(info) = info { + match &mut absorbed.own_deposit { + Deposit::Charge(amount) => + info.storage_deposit = info.storage_deposit.saturating_add(*amount), + Deposit::Refund(amount) => { + // We need to make sure to never refund more than what was deposited and + // still leave the existential deposit inside the contract's account. + // This case can happen when costs change due to a runtime upgrade where + // increased costs could remove an account due to refunds. + let amount = { + let corrected_amount = (*amount).min( + info.storage_deposit.saturating_sub(T::Currency::minimum_balance()), + ); + let correction = (*amount).saturating_sub(corrected_amount); + absorbed.total_deposit = + absorbed.total_deposit.saturating_sub(&Deposit::Refund(correction)); + *amount = corrected_amount; + corrected_amount + }; + info.storage_deposit = info.storage_deposit.saturating_sub(amount); + }, + } + } + + self.total_deposit = self.total_deposit.saturating_add(&absorbed.total_deposit); + if !absorbed.own_deposit.is_zero() { + E::charge(origin, &contract, &absorbed.own_deposit, absorbed.terminated); + } + } + + /// The amount of balance that is still available from the original `limit`. + fn available(&self) -> BalanceOf { + self.total_deposit.available(&self.limit) + } +} + +/// Functions that only apply to the root state. +impl RawMeter +where + T: Config, + T::AccountId: UncheckedFrom + AsRef<[u8]>, + E: Ext, +{ + /// Create new storage meter for the specified `origin` and `limit`. + /// + /// This tries to [`Ext::check_limit`] on `origin` and fails if this is not possible. + pub fn new( + origin: &T::AccountId, + limit: Option>, + min_leftover: BalanceOf, + ) -> Result { + let limit = E::check_limit(&origin, limit, min_leftover)?; + Ok(Self { limit, ..Default::default() }) + } + + /// The total amount of deposit that should change hands as result of the execution + /// that this meter was passed into. + /// + /// This drops the root meter in order to make sure it is only called when the whole + /// execution did finish. + pub fn into_deposit(self) -> DepositOf { + self.total_deposit + } +} + +/// Functions that only apply to the nested state. +impl RawMeter +where + T: Config, + T::AccountId: UncheckedFrom + AsRef<[u8]>, + E: Ext, +{ + /// Try to charge the `diff` from the meter. Fails if this would exceed the original limit. + pub fn charge(&mut self, diff: &Diff) -> Result, DispatchError> { + debug_assert!(!self.terminated); + let deposit = diff.to_deposit::(); + let total_deposit = self.total_deposit.saturating_add(&deposit); + if let Deposit::Charge(amount) = total_deposit { + if amount > self.limit { + return Err(>::StorageDepositLimitExhausted.into()) + } + } + self.total_deposit = total_deposit; + self.own_deposit = self.own_deposit.saturating_add(&deposit); + Ok(deposit) + } + + /// Charge from `origin` a storage deposit for contract instantiation. + /// + /// This immediately transfers the balance in order to create the account. + pub fn charge_instantiate( + &mut self, + origin: &T::AccountId, + contract: &T::AccountId, + info: &mut ContractInfo, + ) -> Result, DispatchError> { + debug_assert!(!self.terminated); + let deposit = Diff { + bytes_added: info.encoded_size() as u32, + items_added: 1, + require_ed: true, + ..Default::default() + } + .to_deposit::(); + debug_assert!(matches!(deposit, Deposit::Charge(_))); + // We do not increase `own_deposit` because this will be charged later when the contract + // execution does conclude. + let total_deposit = self.total_deposit.saturating_add(&deposit); + if let Deposit::Charge(amount) = &total_deposit { + if amount > &self.limit { + return Err(>::StorageDepositLimitExhausted.into()) + } + } + info.storage_deposit = info.storage_deposit.saturating_add(deposit.charge_or_zero()); + self.total_deposit = total_deposit; + if !deposit.is_zero() { + // We need to charge immediately so that the account is created before the `value` + // is transferred from the caller to the contract. + E::charge(origin, contract, &deposit, false); + } + Ok(deposit) + } + + /// Call to tell the meter that the currently executing contract was executed. + /// + /// This will manipulate the meter so that all storage deposit accumulated in + /// `contract_info` will be refunded to the `origin` of the meter. + pub fn terminate(&mut self, contract_info: &ContractInfo) { + debug_assert!(!self.terminated); + let refund = Deposit::Refund(contract_info.storage_deposit); + + // The deposit for `own_deposit` isn't persisted into the contract info until the current + // frame is dropped. This means that whatever changes were introduced during the + // current frame are dicarded when terminating. + self.total_deposit = + self.total_deposit.saturating_add(&refund).saturating_sub(&self.own_deposit); + self.own_deposit = refund; + self.terminated = true; + } +} + +impl Ext for ReservingExt { + fn check_limit( + origin: &T::AccountId, + limit: Option>, + min_leftover: BalanceOf, + ) -> Result, DispatchError> { + let max = T::Currency::free_balance(origin) + .saturating_sub(T::Currency::minimum_balance()) + .saturating_sub(min_leftover); + match limit { + Some(limit) if limit <= max => Ok(limit), + None => Ok(max), + _ => Err(>::StorageDepositNotEnoughFunds.into()), + } + } + + fn charge( + origin: &T::AccountId, + contract: &T::AccountId, + amount: &DepositOf, + terminated: bool, + ) { + // There is nothing we can do when this fails as this constitutes a bug in the runtime: + // Either the runtime does not hold up the invariant of never deleting a contract's account + // or it does not honor reserved balances. We need to settle for emitting an error log + // in this case. + match amount { + Deposit::Charge(amount) => { + // This will never fail because a contract's account is required to exist + // at all times. The pallet enforces this invariant by depositing at least the + // existential deposit when instantiating and never refunds it unless the contract + // is removed. This means the receiver always exists except when instantiating a + // contract. In this case we made sure that at least the existential deposit is + // sent. The sender always has enough balance because we checked that it had enough + // balance when instantiating the storage meter. + let result = T::Currency::transfer( + origin, + contract, + *amount, + ExistenceRequirement::KeepAlive, + ) + .and_then(|_| T::Currency::reserve(contract, *amount)); + if let Err(err) = result { + log::error!( + target: "runtime::contracts", + "Failed to transfer storage deposit {:?} from origin {:?} to contract {:?}: {:?}", + amount, origin, contract, err, + ); + } + }, + // For `Refund(_)` no error happen because the initial value transfer from the + // origin to the contract has a keep alive existence requirement and when reserving we + // make sure to leave at least the ed in the free balance. Therefore the receiver always + // exists because there is no way for it to be removed in between. The sender always has + // enough reserved balance because we track it in the `ContractInfo` and never send more + // back than we have. + Deposit::Refund(amount) => { + let amount = if terminated { + *amount + } else { + // This is necessary when the `storage_deposit` tracked inside the account + // info is out of sync with the actual balance. That can only happen due to + // slashing. We make sure to never dust the contract's account through a + // refund because we consider this unexpected behaviour. + *amount.min( + &T::Currency::reserved_balance(contract) + .saturating_sub(T::Currency::minimum_balance()), + ) + }; + let result = + T::Currency::repatriate_reserved(contract, origin, amount, BalanceStatus::Free); + if matches!(result, Ok(val) if !val.is_zero()) || matches!(result, Err(_)) { + log::error!( + target: "runtime::contracts", + "Failed to repatriate storage deposit {:?} from contract {:?} to origin {:?}: {:?}", + amount, contract, origin, result, + ); + } + }, + }; + } +} + +mod private { + pub trait Sealed {} + impl Sealed for super::Root {} + impl Sealed for super::Nested {} +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + exec::AccountIdOf, + tests::{Test, ALICE, BOB, CHARLIE}, + }; + use pretty_assertions::assert_eq; + use std::cell::RefCell; + + type TestMeter = RawMeter; + + thread_local! { + static TEST_EXT: RefCell = RefCell::new(Default::default()); + } + + #[derive(Debug, PartialEq, Eq)] + struct LimitCheck { + origin: AccountIdOf, + limit: BalanceOf, + min_leftover: BalanceOf, + } + + #[derive(Debug, PartialEq, Eq)] + struct Charge { + origin: AccountIdOf, + contract: AccountIdOf, + amount: DepositOf, + terminated: bool, + } + + #[derive(Default, Debug, PartialEq, Eq)] + struct TestExt { + limit_checks: Vec, + charges: Vec, + } + + impl TestExt { + fn clear(&mut self) { + self.limit_checks.clear(); + self.charges.clear(); + } + } + + impl Ext for TestExt { + fn check_limit( + origin: &AccountIdOf, + limit: Option>, + min_leftover: BalanceOf, + ) -> Result, DispatchError> { + let limit = limit.unwrap_or(42); + TEST_EXT.with(|ext| { + ext.borrow_mut().limit_checks.push(LimitCheck { + origin: origin.clone(), + limit, + min_leftover, + }) + }); + Ok(limit) + } + + fn charge( + origin: &AccountIdOf, + contract: &AccountIdOf, + amount: &DepositOf, + terminated: bool, + ) { + TEST_EXT.with(|ext| { + ext.borrow_mut().charges.push(Charge { + origin: origin.clone(), + contract: contract.clone(), + amount: amount.clone(), + terminated, + }) + }); + } + } + + fn clear_ext() { + TEST_EXT.with(|ext| ext.borrow_mut().clear()) + } + + fn new_info(deposit: BalanceOf) -> ContractInfo { + use crate::storage::Storage; + use sp_runtime::traits::Hash; + + ContractInfo:: { + trie_id: >::generate_trie_id(&ALICE, 42), + code_hash: ::Hashing::hash(b"42"), + storage_deposit: deposit, + } + } + + #[test] + fn new_reserves_balance_works() { + clear_ext(); + + TestMeter::new(&ALICE, Some(1_000), 0).unwrap(); + + TEST_EXT.with(|ext| { + assert_eq!( + *ext.borrow(), + TestExt { + limit_checks: vec![LimitCheck { origin: ALICE, limit: 1_000, min_leftover: 0 }], + ..Default::default() + } + ) + }); + } + + #[test] + fn empty_charge_works() { + clear_ext(); + + let mut meter = TestMeter::new(&ALICE, Some(1_000), 0).unwrap(); + assert_eq!(meter.available(), 1_000); + + // an empty charge foes not create a `Charge` entry + let mut nested0 = meter.nested(); + nested0.charge(&Default::default()).unwrap(); + meter.absorb(nested0, &ALICE, &BOB, None); + + TEST_EXT.with(|ext| { + assert_eq!( + *ext.borrow(), + TestExt { + limit_checks: vec![LimitCheck { origin: ALICE, limit: 1_000, min_leftover: 0 }], + ..Default::default() + } + ) + }); + } + + #[test] + fn existential_deposit_works() { + clear_ext(); + + let mut meter = TestMeter::new(&ALICE, Some(1_000), 0).unwrap(); + assert_eq!(meter.available(), 1_000); + + // a `Refund` will be turned into a `Charge(ed)` which is intended behaviour + let mut nested0 = meter.nested(); + nested0.charge(&Diff { require_ed: true, ..Default::default() }).unwrap(); + nested0 + .charge(&Diff { bytes_removed: 1, require_ed: true, ..Default::default() }) + .unwrap(); + meter.absorb(nested0, &ALICE, &BOB, None); + + TEST_EXT.with(|ext| { + assert_eq!( + *ext.borrow(), + TestExt { + limit_checks: vec![LimitCheck { origin: ALICE, limit: 1_000, min_leftover: 0 }], + charges: vec![Charge { + origin: ALICE, + contract: BOB, + amount: Deposit::Charge(::Currency::minimum_balance() * 2), + terminated: false, + }] + } + ) + }); + } + + #[test] + fn charging_works() { + clear_ext(); + + let min_balance = ::Currency::minimum_balance(); + + let mut meter = TestMeter::new(&ALICE, Some(1_000), 0).unwrap(); + assert_eq!(meter.available(), 1_000); + + let mut nested0_info = new_info(100); + let mut nested0 = meter.nested(); + nested0 + .charge(&Diff { + bytes_added: 10, + bytes_removed: 5, + items_added: 1, + items_removed: 2, + ..Default::default() + }) + .unwrap(); + nested0.charge(&Diff { bytes_removed: 1, ..Default::default() }).unwrap(); + + let mut nested1_info = new_info(50); + let mut nested1 = nested0.nested(); + nested1.charge(&Diff { items_removed: 5, ..Default::default() }).unwrap(); + nested0.absorb(nested1, &ALICE, &CHARLIE, Some(&mut nested1_info)); + + // Trying to refund more than is available in the contract will cap the charge + // to (deposit_in_contract - ed). + let mut nested2_info = new_info(5); + let mut nested2 = nested0.nested(); + nested2.charge(&Diff { bytes_removed: 7, ..Default::default() }).unwrap(); + nested0.absorb(nested2, &ALICE, &CHARLIE, Some(&mut nested2_info)); + + meter.absorb(nested0, &ALICE, &BOB, Some(&mut nested0_info)); + + assert_eq!(nested0_info.storage_deposit, 102); + assert_eq!(nested1_info.storage_deposit, 40); + assert_eq!(nested2_info.storage_deposit, min_balance); + + TEST_EXT.with(|ext| { + assert_eq!( + *ext.borrow(), + TestExt { + limit_checks: vec![LimitCheck { origin: ALICE, limit: 1_000, min_leftover: 0 }], + charges: vec![ + Charge { + origin: ALICE, + contract: CHARLIE, + amount: Deposit::Refund(10), + terminated: false + }, + Charge { + origin: ALICE, + contract: CHARLIE, + amount: Deposit::Refund(4), + terminated: false + }, + Charge { + origin: ALICE, + contract: BOB, + amount: Deposit::Charge(2), + terminated: false + } + ] + } + ) + }); + } + + #[test] + fn termination_works() { + clear_ext(); + + let mut meter = TestMeter::new(&ALICE, Some(1_000), 0).unwrap(); + assert_eq!(meter.available(), 1_000); + + let mut nested0 = meter.nested(); + nested0 + .charge(&Diff { + bytes_added: 5, + bytes_removed: 1, + items_added: 3, + items_removed: 1, + ..Default::default() + }) + .unwrap(); + nested0.charge(&Diff { items_added: 2, ..Default::default() }).unwrap(); + + let nested1_info = new_info(400); + let mut nested1 = nested0.nested(); + nested1.charge(&Diff { items_removed: 5, ..Default::default() }).unwrap(); + nested1.charge(&Diff { bytes_added: 20, ..Default::default() }).unwrap(); + nested1.terminate(&nested1_info); + nested0.absorb(nested1, &ALICE, &CHARLIE, None); + + meter.absorb(nested0, &ALICE, &BOB, None); + drop(meter); + + TEST_EXT.with(|ext| { + assert_eq!( + *ext.borrow(), + TestExt { + limit_checks: vec![LimitCheck { origin: ALICE, limit: 1_000, min_leftover: 0 }], + charges: vec![ + Charge { + origin: ALICE, + contract: CHARLIE, + amount: Deposit::Refund(400), + terminated: true + }, + Charge { + origin: ALICE, + contract: BOB, + amount: Deposit::Charge(12), + terminated: false + } + ] + } + ) + }); + } +} diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index f5b95c192c42..ce59f5bb858a 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,31 +21,35 @@ use crate::{ ReturnFlags, SysConfig, UncheckedFrom, }, exec::Frame, - storage::{RawContractInfo, Storage}, + storage::Storage, wasm::{PrefabWasmModule, ReturnCode as RuntimeReturnCode}, weights::WeightInfo, - BalanceOf, Config, ContractInfoOf, Error, Pallet, Schedule, + BalanceOf, Code, CodeStorage, Config, ContractInfoOf, DefaultAddressGenerator, Error, Pallet, + Schedule, }; use assert_matches::assert_matches; use codec::Encode; use frame_support::{ - assert_err, assert_err_ignore_postinfo, assert_ok, + assert_err, assert_err_ignore_postinfo, assert_noop, assert_ok, dispatch::DispatchErrorWithPostInfo, parameter_types, storage::child, - traits::{Contains, Currency, OnInitialize, ReservableCurrency}, + traits::{ + BalanceStatus, ConstU32, ConstU64, Contains, Currency, OnInitialize, ReservableCurrency, + }, weights::{constants::WEIGHT_PER_SECOND, DispatchClass, PostDispatchInfo, Weight}, }; use frame_system::{self as system, EventRecord, Phase}; use pretty_assertions::assert_eq; use sp_core::Bytes; use sp_io::hashing::blake2_256; +use sp_keystore::{testing::KeyStore, KeystoreExt}; use sp_runtime::{ testing::{Header, H256}, traits::{BlakeTwo256, Convert, Hash, IdentityLookup}, AccountId32, }; -use std::cell::RefCell; +use std::{cell::RefCell, sync::Arc}; use crate as pallet_contracts; @@ -70,31 +74,16 @@ frame_support::construct_runtime!( #[macro_use] pub mod test_utils { use super::{Balances, Test}; - use crate::{ - exec::{AccountIdOf, StorageKey}, - storage::Storage, - AccountCounter, CodeHash, ContractInfoOf, Pallet as Contracts, TrieId, - }; + use crate::{exec::AccountIdOf, storage::Storage, CodeHash, Config, ContractInfoOf, Nonce}; use frame_support::traits::Currency; - pub fn set_storage(addr: &AccountIdOf, key: &StorageKey, value: Option>) { - let mut contract_info = >::get(&addr).unwrap(); - Storage::::write(&mut contract_info, key, value).unwrap(); - } - pub fn get_storage(addr: &AccountIdOf, key: &StorageKey) -> Option> { - let contract_info = >::get(&addr).unwrap(); - Storage::::read(&contract_info.trie_id, key) - } - pub fn generate_trie_id(address: &AccountIdOf) -> TrieId { - let seed = >::mutate(|counter| { + pub fn place_contract(address: &AccountIdOf, code_hash: CodeHash) { + let nonce = >::mutate(|counter| { *counter += 1; *counter }); - Storage::::generate_trie_id(address, seed) - } - pub fn place_contract(address: &AccountIdOf, code_hash: CodeHash) { - let trie_id = generate_trie_id(address); - set_balance(address, Contracts::::subsistence_threshold() * 10); + let trie_id = Storage::::generate_trie_id(address, nonce); + set_balance(address, ::Currency::minimum_balance() * 10); let contract = Storage::::new_contract(&address, trie_id, code_hash).unwrap(); >::insert(address, contract); } @@ -107,13 +96,12 @@ pub mod test_utils { } macro_rules! assert_return_code { ( $x:expr , $y:expr $(,)? ) => {{ - use sp_std::convert::TryInto; assert_eq!(u32::from_le_bytes($x.data[..].try_into().unwrap()), $y as u32); }}; } macro_rules! assert_refcount { ( $code_hash:expr , $should:expr $(,)? ) => {{ - let is = crate::CodeStorage::::get($code_hash).map(|m| m.refcount()).unwrap_or(0); + let is = crate::OwnerInfoOf::::get($code_hash).map(|m| m.refcount()).unwrap(); assert_eq!(is, $should); }}; } @@ -190,10 +178,9 @@ impl ChainExtension for TestExtension { } parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(2 * WEIGHT_PER_SECOND); - pub static ExistentialDeposit: u64 = 0; + pub static ExistentialDeposit: u64 = 1; } impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; @@ -210,7 +197,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -219,6 +206,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } impl pallet_randomness_collective_flip::Config for Test {} impl pallet_balances::Config for Test { @@ -232,28 +220,33 @@ impl pallet_balances::Config for Test { type AccountStore = System; type WeightInfo = (); } -parameter_types! { - pub const MinimumPeriod: u64 = 1; -} + impl pallet_timestamp::Config for Test { type Moment = u64; type OnTimestampSet = (); - type MinimumPeriod = MinimumPeriod; + type MinimumPeriod = ConstU64<1>; type WeightInfo = (); } impl pallet_utility::Config for Test { type Event = Event; type Call = Call; + type PalletsOrigin = OriginCaller; type WeightInfo = (); } parameter_types! { - pub const ContractDeposit: u64 = 16; pub const MaxValueSize: u32 = 16_384; - pub const DeletionQueueDepth: u32 = 1024; pub const DeletionWeightLimit: Weight = 500_000_000_000; pub const MaxCodeSize: u32 = 2 * 1024; - pub MySchedule: Schedule = >::default(); + pub MySchedule: Schedule = { + let mut schedule = >::default(); + // We want stack height to be always enabled for tests so that this + // instrumentation path is always tested implicitly. + schedule.limits.stack_height = Some(512); + schedule + }; pub const TransactionByteFee: u64 = 0; + pub static DepositPerByte: BalanceOf = 1; + pub const DepositPerItem: BalanceOf = 2; } impl Convert> for Test { @@ -288,14 +281,16 @@ impl Config for Test { type Event = Event; type Call = Call; type CallFilter = TestFilter; - type ContractDeposit = ContractDeposit; type CallStack = [Frame; 31]; type WeightPrice = Self; type WeightInfo = (); type ChainExtension = TestExtension; - type DeletionQueueDepth = DeletionQueueDepth; + type DeletionQueueDepth = ConstU32<1024>; type DeletionWeightLimit = DeletionWeightLimit; type Schedule = MySchedule; + type DepositPerByte = DepositPerByte; + type DepositPerItem = DepositPerItem; + type AddressGenerator = DefaultAddressGenerator; } pub const ALICE: AccountId32 = AccountId32::new([1u8; 32]); @@ -303,14 +298,14 @@ pub const BOB: AccountId32 = AccountId32::new([2u8; 32]); pub const CHARLIE: AccountId32 = AccountId32::new([3u8; 32]); pub const DJANGO: AccountId32 = AccountId32::new([4u8; 32]); -const GAS_LIMIT: Weight = 10_000_000_000; +pub const GAS_LIMIT: Weight = 100_000_000_000; pub struct ExtBuilder { existential_deposit: u64, } impl Default for ExtBuilder { fn default() -> Self { - Self { existential_deposit: 1 } + Self { existential_deposit: ExistentialDeposit::get() } } } impl ExtBuilder { @@ -322,12 +317,16 @@ impl ExtBuilder { EXISTENTIAL_DEPOSIT.with(|v| *v.borrow_mut() = self.existential_deposit); } pub fn build(self) -> sp_io::TestExternalities { + use env_logger::{Builder, Env}; + let env = Env::new().default_filter_or("runtime=debug"); + let _ = Builder::from_env(env).is_test(true).try_init(); self.set_associated_consts(); let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { balances: vec![] } .assimilate_storage(&mut t) .unwrap(); let mut ext = sp_io::TestExternalities::new(t); + ext.register_extension(KeystoreExt(Arc::new(KeyStore::new()))); ext.execute_with(|| System::set_block_number(1)); ext } @@ -347,6 +346,11 @@ where Ok((wasm_binary, code_hash)) } +fn initialize_block(number: u64) { + System::reset_events(); + System::initialize(&number, &[0u8; 32].into(), &Default::default()); +} + // Perform a call to a plain account. // The actual transfer fails because we can only call contracts. // Then we check that at least the base costs where charged (no runtime gas costs.) @@ -357,7 +361,7 @@ fn calling_plain_account_fails() { let base_cost = <::WeightInfo as WeightInfo>::call(); assert_eq!( - Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, Vec::new()), + Contracts::call(Origin::signed(ALICE), BOB, 0, GAS_LIMIT, None, Vec::new()), Err(DispatchErrorWithPostInfo { error: Error::::ContractNotFound.into(), post_info: PostDispatchInfo { @@ -369,136 +373,95 @@ fn calling_plain_account_fails() { }); } -#[test] -fn account_removal_does_not_remove_storage() { - use self::test_utils::{get_storage, set_storage}; - - ExtBuilder::default().existential_deposit(100).build().execute_with(|| { - let trie_id1 = test_utils::generate_trie_id(&ALICE); - let trie_id2 = test_utils::generate_trie_id(&BOB); - let key1 = &[1; 32]; - let key2 = &[2; 32]; - - // Set up two accounts with free balance above the existential threshold. - { - let alice_contract_info = RawContractInfo { - trie_id: trie_id1.clone(), - code_hash: H256::repeat_byte(1), - _reserved: None, - }; - let _ = Balances::deposit_creating(&ALICE, 110); - ContractInfoOf::::insert(ALICE, &alice_contract_info); - set_storage(&ALICE, &key1, Some(b"1".to_vec())); - set_storage(&ALICE, &key2, Some(b"2".to_vec())); - - let bob_contract_info = RawContractInfo { - trie_id: trie_id2.clone(), - code_hash: H256::repeat_byte(2), - _reserved: None, - }; - let _ = Balances::deposit_creating(&BOB, 110); - ContractInfoOf::::insert(BOB, &bob_contract_info); - set_storage(&BOB, &key1, Some(b"3".to_vec())); - set_storage(&BOB, &key2, Some(b"4".to_vec())); - } - - // Transfer funds from ALICE account of such amount that after this transfer - // the balance of the ALICE account will be below the existential threshold. - // - // This does not remove the contract storage as we are not notified about a - // account removal. This cannot happen in reality because a contract can only - // remove itself by `seal_terminate`. There is no external event that can remove - // the account appart from that. - assert_ok!(Balances::transfer(Origin::signed(ALICE), BOB, 20)); - - // Verify that no entries are removed. - { - assert_eq!(get_storage(&ALICE, key1), Some(b"1".to_vec())); - assert_eq!(get_storage(&ALICE, key2), Some(b"2".to_vec())); - - assert_eq!(get_storage(&BOB, key1), Some(b"3".to_vec())); - assert_eq!(get_storage(&BOB, key2), Some(b"4".to_vec())); - } - }); -} - #[test] fn instantiate_and_call_and_deposit_event() { let (wasm, code_hash) = compile_module::("return_from_start_fn").unwrap(); - ExtBuilder::default().existential_deposit(100).build().execute_with(|| { + ExtBuilder::default().existential_deposit(500).build().execute_with(|| { let _ = Balances::deposit_creating(&ALICE, 1_000_000); - let subsistence = Pallet::::subsistence_threshold(); + let min_balance = ::Currency::minimum_balance(); + let value = 100; + + // We determine the storage deposit limit after uploading because it depends on ALICEs free + // balance which is changed by uploading a module. + assert_ok!(Contracts::upload_code(Origin::signed(ALICE), wasm, None)); + + // Drop previous events + initialize_block(2); // Check at the end to get hash on error easily - let creation = Contracts::instantiate_with_code( + assert_ok!(Contracts::instantiate( Origin::signed(ALICE), - subsistence * 100, + value, GAS_LIMIT, - wasm, + None, + code_hash, vec![], vec![], - ); + )); let addr = Contracts::contract_address(&ALICE, &code_hash, &[]); + assert!(ContractInfoOf::::contains_key(&addr)); assert_eq!( System::events(), vec![ EventRecord { phase: Phase::Initialization, - event: Event::System(frame_system::Event::NewAccount(ALICE.clone())), - topics: vec![], - }, - EventRecord { - phase: Phase::Initialization, - event: Event::Balances(pallet_balances::Event::Endowed(ALICE, 1_000_000)), + event: Event::System(frame_system::Event::NewAccount { account: addr.clone() }), topics: vec![], }, EventRecord { phase: Phase::Initialization, - event: Event::System(frame_system::Event::NewAccount(addr.clone())), + event: Event::Balances(pallet_balances::Event::Endowed { + account: addr.clone(), + free_balance: min_balance, + }), topics: vec![], }, EventRecord { phase: Phase::Initialization, - event: Event::Balances(pallet_balances::Event::Endowed( - addr.clone(), - subsistence * 100 - )), + event: Event::Balances(pallet_balances::Event::Transfer { + from: ALICE, + to: addr.clone(), + amount: min_balance, + }), topics: vec![], }, EventRecord { phase: Phase::Initialization, - event: Event::Balances(pallet_balances::Event::Transfer( - ALICE, - addr.clone(), - subsistence * 100 - )), + event: Event::Balances(pallet_balances::Event::Reserved { + who: addr.clone(), + amount: min_balance, + }), topics: vec![], }, EventRecord { phase: Phase::Initialization, - event: Event::Contracts(crate::Event::CodeStored(code_hash.into())), + event: Event::Balances(pallet_balances::Event::Transfer { + from: ALICE, + to: addr.clone(), + amount: value, + }), topics: vec![], }, EventRecord { phase: Phase::Initialization, - event: Event::Contracts(crate::Event::ContractEmitted( - addr.clone(), - vec![1, 2, 3, 4] - )), + event: Event::Contracts(crate::Event::ContractEmitted { + contract: addr.clone(), + data: vec![1, 2, 3, 4] + }), topics: vec![], }, EventRecord { phase: Phase::Initialization, - event: Event::Contracts(crate::Event::Instantiated(ALICE, addr.clone())), + event: Event::Contracts(crate::Event::Instantiated { + deployer: ALICE, + contract: addr.clone() + }), topics: vec![], }, ] ); - - assert_ok!(creation); - assert!(ContractInfoOf::::contains_key(&addr)); }); } @@ -513,6 +476,7 @@ fn deposit_event_max_value_limit() { Origin::signed(ALICE), 30_000, GAS_LIMIT, + None, wasm, vec![], vec![], @@ -525,6 +489,7 @@ fn deposit_event_max_value_limit() { addr.clone(), 0, GAS_LIMIT * 2, // we are copying a huge buffer, + None, ::Schedule::get().limits.payload_len.encode(), )); @@ -535,6 +500,7 @@ fn deposit_event_max_value_limit() { addr, 0, GAS_LIMIT, + None, (::Schedule::get().limits.payload_len + 1).encode(), ), Error::::ValueTooLarge, @@ -545,15 +511,15 @@ fn deposit_event_max_value_limit() { #[test] fn run_out_of_gas() { let (wasm, code_hash) = compile_module::("run_out_of_gas").unwrap(); - let subsistence = Pallet::::subsistence_threshold(); - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { + let min_balance = ::Currency::minimum_balance(); let _ = Balances::deposit_creating(&ALICE, 1_000_000); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - 100 * subsistence, + 100 * min_balance, GAS_LIMIT, + None, wasm, vec![], vec![], @@ -568,6 +534,7 @@ fn run_out_of_gas() { addr, // newly created account 0, 1_000_000_000_000, + None, vec![], ), Error::::OutOfGas, @@ -575,8 +542,67 @@ fn run_out_of_gas() { }); } -fn initialize_block(number: u64) { - System::initialize(&number, &[0u8; 32].into(), &Default::default(), Default::default()); +/// Check that contracts with the same account id have different trie ids. +/// Check the `Nonce` storage item for more information. +#[test] +fn instantiate_unique_trie_id() { + let (wasm, code_hash) = compile_module::("self_destruct").unwrap(); + + ExtBuilder::default().existential_deposit(500).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + Contracts::upload_code(Origin::signed(ALICE), wasm, None).unwrap(); + let addr = Contracts::contract_address(&ALICE, &code_hash, &[]); + + // Instantiate the contract and store its trie id for later comparison. + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 0, + GAS_LIMIT, + None, + code_hash, + vec![], + vec![], + )); + let trie_id = ContractInfoOf::::get(&addr).unwrap().trie_id; + + // Try to instantiate it again without termination should yield an error. + assert_err_ignore_postinfo!( + Contracts::instantiate( + Origin::signed(ALICE), + 0, + GAS_LIMIT, + None, + code_hash, + vec![], + vec![], + ), + >::DuplicateContract, + ); + + // Terminate the contract. + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + vec![] + )); + + // Re-Instantiate after termination. + assert_ok!(Contracts::instantiate( + Origin::signed(ALICE), + 0, + GAS_LIMIT, + None, + code_hash, + vec![], + vec![], + )); + + // Trie ids shouldn't match or we might have a collision + assert_ne!(trie_id, ContractInfoOf::::get(&addr).unwrap().trie_id); + }); } #[test] @@ -590,6 +616,7 @@ fn storage_max_value_limit() { Origin::signed(ALICE), 30_000, GAS_LIMIT, + None, wasm, vec![], vec![], @@ -603,6 +630,7 @@ fn storage_max_value_limit() { addr.clone(), 0, GAS_LIMIT * 2, // we are copying a huge buffer + None, ::Schedule::get().limits.payload_len.encode(), )); @@ -613,6 +641,7 @@ fn storage_max_value_limit() { addr, 0, GAS_LIMIT, + None, (::Schedule::get().limits.payload_len + 1).encode(), ), Error::::ValueTooLarge, @@ -622,16 +651,21 @@ fn storage_max_value_limit() { #[test] fn deploy_and_call_other_contract() { - let (callee_wasm, callee_code_hash) = compile_module::("return_with_data").unwrap(); let (caller_wasm, caller_code_hash) = compile_module::("caller_contract").unwrap(); + let (callee_wasm, callee_code_hash) = compile_module::("return_with_data").unwrap(); + let caller_addr = Contracts::contract_address(&ALICE, &caller_code_hash, &[]); + let callee_addr = Contracts::contract_address(&caller_addr, &callee_code_hash, &[]); + + ExtBuilder::default().existential_deposit(500).build().execute_with(|| { + let min_balance = ::Currency::minimum_balance(); - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { // Create let _ = Balances::deposit_creating(&ALICE, 1_000_000); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), 100_000, GAS_LIMIT, + None, caller_wasm, vec![], vec![], @@ -640,18 +674,124 @@ fn deploy_and_call_other_contract() { Origin::signed(ALICE), 100_000, GAS_LIMIT, + None, callee_wasm, 0u32.to_le_bytes().encode(), vec![42], )); + // Drop previous events + initialize_block(2); + // Call BOB contract, which attempts to instantiate and call the callee contract and // makes various assertions on the results from those calls. assert_ok!(Contracts::call( Origin::signed(ALICE), - Contracts::contract_address(&ALICE, &caller_code_hash, &[]), + caller_addr.clone(), 0, GAS_LIMIT, + None, + callee_code_hash.as_ref().to_vec(), + )); + + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::Initialization, + event: Event::System(frame_system::Event::NewAccount { + account: callee_addr.clone() + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Endowed { + account: callee_addr.clone(), + free_balance: min_balance, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Transfer { + from: ALICE, + to: callee_addr.clone(), + amount: min_balance, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Reserved { + who: callee_addr.clone(), + amount: min_balance, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Transfer { + from: caller_addr.clone(), + to: callee_addr.clone(), + amount: 32768, // hard coded in wasm + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Contracts(crate::Event::Instantiated { + deployer: caller_addr.clone(), + contract: callee_addr.clone(), + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Transfer { + from: caller_addr.clone(), + to: callee_addr.clone(), + amount: 32768, + }), + topics: vec![], + }, + ] + ); + }); +} + +#[test] +fn delegate_call() { + let (caller_wasm, caller_code_hash) = compile_module::("delegate_call").unwrap(); + let (callee_wasm, callee_code_hash) = compile_module::("delegate_call_lib").unwrap(); + let caller_addr = Contracts::contract_address(&ALICE, &caller_code_hash, &[]); + + ExtBuilder::default().existential_deposit(500).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + + // Instantiate the 'caller' + assert_ok!(Contracts::instantiate_with_code( + Origin::signed(ALICE), + 300_000, + GAS_LIMIT, + None, + caller_wasm, + vec![], + vec![], + )); + // Only upload 'callee' code + assert_ok!(Contracts::upload_code( + Origin::signed(ALICE), + callee_wasm, + Some(codec::Compact(100_000)), + )); + + assert_ok!(Contracts::call( + Origin::signed(ALICE), + caller_addr.clone(), + 1337, + GAS_LIMIT, + None, callee_code_hash.as_ref().to_vec(), )); }); @@ -660,14 +800,15 @@ fn deploy_and_call_other_contract() { #[test] fn cannot_self_destruct_through_draning() { let (wasm, code_hash) = compile_module::("drain").unwrap(); - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { + ExtBuilder::default().existential_deposit(200).build().execute_with(|| { let _ = Balances::deposit_creating(&ALICE, 1_000_000); // Instantiate the BOB contract. assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - 100_000, + 1_000, GAS_LIMIT, + None, wasm, vec![], vec![], @@ -678,8 +819,153 @@ fn cannot_self_destruct_through_draning() { assert_matches!(ContractInfoOf::::get(&addr), Some(_)); // Call BOB which makes it send all funds to the zero address - // The contract code asserts that the correct error value is returned. - assert_ok!(Contracts::call(Origin::signed(ALICE), addr, 0, GAS_LIMIT, vec![])); + // The contract code asserts that the transfer was successful + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + vec![] + )); + + // Make sure the account wasn't remove by sending all free balance away. + assert_eq!( + ::Currency::total_balance(&addr), + ::Currency::minimum_balance(), + ); + }); +} + +#[test] +fn cannot_self_destruct_through_storage_refund_after_price_change() { + let (wasm, code_hash) = compile_module::("store").unwrap(); + ExtBuilder::default().existential_deposit(200).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + let min_balance = ::Currency::minimum_balance(); + + // Instantiate the BOB contract. + assert_ok!(Contracts::instantiate_with_code( + Origin::signed(ALICE), + 0, + GAS_LIMIT, + None, + wasm, + vec![], + vec![], + )); + let addr = Contracts::contract_address(&ALICE, &code_hash, &[]); + + // Check that the BOB contract has been instantiated and has the minimum balance + let info = ContractInfoOf::::get(&addr).unwrap(); + assert_eq!(info.storage_deposit, min_balance); + assert_eq!(::Currency::total_balance(&addr), min_balance); + + // Create 100 bytes of storage with a price of per byte + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + 100u32.to_le_bytes().to_vec() + )); + + // Increase the byte price and trigger a refund. This could potentially destroy the account + // because the refund removes the reserved existential deposit. This should not happen. + DEPOSIT_PER_BYTE.with(|c| *c.borrow_mut() = 500); + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + 0u32.to_le_bytes().to_vec() + )); + + // Make sure the account wasn't removed by the refund + assert_eq!( + ::Currency::total_balance(&addr), + ::Currency::minimum_balance(), + ); + }); +} + +#[test] +fn cannot_self_destruct_by_refund_after_slash() { + let (wasm, code_hash) = compile_module::("store").unwrap(); + ExtBuilder::default().existential_deposit(500).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + let min_balance = ::Currency::minimum_balance(); + + assert_ok!(Contracts::instantiate_with_code( + Origin::signed(ALICE), + 0, + GAS_LIMIT, + None, + wasm, + vec![], + vec![], + )); + let addr = Contracts::contract_address(&ALICE, &code_hash, &[]); + + // create 100 more reserved balance + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + 98u32.encode(), + )); + + // Drop previous events + initialize_block(2); + + // slash parts of the 100 so that the next refund ould remove the account + // because it the value it stored for `storage_deposit` becomes out of sync + let _ = ::Currency::slash(&addr, 90); + assert_eq!(::Currency::total_balance(&addr), min_balance + 10); + + // trigger a refund of 50 which would bring the contract below min when actually refunded + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + 48u32.encode(), + )); + + // Make sure the account kept the minimum balance and was not destroyed + assert_eq!(::Currency::total_balance(&addr), min_balance); + + // even though it was not charged it is still substracted from the storage deposit tracker + assert_eq!(ContractInfoOf::::get(&addr).unwrap().storage_deposit, 550); + + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Slashed { + who: addr.clone(), + amount: 90, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::ReserveRepatriated { + from: addr.clone(), + to: ALICE, + amount: 10, + destination_status: BalanceStatus::Free, + }), + topics: vec![], + }, + ] + ); }); } @@ -694,6 +980,7 @@ fn cannot_self_destruct_while_live() { Origin::signed(ALICE), 100_000, GAS_LIMIT, + None, wasm, vec![], vec![], @@ -706,7 +993,7 @@ fn cannot_self_destruct_while_live() { // Call BOB with input data, forcing it make a recursive call to itself to // self-destruct, resulting in a trap. assert_err_ignore_postinfo!( - Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, vec![0],), + Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, None, vec![0],), Error::::ContractTrapped, ); @@ -718,7 +1005,7 @@ fn cannot_self_destruct_while_live() { #[test] fn self_destruct_works() { let (wasm, code_hash) = compile_module::("self_destruct").unwrap(); - ExtBuilder::default().existential_deposit(50).build().execute_with(|| { + ExtBuilder::default().existential_deposit(1_000).build().execute_with(|| { let _ = Balances::deposit_creating(&ALICE, 1_000_000); let _ = Balances::deposit_creating(&DJANGO, 1_000_000); @@ -727,6 +1014,7 @@ fn self_destruct_works() { Origin::signed(ALICE), 100_000, GAS_LIMIT, + None, wasm, vec![], vec![], @@ -741,46 +1029,59 @@ fn self_destruct_works() { // Call BOB without input data which triggers termination. assert_matches!( - Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, vec![],), + Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, None, vec![],), Ok(_) ); + // Check that code is still there but refcount dropped to zero. + assert_refcount!(&code_hash, 0); + + // Check that account is gone + assert!(ContractInfoOf::::get(&addr).is_none()); + assert_eq!(Balances::total_balance(&addr), 0); + + // check that the beneficiary (django) got remaining balance + assert_eq!(Balances::free_balance(DJANGO), 1_000_000 + 100_000); + pretty_assertions::assert_eq!( System::events(), vec![ EventRecord { phase: Phase::Initialization, - event: Event::System(frame_system::Event::KilledAccount(addr.clone())), + event: Event::Balances(pallet_balances::Event::Transfer { + from: addr.clone(), + to: DJANGO, + amount: 100_000, + }), topics: vec![], }, EventRecord { phase: Phase::Initialization, - event: Event::Balances(pallet_balances::Event::Transfer( - addr.clone(), - DJANGO, - 100_000, - )), + event: Event::Contracts(crate::Event::Terminated { + contract: addr.clone(), + beneficiary: DJANGO + }), topics: vec![], }, EventRecord { phase: Phase::Initialization, - event: Event::Contracts(crate::Event::CodeRemoved(code_hash)), + event: Event::System(frame_system::Event::KilledAccount { + account: addr.clone() + }), topics: vec![], }, EventRecord { phase: Phase::Initialization, - event: Event::Contracts(crate::Event::Terminated(addr.clone(), DJANGO)), + event: Event::Balances(pallet_balances::Event::ReserveRepatriated { + from: addr.clone(), + to: ALICE, + amount: 1_000, + destination_status: BalanceStatus::Free, + }), topics: vec![], }, ], ); - - // Check that account is gone - assert!(ContractInfoOf::::get(&addr).is_none()); - - // check that the beneficiary (django) got remaining balance - // some rent was deducted before termination - assert_eq!(Balances::free_balance(DJANGO), 1_000_000 + 100_000); }); } @@ -798,6 +1099,7 @@ fn destroy_contract_and_transfer_funds() { Origin::signed(ALICE), 200_000, GAS_LIMIT, + None, callee_wasm, vec![], vec![42] @@ -809,6 +1111,7 @@ fn destroy_contract_and_transfer_funds() { Origin::signed(ALICE), 200_000, GAS_LIMIT, + None, caller_wasm, callee_code_hash.as_ref().to_vec(), vec![], @@ -825,6 +1128,7 @@ fn destroy_contract_and_transfer_funds() { addr_bob, 0, GAS_LIMIT, + None, addr_charlie.encode(), )); @@ -845,6 +1149,7 @@ fn cannot_self_destruct_in_constructor() { Origin::signed(ALICE), 100_000, GAS_LIMIT, + None, wasm, vec![], vec![], @@ -866,6 +1171,7 @@ fn crypto_hashes() { Origin::signed(ALICE), 100_000, GAS_LIMIT, + None, wasm, vec![], vec![], @@ -893,10 +1199,10 @@ fn crypto_hashes() { let mut params = vec![(n + 1) as u8]; params.extend_from_slice(input); let result = - >::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, params, false) + >::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, params, false) .result .unwrap(); - assert!(result.is_success()); + assert!(!result.did_revert()); let expected = hash_fn(input.as_ref()); assert_eq!(&result.data[..*expected_size], &*expected); } @@ -907,33 +1213,36 @@ fn crypto_hashes() { fn transfer_return_code() { let (wasm, code_hash) = compile_module::("transfer_return_code").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, wasm, vec![], vec![], ),); let addr = Contracts::contract_address(&ALICE, &code_hash, &[]); - // Contract has only the minimal balance so any transfer will return BelowSubsistence. - Balances::make_free_balance_be(&addr, subsistence); - let result = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, vec![], false) + // Contract has only the minimal balance so any transfer will fail. + Balances::make_free_balance_be(&addr, min_balance); + let result = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, vec![], false) .result .unwrap(); - assert_return_code!(result, RuntimeReturnCode::BelowSubsistenceThreshold); + assert_return_code!(result, RuntimeReturnCode::TransferFailed); - // Contract has enough total balance in order to not go below the subsistence + // Contract has enough total balance in order to not go below the min balance // threshold when transfering 100 balance but this balance is reserved so - // the transfer still fails but with another return code. - Balances::make_free_balance_be(&addr, subsistence + 100); - Balances::reserve(&addr, subsistence + 100).unwrap(); - let result = Contracts::bare_call(ALICE, addr, 0, GAS_LIMIT, vec![], false).result.unwrap(); - assert_return_code!(result, RuntimeReturnCode::TransferFailed); + // the transfer still fails. + Balances::make_free_balance_be(&addr, min_balance + 100); + Balances::reserve(&addr, min_balance + 100).unwrap(); + let result = Contracts::bare_call(ALICE, addr, 0, GAS_LIMIT, None, vec![], false) + .result + .unwrap(); + assert_return_code!(result, RuntimeReturnCode::TransferFailed); }); } @@ -942,20 +1251,21 @@ fn call_return_code() { let (caller_code, caller_hash) = compile_module::("call_return_code").unwrap(); let (callee_code, callee_hash) = compile_module::("ok_trap_revert").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); - let _ = Balances::deposit_creating(&CHARLIE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); + let _ = Balances::deposit_creating(&CHARLIE, 1000 * min_balance); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, caller_code, vec![0], vec![], ),); let addr_bob = Contracts::contract_address(&ALICE, &caller_hash, &[]); - Balances::make_free_balance_be(&addr_bob, subsistence); + Balances::make_free_balance_be(&addr_bob, min_balance); // Contract calls into Django which is no valid contract let result = Contracts::bare_call( @@ -963,6 +1273,7 @@ fn call_return_code() { addr_bob.clone(), 0, GAS_LIMIT, + None, AsRef::<[u8]>::as_ref(&DJANGO).to_vec(), false, ) @@ -972,21 +1283,23 @@ fn call_return_code() { assert_ok!(Contracts::instantiate_with_code( Origin::signed(CHARLIE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, callee_code, vec![0], vec![], ),); let addr_django = Contracts::contract_address(&CHARLIE, &callee_hash, &[]); - Balances::make_free_balance_be(&addr_django, subsistence); + Balances::make_free_balance_be(&addr_django, min_balance); - // Contract has only the minimal balance so any transfer will return BelowSubsistence. + // Contract has only the minimal balance so any transfer will fail. let result = Contracts::bare_call( ALICE, addr_bob.clone(), 0, GAS_LIMIT, + None, AsRef::<[u8]>::as_ref(&addr_django) .iter() .chain(&0u32.to_le_bytes()) @@ -996,18 +1309,19 @@ fn call_return_code() { ) .result .unwrap(); - assert_return_code!(result, RuntimeReturnCode::BelowSubsistenceThreshold); + assert_return_code!(result, RuntimeReturnCode::TransferFailed); - // Contract has enough total balance in order to not go below the subsistence + // Contract has enough total balance in order to not go below the min balance // threshold when transfering 100 balance but this balance is reserved so - // the transfer still fails but with another return code. - Balances::make_free_balance_be(&addr_bob, subsistence + 100); - Balances::reserve(&addr_bob, subsistence + 100).unwrap(); + // the transfer still fails. + Balances::make_free_balance_be(&addr_bob, min_balance + 100); + Balances::reserve(&addr_bob, min_balance + 100).unwrap(); let result = Contracts::bare_call( ALICE, addr_bob.clone(), 0, GAS_LIMIT, + None, AsRef::<[u8]>::as_ref(&addr_django) .iter() .chain(&0u32.to_le_bytes()) @@ -1020,12 +1334,13 @@ fn call_return_code() { assert_return_code!(result, RuntimeReturnCode::TransferFailed); // Contract has enough balance but callee reverts because "1" is passed. - Balances::make_free_balance_be(&addr_bob, subsistence + 1000); + Balances::make_free_balance_be(&addr_bob, min_balance + 1000); let result = Contracts::bare_call( ALICE, addr_bob.clone(), 0, GAS_LIMIT, + None, AsRef::<[u8]>::as_ref(&addr_django) .iter() .chain(&1u32.to_le_bytes()) @@ -1043,6 +1358,7 @@ fn call_return_code() { addr_bob, 0, GAS_LIMIT, + None, AsRef::<[u8]>::as_ref(&addr_django) .iter() .chain(&2u32.to_le_bytes()) @@ -1061,15 +1377,16 @@ fn instantiate_return_code() { let (caller_code, caller_hash) = compile_module::("instantiate_return_code").unwrap(); let (callee_code, callee_hash) = compile_module::("ok_trap_revert").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); - let _ = Balances::deposit_creating(&CHARLIE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); + let _ = Balances::deposit_creating(&CHARLIE, 1000 * min_balance); let callee_hash = callee_hash.as_ref().to_vec(); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, callee_code, vec![], vec![], @@ -1077,38 +1394,54 @@ fn instantiate_return_code() { assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, caller_code, vec![], vec![], ),); let addr = Contracts::contract_address(&ALICE, &caller_hash, &[]); - // Contract has only the minimal balance so any transfer will return BelowSubsistence. - Balances::make_free_balance_be(&addr, subsistence); - let result = - Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, callee_hash.clone(), false) - .result - .unwrap(); - assert_return_code!(result, RuntimeReturnCode::BelowSubsistenceThreshold); + // Contract has only the minimal balance so any transfer will fail. + Balances::make_free_balance_be(&addr, min_balance); + let result = Contracts::bare_call( + ALICE, + addr.clone(), + 0, + GAS_LIMIT, + None, + callee_hash.clone(), + false, + ) + .result + .unwrap(); + assert_return_code!(result, RuntimeReturnCode::TransferFailed); - // Contract has enough total balance in order to not go below the subsistence + // Contract has enough total balance in order to not go below the min_balance // threshold when transfering the balance but this balance is reserved so - // the transfer still fails but with another return code. - Balances::make_free_balance_be(&addr, subsistence + 10_000); - Balances::reserve(&addr, subsistence + 10_000).unwrap(); - let result = - Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, callee_hash.clone(), false) - .result - .unwrap(); + // the transfer still fails. + Balances::make_free_balance_be(&addr, min_balance + 10_000); + Balances::reserve(&addr, min_balance + 10_000).unwrap(); + let result = Contracts::bare_call( + ALICE, + addr.clone(), + 0, + GAS_LIMIT, + None, + callee_hash.clone(), + false, + ) + .result + .unwrap(); assert_return_code!(result, RuntimeReturnCode::TransferFailed); // Contract has enough balance but the passed code hash is invalid - Balances::make_free_balance_be(&addr, subsistence + 10_000); - let result = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, vec![0; 33], false) - .result - .unwrap(); + Balances::make_free_balance_be(&addr, min_balance + 10_000); + let result = + Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, vec![0; 33], false) + .result + .unwrap(); assert_return_code!(result, RuntimeReturnCode::CodeNotFound); // Contract has enough balance but callee reverts because "1" is passed. @@ -1117,6 +1450,7 @@ fn instantiate_return_code() { addr.clone(), 0, GAS_LIMIT, + None, callee_hash.iter().chain(&1u32.to_le_bytes()).cloned().collect(), false, ) @@ -1130,6 +1464,7 @@ fn instantiate_return_code() { addr, 0, GAS_LIMIT, + None, callee_hash.iter().chain(&2u32.to_le_bytes()).cloned().collect(), false, ) @@ -1143,19 +1478,20 @@ fn instantiate_return_code() { fn disabled_chain_extension_wont_deploy() { let (code, _hash) = compile_module::("chain_extension").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); TestExtension::disable(); assert_err_ignore_postinfo!( Contracts::instantiate_with_code( Origin::signed(ALICE), - 3 * subsistence, + 3 * min_balance, GAS_LIMIT, + None, code, vec![], vec![], ), - "module uses chain extensions but chain extensions are disabled", + >::CodeRejected, ); }); } @@ -1164,12 +1500,13 @@ fn disabled_chain_extension_wont_deploy() { fn disabled_chain_extension_errors_on_call() { let (code, hash) = compile_module::("chain_extension").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, code, vec![], vec![], @@ -1177,7 +1514,7 @@ fn disabled_chain_extension_errors_on_call() { let addr = Contracts::contract_address(&ALICE, &hash, &[]); TestExtension::disable(); assert_err_ignore_postinfo!( - Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, vec![],), + Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, None, vec![],), Error::::NoChainExtension, ); }); @@ -1187,12 +1524,13 @@ fn disabled_chain_extension_errors_on_call() { fn chain_extension_works() { let (code, hash) = compile_module::("chain_extension").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, code, vec![], vec![], @@ -1204,25 +1542,27 @@ fn chain_extension_works() { // func_id. // 0 = read input buffer and pass it through as output - let result = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, vec![0, 99], false); + let result = + Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, vec![0, 99], false); let gas_consumed = result.gas_consumed; assert_eq!(TestExtension::last_seen_buffer(), vec![0, 99]); assert_eq!(result.result.unwrap().data, Bytes(vec![0, 99])); // 1 = treat inputs as integer primitives and store the supplied integers - Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, vec![1], false) + Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, vec![1], false) .result .unwrap(); // those values passed in the fixture assert_eq!(TestExtension::last_seen_inputs(), (4, 1, 16, 12)); // 2 = charge some extra weight (amount supplied in second byte) - let result = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, vec![2, 42], false); + let result = + Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, vec![2, 42], false); assert_ok!(result.result); assert_eq!(result.gas_consumed, gas_consumed + 42); // 3 = diverging chain extension call that sets flags to 0x1 and returns a fixed buffer - let result = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, vec![3], false) + let result = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, vec![3], false) .result .unwrap(); assert_eq!(result.flags, ReturnFlags::REVERT); @@ -1234,13 +1574,14 @@ fn chain_extension_works() { fn lazy_removal_works() { let (code, hash) = compile_module::("self_destruct").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, code, vec![], vec![], @@ -1254,7 +1595,14 @@ fn lazy_removal_works() { child::put(trie, &[99], &42); // Terminate the contract - assert_ok!(Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, vec![])); + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + vec![] + )); // Contract info should be gone assert!(!>::contains_key(&addr)); @@ -1270,6 +1618,59 @@ fn lazy_removal_works() { }); } +#[test] +fn lazy_batch_removal_works() { + let (code, hash) = compile_module::("self_destruct").unwrap(); + ExtBuilder::default().existential_deposit(50).build().execute_with(|| { + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); + let mut tries: Vec = vec![]; + + for i in 0..3u8 { + assert_ok!(Contracts::instantiate_with_code( + Origin::signed(ALICE), + min_balance * 100, + GAS_LIMIT, + None, + code.clone(), + vec![], + vec![i], + ),); + + let addr = Contracts::contract_address(&ALICE, &hash, &[i]); + let info = >::get(&addr).unwrap(); + let trie = &info.child_trie_info(); + + // Put value into the contracts child trie + child::put(trie, &[99], &42); + + // Terminate the contract. Contract info should be gone, but value should be still there + // as the lazy removal did not run, yet. + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + vec![] + )); + + assert!(!>::contains_key(&addr)); + assert_matches!(child::get(trie, &[99]), Some(42)); + + tries.push(trie.clone()) + } + + // Run single lazy removal + Contracts::on_initialize(Weight::max_value()); + + // The single lazy removal should have removed all queued tries + for trie in tries.iter() { + assert_matches!(child::get::(trie, &[99]), None); + } + }); +} + #[test] fn lazy_removal_partial_remove_works() { let (code, hash) = compile_module::("self_destruct").unwrap(); @@ -1285,29 +1686,38 @@ fn lazy_removal_partial_remove_works() { let mut ext = ExtBuilder::default().existential_deposit(50).build(); let trie = ext.execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, code, vec![], vec![], ),); let addr = Contracts::contract_address(&ALICE, &hash, &[]); - let mut info = >::get(&addr).unwrap(); + let info = >::get(&addr).unwrap(); // Put value into the contracts child trie for val in &vals { - Storage::::write(&mut info, &val.0, Some(val.2.clone())).unwrap(); + Storage::::write(&info.trie_id, &val.0, Some(val.2.clone()), None, false) + .unwrap(); } >::insert(&addr, info.clone()); // Terminate the contract - assert_ok!(Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, vec![])); + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + vec![] + )); // Contract info should be gone assert!(!>::contains_key(&addr)); @@ -1355,20 +1765,21 @@ fn lazy_removal_partial_remove_works() { fn lazy_removal_does_no_run_on_full_block() { let (code, hash) = compile_module::("self_destruct").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, code, vec![], vec![], ),); let addr = Contracts::contract_address(&ALICE, &hash, &[]); - let mut info = >::get(&addr).unwrap(); + let info = >::get(&addr).unwrap(); let max_keys = 30; // Create some storage items for the contract. @@ -1378,12 +1789,20 @@ fn lazy_removal_does_no_run_on_full_block() { // Put value into the contracts child trie for val in &vals { - Storage::::write(&mut info, &val.0, Some(val.2.clone())).unwrap(); + Storage::::write(&info.trie_id, &val.0, Some(val.2.clone()), None, false) + .unwrap(); } >::insert(&addr, info.clone()); // Terminate the contract - assert_ok!(Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, vec![])); + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + vec![] + )); // Contract info should be gone assert!(!>::contains_key(&addr)); @@ -1430,20 +1849,21 @@ fn lazy_removal_does_not_use_all_weight() { let mut ext = ExtBuilder::default().existential_deposit(50).build(); let (trie, vals, weight_per_key) = ext.execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, code, vec![], vec![], ),); let addr = Contracts::contract_address(&ALICE, &hash, &[]); - let mut info = >::get(&addr).unwrap(); + let info = >::get(&addr).unwrap(); let (weight_per_key, max_keys) = Storage::::deletion_budget(1, weight_limit); // We create a contract with one less storage item than we can remove within the limit @@ -1453,12 +1873,20 @@ fn lazy_removal_does_not_use_all_weight() { // Put value into the contracts child trie for val in &vals { - Storage::::write(&mut info, &val.0, Some(val.2.clone())).unwrap(); + Storage::::write(&info.trie_id, &val.0, Some(val.2.clone()), None, false) + .unwrap(); } >::insert(&addr, info.clone()); // Terminate the contract - assert_ok!(Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, vec![])); + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + vec![] + )); // Contract info should be gone assert!(!>::contains_key(&addr)); @@ -1495,13 +1923,14 @@ fn lazy_removal_does_not_use_all_weight() { fn deletion_queue_full() { let (code, hash) = compile_module::("self_destruct").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, code, vec![], vec![], @@ -1514,7 +1943,7 @@ fn deletion_queue_full() { // Terminate the contract should fail assert_err_ignore_postinfo!( - Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, vec![],), + Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, None, vec![],), Error::::DeletionQueueFull, ); @@ -1528,21 +1957,23 @@ fn refcounter() { let (wasm, code_hash) = compile_module::("self_destruct").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { let _ = Balances::deposit_creating(&ALICE, 1_000_000); - let subsistence = Pallet::::subsistence_threshold(); + let min_balance = ::Currency::minimum_balance(); // Create two contracts with the same code and check that they do in fact share it. assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, wasm.clone(), vec![], vec![0], )); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, wasm.clone(), vec![], vec![1], @@ -1552,8 +1983,9 @@ fn refcounter() { // Sharing should also work with the usual instantiate call assert_ok!(Contracts::instantiate( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, code_hash, vec![], vec![2], @@ -1566,23 +1998,23 @@ fn refcounter() { let addr2 = Contracts::contract_address(&ALICE, &code_hash, &[2]); // Terminating one contract should decrement the refcount - assert_ok!(Contracts::call(Origin::signed(ALICE), addr0, 0, GAS_LIMIT, vec![])); + assert_ok!(Contracts::call(Origin::signed(ALICE), addr0, 0, GAS_LIMIT, None, vec![])); assert_refcount!(code_hash, 2); // remove another one - assert_ok!(Contracts::call(Origin::signed(ALICE), addr1, 0, GAS_LIMIT, vec![])); + assert_ok!(Contracts::call(Origin::signed(ALICE), addr1, 0, GAS_LIMIT, None, vec![])); assert_refcount!(code_hash, 1); // Pristine code should still be there crate::PristineCode::::get(code_hash).unwrap(); // remove the last contract - assert_ok!(Contracts::call(Origin::signed(ALICE), addr2, 0, GAS_LIMIT, vec![])); + assert_ok!(Contracts::call(Origin::signed(ALICE), addr2, 0, GAS_LIMIT, None, vec![])); assert_refcount!(code_hash, 0); - // all code should be gone - assert_matches!(crate::PristineCode::::get(code_hash), None); - assert_matches!(crate::CodeStorage::::get(code_hash), None); + // refcount is `0` but code should still exists because it needs to be removed manually + assert!(crate::PristineCode::::contains_key(&code_hash)); + assert!(crate::CodeStorage::::contains_key(&code_hash)); }); } @@ -1591,14 +2023,15 @@ fn reinstrument_does_charge() { let (wasm, code_hash) = compile_module::("return_with_data").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { let _ = Balances::deposit_creating(&ALICE, 1_000_000); - let subsistence = Pallet::::subsistence_threshold(); + let min_balance = ::Currency::minimum_balance(); let zero = 0u32.to_le_bytes().encode(); let code_len = wasm.len() as u32; assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, wasm, zero.clone(), vec![], @@ -1608,11 +2041,13 @@ fn reinstrument_does_charge() { // Call the contract two times without reinstrument - let result0 = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, zero.clone(), false); - assert!(result0.result.unwrap().is_success()); + let result0 = + Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, zero.clone(), false); + assert!(!result0.result.unwrap().did_revert()); - let result1 = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, zero.clone(), false); - assert!(result1.result.unwrap().is_success()); + let result1 = + Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, zero.clone(), false); + assert!(!result1.result.unwrap().did_revert()); // They should match because both where called with the same schedule. assert_eq!(result0.gas_consumed, result1.gas_consumed); @@ -1624,12 +2059,13 @@ fn reinstrument_does_charge() { }); // This call should trigger reinstrumentation - let result2 = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, zero.clone(), false); - assert!(result2.result.unwrap().is_success()); + let result2 = + Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, zero.clone(), false); + assert!(!result2.result.unwrap().did_revert()); assert!(result2.gas_consumed > result1.gas_consumed); assert_eq!( result2.gas_consumed, - result1.gas_consumed + ::WeightInfo::instrument(code_len / 1024), + result1.gas_consumed + ::WeightInfo::reinstrument(code_len), ); }); } @@ -1644,12 +2080,13 @@ fn debug_message_works() { Origin::signed(ALICE), 30_000, GAS_LIMIT, + None, wasm, vec![], vec![], ),); let addr = Contracts::contract_address(&ALICE, &code_hash, &[]); - let result = Contracts::bare_call(ALICE, addr, 0, GAS_LIMIT, vec![], true); + let result = Contracts::bare_call(ALICE, addr, 0, GAS_LIMIT, None, vec![], true); assert_matches!(result.result, Ok(_)); assert_eq!(std::str::from_utf8(&result.debug_message).unwrap(), "Hello World!"); @@ -1666,16 +2103,17 @@ fn debug_message_logging_disabled() { Origin::signed(ALICE), 30_000, GAS_LIMIT, + None, wasm, vec![], vec![], ),); let addr = Contracts::contract_address(&ALICE, &code_hash, &[]); // disable logging by passing `false` - let result = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, vec![], false); + let result = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, vec![], false); assert_matches!(result.result, Ok(_)); // the dispatchables always run without debugging - assert_ok!(Contracts::call(Origin::signed(ALICE), addr, 0, GAS_LIMIT, vec![])); + assert_ok!(Contracts::call(Origin::signed(ALICE), addr, 0, GAS_LIMIT, None, vec![])); assert!(result.debug_message.is_empty()); }); } @@ -1690,12 +2128,13 @@ fn debug_message_invalid_utf8() { Origin::signed(ALICE), 30_000, GAS_LIMIT, + None, wasm, vec![], vec![], ),); let addr = Contracts::contract_address(&ALICE, &code_hash, &[]); - let result = Contracts::bare_call(ALICE, addr, 0, GAS_LIMIT, vec![], true); + let result = Contracts::bare_call(ALICE, addr, 0, GAS_LIMIT, None, vec![], true); assert_err!(result.result, >::DebugMessageInvalidUTF8); }); } @@ -1705,14 +2144,15 @@ fn gas_estimation_nested_call_fixed_limit() { let (caller_code, caller_hash) = compile_module::("call_with_limit").unwrap(); let (callee_code, callee_hash) = compile_module::("dummy").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); - let _ = Balances::deposit_creating(&CHARLIE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); + let _ = Balances::deposit_creating(&CHARLIE, 1000 * min_balance); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, caller_code, vec![], vec![0], @@ -1721,8 +2161,9 @@ fn gas_estimation_nested_call_fixed_limit() { assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, callee_code, vec![], vec![1], @@ -1736,15 +2177,32 @@ fn gas_estimation_nested_call_fixed_limit() { .collect(); // Call in order to determine the gas that is required for this call - let result = - Contracts::bare_call(ALICE, addr_caller.clone(), 0, GAS_LIMIT, input.clone(), false); + let result = Contracts::bare_call( + ALICE, + addr_caller.clone(), + 0, + GAS_LIMIT, + None, + input.clone(), + false, + ); assert_ok!(&result.result); + // We have a subcall with a fixed gas limit. This constitutes precharging. assert!(result.gas_required > result.gas_consumed); // Make the same call using the estimated gas. Should succeed. assert_ok!( - Contracts::bare_call(ALICE, addr_caller, 0, result.gas_required, input, false,).result + Contracts::bare_call( + ALICE, + addr_caller, + 0, + result.gas_required, + Some(result.storage_deposit.charge_or_zero()), + input, + false, + ) + .result ); }); } @@ -1755,14 +2213,15 @@ fn gas_estimation_call_runtime() { let (caller_code, caller_hash) = compile_module::("call_runtime").unwrap(); let (callee_code, callee_hash) = compile_module::("dummy").unwrap(); ExtBuilder::default().existential_deposit(50).build().execute_with(|| { - let subsistence = Pallet::::subsistence_threshold(); - let _ = Balances::deposit_creating(&ALICE, 1000 * subsistence); - let _ = Balances::deposit_creating(&CHARLIE, 1000 * subsistence); + let min_balance = ::Currency::minimum_balance(); + let _ = Balances::deposit_creating(&ALICE, 1000 * min_balance); + let _ = Balances::deposit_creating(&CHARLIE, 1000 * min_balance); assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, caller_code, vec![], vec![0], @@ -1771,8 +2230,9 @@ fn gas_estimation_call_runtime() { assert_ok!(Contracts::instantiate_with_code( Origin::signed(ALICE), - subsistence * 100, + min_balance * 100, GAS_LIMIT, + None, callee_code, vec![], vec![1], @@ -1785,18 +2245,34 @@ fn gas_estimation_call_runtime() { dest: addr_callee, value: 0, gas_limit: GAS_LIMIT / 3, + storage_deposit_limit: None, data: vec![], }); - let result = - Contracts::bare_call(ALICE, addr_caller.clone(), 0, GAS_LIMIT, call.encode(), false); + let result = Contracts::bare_call( + ALICE, + addr_caller.clone(), + 0, + GAS_LIMIT, + None, + call.encode(), + false, + ); assert_ok!(&result.result); assert!(result.gas_required > result.gas_consumed); // Make the same call using the required gas. Should succeed. assert_ok!( - Contracts::bare_call(ALICE, addr_caller, 0, result.gas_required, call.encode(), false,) - .result + Contracts::bare_call( + ALICE, + addr_caller, + 0, + result.gas_required, + None, + call.encode(), + false, + ) + .result ); }); } @@ -1814,6 +2290,7 @@ fn ecdsa_recover() { Origin::signed(ALICE), 100_000, GAS_LIMIT, + None, wasm, vec![], vec![], @@ -1843,10 +2320,834 @@ fn ecdsa_recover() { params.extend_from_slice(&signature); params.extend_from_slice(&message_hash); assert!(params.len() == 65 + 32); - let result = >::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, params, false) - .result - .unwrap(); - assert!(result.is_success()); + let result = + >::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, params, false) + .result + .unwrap(); + assert!(!result.did_revert()); assert_eq!(result.data.as_ref(), &EXPECTED_COMPRESSED_PUBLIC_KEY); }) } + +#[test] +fn upload_code_works() { + let (wasm, code_hash) = compile_module::("dummy").unwrap(); + + ExtBuilder::default().existential_deposit(100).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + + // Drop previous events + initialize_block(2); + + assert!(!>::contains_key(code_hash)); + assert_ok!(Contracts::upload_code( + Origin::signed(ALICE), + wasm, + Some(codec::Compact(1_000)) + )); + assert!(>::contains_key(code_hash)); + + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Reserved { + who: ALICE, + amount: 240, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Contracts(crate::Event::CodeStored { code_hash }), + topics: vec![], + }, + ] + ); + }); +} + +#[test] +fn upload_code_limit_too_low() { + let (wasm, _code_hash) = compile_module::("dummy").unwrap(); + + ExtBuilder::default().existential_deposit(100).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + + // Drop previous events + initialize_block(2); + + assert_noop!( + Contracts::upload_code(Origin::signed(ALICE), wasm, Some(codec::Compact(100))), + >::StorageDepositLimitExhausted, + ); + + assert_eq!(System::events(), vec![]); + }); +} + +#[test] +fn upload_code_not_enough_balance() { + let (wasm, _code_hash) = compile_module::("dummy").unwrap(); + + ExtBuilder::default().existential_deposit(100).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 150); + + // Drop previous events + initialize_block(2); + + assert_noop!( + Contracts::upload_code(Origin::signed(ALICE), wasm, Some(codec::Compact(1_000))), + >::StorageDepositNotEnoughFunds, + ); + + assert_eq!(System::events(), vec![]); + }); +} + +#[test] +fn remove_code_works() { + let (wasm, code_hash) = compile_module::("dummy").unwrap(); + + ExtBuilder::default().existential_deposit(100).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + + // Drop previous events + initialize_block(2); + + assert_ok!(Contracts::upload_code( + Origin::signed(ALICE), + wasm, + Some(codec::Compact(1_000)) + )); + + assert!(>::contains_key(code_hash)); + assert_ok!(Contracts::remove_code(Origin::signed(ALICE), code_hash)); + assert!(!>::contains_key(code_hash)); + + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Reserved { + who: ALICE, + amount: 240, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Contracts(crate::Event::CodeStored { code_hash }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Unreserved { + who: ALICE, + amount: 240, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Contracts(crate::Event::CodeRemoved { code_hash }), + topics: vec![], + }, + ] + ); + }); +} + +#[test] +fn remove_code_wrong_origin() { + let (wasm, code_hash) = compile_module::("dummy").unwrap(); + + ExtBuilder::default().existential_deposit(100).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + + // Drop previous events + initialize_block(2); + + assert_ok!(Contracts::upload_code( + Origin::signed(ALICE), + wasm, + Some(codec::Compact(1_000)) + )); + + assert_noop!( + Contracts::remove_code(Origin::signed(BOB), code_hash), + sp_runtime::traits::BadOrigin, + ); + + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Reserved { + who: ALICE, + amount: 240, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Contracts(crate::Event::CodeStored { code_hash }), + topics: vec![], + }, + ] + ); + }); +} + +#[test] +fn remove_code_in_use() { + let (wasm, code_hash) = compile_module::("dummy").unwrap(); + + ExtBuilder::default().existential_deposit(100).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + + assert_ok!(Contracts::instantiate_with_code( + Origin::signed(ALICE), + 0, + GAS_LIMIT, + None, + wasm, + vec![], + vec![], + )); + + // Drop previous events + initialize_block(2); + + assert_noop!( + Contracts::remove_code(Origin::signed(ALICE), code_hash), + >::CodeInUse, + ); + + assert_eq!(System::events(), vec![]); + }); +} + +#[test] +fn remove_code_not_found() { + let (_wasm, code_hash) = compile_module::("dummy").unwrap(); + + ExtBuilder::default().existential_deposit(100).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + + // Drop previous events + initialize_block(2); + + assert_noop!( + Contracts::remove_code(Origin::signed(ALICE), code_hash), + >::CodeNotFound, + ); + + assert_eq!(System::events(), vec![]); + }); +} + +#[test] +fn instantiate_with_zero_balance_works() { + let (wasm, code_hash) = compile_module::("dummy").unwrap(); + ExtBuilder::default().existential_deposit(200).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + let min_balance = ::Currency::minimum_balance(); + + // Drop previous events + initialize_block(2); + + // Instantiate the BOB contract. + assert_ok!(Contracts::instantiate_with_code( + Origin::signed(ALICE), + 0, + GAS_LIMIT, + None, + wasm, + vec![], + vec![], + )); + let addr = Contracts::contract_address(&ALICE, &code_hash, &[]); + + // Check that the BOB contract has been instantiated. + assert_matches!(ContractInfoOf::::get(&addr), Some(_)); + + // Make sure the account exists even though no free balance was send + assert_eq!(::Currency::free_balance(&addr), 0,); + assert_eq!( + ::Currency::total_balance(&addr), + ::Currency::minimum_balance(), + ); + + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::Initialization, + event: Event::System(frame_system::Event::NewAccount { account: addr.clone() }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Endowed { + account: addr.clone(), + free_balance: min_balance, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Transfer { + from: ALICE, + to: addr.clone(), + amount: min_balance, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Reserved { + who: addr.clone(), + amount: min_balance, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Reserved { + who: ALICE, + amount: 240, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Contracts(crate::Event::CodeStored { code_hash }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Contracts(crate::Event::Instantiated { + deployer: ALICE, + contract: addr.clone(), + }), + topics: vec![], + }, + ] + ); + }); +} + +#[test] +fn instantiate_with_below_existential_deposit_works() { + let (wasm, code_hash) = compile_module::("dummy").unwrap(); + ExtBuilder::default().existential_deposit(200).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + let min_balance = ::Currency::minimum_balance(); + + // Drop previous events + initialize_block(2); + + // Instantiate the BOB contract. + assert_ok!(Contracts::instantiate_with_code( + Origin::signed(ALICE), + 50, + GAS_LIMIT, + None, + wasm, + vec![], + vec![], + )); + let addr = Contracts::contract_address(&ALICE, &code_hash, &[]); + + // Check that the BOB contract has been instantiated. + assert_matches!(ContractInfoOf::::get(&addr), Some(_)); + + // Make sure the account exists even though no free balance was send + assert_eq!(::Currency::free_balance(&addr), 50,); + assert_eq!( + ::Currency::total_balance(&addr), + ::Currency::minimum_balance() + 50, + ); + + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::Initialization, + event: Event::System(frame_system::Event::NewAccount { account: addr.clone() }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Endowed { + account: addr.clone(), + free_balance: min_balance, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Transfer { + from: ALICE, + to: addr.clone(), + amount: min_balance, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Reserved { + who: addr.clone(), + amount: min_balance, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Transfer { + from: ALICE, + to: addr.clone(), + amount: 50, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Reserved { + who: ALICE, + amount: 240, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Contracts(crate::Event::CodeStored { code_hash }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Contracts(crate::Event::Instantiated { + deployer: ALICE, + contract: addr.clone(), + }), + topics: vec![], + }, + ] + ); + }); +} + +#[test] +fn storage_deposit_works() { + let (wasm, code_hash) = compile_module::("multi_store").unwrap(); + ExtBuilder::default().existential_deposit(200).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + let mut deposit = ::Currency::minimum_balance(); + + assert_ok!(Contracts::instantiate_with_code( + Origin::signed(ALICE), + 0, + GAS_LIMIT, + None, + wasm, + vec![], + vec![], + )); + let addr = Contracts::contract_address(&ALICE, &code_hash, &[]); + + // Drop previous events + initialize_block(2); + + // Create storage + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 42, + GAS_LIMIT, + None, + (1_000u32, 5_000u32).encode(), + )); + // 4 is for creating 2 storage items + let charged0 = 4 + 1_000 + 5_000; + deposit += charged0; + assert_eq!(>::get(&addr).unwrap().storage_deposit, deposit); + + // Add more storage (but also remove some) + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + (2_000u32, 4_900u32).encode(), + )); + let charged1 = 1_000 - 100; + deposit += charged1; + assert_eq!(>::get(&addr).unwrap().storage_deposit, deposit); + + // Remove more storage (but also add some) + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + (2_100u32, 900u32).encode(), + )); + let refunded0 = 4_000 - 100; + deposit -= refunded0; + assert_eq!(>::get(&addr).unwrap().storage_deposit, deposit); + + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Transfer { + from: ALICE, + to: addr.clone(), + amount: 42, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Transfer { + from: ALICE, + to: addr.clone(), + amount: charged0, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Reserved { + who: addr.clone(), + amount: charged0, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Transfer { + from: ALICE, + to: addr.clone(), + amount: charged1, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Reserved { + who: addr.clone(), + amount: charged1, + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::ReserveRepatriated { + from: addr.clone(), + to: ALICE, + amount: refunded0, + destination_status: BalanceStatus::Free, + }), + topics: vec![], + }, + ] + ); + }); +} + +#[test] +fn call_after_killed_account_needs_funding() { + let (wasm, code_hash) = compile_module::("dummy").unwrap(); + ExtBuilder::default().existential_deposit(200).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + let min_balance = ::Currency::minimum_balance(); + + assert_ok!(Contracts::instantiate_with_code( + Origin::signed(ALICE), + 700, + GAS_LIMIT, + None, + wasm, + vec![], + vec![], + )); + let addr = Contracts::contract_address(&ALICE, &code_hash, &[]); + + // Drop previous events + initialize_block(2); + + // Destroy the account of the contract by slashing. + // Slashing can actually happen if the contract takes part in staking. + // It is a corner case and we except the destruction of the account. + let _ = ::Currency::slash( + &addr, + ::Currency::total_balance(&addr), + ); + + // Sending below the minimum balance will fail the call because it needs to create the + // account in order to send balance there. + assert_err_ignore_postinfo!( + Contracts::call( + Origin::signed(ALICE), + addr.clone(), + min_balance - 1, + GAS_LIMIT, + None, + vec![], + ), + >::TransferFailed + ); + + // Sending zero should work as it does not do a transfer + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + 0, + GAS_LIMIT, + None, + vec![], + )); + + // Sending minimum balance should work + assert_ok!(Contracts::call( + Origin::signed(ALICE), + addr.clone(), + min_balance, + GAS_LIMIT, + None, + vec![], + )); + + assert_eq!( + System::events(), + vec![ + EventRecord { + phase: Phase::Initialization, + event: Event::System(frame_system::Event::KilledAccount { + account: addr.clone() + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Slashed { + who: addr.clone(), + amount: min_balance + 700 + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::System(frame_system::Event::NewAccount { account: addr.clone() }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Endowed { + account: addr.clone(), + free_balance: min_balance + }), + topics: vec![], + }, + EventRecord { + phase: Phase::Initialization, + event: Event::Balances(pallet_balances::Event::Transfer { + from: ALICE, + to: addr.clone(), + amount: min_balance + }), + topics: vec![], + }, + ] + ); + }); +} + +#[test] +fn contract_reverted() { + let (wasm, code_hash) = compile_module::("return_with_data").unwrap(); + + ExtBuilder::default().existential_deposit(100).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + let flags = ReturnFlags::REVERT; + let buffer = [4u8, 8, 15, 16, 23, 42]; + let input = (flags.bits(), buffer).encode(); + + // We just upload the code for later use + assert_ok!(Contracts::upload_code(Origin::signed(ALICE), wasm.clone(), None)); + + // Calling extrinsic: revert leads to an error + assert_err_ignore_postinfo!( + Contracts::instantiate( + Origin::signed(ALICE), + 0, + GAS_LIMIT, + None, + code_hash, + input.clone(), + vec![], + ), + >::ContractReverted, + ); + + // Calling extrinsic: revert leads to an error + assert_err_ignore_postinfo!( + Contracts::instantiate_with_code( + Origin::signed(ALICE), + 0, + GAS_LIMIT, + None, + wasm, + input.clone(), + vec![], + ), + >::ContractReverted, + ); + + // Calling directly: revert leads to success but the flags indicate the error + // This is just a different way of transporting the error that allows the read out + // the `data` which is only there on success. Obviously, the contract isn't + // instantiated. + let result = Contracts::bare_instantiate( + ALICE, + 0, + GAS_LIMIT, + None, + Code::Existing(code_hash), + input.clone(), + vec![], + false, + ) + .result + .unwrap(); + assert_eq!(result.result.flags, flags); + assert_eq!(result.result.data.0, buffer); + assert!(!>::contains_key(result.account_id)); + + // Pass empty flags and therefore successfully instantiate the contract for later use. + let addr = Contracts::bare_instantiate( + ALICE, + 0, + GAS_LIMIT, + None, + Code::Existing(code_hash), + ReturnFlags::empty().bits().encode(), + vec![], + false, + ) + .result + .unwrap() + .account_id; + + // Calling extrinsic: revert leads to an error + assert_err_ignore_postinfo!( + Contracts::call(Origin::signed(ALICE), addr.clone(), 0, GAS_LIMIT, None, input.clone()), + >::ContractReverted, + ); + + // Calling directly: revert leads to success but the flags indicate the error + let result = Contracts::bare_call(ALICE, addr.clone(), 0, GAS_LIMIT, None, input, false) + .result + .unwrap(); + assert_eq!(result.flags, flags); + assert_eq!(result.data.0, buffer); + }); +} + +#[test] +fn code_rejected_error_works() { + let (wasm, _) = compile_module::("invalid_import").unwrap(); + ExtBuilder::default().existential_deposit(200).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + + assert_noop!( + Contracts::upload_code(Origin::signed(ALICE), wasm.clone(), None), + >::CodeRejected, + ); + + let result = Contracts::bare_instantiate( + ALICE, + 0, + GAS_LIMIT, + None, + Code::Upload(Bytes(wasm)), + vec![], + vec![], + true, + ); + assert_err!(result.result, >::CodeRejected); + assert_eq!( + std::str::from_utf8(&result.debug_message).unwrap(), + "module imports a non-existent function" + ); + }); +} + +#[test] +#[cfg(feature = "unstable-interface")] +fn set_code_hash() { + let (wasm, code_hash) = compile_module::("set_code_hash").unwrap(); + let (new_wasm, new_code_hash) = compile_module::("new_set_code_hash_contract").unwrap(); + + let contract_addr = Contracts::contract_address(&ALICE, &code_hash, &[]); + + ExtBuilder::default().existential_deposit(100).build().execute_with(|| { + let _ = Balances::deposit_creating(&ALICE, 1_000_000); + + // Instantiate the 'caller' + assert_ok!(Contracts::instantiate_with_code( + Origin::signed(ALICE), + 300_000, + GAS_LIMIT, + None, + wasm, + vec![], + vec![], + )); + // upload new code + assert_ok!(Contracts::upload_code(Origin::signed(ALICE), new_wasm.clone(), None)); + + // First call sets new code_hash and returns 1 + let result = Contracts::bare_call( + ALICE, + contract_addr.clone(), + 0, + GAS_LIMIT, + None, + new_code_hash.as_ref().to_vec(), + true, + ) + .result + .unwrap(); + assert_return_code!(result, 1); + + // Second calls new contract code that returns 2 + let result = + Contracts::bare_call(ALICE, contract_addr.clone(), 0, GAS_LIMIT, None, vec![], true) + .result + .unwrap(); + assert_return_code!(result, 2); + + // Checking for the last event only + assert_eq!( + System::events().pop().unwrap(), + EventRecord { + phase: Phase::Initialization, + event: Event::Contracts(crate::Event::ContractCodeUpdated { + contract: contract_addr.clone(), + new_code_hash: new_code_hash.clone(), + old_code_hash: code_hash.clone(), + }), + topics: vec![], + }, + ); + }); +} diff --git a/frame/contracts/src/wasm/code_cache.rs b/frame/contracts/src/wasm/code_cache.rs index 08a7449683ed..ee5cdd530721 100644 --- a/frame/contracts/src/wasm/code_cache.rs +++ b/frame/contracts/src/wasm/code_cache.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,54 +28,103 @@ //! this guarantees that every instrumented contract code in cache cannot have the version equal to //! the current one. Thus, before executing a contract it should be reinstrument with new schedule. -#[cfg(feature = "runtime-benchmarks")] -pub use self::private::reinstrument; use crate::{ gas::{GasMeter, Token}, wasm::{prepare, PrefabWasmModule}, weights::WeightInfo, - CodeHash, CodeStorage, Config, Error, Event, Pallet as Contracts, PristineCode, Schedule, + CodeHash, CodeStorage, Config, Error, Event, OwnerInfoOf, Pallet, PristineCode, Schedule, Weight, }; -use frame_support::dispatch::DispatchError; +use frame_support::{ + dispatch::{DispatchError, DispatchResult}, + ensure, + traits::ReservableCurrency, +}; use sp_core::crypto::UncheckedFrom; +use sp_runtime::traits::BadOrigin; /// Put the instrumented module in storage. /// /// Increments the refcount of the in-storage `prefab_module` if it already exists in storage /// under the specified `code_hash`. -pub fn store(mut prefab_module: PrefabWasmModule) +pub fn store(mut module: PrefabWasmModule, instantiated: bool) -> DispatchResult where T::AccountId: UncheckedFrom + AsRef<[u8]>, { - let code_hash = sp_std::mem::take(&mut prefab_module.code_hash); - - // original_code is only `Some` if the contract was instantiated from a new code - // but `None` if it was loaded from storage. - if let Some(code) = prefab_module.original_code.take() { - >::insert(&code_hash, code); - } + let code_hash = sp_std::mem::take(&mut module.code_hash); >::mutate(&code_hash, |existing| match existing { - Some(module) => increment_64(&mut module.refcount), + Some(existing) => { + // We instrument any uploaded contract anyways. We might as well store it to save + // a potential re-instrumentation later. + existing.code = module.code; + existing.instruction_weights_version = module.instruction_weights_version; + // When the code was merely uploaded but not instantiated we can skip this. + if instantiated { + >::mutate(&code_hash, |owner_info| { + if let Some(owner_info) = owner_info { + owner_info.refcount = owner_info.refcount.checked_add(1).expect( + " + refcount is 64bit. Generating this overflow would require to store + _at least_ 18 exabyte of data assuming that a contract consumes only + one byte of data. Any node would run out of storage space before hitting + this overflow. + qed + ", + ); + } + }) + } + Ok(()) + }, None => { - *existing = Some(prefab_module); - Contracts::::deposit_event(Event::CodeStored(code_hash)) + let orig_code = module.original_code.take().expect( + " + If an executable isn't in storage it was uploaded. + If it was uploaded the original code must exist. qed + ", + ); + let mut owner_info = module.owner_info.take().expect( + "If an executable isn't in storage it was uploaded. + If it was uploaded the owner info was generated and attached. qed + ", + ); + // This `None` case happens only in freshly uploaded modules. This means that + // the `owner` is always the origin of the current transaction. + T::Currency::reserve(&owner_info.owner, owner_info.deposit) + .map_err(|_| >::StorageDepositNotEnoughFunds)?; + owner_info.refcount = if instantiated { 1 } else { 0 }; + >::insert(&code_hash, orig_code); + >::insert(&code_hash, owner_info); + *existing = Some(module); + >::deposit_event(Event::CodeStored { code_hash }); + Ok(()) }, + }) +} + +/// Decrement the refcount of a code in-storage by one. +/// +/// # Note +/// +/// A contract whose refcount dropped to zero isn't automatically removed. A `remove_code` +/// transaction must be submitted by the original uploader to do so. +pub fn decrement_refcount(code_hash: CodeHash) { + >::mutate(code_hash, |existing| { + if let Some(info) = existing { + info.refcount = info.refcount.saturating_sub(1); + } }); } /// Increment the refcount of a code in-storage by one. -pub fn increment_refcount( - code_hash: CodeHash, - gas_meter: &mut GasMeter, -) -> Result<(), DispatchError> -where - T::AccountId: UncheckedFrom + AsRef<[u8]>, -{ - gas_meter.charge(CodeToken::UpdateRefcount(estimate_code_size::(&code_hash)?))?; - >::mutate(code_hash, |existing| { - if let Some(module) = existing { - increment_64(&mut module.refcount); +/// +/// # Errors +/// +/// [`Error::CodeNotFound`] is returned if the specified `code_hash` does not exist. +pub fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError> { + >::mutate(code_hash, |existing| -> Result<(), DispatchError> { + if let Some(info) = existing { + info.refcount = info.refcount.saturating_add(1); Ok(()) } else { Err(Error::::CodeNotFound.into()) @@ -83,27 +132,22 @@ where }) } -/// Decrement the refcount of a code in-storage by one and remove the code when it drops to zero. -pub fn decrement_refcount( - code_hash: CodeHash, - gas_meter: &mut GasMeter, -) -> Result<(), DispatchError> -where - T::AccountId: UncheckedFrom + AsRef<[u8]>, -{ - if let Ok(len) = estimate_code_size::(&code_hash) { - gas_meter.charge(CodeToken::UpdateRefcount(len))?; - } - >::mutate_exists(code_hash, |existing| { - if let Some(module) = existing { - module.refcount = module.refcount.saturating_sub(1); - if module.refcount == 0 { - *existing = None; - finish_removal::(code_hash); - } +/// Try to remove code together with all associated information. +pub fn try_remove(origin: &T::AccountId, code_hash: CodeHash) -> DispatchResult { + >::try_mutate_exists(&code_hash, |existing| { + if let Some(owner_info) = existing { + ensure!(owner_info.refcount == 0, >::CodeInUse); + ensure!(&owner_info.owner == origin, BadOrigin); + T::Currency::unreserve(&owner_info.owner, owner_info.deposit); + *existing = None; + >::remove(&code_hash); + >::remove(&code_hash); + >::deposit_event(Event::CodeRemoved { code_hash }); + Ok(()) + } else { + Err(>::CodeNotFound.into()) } - }); - Ok(()) + }) } /// Load code with the given code hash. @@ -111,129 +155,70 @@ where /// If the module was instrumented with a lower version of schedule than /// the current one given as an argument, then this function will perform /// re-instrumentation and update the cache in the storage. -/// -/// # Note -/// -/// If `reinstrument` is set it is assumed that the load is performed in the context of -/// a contract call: This means we charge the size based cased for loading the contract. pub fn load( code_hash: CodeHash, - mut reinstrument: Option<(&Schedule, &mut GasMeter)>, + schedule: &Schedule, + gas_meter: &mut GasMeter, ) -> Result, DispatchError> where T::AccountId: UncheckedFrom + AsRef<[u8]>, { - // The reinstrument case coincides with the cases where we need to charge extra - // based upon the code size: On-chain execution. - if let Some((_, gas_meter)) = &mut reinstrument { - gas_meter.charge(CodeToken::Load(estimate_code_size::(&code_hash)?))?; - } + let charged = gas_meter.charge(CodeToken::Load(schedule.limits.code_len))?; let mut prefab_module = >::get(code_hash).ok_or_else(|| Error::::CodeNotFound)?; + gas_meter.adjust_gas(charged, CodeToken::Load(prefab_module.code.len() as u32)); prefab_module.code_hash = code_hash; - if let Some((schedule, gas_meter)) = reinstrument { - if prefab_module.instruction_weights_version < schedule.instruction_weights.version { - // The instruction weights have changed. - // We need to re-instrument the code with the new instruction weights. - gas_meter.charge(CodeToken::Instrument(prefab_module.original_code_len))?; - private::reinstrument(&mut prefab_module, schedule)?; - } - } - Ok(prefab_module) -} - -mod private { - use super::*; - - /// Instruments the passed prefab wasm module with the supplied schedule. - pub fn reinstrument( - prefab_module: &mut PrefabWasmModule, - schedule: &Schedule, - ) -> Result<(), DispatchError> - where - T::AccountId: UncheckedFrom + AsRef<[u8]>, - { - let original_code = >::get(&prefab_module.code_hash) - .ok_or_else(|| Error::::CodeNotFound)?; - prefab_module.code = prepare::reinstrument_contract::(original_code, schedule)?; - prefab_module.instruction_weights_version = schedule.instruction_weights.version; - >::insert(&prefab_module.code_hash, &*prefab_module); - Ok(()) + if prefab_module.instruction_weights_version < schedule.instruction_weights.version { + // The instruction weights have changed. + // We need to re-instrument the code with the new instruction weights. + let charged = gas_meter.charge(CodeToken::Reinstrument(schedule.limits.code_len))?; + let code_size = reinstrument(&mut prefab_module, schedule)?; + gas_meter.adjust_gas(charged, CodeToken::Reinstrument(code_size)); } -} -/// Finish removal of a code by deleting the pristine code and emitting an event. -fn finish_removal(code_hash: CodeHash) -where - T::AccountId: UncheckedFrom + AsRef<[u8]>, -{ - >::remove(code_hash); - Contracts::::deposit_event(Event::CodeRemoved(code_hash)) -} - -/// Increment the refcount panicking if it should ever overflow (which will not happen). -/// -/// We try hard to be infallible here because otherwise more storage transactions would be -/// necessary to account for failures in storing code for an already instantiated contract. -fn increment_64(refcount: &mut u64) { - *refcount = refcount.checked_add(1).expect( - " - refcount is 64bit. Generating this overflow would require to store - _at least_ 18 exabyte of data assuming that a contract consumes only - one byte of data. Any node would run out of storage space before hitting - this overflow. - qed - ", - ); + Ok(prefab_module) } -/// Get the size of the instrumented code stored at `code_hash` without loading it. +/// Instruments the passed prefab wasm module with the supplied schedule. /// -/// The returned value is slightly too large because it also contains the fields apart from -/// `code` which are located inside [`PrefabWasmModule`]. However, those are negligible when -/// compared to the code size. Additionally, charging too much weight is completely safe. -fn estimate_code_size(code_hash: &CodeHash) -> Result -where - T::AccountId: UncheckedFrom + AsRef<[u8]>, -{ - let key = >::hashed_key_for(code_hash); - let mut data = [0u8; 0]; - let len = sp_io::storage::read(&key, &mut data, 0).ok_or_else(|| Error::::CodeNotFound)?; - Ok(len) +/// Returns the size in bytes of the uninstrumented code. +pub fn reinstrument( + prefab_module: &mut PrefabWasmModule, + schedule: &Schedule, +) -> Result { + let original_code = + >::get(&prefab_module.code_hash).ok_or_else(|| Error::::CodeNotFound)?; + let original_code_len = original_code.len(); + prefab_module.code = prepare::reinstrument_contract::(original_code, schedule)?; + prefab_module.instruction_weights_version = schedule.instruction_weights.version; + >::insert(&prefab_module.code_hash, &*prefab_module); + Ok(original_code_len as u32) } /// Costs for operations that are related to code handling. #[cfg_attr(test, derive(Debug, PartialEq, Eq))] #[derive(Clone, Copy)] enum CodeToken { - /// Weight for instrumenting a contract contract of the supplied size in bytes. - Instrument(u32), - /// Weight for loading a contract per kilobyte. + /// Weight for reinstrumenting a contract contract of the supplied size in bytes. + Reinstrument(u32), + /// Weight for loading a contract per byte. Load(u32), - /// Weight for changing the refcount of a contract per kilobyte. - UpdateRefcount(u32), } -impl Token for CodeToken -where - T: Config, - T::AccountId: UncheckedFrom + AsRef<[u8]>, -{ +impl Token for CodeToken { fn weight(&self) -> Weight { use self::CodeToken::*; - // In case of `Load` and `UpdateRefcount` we already covered the general costs of - // accessing the storage but still need to account for the actual size of the + // In case of `Load` we already covered the general costs of + // calling the storage but still need to account for the actual size of the // contract code. This is why we substract `T::*::(0)`. We need to do this at this - // point because when charging the general weight we do not know the size of - // the contract. + // point because when charging the general weight for calling the contract we not know the + // size of the contract. match *self { - Instrument(len) => T::WeightInfo::instrument(len / 1024), - Load(len) => - T::WeightInfo::code_load(len / 1024).saturating_sub(T::WeightInfo::code_load(0)), - UpdateRefcount(len) => T::WeightInfo::code_refcount(len / 1024) - .saturating_sub(T::WeightInfo::code_refcount(0)), + Reinstrument(len) => T::WeightInfo::reinstrument(len), + Load(len) => T::WeightInfo::call_with_code_per_byte(len) + .saturating_sub(T::WeightInfo::call_with_code_per_byte(0)), } } } diff --git a/frame/contracts/src/wasm/env_def/macros.rs b/frame/contracts/src/wasm/env_def/macros.rs index ea7f51da7526..aa5a1626681f 100644 --- a/frame/contracts/src/wasm/env_def/macros.rs +++ b/frame/contracts/src/wasm/env_def/macros.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,7 +28,7 @@ macro_rules! convert_args { macro_rules! gen_signature { ( ( $( $params: ty ),* ) ) => ( { - pwasm_utils::parity_wasm::elements::FunctionType::new( + wasm_instrument::parity_wasm::elements::FunctionType::new( convert_args!($($params),*), vec![], ) } @@ -36,7 +36,7 @@ macro_rules! gen_signature { ( ( $( $params: ty ),* ) -> $returns: ty ) => ( { - pwasm_utils::parity_wasm::elements::FunctionType::new( + wasm_instrument::parity_wasm::elements::FunctionType::new( convert_args!($($params),*), vec![{use $crate::wasm::env_def::ConvertibleToWasm; <$returns>::VALUE_TYPE}], ) @@ -220,7 +220,7 @@ macro_rules! define_env { fn can_satisfy( module: &[u8], name: &[u8], - func_type: &pwasm_utils::parity_wasm::elements::FunctionType, + func_type: &wasm_instrument::parity_wasm::elements::FunctionType, ) -> bool { #[cfg(not(feature = "unstable-interface"))] @@ -260,9 +260,9 @@ mod tests { wasm::{runtime::TrapReason, tests::MockExt, Runtime}, Weight, }; - use pwasm_utils::parity_wasm::elements::{FunctionType, ValueType}; use sp_runtime::traits::Zero; use sp_sandbox::{ReturnValue, Value}; + use wasm_instrument::parity_wasm::elements::{FunctionType, ValueType}; struct TestRuntime { value: u32, diff --git a/frame/contracts/src/wasm/env_def/mod.rs b/frame/contracts/src/wasm/env_def/mod.rs index 6a55677f69a0..b4c5ffe81e7c 100644 --- a/frame/contracts/src/wasm/env_def/mod.rs +++ b/frame/contracts/src/wasm/env_def/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,8 +18,8 @@ use super::Runtime; use crate::exec::Ext; -use pwasm_utils::parity_wasm::elements::{FunctionType, ValueType}; use sp_sandbox::Value; +use wasm_instrument::parity_wasm::elements::{FunctionType, ValueType}; #[macro_use] pub mod macros; diff --git a/frame/contracts/src/wasm/mod.rs b/frame/contracts/src/wasm/mod.rs index 855cb6e45091..c38613cb6810 100644 --- a/frame/contracts/src/wasm/mod.rs +++ b/frame/contracts/src/wasm/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,17 +25,18 @@ mod prepare; mod runtime; #[cfg(feature = "runtime-benchmarks")] -pub use self::code_cache::reinstrument; -pub use self::runtime::{ReturnCode, Runtime, RuntimeCosts}; +pub use crate::wasm::code_cache::reinstrument; +pub use crate::wasm::runtime::{CallFlags, ReturnCode, Runtime, RuntimeCosts}; use crate::{ exec::{ExecResult, Executable, ExportedFunction, Ext}, gas::GasMeter, wasm::env_def::FunctionImplProvider, - CodeHash, Config, Schedule, + AccountIdOf, BalanceOf, CodeHash, CodeStorage, Config, Schedule, }; -use codec::{Decode, Encode}; -use frame_support::dispatch::DispatchError; +use codec::{Decode, Encode, MaxEncodedLen}; +use frame_support::dispatch::{DispatchError, DispatchResult}; use sp_core::crypto::UncheckedFrom; +use sp_sandbox::{SandboxEnvironmentBuilder, SandboxInstance, SandboxMemory}; use sp_std::prelude::*; #[cfg(test)] pub use tests::MockExt; @@ -45,10 +46,9 @@ pub use tests::MockExt; /// # Note /// /// This data structure is mostly immutable once created and stored. The exceptions that -/// can be changed by calling a contract are `refcount`, `instruction_weights_version` and `code`. -/// `refcount` can change when a contract instantiates a new contract or self terminates. -/// `instruction_weights_version` and `code` when a contract with an outdated instrumention is -/// called. Therefore one must be careful when holding any in-memory representation of this +/// can be changed by calling a contract are `instruction_weights_version` and `code`. +/// `instruction_weights_version` and `code` change when a contract with an outdated instrumentation +/// is called. Therefore one must be careful when holding any in-memory representation of this /// type while calling into a contract as those fields can get out of date. #[derive(Clone, Encode, Decode, scale_info::TypeInfo)] #[scale_info(skip_type_params(T))] @@ -62,26 +62,8 @@ pub struct PrefabWasmModule { /// The maximum memory size of a contract's sandbox. #[codec(compact)] maximum: u32, - /// The number of contracts that use this as their contract code. - /// - /// If this number drops to zero this module is removed from storage. - #[codec(compact)] - refcount: u64, - /// This field is reserved for future evolution of format. - /// - /// For now this field is serialized as `None`. In the future we are able to change the - /// type parameter to a new struct that contains the fields that we want to add. - /// That new struct would also contain a reserved field for its future extensions. - /// This works because in SCALE `None` is encoded independently from the type parameter - /// of the option. - _reserved: Option<()>, /// Code instrumented with the latest schedule. code: Vec, - /// The size of the uninstrumented code. - /// - /// We cache this value here in order to avoid the need to pull the pristine code - /// from storage when we only need its length for rent calculations. - original_code_len: u32, /// The uninstrumented, pristine version of the code. /// /// It is not stored because the pristine code has its own storage item. The value @@ -95,6 +77,27 @@ pub struct PrefabWasmModule { /// when loading the module from storage. #[codec(skip)] code_hash: CodeHash, + // This isn't needed for contract execution and does not get loaded from storage by default. + // It is `Some` if and only if this struct was generated from code. + #[codec(skip)] + owner_info: Option>, +} + +/// Information that belongs to a [`PrefabWasmModule`] but is stored separately. +/// +/// It is stored in a separate storage entry to avoid loading the code when not necessary. +#[derive(Clone, Encode, Decode, scale_info::TypeInfo, MaxEncodedLen)] +#[codec(mel_bound())] +#[scale_info(skip_type_params(T))] +pub struct OwnerInfo { + /// The account that has deployed the contract and hence is allowed to remove it. + owner: AccountIdOf, + /// The amount of balance that was deposited by the owner in order to deploy it. + #[codec(compact)] + deposit: BalanceOf, + /// The number of contracts that use this as their code. + #[codec(compact)] + refcount: u64, } impl ExportedFunction { @@ -112,11 +115,43 @@ where T::AccountId: UncheckedFrom + AsRef<[u8]>, { /// Create the module by checking and instrumenting `original_code`. + /// + /// This does **not** store the module. For this one need to either call [`Self::store`] + /// or [`::execute`]. pub fn from_code( original_code: Vec, schedule: &Schedule, - ) -> Result { - prepare::prepare_contract(original_code, schedule).map_err(Into::into) + owner: AccountIdOf, + ) -> Result { + prepare::prepare_contract(original_code, schedule, owner) + } + + /// Store the code without instantiating it. + /// + /// Otherwise the code is stored when [`::execute`] is called. + pub fn store(self) -> DispatchResult { + code_cache::store(self, false) + } + + /// Remove the code from storage and refund the deposit to its owner. + /// + /// Applies all necessary checks before removing the code. + pub fn remove(origin: &T::AccountId, code_hash: CodeHash) -> DispatchResult { + code_cache::try_remove::(origin, code_hash) + } + + /// Returns whether there is a deposit to be payed for this module. + /// + /// Returns `0` if the module is already in storage and hence no deposit will + /// be charged when storing it. + pub fn open_deposit(&self) -> BalanceOf { + if >::contains_key(&self.code_hash) { + 0u32.into() + } else { + // Only already in-storage contracts have their `owner_info` set to `None`. + // Therefore it is correct to return `0` in this case. + self.owner_info.as_ref().map(|i| i.deposit).unwrap_or_default() + } } /// Create and store the module without checking nor instrumenting the passed code. @@ -124,22 +159,16 @@ where /// # Note /// /// This is useful for benchmarking where we don't want instrumentation to skew - /// our results. + /// our results. This also does not collect any deposit from the `owner`. #[cfg(feature = "runtime-benchmarks")] pub fn store_code_unchecked( original_code: Vec, schedule: &Schedule, - ) -> Result<(), DispatchError> { - let executable = prepare::benchmarking::prepare_contract(original_code, schedule) + owner: T::AccountId, + ) -> DispatchResult { + let executable = prepare::benchmarking::prepare_contract(original_code, schedule, owner) .map_err::(Into::into)?; - code_cache::store(executable); - Ok(()) - } - - /// Return the refcount of the module. - #[cfg(test)] - pub fn refcount(&self) -> u64 { - self.refcount + code_cache::store(executable, false) } /// Decrement instruction_weights_version by 1. Panics if it is already 0. @@ -149,6 +178,14 @@ where } } +impl OwnerInfo { + /// Return the refcount of the module. + #[cfg(test)] + pub fn refcount(&self) -> u64 { + self.refcount + } +} + impl Executable for PrefabWasmModule where T::AccountId: UncheckedFrom + AsRef<[u8]>, @@ -158,22 +195,15 @@ where schedule: &Schedule, gas_meter: &mut GasMeter, ) -> Result { - code_cache::load(code_hash, Some((schedule, gas_meter))) + code_cache::load(code_hash, schedule, gas_meter) } - fn from_storage_noinstr(code_hash: CodeHash) -> Result { - code_cache::load(code_hash, None) + fn add_user(code_hash: CodeHash) -> Result<(), DispatchError> { + code_cache::increment_refcount::(code_hash) } - fn add_user(code_hash: CodeHash, gas_meter: &mut GasMeter) -> Result<(), DispatchError> { - code_cache::increment_refcount::(code_hash, gas_meter) - } - - fn remove_user( - code_hash: CodeHash, - gas_meter: &mut GasMeter, - ) -> Result<(), DispatchError> { - code_cache::decrement_refcount::(code_hash, gas_meter) + fn remove_user(code_hash: CodeHash) { + code_cache::decrement_refcount::(code_hash) } fn execute>( @@ -182,8 +212,8 @@ where function: &ExportedFunction, input_data: Vec, ) -> ExecResult { - let memory = - sp_sandbox::Memory::new(self.initial, Some(self.maximum)).unwrap_or_else(|_| { + let memory = sp_sandbox::default_executor::Memory::new(self.initial, Some(self.maximum)) + .unwrap_or_else(|_| { // unlike `.expect`, explicit panic preserves the source location. // Needed as we can't use `RUST_BACKTRACE` in here. panic!( @@ -193,23 +223,22 @@ where ) }); - let mut imports = sp_sandbox::EnvironmentDefinitionBuilder::new(); + let mut imports = sp_sandbox::default_executor::EnvironmentDefinitionBuilder::new(); imports.add_memory(self::prepare::IMPORT_MODULE_MEMORY, "memory", memory.clone()); runtime::Env::impls(&mut |module, name, func_ptr| { imports.add_host_func(module, name, func_ptr); }); - let mut runtime = Runtime::new(ext, input_data, memory); - // We store before executing so that the code hash is available in the constructor. let code = self.code.clone(); if let &ExportedFunction::Constructor = function { - code_cache::store(self) + code_cache::store(self, true)?; } // Instantiate the instance from the instrumented module code and invoke the contract // entrypoint. - let result = sp_sandbox::Instance::new(&code, &imports, &mut runtime) + let mut runtime = Runtime::new(ext, input_data, memory); + let result = sp_sandbox::default_executor::Instance::new(&code, &imports, &mut runtime) .and_then(|mut instance| instance.invoke(function.identifier(), &[], &mut runtime)); runtime.to_execution_result(result) @@ -222,14 +251,6 @@ where fn code_len(&self) -> u32 { self.code.len() as u32 } - - fn aggregate_code_len(&self) -> u32 { - self.original_code_len.saturating_add(self.code_len()) - } - - fn refcount(&self) -> u32 { - self.refcount as u32 - } } #[cfg(test)] @@ -240,26 +261,27 @@ mod tests { AccountIdOf, BlockNumberOf, ErrorOrigin, ExecError, Executable, Ext, SeedOf, StorageKey, }, gas::GasMeter, + storage::WriteOutcome, tests::{Call, Test, ALICE, BOB}, BalanceOf, CodeHash, Error, Pallet as Contracts, }; use assert_matches::assert_matches; - use frame_support::{ - assert_ok, - dispatch::{DispatchResult, DispatchResultWithPostInfo}, - weights::Weight, - }; + use frame_support::{assert_ok, dispatch::DispatchResultWithPostInfo, weights::Weight}; use hex_literal::hex; use pallet_contracts_primitives::{ExecReturnValue, ReturnFlags}; use pretty_assertions::assert_eq; use sp_core::{Bytes, H256}; use sp_runtime::DispatchError; - use std::{borrow::BorrowMut, cell::RefCell, collections::HashMap}; + use std::{ + borrow::BorrowMut, + cell::RefCell, + collections::hash_map::{Entry, HashMap}, + }; #[derive(Debug, PartialEq, Eq)] struct InstantiateEntry { code_hash: H256, - endowment: u64, + value: u64, data: Vec, gas_left: u64, salt: Vec, @@ -284,11 +306,18 @@ mod tests { allows_reentry: bool, } + #[derive(Debug, PartialEq, Eq)] + struct CallCodeEntry { + code_hash: H256, + data: Vec, + } + pub struct MockExt { storage: HashMap>, instantiates: Vec, terminations: Vec, calls: Vec, + code_calls: Vec, transfers: Vec, // (topics, data) events: Vec<(Vec, Vec)>, @@ -297,6 +326,7 @@ mod tests { gas_meter: GasMeter, debug_buffer: Vec, ecdsa_recover: RefCell>, + code_hashes: Vec>, } /// The call is mocked and just returns this hardcoded value. @@ -307,10 +337,12 @@ mod tests { impl Default for MockExt { fn default() -> Self { Self { + code_hashes: Default::default(), storage: Default::default(), instantiates: Default::default(), terminations: Default::default(), calls: Default::default(), + code_calls: Default::default(), transfers: Default::default(), events: Default::default(), runtime_calls: Default::default(), @@ -336,17 +368,25 @@ mod tests { self.calls.push(CallEntry { to, value, data, allows_reentry }); Ok(ExecReturnValue { flags: ReturnFlags::empty(), data: call_return_data() }) } + fn delegate_call( + &mut self, + code_hash: CodeHash, + data: Vec, + ) -> Result { + self.code_calls.push(CallCodeEntry { code_hash, data }); + Ok(ExecReturnValue { flags: ReturnFlags::empty(), data: call_return_data() }) + } fn instantiate( &mut self, gas_limit: Weight, code_hash: CodeHash, - endowment: u64, + value: u64, data: Vec, salt: &[u8], ) -> Result<(AccountIdOf, ExecReturnValue), ExecError> { self.instantiates.push(InstantiateEntry { code_hash: code_hash.clone(), - endowment, + value, data: data.to_vec(), gas_left: gas_limit, salt: salt.to_vec(), @@ -356,6 +396,10 @@ mod tests { ExecReturnValue { flags: ReturnFlags::empty(), data: Bytes(Vec::new()) }, )) } + fn set_code_hash(&mut self, hash: CodeHash) -> Result<(), DispatchError> { + self.code_hashes.push(hash); + Ok(()) + } fn transfer(&mut self, to: &AccountIdOf, value: u64) -> Result<(), DispatchError> { self.transfers.push(TransferEntry { to: to.clone(), value }); Ok(()) @@ -367,13 +411,43 @@ mod tests { fn get_storage(&mut self, key: &StorageKey) -> Option> { self.storage.get(key).cloned() } - fn set_storage(&mut self, key: StorageKey, value: Option>) -> DispatchResult { - *self.storage.entry(key).or_insert(Vec::new()) = value.unwrap_or(Vec::new()); - Ok(()) + fn get_storage_size(&mut self, key: &StorageKey) -> Option { + self.storage.get(key).map(|val| val.len() as u32) + } + fn set_storage( + &mut self, + key: StorageKey, + value: Option>, + take_old: bool, + ) -> Result { + let entry = self.storage.entry(key); + let result = match (entry, take_old) { + (Entry::Vacant(_), _) => WriteOutcome::New, + (Entry::Occupied(entry), false) => + WriteOutcome::Overwritten(entry.remove().len() as u32), + (Entry::Occupied(entry), true) => WriteOutcome::Taken(entry.remove()), + }; + if let Some(value) = value { + self.storage.insert(key, value); + } + Ok(result) } fn caller(&self) -> &AccountIdOf { &ALICE } + fn is_contract(&self, _address: &AccountIdOf) -> bool { + true + } + fn code_hash(&self, _address: &AccountIdOf) -> Option> { + Some(H256::from_slice(&[0x11; 32])) + } + fn own_code_hash(&mut self) -> &CodeHash { + const HASH: H256 = H256::repeat_byte(0x10); + &HASH + } + fn caller_is_origin(&self) -> bool { + false + } fn address(&self) -> &AccountIdOf { &BOB } @@ -389,9 +463,6 @@ mod tests { fn minimum_balance(&self) -> u64 { 666 } - fn contract_deposit(&self) -> u64 { - 16 - } fn random(&self, subject: &[u8]) -> (SeedOf, BlockNumberOf) { (H256::from_slice(subject), 42) } @@ -421,7 +492,6 @@ mod tests { self.runtime_calls.borrow_mut().push(call); Ok(Default::default()) } - fn ecdsa_recover( &self, signature: &[u8; 65], @@ -430,13 +500,16 @@ mod tests { self.ecdsa_recover.borrow_mut().push((signature.clone(), message_hash.clone())); Ok([3; 33]) } + fn contract_info(&mut self) -> &mut crate::ContractInfo { + unimplemented!() + } } fn execute>(wat: &str, input_data: Vec, mut ext: E) -> ExecResult { let wasm = wat::parse_str(wat).unwrap(); let schedule = crate::Schedule::default(); let executable = - PrefabWasmModule::<::T>::from_code(wasm, &schedule).unwrap(); + PrefabWasmModule::<::T>::from_code(wasm, &schedule, ALICE).unwrap(); executable.execute(ext.borrow_mut(), &ExportedFunction::Call, input_data) } @@ -541,10 +614,56 @@ mod tests { #[test] #[cfg(feature = "unstable-interface")] + fn contract_delegate_call() { + const CODE: &str = r#" +(module + ;; seal_delegate_call( + ;; flags: u32, + ;; code_hash_ptr: u32, + ;; input_data_ptr: u32, + ;; input_data_len: u32, + ;; output_ptr: u32, + ;; output_len_ptr: u32 + ;;) -> u32 + (import "seal0" "seal_delegate_call" (func $seal_delegate_call (param i32 i32 i32 i32 i32 i32) (result i32))) + (import "env" "memory" (memory 1 1)) + (func (export "call") + (drop + (call $seal_delegate_call + (i32.const 0) ;; No flags are set + (i32.const 4) ;; Pointer to "callee" code_hash. + (i32.const 36) ;; Pointer to input data buffer address + (i32.const 4) ;; Length of input data buffer + (i32.const 4294967295) ;; u32 max value is the sentinel value: do not copy output + (i32.const 0) ;; Length is ignored in this case + ) + ) + ) + (func (export "deploy")) + + ;; Callee code_hash + (data (i32.const 4) + "\11\11\11\11\11\11\11\11\11\11\11\11\11\11\11\11" + "\11\11\11\11\11\11\11\11\11\11\11\11\11\11\11\11" + ) + + (data (i32.const 36) "\01\02\03\04") +) +"#; + let mut mock_ext = MockExt::default(); + assert_ok!(execute(CODE, vec![], &mut mock_ext)); + + assert_eq!( + &mock_ext.code_calls, + &[CallCodeEntry { code_hash: [0x11; 32].into(), data: vec![1, 2, 3, 4] }] + ); + } + + #[test] fn contract_call_forward_input() { const CODE: &str = r#" (module - (import "__unstable__" "seal_call" (func $seal_call (param i32 i32 i64 i32 i32 i32 i32 i32) (result i32))) + (import "seal1" "seal_call" (func $seal_call (param i32 i32 i64 i32 i32 i32 i32 i32) (result i32))) (import "seal0" "seal_input" (func $seal_input (param i32 i32))) (import "env" "memory" (memory 1 1)) (func (export "call") @@ -595,11 +714,10 @@ mod tests { } #[test] - #[cfg(feature = "unstable-interface")] fn contract_call_clone_input() { const CODE: &str = r#" (module - (import "__unstable__" "seal_call" (func $seal_call (param i32 i32 i64 i32 i32 i32 i32 i32) (result i32))) + (import "seal1" "seal_call" (func $seal_call (param i32 i32 i64 i32 i32 i32 i32 i32) (result i32))) (import "seal0" "seal_input" (func $seal_input (param i32 i32))) (import "seal0" "seal_return" (func $seal_return (param i32 i32 i32))) (import "env" "memory" (memory 1 1)) @@ -651,11 +769,10 @@ mod tests { } #[test] - #[cfg(feature = "unstable-interface")] fn contract_call_tail_call() { const CODE: &str = r#" (module - (import "__unstable__" "seal_call" (func $seal_call (param i32 i32 i64 i32 i32 i32 i32 i32) (result i32))) + (import "seal1" "seal_call" (func $seal_call (param i32 i32 i64 i32 i32 i32 i32 i32) (result i32))) (import "env" "memory" (memory 1 1)) (func (export "call") (drop @@ -698,6 +815,67 @@ mod tests { ); } + #[test] + #[cfg(feature = "unstable-interface")] + fn contains_storage_works() { + const CODE: &str = r#" +(module + (import "seal0" "seal_return" (func $seal_return (param i32 i32 i32))) + (import "seal0" "seal_input" (func $seal_input (param i32 i32))) + (import "__unstable__" "seal_contains_storage" (func $seal_contains_storage (param i32) (result i32))) + (import "env" "memory" (memory 1 1)) + + ;; [0, 4) size of input buffer (32 byte as we copy the key here) + (data (i32.const 0) "\20") + + ;; [4, 36) input buffer + + ;; [36, inf) output buffer + + (func (export "call") + ;; Receive key + (call $seal_input + (i32.const 4) ;; Pointer to the input buffer + (i32.const 0) ;; Size of the length buffer + ) + + ;; Load the return value into the output buffer + (i32.store (i32.const 36) + (call $seal_contains_storage + (i32.const 4) ;; The pointer to the storage key to fetch + ) + ) + + ;; Return the contents of the buffer + (call $seal_return + (i32.const 0) ;; flags + (i32.const 36) ;; output buffer ptr + (i32.const 4) ;; result is integer (4 bytes) + ) + ) + + (func (export "deploy")) +) +"#; + + let mut ext = MockExt::default(); + + ext.storage.insert([1u8; 32], vec![42u8]); + ext.storage.insert([2u8; 32], vec![]); + + // value does not exist -> sentinel value returned + let result = execute(CODE, [3u8; 32].encode(), &mut ext).unwrap(); + assert_eq!(u32::from_le_bytes(result.data.0.try_into().unwrap()), crate::SENTINEL); + + // value did exist -> success + let result = execute(CODE, [1u8; 32].encode(), &mut ext).unwrap(); + assert_eq!(u32::from_le_bytes(result.data.0.try_into().unwrap()), 1,); + + // value did exist -> success (zero sized type) + let result = execute(CODE, [2u8; 32].encode(), &mut ext).unwrap(); + assert_eq!(u32::from_le_bytes(result.data.0.try_into().unwrap()), 0,); + } + const CODE_INSTANTIATE: &str = r#" (module ;; seal_instantiate( @@ -763,7 +941,7 @@ mod tests { &mock_ext.instantiates[..], [InstantiateEntry { code_hash, - endowment: 3, + value: 3, data, gas_left: _, salt, @@ -919,8 +1097,8 @@ mod tests { "\11\11\11\11\11\11\11\11\11\11\11\11\11\11\11\11" ) - ;; [32, 36) buffer size = 128 bytes - (data (i32.const 32) "\80") + ;; [32, 36) buffer size = 4k in little endian + (data (i32.const 32) "\00\10") ;; [36; inf) buffer where the result is copied @@ -984,7 +1162,7 @@ mod tests { ); } - /// calls `seal_caller` and compares the result with the constant 42. + /// calls `seal_caller` and compares the result with the constant (ALICE's address part). const CODE_CALLER: &str = r#" (module (import "seal0" "seal_caller" (func $seal_caller (param i32 i32))) @@ -1014,7 +1192,7 @@ mod tests { ) ) - ;; assert that the first 64 byte are the beginning of "ALICE" + ;; assert that the first 8 bytes are the beginning of "ALICE" (call $assert (i64.eq (i64.load (i32.const 0)) @@ -1032,7 +1210,7 @@ mod tests { assert_ok!(execute(CODE_CALLER, vec![], MockExt::default())); } - /// calls `seal_address` and compares the result with the constant 69. + /// calls `seal_address` and compares the result with the constant (BOB's address part). const CODE_ADDRESS: &str = r#" (module (import "seal0" "seal_address" (func $seal_address (param i32 i32))) @@ -1062,7 +1240,7 @@ mod tests { ) ) - ;; assert that the first 64 byte are the beginning of "BOB" + ;; assert that the first 8 bytes are the beginning of "BOB" (call $assert (i64.eq (i64.load (i32.const 0)) @@ -1395,51 +1573,6 @@ mod tests { assert_ok!(execute(CODE_MINIMUM_BALANCE, vec![], MockExt::default())); } - const CODE_CONTRACT_DEPOSIT: &str = r#" -(module - (import "seal0" "seal_contract_deposit" (func $seal_contract_deposit (param i32 i32))) - (import "env" "memory" (memory 1 1)) - - ;; size of our buffer is 32 bytes - (data (i32.const 32) "\20") - - (func $assert (param i32) - (block $ok - (br_if $ok - (get_local 0) - ) - (unreachable) - ) - ) - - (func (export "call") - (call $seal_contract_deposit (i32.const 0) (i32.const 32)) - - ;; assert len == 8 - (call $assert - (i32.eq - (i32.load (i32.const 32)) - (i32.const 8) - ) - ) - - ;; assert that contents of the buffer is equal to the i64 value of 16. - (call $assert - (i64.eq - (i64.load (i32.const 0)) - (i64.const 16) - ) - ) - ) - (func (export "deploy")) -) -"#; - - #[test] - fn contract_deposit() { - assert_ok!(execute(CODE_CONTRACT_DEPOSIT, vec![], MockExt::default())); - } - const CODE_RANDOM: &str = r#" (module (import "seal0" "seal_random" (func $seal_random (param i32 i32 i32 i32))) @@ -1797,7 +1930,7 @@ mod tests { data: Bytes(hex!("445566778899").to_vec()), } ); - assert!(output.is_success()); + assert!(!output.did_revert()); } #[test] @@ -1813,7 +1946,7 @@ mod tests { data: Bytes(hex!("5566778899").to_vec()), } ); - assert!(!output.is_success()); + assert!(output.did_revert()); } const CODE_OUT_OF_BOUNDS_ACCESS: &str = r#" @@ -1967,7 +2100,6 @@ mod tests { #[test] #[cfg(feature = "unstable-interface")] fn call_runtime_works() { - use std::convert::TryInto; let call = Call::System(frame_system::Call::remark { remark: b"Hello World".to_vec() }); let mut ext = MockExt::default(); let result = execute(CODE_CALL_RUNTIME, call.encode(), &mut ext).unwrap(); @@ -1990,4 +2122,428 @@ mod tests { ); assert_eq!(*ext.runtime_calls.borrow(), vec![]); } + + #[test] + #[cfg(feature = "unstable-interface")] + fn set_storage_works() { + const CODE: &str = r#" +(module + (import "seal0" "seal_input" (func $seal_input (param i32 i32))) + (import "seal0" "seal_return" (func $seal_return (param i32 i32 i32))) + (import "__unstable__" "seal_set_storage" (func $seal_set_storage (param i32 i32 i32) (result i32))) + (import "env" "memory" (memory 1 1)) + + ;; 0x1000 = 4k in little endian + ;; size of input buffer + (data (i32.const 0) "\00\10") + + (func (export "call") + ;; Receive (key ++ value_to_write) + (call $seal_input + (i32.const 4) ;; Pointer to the input buffer + (i32.const 0) ;; Size of the length buffer + ) + ;; Store the passed value to the passed key and store result to memory + (i32.store (i32.const 0) + (call $seal_set_storage + (i32.const 4) ;; key_ptr + (i32.const 36) ;; value_ptr + (i32.sub ;; value_len (input_size - key_size) + (i32.load (i32.const 0)) + (i32.const 32) + ) + ) + ) + (call $seal_return + (i32.const 0) ;; flags + (i32.const 0) ;; returned value + (i32.const 4) ;; length of returned value + ) + ) + + (func (export "deploy")) +) +"#; + + let mut ext = MockExt::default(); + + // value did not exist before -> sentinel returned + let input = ([1u8; 32], [42u8, 48]).encode(); + let result = execute(CODE, input, &mut ext).unwrap(); + assert_eq!(u32::from_le_bytes(result.data.0.try_into().unwrap()), crate::SENTINEL); + assert_eq!(ext.storage.get(&[1u8; 32]).unwrap(), &[42u8, 48]); + + // value do exist -> length of old value returned + let input = ([1u8; 32], [0u8; 0]).encode(); + let result = execute(CODE, input, &mut ext).unwrap(); + assert_eq!(u32::from_le_bytes(result.data.0.try_into().unwrap()), 2); + assert_eq!(ext.storage.get(&[1u8; 32]).unwrap(), &[0u8; 0]); + + // value do exist -> length of old value returned (test for zero sized val) + let input = ([1u8; 32], [99u8]).encode(); + let result = execute(CODE, input, &mut ext).unwrap(); + assert_eq!(u32::from_le_bytes(result.data.0.try_into().unwrap()), 0); + assert_eq!(ext.storage.get(&[1u8; 32]).unwrap(), &[99u8]); + } + + #[test] + #[cfg(feature = "unstable-interface")] + fn clear_storage_works() { + const CODE: &str = r#" +(module + (import "seal0" "seal_input" (func $seal_input (param i32 i32))) + (import "seal0" "seal_return" (func $seal_return (param i32 i32 i32))) + (import "__unstable__" "seal_clear_storage" (func $seal_clear_storage (param i32) (result i32))) + (import "env" "memory" (memory 1 1)) + + ;; 0x1000 = 4k in little endian + ;; size of input buffer + (data (i32.const 0) "\00\10") + + (func (export "call") + ;; Receive key + (call $seal_input + (i32.const 4) ;; Pointer to the input buffer + (i32.const 0) ;; Size of the length buffer + ) + ;; Store the passed value to the passed key and store result to memory + (i32.store (i32.const 0) + (call $seal_clear_storage + (i32.const 4) ;; key_ptr + ) + ) + (call $seal_return + (i32.const 0) ;; flags + (i32.const 0) ;; returned value + (i32.const 4) ;; length of returned value + ) + ) + + (func (export "deploy")) +) +"#; + + let mut ext = MockExt::default(); + + ext.storage.insert([1u8; 32], vec![42u8]); + ext.storage.insert([2u8; 32], vec![]); + + // value does not exist -> sentinel returned + let result = execute(CODE, [3u8; 32].encode(), &mut ext).unwrap(); + assert_eq!(u32::from_le_bytes(result.data.0.try_into().unwrap()), crate::SENTINEL); + assert_eq!(ext.storage.get(&[3u8; 32]), None); + + // value did exist -> length returned + let result = execute(CODE, [1u8; 32].encode(), &mut ext).unwrap(); + assert_eq!(u32::from_le_bytes(result.data.0.try_into().unwrap()), 1); + assert_eq!(ext.storage.get(&[1u8; 32]), None); + + // value did exist -> length returned (test for 0 sized) + let result = execute(CODE, [2u8; 32].encode(), &mut ext).unwrap(); + assert_eq!(u32::from_le_bytes(result.data.0.try_into().unwrap()), 0); + assert_eq!(ext.storage.get(&[2u8; 32]), None); + } + + #[test] + #[cfg(feature = "unstable-interface")] + fn take_storage_works() { + const CODE: &str = r#" +(module + (import "seal0" "seal_return" (func $seal_return (param i32 i32 i32))) + (import "seal0" "seal_input" (func $seal_input (param i32 i32))) + (import "__unstable__" "seal_take_storage" (func $seal_take_storage (param i32 i32 i32) (result i32))) + (import "env" "memory" (memory 1 1)) + + ;; [0, 32) size of input buffer (32 byte as we copy the key here) + (data (i32.const 0) "\20") + + ;; [32, 64) size of output buffer + ;; 4k in little endian + (data (i32.const 32) "\00\10") + + ;; [64, 96) input buffer + + ;; [96, inf) output buffer + + (func (export "call") + ;; Receive key + (call $seal_input + (i32.const 64) ;; Pointer to the input buffer + (i32.const 0) ;; Size of the length buffer + ) + + ;; Load a storage value and result of this call into the output buffer + (i32.store (i32.const 96) + (call $seal_take_storage + (i32.const 64) ;; The pointer to the storage key to fetch + (i32.const 100) ;; Pointer to the output buffer + (i32.const 32) ;; Pointer to the size of the buffer + ) + ) + + ;; Return the contents of the buffer + (call $seal_return + (i32.const 0) ;; flags + (i32.const 96) ;; output buffer ptr + (i32.add ;; length: storage size + 4 (retval) + (i32.load (i32.const 32)) + (i32.const 4) + ) + ) + ) + + (func (export "deploy")) +) +"#; + + let mut ext = MockExt::default(); + + ext.storage.insert([1u8; 32], vec![42u8]); + ext.storage.insert([2u8; 32], vec![]); + + // value does not exist -> error returned + let result = execute(CODE, [3u8; 32].encode(), &mut ext).unwrap(); + assert_eq!( + u32::from_le_bytes(result.data.0[0..4].try_into().unwrap()), + ReturnCode::KeyNotFound as u32 + ); + + // value did exist -> value returned + let result = execute(CODE, [1u8; 32].encode(), &mut ext).unwrap(); + assert_eq!( + u32::from_le_bytes(result.data.0[0..4].try_into().unwrap()), + ReturnCode::Success as u32 + ); + assert_eq!(ext.storage.get(&[1u8; 32]), None); + assert_eq!(&result.data.0[4..], &[42u8]); + + // value did exist -> length returned (test for 0 sized) + let result = execute(CODE, [2u8; 32].encode(), &mut ext).unwrap(); + assert_eq!( + u32::from_le_bytes(result.data.0[0..4].try_into().unwrap()), + ReturnCode::Success as u32 + ); + assert_eq!(ext.storage.get(&[2u8; 32]), None); + assert_eq!(&result.data.0[4..], &[0u8; 0]); + } + + #[test] + fn is_contract_works() { + const CODE_IS_CONTRACT: &str = r#" +;; This runs `is_contract` check on zero account address +(module + (import "seal0" "seal_is_contract" (func $seal_is_contract (param i32) (result i32))) + (import "seal0" "seal_return" (func $seal_return (param i32 i32 i32))) + (import "env" "memory" (memory 1 1)) + + ;; [0, 32) zero-adress + (data (i32.const 0) + "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00" + "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00" + ) + + ;; [32, 36) here we store the return code of the `seal_is_contract` + + (func (export "deploy")) + + (func (export "call") + (i32.store + (i32.const 32) + (call $seal_is_contract + (i32.const 0) ;; ptr to destination address + ) + ) + ;; exit with success and take `seal_is_contract` return code to the output buffer + (call $seal_return (i32.const 0) (i32.const 32) (i32.const 4)) + ) +) +"#; + let output = execute(CODE_IS_CONTRACT, vec![], MockExt::default()).unwrap(); + + // The mock ext just always returns 1u32 (`true`). + assert_eq!( + output, + ExecReturnValue { flags: ReturnFlags::empty(), data: Bytes(1u32.encode()) }, + ); + } + + #[test] + #[cfg(feature = "unstable-interface")] + fn code_hash_works() { + /// calls `seal_code_hash` and compares the result with the constant. + const CODE_CODE_HASH: &str = r#" +(module + (import "__unstable__" "seal_code_hash" (func $seal_code_hash (param i32 i32 i32) (result i32))) + (import "env" "memory" (memory 1 1)) + + ;; size of our buffer is 32 bytes + (data (i32.const 32) "\20") + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "call") + ;; fill the buffer with the code hash. + (call $seal_code_hash + (i32.const 0) ;; input: address_ptr (before call) + (i32.const 0) ;; output: code_hash_ptr (after call) + (i32.const 32) ;; same 32 bytes length for input and output + ) + + ;; assert size == 32 + (call $assert + (i32.eq + (i32.load (i32.const 32)) + (i32.const 32) + ) + ) + + ;; assert that the first 8 bytes are "1111111111111111" + (call $assert + (i64.eq + (i64.load (i32.const 0)) + (i64.const 0x1111111111111111) + ) + ) + drop + ) + + (func (export "deploy")) +) +"#; + assert_ok!(execute(CODE_CODE_HASH, vec![], MockExt::default())); + } + + #[test] + #[cfg(feature = "unstable-interface")] + fn own_code_hash_works() { + /// calls `seal_own_code_hash` and compares the result with the constant. + const CODE_OWN_CODE_HASH: &str = r#" +(module + (import "__unstable__" "seal_own_code_hash" (func $seal_own_code_hash (param i32 i32))) + (import "env" "memory" (memory 1 1)) + + ;; size of our buffer is 32 bytes + (data (i32.const 32) "\20") + + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + + (func (export "call") + ;; fill the buffer with the code hash + (call $seal_own_code_hash + (i32.const 0) ;; output: code_hash_ptr + (i32.const 32) ;; 32 bytes length of code_hash output + ) + + ;; assert size == 32 + (call $assert + (i32.eq + (i32.load (i32.const 32)) + (i32.const 32) + ) + ) + + ;; assert that the first 8 bytes are "1010101010101010" + (call $assert + (i64.eq + (i64.load (i32.const 0)) + (i64.const 0x1010101010101010) + ) + ) + ) + + (func (export "deploy")) +) +"#; + assert_ok!(execute(CODE_OWN_CODE_HASH, vec![], MockExt::default())); + } + + #[test] + #[cfg(feature = "unstable-interface")] + fn caller_is_origin_works() { + const CODE_CALLER_IS_ORIGIN: &str = r#" +;; This runs `caller_is_origin` check on zero account address +(module + (import "seal0" "seal_caller_is_origin" (func $seal_caller_is_origin (result i32))) + (import "seal0" "seal_return" (func $seal_return (param i32 i32 i32))) + (import "env" "memory" (memory 1 1)) + + ;; [0, 4) here the return code of the `seal_caller_is_origin` will be stored + ;; we initialize it with non-zero value to be sure that it's being overwritten below + (data (i32.const 0) "\10\10\10\10") + + (func (export "deploy")) + + (func (export "call") + (i32.store + (i32.const 0) + (call $seal_caller_is_origin) + ) + ;; exit with success and take `seal_caller_is_origin` return code to the output buffer + (call $seal_return (i32.const 0) (i32.const 0) (i32.const 4)) + ) +) +"#; + let output = execute(CODE_CALLER_IS_ORIGIN, vec![], MockExt::default()).unwrap(); + + // The mock ext just always returns 0u32 (`false`) + assert_eq!( + output, + ExecReturnValue { flags: ReturnFlags::empty(), data: Bytes(0u32.encode()) }, + ); + } + + #[test] + #[cfg(feature = "unstable-interface")] + fn set_code_hash() { + const CODE: &str = r#" +(module + (import "__unstable__" "seal_set_code_hash" (func $seal_set_code_hash (param i32) (result i32))) + (import "env" "memory" (memory 1 1)) + (func $assert (param i32) + (block $ok + (br_if $ok + (get_local 0) + ) + (unreachable) + ) + ) + (func (export "call") + (local $exit_code i32) + (set_local $exit_code + (call $seal_set_code_hash (i32.const 0)) + ) + (call $assert + (i32.eq (get_local $exit_code) (i32.const 0)) ;; ReturnCode::Success + ) + ) + + (func (export "deploy")) + + ;; Hash of code. + (data (i32.const 0) + "\11\11\11\11\11\11\11\11\11\11\11\11\11\11\11\11" + "\11\11\11\11\11\11\11\11\11\11\11\11\11\11\11\11" + ) +) +"#; + + let mut mock_ext = MockExt::default(); + execute(CODE, [0u8; 32].encode(), &mut mock_ext).unwrap(); + + assert_eq!(mock_ext.code_hashes.pop().unwrap(), H256::from_slice(&[17u8; 32])); + } } diff --git a/frame/contracts/src/wasm/prepare.rs b/frame/contracts/src/wasm/prepare.rs index c766914f3d46..4571d752a80c 100644 --- a/frame/contracts/src/wasm/prepare.rs +++ b/frame/contracts/src/wasm/prepare.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,12 +21,16 @@ use crate::{ chain_extension::ChainExtension, - wasm::{env_def::ImportSatisfyCheck, PrefabWasmModule}, - Config, Schedule, + storage::meter::Diff, + wasm::{env_def::ImportSatisfyCheck, OwnerInfo, PrefabWasmModule}, + AccountIdOf, Config, Schedule, }; -use pwasm_utils::parity_wasm::elements::{self, External, Internal, MemoryType, Type, ValueType}; +use codec::{Encode, MaxEncodedLen}; use sp_runtime::traits::Hash; use sp_std::prelude::*; +use wasm_instrument::parity_wasm::elements::{ + self, External, Internal, MemoryType, Type, ValueType, +}; /// Imported memory must be located inside this module. The reason for hardcoding is that current /// compiler toolchains might not support specifying other modules than "env" for memory imports. @@ -180,18 +184,20 @@ impl<'a, T: Config> ContractModule<'a, T> { fn inject_gas_metering(self) -> Result { let gas_rules = self.schedule.rules(&self.module); - let contract_module = pwasm_utils::inject_gas_counter(self.module, &gas_rules, "seal0") - .map_err(|_| "gas instrumentation failed")?; + let contract_module = + wasm_instrument::gas_metering::inject(self.module, &gas_rules, "seal0") + .map_err(|_| "gas instrumentation failed")?; Ok(ContractModule { module: contract_module, schedule: self.schedule }) } fn inject_stack_height_metering(self) -> Result { - let contract_module = pwasm_utils::stack_height::inject_limiter( - self.module, - self.schedule.limits.stack_height, - ) - .map_err(|_| "stack height instrumentation failed")?; - Ok(ContractModule { module: contract_module, schedule: self.schedule }) + if let Some(limit) = self.schedule.limits.stack_height { + let contract_module = wasm_instrument::inject_stack_limiter(self.module, limit) + .map_err(|_| "stack height instrumentation failed")?; + Ok(ContractModule { module: contract_module, schedule: self.schedule }) + } else { + Ok(ContractModule { module: self.module, schedule: self.schedule }) + } } /// Check that the module has required exported functions. For now @@ -370,45 +376,68 @@ fn check_and_instrument( original_code: &[u8], schedule: &Schedule, ) -> Result<(Vec, (u32, u32)), &'static str> { - let contract_module = ContractModule::new(&original_code, schedule)?; - contract_module.scan_exports()?; - contract_module.ensure_no_internal_memory()?; - contract_module.ensure_table_size_limit(schedule.limits.table_size)?; - contract_module.ensure_global_variable_limit(schedule.limits.globals)?; - contract_module.ensure_no_floating_types()?; - contract_module.ensure_parameter_limit(schedule.limits.parameters)?; - contract_module.ensure_br_table_size_limit(schedule.limits.br_table_size)?; - - // We disallow importing `gas` function here since it is treated as implementation detail. - let disallowed_imports = [b"gas".as_ref()]; - let memory_limits = - get_memory_limits(contract_module.scan_imports::(&disallowed_imports)?, schedule)?; - - let code = contract_module - .inject_gas_metering()? - .inject_stack_height_metering()? - .into_wasm_code()?; - - Ok((code, memory_limits)) + let result = (|| { + let contract_module = ContractModule::new(&original_code, schedule)?; + contract_module.scan_exports()?; + contract_module.ensure_no_internal_memory()?; + contract_module.ensure_table_size_limit(schedule.limits.table_size)?; + contract_module.ensure_global_variable_limit(schedule.limits.globals)?; + contract_module.ensure_no_floating_types()?; + contract_module.ensure_parameter_limit(schedule.limits.parameters)?; + contract_module.ensure_br_table_size_limit(schedule.limits.br_table_size)?; + + // We disallow importing `gas` function here since it is treated as implementation detail. + let disallowed_imports = [b"gas".as_ref()]; + let memory_limits = + get_memory_limits(contract_module.scan_imports::(&disallowed_imports)?, schedule)?; + + let code = contract_module + .inject_gas_metering()? + .inject_stack_height_metering()? + .into_wasm_code()?; + + Ok((code, memory_limits)) + })(); + + if let Err(msg) = &result { + log::debug!(target: "runtime::contracts", "CodeRejected: {}", msg); + } + + result } fn do_preparation( original_code: Vec, schedule: &Schedule, + owner: AccountIdOf, ) -> Result, &'static str> { let (code, (initial, maximum)) = check_and_instrument::(original_code.as_ref(), schedule)?; - Ok(PrefabWasmModule { + let original_code_len = original_code.len(); + + let mut module = PrefabWasmModule { instruction_weights_version: schedule.instruction_weights.version, initial, maximum, - _reserved: None, code, - original_code_len: original_code.len() as u32, - refcount: 1, code_hash: T::Hashing::hash(&original_code), original_code: Some(original_code), - }) + owner_info: None, + }; + + // We need to add the sizes of the `#[codec(skip)]` fields which are stored in different + // storage items. This is also why we have `3` items added and not only one. + let bytes_added = module + .encoded_size() + .saturating_add(original_code_len) + .saturating_add(>::max_encoded_len()) as u32; + let deposit = Diff { bytes_added, items_added: 3, ..Default::default() } + .to_deposit::() + .charge_or_zero(); + + module.owner_info = Some(OwnerInfo { owner, deposit, refcount: 0 }); + + Ok(module) } /// Loads the given module given in `original_code`, performs some checks on it and @@ -425,8 +454,9 @@ fn do_preparation( pub fn prepare_contract( original_code: Vec, schedule: &Schedule, + owner: AccountIdOf, ) -> Result, &'static str> { - do_preparation::(original_code, schedule) + do_preparation::(original_code, schedule, owner) } /// The same as [`prepare_contract`] but without constructing a new [`PrefabWasmModule`] @@ -461,6 +491,7 @@ pub mod benchmarking { pub fn prepare_contract( original_code: Vec, schedule: &Schedule, + owner: AccountIdOf, ) -> Result, &'static str> { let contract_module = ContractModule::new(&original_code, schedule)?; let memory_limits = get_memory_limits(contract_module.scan_imports::<()>(&[])?, schedule)?; @@ -468,12 +499,15 @@ pub mod benchmarking { instruction_weights_version: schedule.instruction_weights.version, initial: memory_limits.0, maximum: memory_limits.1, - _reserved: None, code: contract_module.into_wasm_code()?, - original_code_len: original_code.len() as u32, - refcount: 1, code_hash: T::Hashing::hash(&original_code), original_code: Some(original_code), + owner_info: Some(OwnerInfo { + owner, + // this is a helper function for benchmarking which skips deposit collection + deposit: Default::default(), + refcount: 0, + }), }) } } @@ -481,10 +515,14 @@ pub mod benchmarking { #[cfg(test)] mod tests { use super::*; - use crate::{exec::Ext, schedule::Limits}; + use crate::{ + exec::Ext, + schedule::Limits, + tests::{Test, ALICE}, + }; use std::fmt; - impl fmt::Debug for PrefabWasmModule { + impl fmt::Debug for PrefabWasmModule { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "PreparedContract {{ .. }}") } @@ -526,7 +564,7 @@ mod tests { }, .. Default::default() }; - let r = do_preparation::(wasm, &schedule); + let r = do_preparation::(wasm, &schedule, ALICE); assert_matches::assert_matches!(r, $($expected)*); } }; diff --git a/frame/contracts/src/wasm/runtime.rs b/frame/contracts/src/wasm/runtime.rs index 52b864bf18ea..975cfcdd12db 100644 --- a/frame/contracts/src/wasm/runtime.rs +++ b/frame/contracts/src/wasm/runtime.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,17 +22,18 @@ use crate::{ gas::{ChargedAmount, Token}, schedule::HostFnWeights, wasm::env_def::ConvertibleToWasm, - BalanceOf, CodeHash, Config, Error, + BalanceOf, CodeHash, Config, Error, SENTINEL, }; use bitflags::bitflags; use codec::{Decode, DecodeAll, Encode, MaxEncodedLen}; use frame_support::{dispatch::DispatchError, ensure, weights::Weight}; use pallet_contracts_primitives::{ExecReturnValue, ReturnFlags}; -use pwasm_utils::parity_wasm::elements::ValueType; use sp_core::{crypto::UncheckedFrom, Bytes}; use sp_io::hashing::{blake2_128, blake2_256, keccak_256, sha2_256}; -use sp_runtime::traits::Bounded; +use sp_runtime::traits::{Bounded, Zero}; +use sp_sandbox::SandboxMemory; use sp_std::prelude::*; +use wasm_instrument::parity_wasm::elements::ValueType; /// Every error that can be returned to a contract when it calls any of the host functions. /// @@ -54,15 +55,12 @@ pub enum ReturnCode { CalleeReverted = 2, /// The passed key does not exist in storage. KeyNotFound = 3, - /// Transfer failed because it would have brought the sender's total balance below the - /// subsistence threshold. - BelowSubsistenceThreshold = 4, - /// Transfer failed for other reasons. Most probably reserved or locked balance of the - /// sender prevents the transfer. + /// Deprecated and no longer returned: There is only the minimum balance. + _BelowSubsistenceThreshold = 4, + /// See [`Error::TransferFailed`]. TransferFailed = 5, - /// The newly created contract is below the subsistence threshold after executing - /// its constructor. - NewContractNotFunded = 6, + /// Deprecated and no longer returned: Endowment is no longer required. + _EndowmentTooLow = 6, /// No code could be found at the supplied code hash. CodeNotFound = 7, /// The contract that was called is no contract (a plain account). @@ -138,8 +136,22 @@ pub enum RuntimeCosts { /// Charge the gas meter with the cost of a metering block. The charged costs are /// the supplied cost of the block plus the overhead of the metering itself. MeteringBlock(u32), + /// Weight charged for copying data from the sandbox. + CopyFromContract(u32), + /// Weight charged for copying data to the sandbox. + CopyToContract(u32), /// Weight of calling `seal_caller`. Caller, + /// Weight of calling `seal_is_contract`. + IsContract, + /// Weight of calling `seal_code_hash`. + #[cfg(feature = "unstable-interface")] + CodeHash, + /// Weight of calling `seal_own_code_hash`. + #[cfg(feature = "unstable-interface")] + OwnCodeHash, + /// Weight of calling `seal_caller_is_origin`. + CallerIsOrigin, /// Weight of calling `seal_address`. Address, /// Weight of calling `seal_gas_left`. @@ -150,8 +162,6 @@ pub enum RuntimeCosts { ValueTransferred, /// Weight of calling `seal_minimum_balance`. MinimumBalance, - /// Weight of calling `seal_contract_deposit`. - ContractDeposit, /// Weight of calling `seal_block_number`. BlockNumber, /// Weight of calling `seal_now`. @@ -160,8 +170,6 @@ pub enum RuntimeCosts { WeightToFee, /// Weight of calling `seal_input` without the weight of copying the input. InputBase, - /// Weight of copying the input data for the given size. - InputCopyOut(u32), /// Weight of calling `seal_return` for the given output size. Return(u32), /// Weight of calling `seal_terminate`. @@ -172,28 +180,32 @@ pub enum RuntimeCosts { DepositEvent { num_topic: u32, len: u32 }, /// Weight of calling `seal_debug_message`. DebugMessage, - /// Weight of calling `seal_set_storage` for the given storage item size. - SetStorage(u32), - /// Weight of calling `seal_clear_storage`. - ClearStorage, - /// Weight of calling `seal_get_storage` without output weight. - GetStorageBase, - /// Weight of an item received via `seal_get_storage` for the given size. - GetStorageCopyOut(u32), + /// Weight of calling `seal_set_storage` for the given storage item sizes. + SetStorage { old_bytes: u32, new_bytes: u32 }, + /// Weight of calling `seal_clear_storage` per cleared byte. + ClearStorage(u32), + /// Weight of calling `seal_contains_storage` per byte of the checked item. + #[cfg(feature = "unstable-interface")] + ContainsStorage(u32), + /// Weight of calling `seal_get_storage` with the specified size in storage. + GetStorage(u32), + /// Weight of calling `seal_take_storage` for the given size. + #[cfg(feature = "unstable-interface")] + TakeStorage(u32), /// Weight of calling `seal_transfer`. Transfer, - /// Weight of calling `seal_call` for the given input size. - CallBase(u32), + /// Base weight of calling `seal_call`. + CallBase, + /// Weight of calling `seal_delegate_call` for the given input size. + DelegateCallBase, /// Weight of the transfer performed during a call. CallSurchargeTransfer, - /// Weight of output received through `seal_call` for the given size. - CallCopyOut(u32), - /// Weight of calling `seal_instantiate` for the given input and salt without output weight. - /// This includes the transfer as an instantiate without a value will always be below - /// the existential deposit and is disregarded as corner case. + /// Weight per byte that is cloned by supplying the `CLONE_INPUT` flag. + CallInputCloned(u32), + /// Weight of calling `seal_instantiate` for the given input length and salt. InstantiateBase { input_data_len: u32, salt_len: u32 }, - /// Weight of output received through `seal_instantiate` for the given size. - InstantiateCopyOut(u32), + /// Weight of the transfer performed during an instantiate. + InstantiateSurchargeTransfer, /// Weight of calling `seal_hash_sha_256` for the given input size. HashSha256(u32), /// Weight of calling `seal_hash_keccak_256` for the given input size. @@ -207,12 +219,12 @@ pub enum RuntimeCosts { EcdsaRecovery, /// Weight charged by a chain extension through `seal_call_chain_extension`. ChainExtension(u64), - /// Weight charged for copying data from the sandbox. - #[cfg(feature = "unstable-interface")] - CopyIn(u32), /// Weight charged for calling into the runtime. #[cfg(feature = "unstable-interface")] CallRuntime(Weight), + /// Weight of calling `seal_set_code_hash` + #[cfg(feature = "unstable-interface")] + SetCodeHash, } impl RuntimeCosts { @@ -224,18 +236,24 @@ impl RuntimeCosts { use self::RuntimeCosts::*; let weight = match *self { MeteringBlock(amount) => s.gas.saturating_add(amount.into()), + CopyFromContract(len) => s.return_per_byte.saturating_mul(len.into()), + CopyToContract(len) => s.input_per_byte.saturating_mul(len.into()), Caller => s.caller, + IsContract => s.is_contract, + #[cfg(feature = "unstable-interface")] + CodeHash => s.code_hash, + #[cfg(feature = "unstable-interface")] + OwnCodeHash => s.own_code_hash, + CallerIsOrigin => s.caller_is_origin, Address => s.address, GasLeft => s.gas_left, Balance => s.balance, ValueTransferred => s.value_transferred, MinimumBalance => s.minimum_balance, - ContractDeposit => s.contract_deposit, BlockNumber => s.block_number, Now => s.now, WeightToFee => s.weight_to_fee, InputBase => s.input, - InputCopyOut(len) => s.input_per_byte.saturating_mul(len.into()), Return(len) => s.r#return.saturating_add(s.return_per_byte.saturating_mul(len.into())), Terminate => s.terminate, Random => s.random, @@ -244,21 +262,33 @@ impl RuntimeCosts { .saturating_add(s.deposit_event_per_topic.saturating_mul(num_topic.into())) .saturating_add(s.deposit_event_per_byte.saturating_mul(len.into())), DebugMessage => s.debug_message, - SetStorage(len) => - s.set_storage.saturating_add(s.set_storage_per_byte.saturating_mul(len.into())), - ClearStorage => s.clear_storage, - GetStorageBase => s.get_storage, - GetStorageCopyOut(len) => s.get_storage_per_byte.saturating_mul(len.into()), + SetStorage { new_bytes, old_bytes } => s + .set_storage + .saturating_add(s.set_storage_per_new_byte.saturating_mul(new_bytes.into())) + .saturating_add(s.set_storage_per_old_byte.saturating_mul(old_bytes.into())), + ClearStorage(len) => s + .clear_storage + .saturating_add(s.clear_storage_per_byte.saturating_mul(len.into())), + #[cfg(feature = "unstable-interface")] + ContainsStorage(len) => s + .contains_storage + .saturating_add(s.contains_storage_per_byte.saturating_mul(len.into())), + GetStorage(len) => + s.get_storage.saturating_add(s.get_storage_per_byte.saturating_mul(len.into())), + #[cfg(feature = "unstable-interface")] + TakeStorage(len) => s + .take_storage + .saturating_add(s.take_storage_per_byte.saturating_mul(len.into())), Transfer => s.transfer, - CallBase(len) => - s.call.saturating_add(s.call_per_input_byte.saturating_mul(len.into())), + CallBase => s.call, + DelegateCallBase => s.delegate_call, CallSurchargeTransfer => s.call_transfer_surcharge, - CallCopyOut(len) => s.call_per_output_byte.saturating_mul(len.into()), + CallInputCloned(len) => s.call_per_cloned_byte.saturating_mul(len.into()), InstantiateBase { input_data_len, salt_len } => s .instantiate - .saturating_add(s.instantiate_per_input_byte.saturating_mul(input_data_len.into())) + .saturating_add(s.return_per_byte.saturating_mul(input_data_len.into())) .saturating_add(s.instantiate_per_salt_byte.saturating_mul(salt_len.into())), - InstantiateCopyOut(len) => s.instantiate_per_output_byte.saturating_mul(len.into()), + InstantiateSurchargeTransfer => s.instantiate_transfer_surcharge, HashSha256(len) => s .hash_sha2_256 .saturating_add(s.hash_sha2_256_per_byte.saturating_mul(len.into())), @@ -274,10 +304,11 @@ impl RuntimeCosts { #[cfg(feature = "unstable-interface")] EcdsaRecovery => s.ecdsa_recover, ChainExtension(amount) => amount, - #[cfg(feature = "unstable-interface")] - CopyIn(len) => s.return_per_byte.saturating_mul(len.into()), + #[cfg(feature = "unstable-interface")] CallRuntime(weight) => weight, + #[cfg(feature = "unstable-interface")] + SetCodeHash => s.set_code_hash, }; RuntimeToken { #[cfg(test)] @@ -287,6 +318,17 @@ impl RuntimeCosts { } } +/// Same as [`Runtime::charge_gas`]. +/// +/// We need this access as a macro because sometimes hiding the lifetimes behind +/// a function won't work out. +macro_rules! charge_gas { + ($runtime:expr, $costs:expr) => {{ + let token = $costs.token(&$runtime.ext.schedule().host_fn_weights); + $runtime.ext.gas_meter().charge(token) + }}; +} + #[cfg_attr(test, derive(Debug, PartialEq, Eq))] #[derive(Copy, Clone)] struct RuntimeToken { @@ -306,8 +348,8 @@ where } bitflags! { - /// Flags used to change the behaviour of `seal_call`. - struct CallFlags: u32 { + /// Flags used to change the behaviour of `seal_call` and `seal_delegate_call`. + pub struct CallFlags: u32 { /// Forward the input of current function to the callee. /// /// Supplied input pointers are ignored when set. @@ -342,10 +384,32 @@ bitflags! { /// Without this flag any reentrancy into the current contract that originates from /// the callee (or any of its callees) is denied. This includes the first callee: /// You cannot call into yourself with this flag set. + /// + /// # Note + /// + /// For `seal_delegate_call` should be always unset, otherwise + /// [`Error::InvalidCallFlags`] is returned. const ALLOW_REENTRY = 0b0000_1000; } } +/// The kind of call that should be performed. +enum CallType { + /// Execute another instantiated contract + Call { callee_ptr: u32, value_ptr: u32, gas: u64 }, + /// Execute deployed code in the context (storage, account ID, value) of the caller contract + DelegateCall { code_hash_ptr: u32 }, +} + +impl CallType { + fn cost(&self) -> RuntimeCosts { + match self { + CallType::Call { .. } => RuntimeCosts::CallBase, + CallType::DelegateCall { .. } => RuntimeCosts::DelegateCallBase, + } + } +} + /// This is only appropriate when writing out data of constant size that does not depend on user /// input. In this case the costs for this copy was already charged as part of the token at /// the beginning of the API entry point. @@ -357,7 +421,7 @@ fn already_charged(_: u32) -> Option { pub struct Runtime<'a, E: Ext + 'a> { ext: &'a mut E, input_data: Option>, - memory: sp_sandbox::Memory, + memory: sp_sandbox::default_executor::Memory, trap_reason: Option, } @@ -367,7 +431,11 @@ where ::AccountId: UncheckedFrom<::Hash> + AsRef<[u8]>, { - pub fn new(ext: &'a mut E, input_data: Vec, memory: sp_sandbox::Memory) -> Self { + pub fn new( + ext: &'a mut E, + input_data: Vec, + memory: sp_sandbox::default_executor::Memory, + ) -> Self { Runtime { ext, input_data: Some(input_data), memory, trap_reason: None } } @@ -386,7 +454,7 @@ where // The trap was the result of the execution `return` host function. TrapReason::Return(ReturnData { flags, data }) => { let flags = ReturnFlags::from_bits(flags) - .ok_or_else(|| "used reserved bit in return flags")?; + .ok_or_else(|| Error::::InvalidCallFlags)?; Ok(ExecReturnValue { flags, data: Bytes(data) }) }, TrapReason::Termination => @@ -433,8 +501,7 @@ where /// /// Returns `Err(HostError)` if there is not enough gas. pub fn charge_gas(&mut self, costs: RuntimeCosts) -> Result { - let token = costs.token(&self.ext.schedule().host_fn_weights); - self.ext.gas_meter().charge(token) + charge_gas!(self, costs) } /// Adjust a previously charged amount down to its actual amount. @@ -519,7 +586,7 @@ where /// length of the buffer located at `out_ptr`. If that buffer is large enough the actual /// `buf.len()` is written to this location. /// - /// If `out_ptr` is set to the sentinel value of `u32::MAX` and `allow_skip` is true the + /// If `out_ptr` is set to the sentinel value of `SENTINEL` and `allow_skip` is true the /// operation is skipped and `Ok` is returned. This is supposed to help callers to make copying /// output optional. For example to skip copying back the output buffer of an `seal_call` /// when the caller is not interested in the result. @@ -538,7 +605,7 @@ where allow_skip: bool, create_token: impl FnOnce(u32) -> Option, ) -> Result<(), DispatchError> { - if allow_skip && out_ptr == u32::MAX { + if allow_skip && out_ptr == SENTINEL { return Ok(()) } @@ -606,16 +673,12 @@ where fn err_into_return_code(from: DispatchError) -> Result { use ReturnCode::*; - let below_sub = Error::::BelowSubsistenceThreshold.into(); let transfer_failed = Error::::TransferFailed.into(); - let not_funded = Error::::NewContractNotFunded.into(); let no_code = Error::::CodeNotFound.into(); let not_found = Error::::ContractNotFound.into(); match from { - x if x == below_sub => Ok(BelowSubsistenceThreshold), x if x == transfer_failed => Ok(TransferFailed), - x if x == not_funded => Ok(NewContractNotFunded), x if x == no_code => Ok(CodeNotFound), x if x == not_found => Ok(NotCallable), err => Err(err), @@ -637,34 +700,83 @@ where } } + fn set_storage( + &mut self, + key_ptr: u32, + value_ptr: u32, + value_len: u32, + ) -> Result { + let max_size = self.ext.max_value_size(); + let charged = self + .charge_gas(RuntimeCosts::SetStorage { new_bytes: value_len, old_bytes: max_size })?; + if value_len > max_size { + Err(Error::::ValueTooLarge)?; + } + let mut key: StorageKey = [0; 32]; + self.read_sandbox_memory_into_buf(key_ptr, &mut key)?; + let value = Some(self.read_sandbox_memory(value_ptr, value_len)?); + let write_outcome = self.ext.set_storage(key, value, false)?; + self.adjust_gas( + charged, + RuntimeCosts::SetStorage { new_bytes: value_len, old_bytes: write_outcome.old_len() }, + ); + Ok(write_outcome.old_len_with_sentinel()) + } + + fn clear_storage(&mut self, key_ptr: u32) -> Result { + let charged = self.charge_gas(RuntimeCosts::ClearStorage(self.ext.max_value_size()))?; + let mut key: StorageKey = [0; 32]; + self.read_sandbox_memory_into_buf(key_ptr, &mut key)?; + let outcome = self.ext.set_storage(key, None, false)?; + self.adjust_gas(charged, RuntimeCosts::ClearStorage(outcome.old_len())); + Ok(outcome.old_len_with_sentinel()) + } + fn call( &mut self, flags: CallFlags, - callee_ptr: u32, - gas: u64, - value_ptr: u32, + call_type: CallType, input_data_ptr: u32, input_data_len: u32, output_ptr: u32, output_len_ptr: u32, ) -> Result { - self.charge_gas(RuntimeCosts::CallBase(input_data_len))?; - let callee: <::T as frame_system::Config>::AccountId = - self.read_sandbox_memory_as(callee_ptr)?; - let value: BalanceOf<::T> = self.read_sandbox_memory_as(value_ptr)?; + self.charge_gas(call_type.cost())?; let input_data = if flags.contains(CallFlags::CLONE_INPUT) { - self.input_data.as_ref().ok_or_else(|| Error::::InputForwarded)?.clone() + let input = self.input_data.as_ref().ok_or_else(|| Error::::InputForwarded)?; + charge_gas!(self, RuntimeCosts::CallInputCloned(input.len() as u32))?; + input.clone() } else if flags.contains(CallFlags::FORWARD_INPUT) { self.input_data.take().ok_or_else(|| Error::::InputForwarded)? } else { + self.charge_gas(RuntimeCosts::CopyFromContract(input_data_len))?; self.read_sandbox_memory(input_data_ptr, input_data_len)? }; - if value > 0u32.into() { - self.charge_gas(RuntimeCosts::CallSurchargeTransfer)?; - } - let ext = &mut self.ext; - let call_outcome = - ext.call(gas, callee, value, input_data, flags.contains(CallFlags::ALLOW_REENTRY)); + + let call_outcome = match call_type { + CallType::Call { callee_ptr, value_ptr, gas } => { + let callee: <::T as frame_system::Config>::AccountId = + self.read_sandbox_memory_as(callee_ptr)?; + let value: BalanceOf<::T> = self.read_sandbox_memory_as(value_ptr)?; + if value > 0u32.into() { + self.charge_gas(RuntimeCosts::CallSurchargeTransfer)?; + } + self.ext.call( + gas, + callee, + value, + input_data, + flags.contains(CallFlags::ALLOW_REENTRY), + ) + }, + CallType::DelegateCall { code_hash_ptr } => { + if flags.contains(CallFlags::ALLOW_REENTRY) { + return Err(Error::::InvalidCallFlags.into()) + } + let code_hash = self.read_sandbox_memory_as(code_hash_ptr)?; + self.ext.delegate_call(code_hash, input_data) + }, + }; // `TAIL_CALL` only matters on an `OK` result. Otherwise the call stack comes to // a halt anyways without anymore code being executed. @@ -679,7 +791,7 @@ where if let Ok(output) = &call_outcome { self.write_sandbox_output(output_ptr, output_len_ptr, &output.data, true, |len| { - Some(RuntimeCosts::CallCopyOut(len)) + Some(RuntimeCosts::CopyToContract(len)) })?; } Ok(Runtime::::exec_into_return_code(call_outcome)?) @@ -700,8 +812,11 @@ where salt_len: u32, ) -> Result { self.charge_gas(RuntimeCosts::InstantiateBase { input_data_len, salt_len })?; - let code_hash: CodeHash<::T> = self.read_sandbox_memory_as(code_hash_ptr)?; let value: BalanceOf<::T> = self.read_sandbox_memory_as(value_ptr)?; + if value > 0u32.into() { + self.charge_gas(RuntimeCosts::InstantiateSurchargeTransfer)?; + } + let code_hash: CodeHash<::T> = self.read_sandbox_memory_as(code_hash_ptr)?; let input_data = self.read_sandbox_memory(input_data_ptr, input_data_len)?; let salt = self.read_sandbox_memory(salt_ptr, salt_len)?; let instantiate_outcome = self.ext.instantiate(gas, code_hash, value, input_data, &salt); @@ -716,7 +831,7 @@ where )?; } self.write_sandbox_output(output_ptr, output_len_ptr, &output.data, true, |len| { - Some(RuntimeCosts::InstantiateCopyOut(len)) + Some(RuntimeCosts::CopyToContract(len)) })?; } Ok(Runtime::::exec_into_return_code(instantiate_outcome.map(|(_, retval)| retval))?) @@ -731,12 +846,7 @@ where } } -// *********************************************************** -// * AFTER MAKING A CHANGE MAKE SURE TO UPDATE COMPLEXITY.MD * -// *********************************************************** - -// Define a function `fn init_env() -> HostFunctionSet` that returns -// a function set which can be imported by an executed contract. +// This is the API exposed to contracts. // // # Note // @@ -755,10 +865,18 @@ define_env!(Env, , Ok(()) }, + // Set the value at the given key in the contract storage. + // + // Equivalent to the newer version of `seal_set_storage` with the exception of the return + // type. Still a valid thing to call when not interested in the return value. + [seal0] seal_set_storage(ctx, key_ptr: u32, value_ptr: u32, value_len: u32) => { + ctx.set_storage(key_ptr, value_ptr, value_len).map(|_| ()) + }, + // Set the value at the given key in the contract storage. // // The value length must not exceed the maximum defined by the contracts module parameters. - // Storing an empty value is disallowed. + // Specifying a `value_len` of zero will store an empty value. // // # Parameters // @@ -766,19 +884,20 @@ define_env!(Env, , // - `value_ptr`: pointer into the linear memory where the value to set is placed. // - `value_len`: the length of the value in bytes. // - // # Traps + // # Return Value // - // - If value length exceeds the configured maximum value length of a storage entry. - // - Upon trying to set an empty storage entry (value length is 0). - [seal0] seal_set_storage(ctx, key_ptr: u32, value_ptr: u32, value_len: u32) => { - ctx.charge_gas(RuntimeCosts::SetStorage(value_len))?; - if value_len > ctx.ext.max_value_size() { - Err(Error::::ValueTooLarge)?; - } - let mut key: StorageKey = [0; 32]; - ctx.read_sandbox_memory_into_buf(key_ptr, &mut key)?; - let value = Some(ctx.read_sandbox_memory(value_ptr, value_len)?); - ctx.ext.set_storage(key, value).map_err(Into::into) + // Returns the size of the pre-existing value at the specified key if any. Otherwise + // `SENTINEL` is returned as a sentinel value. + [__unstable__] seal_set_storage(ctx, key_ptr: u32, value_ptr: u32, value_len: u32) -> u32 => { + ctx.set_storage(key_ptr, value_ptr, value_len) + }, + + // Clear the value at the given key in the contract storage. + // + // Equivalent to the newer version of `seal_clear_storage` with the exception of the return + // type. Still a valid thing to call when not interested in the return value. + [seal0] seal_clear_storage(ctx, key_ptr: u32) => { + ctx.clear_storage(key_ptr).map(|_| ()).map_err(Into::into) }, // Clear the value at the given key in the contract storage. @@ -786,11 +905,13 @@ define_env!(Env, , // # Parameters // // - `key_ptr`: pointer into the linear memory where the location to clear the value is placed. - [seal0] seal_clear_storage(ctx, key_ptr: u32) => { - ctx.charge_gas(RuntimeCosts::ClearStorage)?; - let mut key: StorageKey = [0; 32]; - ctx.read_sandbox_memory_into_buf(key_ptr, &mut key)?; - ctx.ext.set_storage(key, None).map_err(Into::into) + // + // # Return Value + // + // Returns the size of the pre-existing value at the specified key if any. Otherwise + // `SENTINEL` is returned as a sentinel value. + [__unstable__] seal_clear_storage(ctx, key_ptr: u32) -> u32 => { + ctx.clear_storage(key_ptr).map_err(Into::into) }, // Retrieve the value under the given key from storage. @@ -806,15 +927,64 @@ define_env!(Env, , // // `ReturnCode::KeyNotFound` [seal0] seal_get_storage(ctx, key_ptr: u32, out_ptr: u32, out_len_ptr: u32) -> ReturnCode => { - ctx.charge_gas(RuntimeCosts::GetStorageBase)?; + let charged = ctx.charge_gas(RuntimeCosts::GetStorage(ctx.ext.max_value_size()))?; let mut key: StorageKey = [0; 32]; ctx.read_sandbox_memory_into_buf(key_ptr, &mut key)?; if let Some(value) = ctx.ext.get_storage(&key) { - ctx.write_sandbox_output(out_ptr, out_len_ptr, &value, false, |len| { - Some(RuntimeCosts::GetStorageCopyOut(len)) - })?; + ctx.adjust_gas(charged, RuntimeCosts::GetStorage(value.len() as u32)); + ctx.write_sandbox_output(out_ptr, out_len_ptr, &value, false, already_charged)?; + Ok(ReturnCode::Success) + } else { + ctx.adjust_gas(charged, RuntimeCosts::GetStorage(0)); + Ok(ReturnCode::KeyNotFound) + } + }, + + // Checks whether there is a value stored under the given key. + // + // # Parameters + // + // - `key_ptr`: pointer into the linear memory where the key of the requested value is placed. + // + // # Return Value + // + // Returns the size of the pre-existing value at the specified key if any. Otherwise + // `SENTINEL` is returned as a sentinel value. + [__unstable__] seal_contains_storage(ctx, key_ptr: u32) -> u32 => { + let charged = ctx.charge_gas(RuntimeCosts::ContainsStorage(ctx.ext.max_value_size()))?; + let mut key: StorageKey = [0; 32]; + ctx.read_sandbox_memory_into_buf(key_ptr, &mut key)?; + if let Some(len) = ctx.ext.get_storage_size(&key) { + ctx.adjust_gas(charged, RuntimeCosts::ContainsStorage(len)); + Ok(len) + } else { + ctx.adjust_gas(charged, RuntimeCosts::ContainsStorage(0)); + Ok(SENTINEL) + } + }, + + // Retrieve and remove the value under the given key from storage. + // + // # Parameters + // + // - `key_ptr`: pointer into the linear memory where the key of the requested value is placed. + // - `out_ptr`: pointer to the linear memory where the value is written to. + // - `out_len_ptr`: in-out pointer into linear memory where the buffer length + // is read from and the value length is written to. + // + // # Errors + // + // `ReturnCode::KeyNotFound` + [__unstable__] seal_take_storage(ctx, key_ptr: u32, out_ptr: u32, out_len_ptr: u32) -> ReturnCode => { + let charged = ctx.charge_gas(RuntimeCosts::TakeStorage(ctx.ext.max_value_size()))?; + let mut key: StorageKey = [0; 32]; + ctx.read_sandbox_memory_into_buf(key_ptr, &mut key)?; + if let crate::storage::WriteOutcome::Taken(value) = ctx.ext.set_storage(key, None, true)? { + ctx.adjust_gas(charged, RuntimeCosts::TakeStorage(value.len() as u32)); + ctx.write_sandbox_output(out_ptr, out_len_ptr, &value, false, already_charged)?; Ok(ReturnCode::Success) } else { + ctx.adjust_gas(charged, RuntimeCosts::TakeStorage(0)); Ok(ReturnCode::KeyNotFound) } }, @@ -832,7 +1002,6 @@ define_env!(Env, , // // # Errors // - // `ReturnCode::BelowSubsistenceThreshold` // `ReturnCode::TransferFailed` [seal0] seal_transfer( ctx, @@ -883,9 +1052,7 @@ define_env!(Env, , ) -> ReturnCode => { ctx.call( CallFlags::ALLOW_REENTRY, - callee_ptr, - gas, - value_ptr, + CallType::Call{callee_ptr, value_ptr, gas}, input_data_ptr, input_data_len, output_ptr, @@ -897,7 +1064,7 @@ define_env!(Env, , // // The callees output buffer is copied to `output_ptr` and its length to `output_len_ptr`. // The copy of the output buffer can be skipped by supplying the sentinel value - // of `u32::MAX` to `output_ptr`. + // of `SENTINEL` to `output_ptr`. // // # Parameters // @@ -920,10 +1087,9 @@ define_env!(Env, , // // `ReturnCode::CalleeReverted`: Output buffer is returned. // `ReturnCode::CalleeTrapped` - // `ReturnCode::BelowSubsistenceThreshold` // `ReturnCode::TransferFailed` // `ReturnCode::NotCallable` - [__unstable__] seal_call( + [seal1] seal_call( ctx, flags: u32, callee_ptr: u32, @@ -935,10 +1101,51 @@ define_env!(Env, , output_len_ptr: u32 ) -> ReturnCode => { ctx.call( - CallFlags::from_bits(flags).ok_or_else(|| "used rerved bit in CallFlags")?, - callee_ptr, - gas, - value_ptr, + CallFlags::from_bits(flags).ok_or_else(|| Error::::InvalidCallFlags)?, + CallType::Call{callee_ptr, value_ptr, gas}, + input_data_ptr, + input_data_len, + output_ptr, + output_len_ptr, + ) + }, + + // Execute code in the context (storage, caller, value) of the current contract. + // + // Reentrancy protection is always disabled since the callee is allowed + // to modify the callers storage. This makes going through a reentrancy attack + // unnecessary for the callee when it wants to exploit the caller. + // + // # Parameters + // + // - flags: See [`CallFlags`] for a documentation of the supported flags. + // - code_hash: a pointer to the hash of the code to be called. + // - input_data_ptr: a pointer to a buffer to be used as input data to the callee. + // - input_data_len: length of the input data buffer. + // - output_ptr: a pointer where the output buffer is copied to. + // - output_len_ptr: in-out pointer to where the length of the buffer is read from + // and the actual length is written to. + // + // # Errors + // + // An error means that the call wasn't successful and no output buffer is returned unless + // stated otherwise. + // + // `ReturnCode::CalleeReverted`: Output buffer is returned. + // `ReturnCode::CalleeTrapped` + // `ReturnCode::CodeNotFound` + [seal0] seal_delegate_call( + ctx, + flags: u32, + code_hash_ptr: u32, + input_data_ptr: u32, + input_data_len: u32, + output_ptr: u32, + output_len_ptr: u32 + ) -> ReturnCode => { + ctx.call( + CallFlags::from_bits(flags).ok_or_else(|| Error::::InvalidCallFlags)?, + CallType::DelegateCall{code_hash_ptr}, input_data_ptr, input_data_len, output_ptr, @@ -995,11 +1202,10 @@ define_env!(Env, , // by the code hash. The address of this new account is copied to `address_ptr` and its length // to `address_len_ptr`. The constructors output buffer is copied to `output_ptr` and its // length to `output_len_ptr`. The copy of the output buffer and address can be skipped by - // supplying the sentinel value of `u32::MAX` to `output_ptr` or `address_ptr`. + // supplying the sentinel value of `SENTINEL` to `output_ptr` or `address_ptr`. // - // After running the constructor it is verified that the contract account holds at - // least the subsistence threshold. If that is not the case the instantiation fails and - // the contract is not created. + // `value` must be at least the minimum balance. Otherwise the instantiation fails and the + // contract is not created. // // # Parameters // @@ -1028,9 +1234,7 @@ define_env!(Env, , // // `ReturnCode::CalleeReverted`: Output buffer is returned. // `ReturnCode::CalleeTrapped` - // `ReturnCode::BelowSubsistenceThreshold` // `ReturnCode::TransferFailed` - // `ReturnCode::NewContractNotFunded` // `ReturnCode::CodeNotFound` [seal1] seal_instantiate( ctx, @@ -1077,7 +1281,7 @@ define_env!(Env, , ctx.terminate(beneficiary_ptr) }, - // Remove the calling account and transfer remaining balance. + // Remove the calling account and transfer remaining **free** balance. // // This function never returns. Either the termination was successful and the // execution of the destroyed contract is halted. Or it failed during the termination @@ -1110,7 +1314,7 @@ define_env!(Env, , ctx.charge_gas(RuntimeCosts::InputBase)?; if let Some(input) = ctx.input_data.take() { ctx.write_sandbox_output(out_ptr, out_len_ptr, &input, false, |len| { - Some(RuntimeCosts::InputCopyOut(len)) + Some(RuntimeCosts::CopyToContract(len)) })?; ctx.input_data = Some(input); Ok(()) @@ -1161,6 +1365,75 @@ define_env!(Env, , )?) }, + // Checks whether a specified address belongs to a contract. + // + // # Parameters + // + // - account_ptr: a pointer to the address of the beneficiary account + // Should be decodable as an `T::AccountId`. Traps otherwise. + // + // Returned value is a u32-encoded boolean: (0 = false, 1 = true). + [seal0] seal_is_contract(ctx, account_ptr: u32) -> u32 => { + ctx.charge_gas(RuntimeCosts::IsContract)?; + let address: <::T as frame_system::Config>::AccountId = + ctx.read_sandbox_memory_as(account_ptr)?; + + Ok(ctx.ext.is_contract(&address) as u32) + }, + + // Retrieve the code hash for a specified contract address. + // + // # Parameters + // + // - `account_ptr`: a pointer to the address in question. + // Should be decodable as an `T::AccountId`. Traps otherwise. + // - `out_ptr`: pointer to the linear memory where the returning value is written to. + // - `out_len_ptr`: in-out pointer into linear memory where the buffer length + // is read from and the value length is written to. + // + // # Errors + // + // `ReturnCode::KeyNotFound` + [__unstable__] seal_code_hash(ctx, account_ptr: u32, out_ptr: u32, out_len_ptr: u32) -> ReturnCode => { + ctx.charge_gas(RuntimeCosts::CodeHash)?; + let address: <::T as frame_system::Config>::AccountId = + ctx.read_sandbox_memory_as(account_ptr)?; + if let Some(value) = ctx.ext.code_hash(&address) { + ctx.write_sandbox_output(out_ptr, out_len_ptr, &value.encode(), false, already_charged)?; + Ok(ReturnCode::Success) + } else { + Ok(ReturnCode::KeyNotFound) + } + }, + + // Retrieve the code hash of the currently executing contract. + // + // # Parameters + // + // - `out_ptr`: pointer to the linear memory where the returning value is written to. + // - `out_len_ptr`: in-out pointer into linear memory where the buffer length + // is read from and the value length is written to. + [__unstable__] seal_own_code_hash(ctx, out_ptr: u32, out_len_ptr: u32) => { + ctx.charge_gas(RuntimeCosts::OwnCodeHash)?; + let code_hash_encoded = &ctx.ext.own_code_hash().encode(); + Ok(ctx.write_sandbox_output(out_ptr, out_len_ptr, code_hash_encoded, false, already_charged)?) + }, + + // Checks whether the caller of the current contract is the origin of the whole call stack. + // + // Prefer this over `seal_is_contract` when checking whether your contract is being called by a contract + // or a plain account. The reason is that it performs better since it does not need to + // do any storage lookups. + // + // A return value of`true` indicates that this contract is being called by a plain account + // and `false` indicates that the caller is another contract. + // + // Returned value is a u32-encoded boolean: (0 = false, 1 = true). + [seal0] seal_caller_is_origin(ctx) -> u32 => { + ctx.charge_gas(RuntimeCosts::CallerIsOrigin)?; + Ok(ctx.ext.caller_is_origin() as u32) + }, + // Stores the address of the current contract into the supplied buffer. // // The value is stored to linear memory at the address pointed to by `out_ptr`. @@ -1210,7 +1483,7 @@ define_env!(Env, , )?) }, - // Stores the balance of the current account into the supplied buffer. + // Stores the **free* balance of the current account into the supplied buffer. // // The value is stored to linear memory at the address pointed to by `out_ptr`. // `out_len_ptr` must point to a u32 value that describes the available space at @@ -1225,7 +1498,7 @@ define_env!(Env, , )?) }, - // Stores the value transferred along with this call or as endowment into the supplied buffer. + // Stores the value transferred along with this call/instantiate into the supplied buffer. // // The value is stored to linear memory at the address pointed to by `out_ptr`. // `out_len_ptr` must point to a u32 value that describes the available space at @@ -1318,38 +1591,20 @@ define_env!(Env, , )?) }, - // Stores the contract deposit into the supplied buffer. - // - // # Deprecation - // - // This is equivalent to calling `seal_contract_deposit` and only exists for backwards - // compatibility. See that function for documentation. - [seal0] seal_tombstone_deposit(ctx, out_ptr: u32, out_len_ptr: u32) => { - ctx.charge_gas(RuntimeCosts::ContractDeposit)?; - Ok(ctx.write_sandbox_output( - out_ptr, out_len_ptr, &ctx.ext.contract_deposit().encode(), false, already_charged - )?) - }, - - // Stores the contract deposit into the supplied buffer. + // Stores the tombstone deposit into the supplied buffer. // // The value is stored to linear memory at the address pointed to by `out_ptr`. // `out_len_ptr` must point to a u32 value that describes the available space at // `out_ptr`. This call overwrites it with the size of the value. If the available // space at `out_ptr` is less than the size of the value a trap is triggered. // - // The data is encoded as T::Balance. - // - // # Note + // # Deprecation // - // The contract deposit is on top of the existential deposit. The sum - // is commonly referred as subsistence threshold in code. No contract initiated - // balance transfer can go below this threshold. - [seal0] seal_contract_deposit(ctx, out_ptr: u32, out_len_ptr: u32) => { - ctx.charge_gas(RuntimeCosts::ContractDeposit)?; - Ok(ctx.write_sandbox_output( - out_ptr, out_len_ptr, &ctx.ext.contract_deposit().encode(), false, already_charged - )?) + // There is no longer a tombstone deposit. This function always returns 0. + [seal0] seal_tombstone_deposit(ctx, out_ptr: u32, out_len_ptr: u32) => { + ctx.charge_gas(RuntimeCosts::Balance)?; + let deposit = >::zero().encode(); + Ok(ctx.write_sandbox_output(out_ptr, out_len_ptr, &deposit, false, already_charged)?) }, // Was used to restore the given destination contract sacrificing the caller. @@ -1625,7 +1880,7 @@ define_env!(Env, , output_len_ptr: u32 ) -> u32 => { use crate::chain_extension::{ChainExtension, Environment, RetVal}; - if ::ChainExtension::enabled() == false { + if !::ChainExtension::enabled() { Err(Error::::NoChainExtension)?; } let env = Environment::new(ctx, input_ptr, input_len, output_ptr, output_len_ptr); @@ -1706,7 +1961,7 @@ define_env!(Env, , // deploy a contract using it to a production chain. [__unstable__] seal_call_runtime(ctx, call_ptr: u32, call_len: u32) -> ReturnCode => { use frame_support::{dispatch::GetDispatchInfo, weights::extract_actual_weight}; - ctx.charge_gas(RuntimeCosts::CopyIn(call_len))?; + ctx.charge_gas(RuntimeCosts::CopyFromContract(call_len))?; let call: ::Call = ctx.read_sandbox_memory_as_unbounded( call_ptr, call_len )?; @@ -1760,4 +2015,41 @@ define_env!(Env, , Err(_) => Ok(ReturnCode::EcdsaRecoverFailed), } }, + + // Replace the contract code at the specified address with new code. + // + // # Note + // + // There are a couple of important considerations which must be taken into account when + // using this API: + // + // 1. The storage at the code address will remain untouched. This means that contract developers + // must ensure that the storage layout of the new code is compatible with that of the old code. + // + // 2. Contracts using this API can't be assumed as having deterministic addresses. Said another way, + // when using this API you lose the guarantee that an address always identifies a specific code hash. + // + // 3. If a contract calls into itself after changing its code the new call would use + // the new code. However, if the original caller panics after returning from the sub call it + // would revert the changes made by `seal_set_code_hash` and the next caller would use + // the old code. + // + // # Parameters + // + // - code_hash_ptr: A pointer to the buffer that contains the new code hash. + // + // # Errors + // + // `ReturnCode::CodeNotFound` + [__unstable__] seal_set_code_hash(ctx, code_hash_ptr: u32) -> ReturnCode => { + ctx.charge_gas(RuntimeCosts::SetCodeHash)?; + let code_hash: CodeHash<::T> = ctx.read_sandbox_memory_as(code_hash_ptr)?; + match ctx.ext.set_code_hash(code_hash) { + Err(err) => { + let code = Runtime::::err_into_return_code(err)?; + Ok(code) + }, + Ok(()) => Ok(ReturnCode::Success) + } + }, ); diff --git a/frame/contracts/src/weights.rs b/frame/contracts/src/weights.rs index 1cebcb3b5d9a..43f00196ab3b 100644 --- a/frame/contracts/src/weights.rs +++ b/frame/contracts/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_contracts //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-09-09, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-03-21, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -35,7 +35,6 @@ // --output=./frame/contracts/src/weights.rs // --template=./.maintain/frame-weight-template.hbs - #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] #![allow(unused_imports)] @@ -48,19 +47,23 @@ pub trait WeightInfo { fn on_initialize() -> Weight; fn on_initialize_per_trie_key(k: u32, ) -> Weight; fn on_initialize_per_queue_item(q: u32, ) -> Weight; - fn instrument(c: u32, ) -> Weight; - fn code_load(c: u32, ) -> Weight; - fn code_refcount(c: u32, ) -> Weight; + fn reinstrument(c: u32, ) -> Weight; + fn call_with_code_per_byte(c: u32, ) -> Weight; fn instantiate_with_code(c: u32, s: u32, ) -> Weight; fn instantiate(s: u32, ) -> Weight; fn call() -> Weight; + fn upload_code(c: u32, ) -> Weight; + fn remove_code() -> Weight; fn seal_caller(r: u32, ) -> Weight; + fn seal_is_contract(r: u32, ) -> Weight; + fn seal_code_hash(r: u32, ) -> Weight; + fn seal_own_code_hash(r: u32, ) -> Weight; + fn seal_caller_is_origin(r: u32, ) -> Weight; fn seal_address(r: u32, ) -> Weight; fn seal_gas_left(r: u32, ) -> Weight; fn seal_balance(r: u32, ) -> Weight; fn seal_value_transferred(r: u32, ) -> Weight; fn seal_minimum_balance(r: u32, ) -> Weight; - fn seal_tombstone_deposit(r: u32, ) -> Weight; fn seal_block_number(r: u32, ) -> Weight; fn seal_now(r: u32, ) -> Weight; fn seal_weight_to_fee(r: u32, ) -> Weight; @@ -75,15 +78,22 @@ pub trait WeightInfo { fn seal_deposit_event_per_topic_and_kb(t: u32, n: u32, ) -> Weight; fn seal_debug_message(r: u32, ) -> Weight; fn seal_set_storage(r: u32, ) -> Weight; - fn seal_set_storage_per_kb(n: u32, ) -> Weight; + fn seal_set_storage_per_new_kb(n: u32, ) -> Weight; + fn seal_set_storage_per_old_kb(n: u32, ) -> Weight; fn seal_clear_storage(r: u32, ) -> Weight; + fn seal_clear_storage_per_kb(n: u32, ) -> Weight; fn seal_get_storage(r: u32, ) -> Weight; fn seal_get_storage_per_kb(n: u32, ) -> Weight; + fn seal_contains_storage(r: u32, ) -> Weight; + fn seal_contains_storage_per_kb(n: u32, ) -> Weight; + fn seal_take_storage(r: u32, ) -> Weight; + fn seal_take_storage_per_kb(n: u32, ) -> Weight; fn seal_transfer(r: u32, ) -> Weight; fn seal_call(r: u32, ) -> Weight; - fn seal_call_per_transfer_input_output_kb(t: u32, i: u32, o: u32, ) -> Weight; + fn seal_delegate_call(r: u32, ) -> Weight; + fn seal_call_per_transfer_clone_kb(t: u32, c: u32, ) -> Weight; fn seal_instantiate(r: u32, ) -> Weight; - fn seal_instantiate_per_input_output_salt_kb(i: u32, o: u32, s: u32, ) -> Weight; + fn seal_instantiate_per_transfer_salt_kb(t: u32, s: u32, ) -> Weight; fn seal_hash_sha2_256(r: u32, ) -> Weight; fn seal_hash_sha2_256_per_kb(n: u32, ) -> Weight; fn seal_hash_keccak_256(r: u32, ) -> Weight; @@ -93,6 +103,7 @@ pub trait WeightInfo { fn seal_hash_blake2_128(r: u32, ) -> Weight; fn seal_hash_blake2_128_per_kb(n: u32, ) -> Weight; fn seal_ecdsa_recover(r: u32, ) -> Weight; + fn seal_set_code_hash(r: u32, ) -> Weight; fn instr_i64const(r: u32, ) -> Weight; fn instr_i64load(r: u32, ) -> Weight; fn instr_i64store(r: u32, ) -> Weight; @@ -151,761 +162,911 @@ pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { // Storage: Contracts DeletionQueue (r:1 w:0) fn on_initialize() -> Weight { - (3_226_000 as Weight) + (1_569_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) } // Storage: Skipped Metadata (r:0 w:0) fn on_initialize_per_trie_key(k: u32, ) -> Weight { - (0 as Weight) - // Standard Error: 2_000 - .saturating_add((2_178_000 as Weight).saturating_mul(k as Weight)) + (9_620_000 as Weight) + // Standard Error: 0 + .saturating_add((748_000 as Weight).saturating_mul(k as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) } // Storage: Contracts DeletionQueue (r:1 w:0) fn on_initialize_per_queue_item(q: u32, ) -> Weight { - (78_329_000 as Weight) - // Standard Error: 1_000 - .saturating_add((353_000 as Weight).saturating_mul(q as Weight)) + (0 as Weight) + // Standard Error: 4_000 + .saturating_add((1_795_000 as Weight).saturating_mul(q as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Contracts PristineCode (r:1 w:0) // Storage: Contracts CodeStorage (r:0 w:1) - fn instrument(c: u32, ) -> Weight { - (37_190_000 as Weight) - // Standard Error: 80_000 - .saturating_add((72_791_000 as Weight).saturating_mul(c as Weight)) + fn reinstrument(c: u32, ) -> Weight { + (12_256_000 as Weight) + // Standard Error: 0 + .saturating_add((49_000 as Weight).saturating_mul(c as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) - fn code_load(c: u32, ) -> Weight { - (6_191_000 as Weight) + // Storage: Timestamp Now (r:1 w:0) + // Storage: System Account (r:1 w:1) + fn call_with_code_per_byte(c: u32, ) -> Weight { + (213_494_000 as Weight) // Standard Error: 0 - .saturating_add((1_426_000 as Weight).saturating_mul(c as Weight)) - .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add((53_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Contracts CodeStorage (r:1 w:1) - fn code_refcount(c: u32, ) -> Weight { - (10_333_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_275_000 as Weight).saturating_mul(c as Weight)) - .saturating_add(T::DbWeight::get().reads(1 as Weight)) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) - } - // Storage: Contracts AccountCounter (r:1 w:0) + // Storage: Contracts Nonce (r:1 w:1) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Timestamp Now (r:1 w:0) // Storage: System Account (r:1 w:1) - // Storage: Contracts CodeStorage (r:1 w:1) // Storage: Contracts PristineCode (r:0 w:1) + // Storage: Contracts OwnerInfoOf (r:0 w:1) fn instantiate_with_code(c: u32, s: u32, ) -> Weight { - (438_556_000 as Weight) - // Standard Error: 147_000 - .saturating_add((179_307_000 as Weight).saturating_mul(c as Weight)) - // Standard Error: 9_000 - .saturating_add((2_159_000 as Weight).saturating_mul(s as Weight)) + (231_180_000 as Weight) + // Standard Error: 0 + .saturating_add((125_000 as Weight).saturating_mul(c as Weight)) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(5 as Weight)) - .saturating_add(T::DbWeight::get().writes(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) } // Storage: Contracts CodeStorage (r:1 w:1) - // Storage: Contracts AccountCounter (r:1 w:0) + // Storage: Contracts Nonce (r:1 w:1) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Timestamp Now (r:1 w:0) // Storage: System Account (r:1 w:1) + // Storage: Contracts OwnerInfoOf (r:1 w:1) fn instantiate(s: u32, ) -> Weight { - (186_776_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_033_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(5 as Weight)) - .saturating_add(T::DbWeight::get().writes(3 as Weight)) + (172_238_000 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(6 as Weight)) + .saturating_add(T::DbWeight::get().writes(5 as Weight)) } // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) // Storage: System Account (r:1 w:1) fn call() -> Weight { - (159_247_000 as Weight) + (140_912_000 as Weight) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } + // Storage: Contracts CodeStorage (r:1 w:1) + // Storage: Contracts PristineCode (r:0 w:1) + // Storage: Contracts OwnerInfoOf (r:0 w:1) + fn upload_code(c: u32, ) -> Weight { + (42_493_000 as Weight) + // Standard Error: 0 + .saturating_add((49_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Contracts OwnerInfoOf (r:1 w:1) + // Storage: Contracts CodeStorage (r:0 w:1) + // Storage: Contracts PristineCode (r:0 w:1) + fn remove_code() -> Weight { + (24_533_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_caller(r: u32, ) -> Weight { - (422_263_000 as Weight) - // Standard Error: 159_000 - .saturating_add((125_490_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (220_009_000 as Weight) + // Standard Error: 80_000 + .saturating_add((47_887_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - fn seal_address(r: u32, ) -> Weight { - (423_009_000 as Weight) - // Standard Error: 183_000 - .saturating_add((125_795_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + fn seal_is_contract(r: u32, ) -> Weight { + (71_779_000 as Weight) + // Standard Error: 900_000 + .saturating_add((371_278_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - fn seal_gas_left(r: u32, ) -> Weight { - (429_297_000 as Weight) - // Standard Error: 164_000 - .saturating_add((124_324_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + fn seal_code_hash(r: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 2_329_000 + .saturating_add((451_731_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) + fn seal_own_code_hash(r: u32, ) -> Weight { + (227_824_000 as Weight) + // Standard Error: 128_000 + .saturating_add((52_843_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } // Storage: System Account (r:1 w:0) - fn seal_balance(r: u32, ) -> Weight { - (442_330_000 as Weight) - // Standard Error: 187_000 - .saturating_add((354_665_000 as Weight).saturating_mul(r as Weight)) + // Storage: Contracts ContractInfoOf (r:1 w:1) + // Storage: Contracts CodeStorage (r:1 w:0) + // Storage: Timestamp Now (r:1 w:0) + fn seal_caller_is_origin(r: u32, ) -> Weight { + (213_057_000 as Weight) + // Standard Error: 43_000 + .saturating_add((21_023_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - fn seal_value_transferred(r: u32, ) -> Weight { - (411_893_000 as Weight) - // Standard Error: 178_000 - .saturating_add((125_971_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + fn seal_address(r: u32, ) -> Weight { + (219_066_000 as Weight) + // Standard Error: 117_000 + .saturating_add((48_056_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: System Account (r:1 w:0) + // Storage: Contracts ContractInfoOf (r:1 w:1) + // Storage: Contracts CodeStorage (r:1 w:0) + // Storage: Timestamp Now (r:1 w:0) + fn seal_gas_left(r: u32, ) -> Weight { + (218_844_000 as Weight) + // Standard Error: 101_000 + .saturating_add((47_325_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - fn seal_minimum_balance(r: u32, ) -> Weight { - (413_273_000 as Weight) - // Standard Error: 180_000 - .saturating_add((125_103_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + fn seal_balance(r: u32, ) -> Weight { + (219_234_000 as Weight) + // Standard Error: 171_000 + .saturating_add((142_534_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: System Account (r:1 w:0) + // Storage: Contracts ContractInfoOf (r:1 w:1) + // Storage: Contracts CodeStorage (r:1 w:0) + // Storage: Timestamp Now (r:1 w:0) + fn seal_value_transferred(r: u32, ) -> Weight { + (215_128_000 as Weight) + // Standard Error: 119_000 + .saturating_add((48_392_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - fn seal_tombstone_deposit(r: u32, ) -> Weight { - (415_613_000 as Weight) - // Standard Error: 192_000 - .saturating_add((126_106_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + fn seal_minimum_balance(r: u32, ) -> Weight { + (214_603_000 as Weight) + // Standard Error: 115_000 + .saturating_add((48_041_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_block_number(r: u32, ) -> Weight { - (414_718_000 as Weight) - // Standard Error: 170_000 - .saturating_add((124_962_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (214_091_000 as Weight) + // Standard Error: 126_000 + .saturating_add((48_067_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_now(r: u32, ) -> Weight { - (419_120_000 as Weight) - // Standard Error: 178_000 - .saturating_add((125_188_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (214_418_000 as Weight) + // Standard Error: 100_000 + .saturating_add((47_791_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) // Storage: TransactionPayment NextFeeMultiplier (r:1 w:0) fn seal_weight_to_fee(r: u32, ) -> Weight { - (419_125_000 as Weight) - // Standard Error: 216_000 - .saturating_add((290_592_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(4 as Weight)) + (229_261_000 as Weight) + // Standard Error: 150_000 + .saturating_add((121_988_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_gas(r: u32, ) -> Weight { - (149_609_000 as Weight) - // Standard Error: 117_000 - .saturating_add((56_860_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (127_983_000 as Weight) + // Standard Error: 56_000 + .saturating_add((24_016_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_input(r: u32, ) -> Weight { - (423_570_000 as Weight) - // Standard Error: 151_000 - .saturating_add((106_985_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (216_634_000 as Weight) + // Standard Error: 114_000 + .saturating_add((46_864_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_input_per_kb(n: u32, ) -> Weight { - (566_496_000 as Weight) - // Standard Error: 6_000 - .saturating_add((38_091_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (285_180_000 as Weight) + // Standard Error: 4_000 + .saturating_add((11_899_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - fn seal_return(r: u32, ) -> Weight { - (406_811_000 as Weight) - // Standard Error: 1_833_000 - .saturating_add((6_551_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + fn seal_return(_r: u32, ) -> Weight { + (215_379_000 as Weight) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_return_per_kb(n: u32, ) -> Weight { - (412_094_000 as Weight) - // Standard Error: 1_000 - .saturating_add((631_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (213_957_000 as Weight) + // Standard Error: 0 + .saturating_add((201_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) // Storage: Contracts DeletionQueue (r:1 w:1) - // Storage: System Account (r:2 w:2) + // Storage: Contracts OwnerInfoOf (r:1 w:1) fn seal_terminate(r: u32, ) -> Weight { - (415_716_000 as Weight) - // Standard Error: 1_608_000 - .saturating_add((72_648_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(r as Weight))) + (215_782_000 as Weight) + // Standard Error: 149_000 + .saturating_add((52_421_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(r as Weight))) .saturating_add(T::DbWeight::get().writes(1 as Weight)) - .saturating_add(T::DbWeight::get().writes((4 as Weight).saturating_mul(r as Weight))) + .saturating_add(T::DbWeight::get().writes((5 as Weight).saturating_mul(r as Weight))) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) // Storage: RandomnessCollectiveFlip RandomMaterial (r:1 w:0) fn seal_random(r: u32, ) -> Weight { - (421_387_000 as Weight) - // Standard Error: 275_000 - .saturating_add((393_452_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(4 as Weight)) + (217_910_000 as Weight) + // Standard Error: 149_000 + .saturating_add((157_525_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_deposit_event(r: u32, ) -> Weight { - (428_591_000 as Weight) - // Standard Error: 293_000 - .saturating_add((690_833_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (230_787_000 as Weight) + // Standard Error: 210_000 + .saturating_add((296_973_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) // Storage: System EventTopics (r:100 w:100) fn seal_deposit_event_per_topic_and_kb(t: u32, n: u32, ) -> Weight { - (1_245_676_000 as Weight) - // Standard Error: 2_636_000 - .saturating_add((484_691_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 519_000 - .saturating_add((165_836_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (539_238_000 as Weight) + // Standard Error: 1_701_000 + .saturating_add((294_348_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 335_000 + .saturating_add((82_116_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().reads((100 as Weight).saturating_mul(t as Weight))) .saturating_add(T::DbWeight::get().writes(1 as Weight)) .saturating_add(T::DbWeight::get().writes((100 as Weight).saturating_mul(t as Weight))) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_debug_message(r: u32, ) -> Weight { - (162_162_000 as Weight) - // Standard Error: 127_000 - .saturating_add((72_828_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (135_081_000 as Weight) + // Standard Error: 94_000 + .saturating_add((39_247_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Skipped Metadata (r:0 w:0) fn seal_set_storage(r: u32, ) -> Weight { - (399_301_000 as Weight) - // Standard Error: 221_000 - .saturating_add((245_222_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (41_752_000 as Weight) + // Standard Error: 1_107_000 + .saturating_add((403_473_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) .saturating_add(T::DbWeight::get().writes(1 as Weight)) .saturating_add(T::DbWeight::get().writes((100 as Weight).saturating_mul(r as Weight))) } - // Storage: Contracts ContractInfoOf (r:1 w:1) - // Storage: Contracts CodeStorage (r:1 w:0) - // Storage: Timestamp Now (r:1 w:0) - // Storage: unknown [0x7afa01283080ef247df84e0ba38ea5a587d25ce6633a6bfbba02068c14023441] (r:0 w:1) - fn seal_set_storage_per_kb(n: u32, ) -> Weight { - (623_011_000 as Weight) - // Standard Error: 246_000 - .saturating_add((72_051_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + // Storage: Skipped Metadata (r:0 w:0) + fn seal_set_storage_per_new_kb(n: u32, ) -> Weight { + (602_028_000 as Weight) + // Standard Error: 255_000 + .saturating_add((28_303_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(105 as Weight)) + .saturating_add(T::DbWeight::get().writes(103 as Weight)) + } + // Storage: Skipped Metadata (r:0 w:0) + fn seal_set_storage_per_old_kb(n: u32, ) -> Weight { + (620_964_000 as Weight) + // Standard Error: 308_000 + .saturating_add((11_338_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(105 as Weight)) + .saturating_add(T::DbWeight::get().writes(103 as Weight)) } // Storage: Skipped Metadata (r:0 w:0) fn seal_clear_storage(r: u32, ) -> Weight { - (445_102_000 as Weight) - // Standard Error: 247_000 - .saturating_add((224_384_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) + (88_113_000 as Weight) + // Standard Error: 851_000 + .saturating_add((381_671_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) + .saturating_add(T::DbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) .saturating_add(T::DbWeight::get().writes((100 as Weight).saturating_mul(r as Weight))) } // Storage: Skipped Metadata (r:0 w:0) + fn seal_clear_storage_per_kb(n: u32, ) -> Weight { + (603_193_000 as Weight) + // Standard Error: 262_000 + .saturating_add((10_286_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(105 as Weight)) + .saturating_add(T::DbWeight::get().writes(103 as Weight)) + } + // Storage: Skipped Metadata (r:0 w:0) fn seal_get_storage(r: u32, ) -> Weight { - (290_227_000 as Weight) - // Standard Error: 694_000 - .saturating_add((547_193_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (112_477_000 as Weight) + // Standard Error: 666_000 + .saturating_add((324_824_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } - // Storage: Contracts ContractInfoOf (r:1 w:1) - // Storage: Contracts CodeStorage (r:1 w:0) - // Storage: Timestamp Now (r:1 w:0) - // Storage: unknown [0x7afa01283080ef247df84e0ba38ea5a587d25ce6633a6bfbba02068c14023441] (r:1 w:0) + // Storage: Skipped Metadata (r:0 w:0) fn seal_get_storage_per_kb(n: u32, ) -> Weight { - (737_772_000 as Weight) - // Standard Error: 267_000 - .saturating_add((112_216_000 as Weight).saturating_mul(n as Weight)) + (564_781_000 as Weight) + // Standard Error: 403_000 + .saturating_add((63_824_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(104 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Skipped Metadata (r:0 w:0) + fn seal_contains_storage(r: u32, ) -> Weight { + (115_207_000 as Weight) + // Standard Error: 672_000 + .saturating_add((290_919_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Skipped Metadata (r:0 w:0) + fn seal_contains_storage_per_kb(n: u32, ) -> Weight { + (511_026_000 as Weight) + // Standard Error: 224_000 + .saturating_add((10_138_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(104 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: Skipped Metadata (r:0 w:0) + fn seal_take_storage(r: u32, ) -> Weight { + (79_113_000 as Weight) + // Standard Error: 904_000 + .saturating_add((417_022_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) + .saturating_add(T::DbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + .saturating_add(T::DbWeight::get().writes((100 as Weight).saturating_mul(r as Weight))) + } + // Storage: Skipped Metadata (r:0 w:0) + fn seal_take_storage_per_kb(n: u32, ) -> Weight { + (651_769_000 as Weight) + // Standard Error: 338_000 + .saturating_add((65_576_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(105 as Weight)) + .saturating_add(T::DbWeight::get().writes(103 as Weight)) + } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - // Storage: System Account (r:101 w:101) fn seal_transfer(r: u32, ) -> Weight { - (383_402_000 as Weight) - // Standard Error: 2_184_000 - .saturating_add((4_335_681_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(4 as Weight)) + (93_588_000 as Weight) + // Standard Error: 1_444_000 + .saturating_add((1_803_217_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) .saturating_add(T::DbWeight::get().writes((100 as Weight).saturating_mul(r as Weight))) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_call(r: u32, ) -> Weight { (0 as Weight) - // Standard Error: 11_019_000 - .saturating_add((39_806_777_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(4 as Weight)) + // Standard Error: 3_050_000 + .saturating_add((19_925_209_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) .saturating_add(T::DbWeight::get().writes(1 as Weight)) .saturating_add(T::DbWeight::get().writes((100 as Weight).saturating_mul(r as Weight))) } + // Storage: System Account (r:1 w:0) + // Storage: Contracts ContractInfoOf (r:1 w:1) + // Storage: Contracts CodeStorage (r:1 w:0) + // Storage: Timestamp Now (r:1 w:0) + fn seal_delegate_call(r: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 7_377_000 + .saturating_add((19_978_301_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads((99 as Weight).saturating_mul(r as Weight))) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:101 w:101) // Storage: Contracts CodeStorage (r:2 w:0) // Storage: Timestamp Now (r:1 w:0) - // Storage: System Account (r:101 w:101) - fn seal_call_per_transfer_input_output_kb(t: u32, i: u32, o: u32, ) -> Weight { - (38_662_592_000 as Weight) - // Standard Error: 52_762_000 - .saturating_add((3_888_801_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 18_000 - .saturating_add((63_571_000 as Weight).saturating_mul(i as Weight)) - // Standard Error: 20_000 - .saturating_add((101_610_000 as Weight).saturating_mul(o as Weight)) - .saturating_add(T::DbWeight::get().reads(104 as Weight)) + fn seal_call_per_transfer_clone_kb(t: u32, c: u32, ) -> Weight { + (11_124_804_000 as Weight) + // Standard Error: 21_475_000 + .saturating_add((1_635_442_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 9_000 + .saturating_add((11_981_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(T::DbWeight::get().reads(105 as Weight)) .saturating_add(T::DbWeight::get().reads((101 as Weight).saturating_mul(t as Weight))) .saturating_add(T::DbWeight::get().writes(101 as Weight)) .saturating_add(T::DbWeight::get().writes((101 as Weight).saturating_mul(t as Weight))) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - // Storage: Contracts AccountCounter (r:1 w:1) - // Storage: System Account (r:101 w:101) + // Storage: Contracts Nonce (r:1 w:1) + // Storage: Contracts OwnerInfoOf (r:100 w:100) fn seal_instantiate(r: u32, ) -> Weight { - (626_132_000 as Weight) - // Standard Error: 39_245_000 - .saturating_add((46_398_859_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(5 as Weight)) - .saturating_add(T::DbWeight::get().reads((300 as Weight).saturating_mul(r as Weight))) + (0 as Weight) + // Standard Error: 47_682_000 + .saturating_add((27_883_754_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(6 as Weight)) + .saturating_add(T::DbWeight::get().reads((400 as Weight).saturating_mul(r as Weight))) .saturating_add(T::DbWeight::get().writes(3 as Weight)) - .saturating_add(T::DbWeight::get().writes((300 as Weight).saturating_mul(r as Weight))) + .saturating_add(T::DbWeight::get().writes((400 as Weight).saturating_mul(r as Weight))) } + // Storage: System Account (r:101 w:101) // Storage: Contracts ContractInfoOf (r:101 w:101) // Storage: Contracts CodeStorage (r:2 w:1) // Storage: Timestamp Now (r:1 w:0) - // Storage: Contracts AccountCounter (r:1 w:1) - // Storage: System Account (r:101 w:101) - fn seal_instantiate_per_input_output_salt_kb(i: u32, o: u32, s: u32, ) -> Weight { - (46_649_369_000 as Weight) - // Standard Error: 26_000 - .saturating_add((63_469_000 as Weight).saturating_mul(i as Weight)) - // Standard Error: 26_000 - .saturating_add((100_694_000 as Weight).saturating_mul(o as Weight)) - // Standard Error: 26_000 - .saturating_add((201_705_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(206 as Weight)) - .saturating_add(T::DbWeight::get().writes(204 as Weight)) + // Storage: Contracts Nonce (r:1 w:1) + // Storage: Contracts OwnerInfoOf (r:1 w:1) + fn seal_instantiate_per_transfer_salt_kb(t: u32, s: u32, ) -> Weight { + (14_824_308_000 as Weight) + // Standard Error: 39_823_000 + .saturating_add((880_630_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 18_000 + .saturating_add((156_232_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(207 as Weight)) + .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(t as Weight))) + .saturating_add(T::DbWeight::get().writes(205 as Weight)) + .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(t as Weight))) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_sha2_256(r: u32, ) -> Weight { - (417_820_000 as Weight) - // Standard Error: 160_000 - .saturating_add((133_795_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (218_378_000 as Weight) + // Standard Error: 131_000 + .saturating_add((78_260_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_sha2_256_per_kb(n: u32, ) -> Weight { - (609_012_000 as Weight) - // Standard Error: 23_000 - .saturating_add((499_227_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (202_849_000 as Weight) + // Standard Error: 61_000 + .saturating_add((466_532_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_keccak_256(r: u32, ) -> Weight { - (419_043_000 as Weight) - // Standard Error: 177_000 - .saturating_add((140_704_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (220_258_000 as Weight) + // Standard Error: 147_000 + .saturating_add((90_363_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_keccak_256_per_kb(n: u32, ) -> Weight { - (564_451_000 as Weight) - // Standard Error: 19_000 - .saturating_add((346_948_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (232_371_000 as Weight) + // Standard Error: 23_000 + .saturating_add((307_036_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_blake2_256(r: u32, ) -> Weight { - (420_951_000 as Weight) - // Standard Error: 163_000 - .saturating_add((113_596_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (217_991_000 as Weight) + // Standard Error: 124_000 + .saturating_add((62_273_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_blake2_256_per_kb(n: u32, ) -> Weight { - (563_168_000 as Weight) - // Standard Error: 17_000 - .saturating_add((164_114_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (396_282_000 as Weight) + // Standard Error: 13_000 + .saturating_add((119_575_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_blake2_128(r: u32, ) -> Weight { - (418_794_000 as Weight) - // Standard Error: 167_000 - .saturating_add((113_205_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (217_578_000 as Weight) + // Standard Error: 104_000 + .saturating_add((62_189_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_blake2_128_per_kb(n: u32, ) -> Weight { - (584_668_000 as Weight) + (358_167_000 as Weight) // Standard Error: 15_000 - .saturating_add((164_127_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add((119_692_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_ecdsa_recover(r: u32, ) -> Weight { - (435_443_000 as Weight) - // Standard Error: 1_408_000 - .saturating_add((15_624_877_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (292_884_000 as Weight) + // Standard Error: 683_000 + .saturating_add((3_824_902_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) + // Storage: Contracts ContractInfoOf (r:1 w:1) + // Storage: Contracts CodeStorage (r:1 w:0) + // Storage: Timestamp Now (r:1 w:0) + // Storage: Contracts OwnerInfoOf (r:36 w:36) + fn seal_set_code_hash(r: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 2_302_000 + .saturating_add((922_467_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads((99 as Weight).saturating_mul(r as Weight))) + .saturating_add(T::DbWeight::get().writes((99 as Weight).saturating_mul(r as Weight))) + } fn instr_i64const(r: u32, ) -> Weight { - (45_937_000 as Weight) - // Standard Error: 10_000 - .saturating_add((1_108_000 as Weight).saturating_mul(r as Weight)) + (74_516_000 as Weight) + // Standard Error: 1_000 + .saturating_add((592_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64load(r: u32, ) -> Weight { - (44_001_000 as Weight) - // Standard Error: 11_000 - .saturating_add((2_412_000 as Weight).saturating_mul(r as Weight)) + (74_430_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_320_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64store(r: u32, ) -> Weight { - (43_157_000 as Weight) - // Standard Error: 12_000 - .saturating_add((2_677_000 as Weight).saturating_mul(r as Weight)) + (74_440_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_428_000 as Weight).saturating_mul(r as Weight)) } fn instr_select(r: u32, ) -> Weight { - (48_475_000 as Weight) - // Standard Error: 8_000 - .saturating_add((2_604_000 as Weight).saturating_mul(r as Weight)) + (74_151_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_782_000 as Weight).saturating_mul(r as Weight)) } fn instr_if(r: u32, ) -> Weight { - (50_649_000 as Weight) - // Standard Error: 12_000 - .saturating_add((2_553_000 as Weight).saturating_mul(r as Weight)) + (74_225_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_887_000 as Weight).saturating_mul(r as Weight)) } fn instr_br(r: u32, ) -> Weight { - (48_433_000 as Weight) - // Standard Error: 8_000 - .saturating_add((1_670_000 as Weight).saturating_mul(r as Weight)) + (73_987_000 as Weight) + // Standard Error: 1_000 + .saturating_add((898_000 as Weight).saturating_mul(r as Weight)) } fn instr_br_if(r: u32, ) -> Weight { - (49_244_000 as Weight) - // Standard Error: 16_000 - .saturating_add((1_946_000 as Weight).saturating_mul(r as Weight)) + (73_305_000 as Weight) + // Standard Error: 4_000 + .saturating_add((1_465_000 as Weight).saturating_mul(r as Weight)) } fn instr_br_table(r: u32, ) -> Weight { - (46_117_000 as Weight) - // Standard Error: 17_000 - .saturating_add((2_387_000 as Weight).saturating_mul(r as Weight)) + (73_037_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_605_000 as Weight).saturating_mul(r as Weight)) } - fn instr_br_table_per_entry(_e: u32, ) -> Weight { - (55_204_000 as Weight) + fn instr_br_table_per_entry(e: u32, ) -> Weight { + (76_434_000 as Weight) + // Standard Error: 0 + .saturating_add((4_000 as Weight).saturating_mul(e as Weight)) } fn instr_call(r: u32, ) -> Weight { - (43_651_000 as Weight) - // Standard Error: 26_000 - .saturating_add((19_163_000 as Weight).saturating_mul(r as Weight)) + (75_461_000 as Weight) + // Standard Error: 10_000 + .saturating_add((7_446_000 as Weight).saturating_mul(r as Weight)) } fn instr_call_indirect(r: u32, ) -> Weight { - (54_063_000 as Weight) - // Standard Error: 32_000 - .saturating_add((27_970_000 as Weight).saturating_mul(r as Weight)) + (87_222_000 as Weight) + // Standard Error: 15_000 + .saturating_add((9_406_000 as Weight).saturating_mul(r as Weight)) } fn instr_call_indirect_per_param(p: u32, ) -> Weight { - (88_527_000 as Weight) - // Standard Error: 6_000 - .saturating_add((958_000 as Weight).saturating_mul(p as Weight)) + (97_204_000 as Weight) + // Standard Error: 1_000 + .saturating_add((472_000 as Weight).saturating_mul(p as Weight)) } fn instr_local_get(r: u32, ) -> Weight { - (55_066_000 as Weight) - // Standard Error: 12_000 - .saturating_add((682_000 as Weight).saturating_mul(r as Weight)) + (75_299_000 as Weight) + // Standard Error: 1_000 + .saturating_add((601_000 as Weight).saturating_mul(r as Weight)) } fn instr_local_set(r: u32, ) -> Weight { - (55_298_000 as Weight) - // Standard Error: 13_000 - .saturating_add((778_000 as Weight).saturating_mul(r as Weight)) + (74_827_000 as Weight) + // Standard Error: 3_000 + .saturating_add((686_000 as Weight).saturating_mul(r as Weight)) } fn instr_local_tee(r: u32, ) -> Weight { - (56_302_000 as Weight) - // Standard Error: 11_000 - .saturating_add((1_079_000 as Weight).saturating_mul(r as Weight)) + (74_624_000 as Weight) + // Standard Error: 2_000 + .saturating_add((895_000 as Weight).saturating_mul(r as Weight)) } fn instr_global_get(r: u32, ) -> Weight { - (71_567_000 as Weight) - // Standard Error: 11_000 - .saturating_add((1_107_000 as Weight).saturating_mul(r as Weight)) + (77_435_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_201_000 as Weight).saturating_mul(r as Weight)) } fn instr_global_set(r: u32, ) -> Weight { - (71_186_000 as Weight) - // Standard Error: 12_000 - .saturating_add((1_151_000 as Weight).saturating_mul(r as Weight)) + (76_693_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_410_000 as Weight).saturating_mul(r as Weight)) } fn instr_memory_current(r: u32, ) -> Weight { - (46_240_000 as Weight) - // Standard Error: 10_000 - .saturating_add((1_044_000 as Weight).saturating_mul(r as Weight)) + (74_244_000 as Weight) + // Standard Error: 1_000 + .saturating_add((660_000 as Weight).saturating_mul(r as Weight)) } fn instr_memory_grow(r: u32, ) -> Weight { - (52_369_000 as Weight) - // Standard Error: 2_508_000 - .saturating_add((615_448_000 as Weight).saturating_mul(r as Weight)) + (73_527_000 as Weight) + // Standard Error: 931_000 + .saturating_add((184_946_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64clz(r: u32, ) -> Weight { - (47_623_000 as Weight) - // Standard Error: 9_000 - .saturating_add((1_583_000 as Weight).saturating_mul(r as Weight)) + (74_181_000 as Weight) + // Standard Error: 6_000 + .saturating_add((906_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64ctz(r: u32, ) -> Weight { - (47_670_000 as Weight) - // Standard Error: 9_000 - .saturating_add((1_583_000 as Weight).saturating_mul(r as Weight)) + (74_339_000 as Weight) + // Standard Error: 1_000 + .saturating_add((896_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64popcnt(r: u32, ) -> Weight { - (47_508_000 as Weight) - // Standard Error: 9_000 - .saturating_add((1_583_000 as Weight).saturating_mul(r as Weight)) + (74_444_000 as Weight) + // Standard Error: 3_000 + .saturating_add((889_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64eqz(r: u32, ) -> Weight { - (48_109_000 as Weight) - // Standard Error: 9_000 - .saturating_add((1_580_000 as Weight).saturating_mul(r as Weight)) + (74_572_000 as Weight) + // Standard Error: 1_000 + .saturating_add((908_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64extendsi32(r: u32, ) -> Weight { - (55_270_000 as Weight) - // Standard Error: 9_000 - .saturating_add((1_102_000 as Weight).saturating_mul(r as Weight)) + (74_349_000 as Weight) + // Standard Error: 2_000 + .saturating_add((881_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64extendui32(r: u32, ) -> Weight { - (55_093_000 as Weight) - // Standard Error: 9_000 - .saturating_add((1_108_000 as Weight).saturating_mul(r as Weight)) + (74_426_000 as Weight) + // Standard Error: 1_000 + .saturating_add((875_000 as Weight).saturating_mul(r as Weight)) } fn instr_i32wrapi64(r: u32, ) -> Weight { - (48_265_000 as Weight) - // Standard Error: 10_000 - .saturating_add((1_573_000 as Weight).saturating_mul(r as Weight)) + (74_172_000 as Weight) + // Standard Error: 2_000 + .saturating_add((906_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64eq(r: u32, ) -> Weight { - (48_733_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_088_000 as Weight).saturating_mul(r as Weight)) + (74_169_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_362_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64ne(r: u32, ) -> Weight { - (48_831_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_085_000 as Weight).saturating_mul(r as Weight)) + (74_205_000 as Weight) + // Standard Error: 4_000 + .saturating_add((1_360_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64lts(r: u32, ) -> Weight { - (49_147_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_056_000 as Weight).saturating_mul(r as Weight)) + (74_237_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_356_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64ltu(r: u32, ) -> Weight { - (49_596_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_049_000 as Weight).saturating_mul(r as Weight)) + (74_181_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_357_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64gts(r: u32, ) -> Weight { - (49_872_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_038_000 as Weight).saturating_mul(r as Weight)) + (74_038_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_360_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64gtu(r: u32, ) -> Weight { - (48_843_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_081_000 as Weight).saturating_mul(r as Weight)) + (73_881_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_372_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64les(r: u32, ) -> Weight { - (48_765_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_089_000 as Weight).saturating_mul(r as Weight)) + (73_969_000 as Weight) + // Standard Error: 0 + .saturating_add((1_361_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64leu(r: u32, ) -> Weight { - (48_720_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_083_000 as Weight).saturating_mul(r as Weight)) + (74_497_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_359_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64ges(r: u32, ) -> Weight { - (48_736_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_097_000 as Weight).saturating_mul(r as Weight)) + (74_275_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_354_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64geu(r: u32, ) -> Weight { - (48_772_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_093_000 as Weight).saturating_mul(r as Weight)) + (74_349_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_357_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64add(r: u32, ) -> Weight { - (48_827_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_082_000 as Weight).saturating_mul(r as Weight)) + (74_192_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_333_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64sub(r: u32, ) -> Weight { - (48_961_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_072_000 as Weight).saturating_mul(r as Weight)) + (74_271_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_340_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64mul(r: u32, ) -> Weight { - (49_069_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_067_000 as Weight).saturating_mul(r as Weight)) + (73_971_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_340_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64divs(r: u32, ) -> Weight { - (49_035_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_677_000 as Weight).saturating_mul(r as Weight)) + (74_546_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_995_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64divu(r: u32, ) -> Weight { - (48_842_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_449_000 as Weight).saturating_mul(r as Weight)) + (74_194_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_050_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64rems(r: u32, ) -> Weight { - (48_536_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_723_000 as Weight).saturating_mul(r as Weight)) + (74_106_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_997_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64remu(r: u32, ) -> Weight { - (48_851_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_432_000 as Weight).saturating_mul(r as Weight)) + (74_219_000 as Weight) + // Standard Error: 5_000 + .saturating_add((2_061_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64and(r: u32, ) -> Weight { - (48_624_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_093_000 as Weight).saturating_mul(r as Weight)) + (74_157_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_338_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64or(r: u32, ) -> Weight { - (49_348_000 as Weight) - // Standard Error: 8_000 - .saturating_add((2_073_000 as Weight).saturating_mul(r as Weight)) + (74_135_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_336_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64xor(r: u32, ) -> Weight { - (49_112_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_055_000 as Weight).saturating_mul(r as Weight)) + (74_038_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_345_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64shl(r: u32, ) -> Weight { - (49_654_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_051_000 as Weight).saturating_mul(r as Weight)) + (74_011_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_362_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64shrs(r: u32, ) -> Weight { - (48_848_000 as Weight) - // Standard Error: 8_000 - .saturating_add((2_089_000 as Weight).saturating_mul(r as Weight)) + (74_054_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_356_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64shru(r: u32, ) -> Weight { - (49_455_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_054_000 as Weight).saturating_mul(r as Weight)) + (73_900_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_358_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64rotl(r: u32, ) -> Weight { - (49_640_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_048_000 as Weight).saturating_mul(r as Weight)) + (73_948_000 as Weight) + // Standard Error: 0 + .saturating_add((1_357_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64rotr(r: u32, ) -> Weight { - (49_498_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_068_000 as Weight).saturating_mul(r as Weight)) + (73_972_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_358_000 as Weight).saturating_mul(r as Weight)) } } @@ -913,760 +1074,910 @@ impl WeightInfo for SubstrateWeight { impl WeightInfo for () { // Storage: Contracts DeletionQueue (r:1 w:0) fn on_initialize() -> Weight { - (3_226_000 as Weight) + (1_569_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) } // Storage: Skipped Metadata (r:0 w:0) fn on_initialize_per_trie_key(k: u32, ) -> Weight { - (0 as Weight) - // Standard Error: 2_000 - .saturating_add((2_178_000 as Weight).saturating_mul(k as Weight)) + (9_620_000 as Weight) + // Standard Error: 0 + .saturating_add((748_000 as Weight).saturating_mul(k as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) } // Storage: Contracts DeletionQueue (r:1 w:0) fn on_initialize_per_queue_item(q: u32, ) -> Weight { - (78_329_000 as Weight) - // Standard Error: 1_000 - .saturating_add((353_000 as Weight).saturating_mul(q as Weight)) + (0 as Weight) + // Standard Error: 4_000 + .saturating_add((1_795_000 as Weight).saturating_mul(q as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Contracts PristineCode (r:1 w:0) // Storage: Contracts CodeStorage (r:0 w:1) - fn instrument(c: u32, ) -> Weight { - (37_190_000 as Weight) - // Standard Error: 80_000 - .saturating_add((72_791_000 as Weight).saturating_mul(c as Weight)) + fn reinstrument(c: u32, ) -> Weight { + (12_256_000 as Weight) + // Standard Error: 0 + .saturating_add((49_000 as Weight).saturating_mul(c as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) - fn code_load(c: u32, ) -> Weight { - (6_191_000 as Weight) + // Storage: Timestamp Now (r:1 w:0) + // Storage: System Account (r:1 w:1) + fn call_with_code_per_byte(c: u32, ) -> Weight { + (213_494_000 as Weight) // Standard Error: 0 - .saturating_add((1_426_000 as Weight).saturating_mul(c as Weight)) - .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add((53_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Contracts CodeStorage (r:1 w:1) - fn code_refcount(c: u32, ) -> Weight { - (10_333_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_275_000 as Weight).saturating_mul(c as Weight)) - .saturating_add(RocksDbWeight::get().reads(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) - } - // Storage: Contracts AccountCounter (r:1 w:0) + // Storage: Contracts Nonce (r:1 w:1) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Timestamp Now (r:1 w:0) // Storage: System Account (r:1 w:1) - // Storage: Contracts CodeStorage (r:1 w:1) // Storage: Contracts PristineCode (r:0 w:1) + // Storage: Contracts OwnerInfoOf (r:0 w:1) fn instantiate_with_code(c: u32, s: u32, ) -> Weight { - (438_556_000 as Weight) - // Standard Error: 147_000 - .saturating_add((179_307_000 as Weight).saturating_mul(c as Weight)) - // Standard Error: 9_000 - .saturating_add((2_159_000 as Weight).saturating_mul(s as Weight)) + (231_180_000 as Weight) + // Standard Error: 0 + .saturating_add((125_000 as Weight).saturating_mul(c as Weight)) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) - .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } // Storage: Contracts CodeStorage (r:1 w:1) - // Storage: Contracts AccountCounter (r:1 w:0) + // Storage: Contracts Nonce (r:1 w:1) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Timestamp Now (r:1 w:0) // Storage: System Account (r:1 w:1) + // Storage: Contracts OwnerInfoOf (r:1 w:1) fn instantiate(s: u32, ) -> Weight { - (186_776_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_033_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(5 as Weight)) - .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + (172_238_000 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(6 as Weight)) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) // Storage: System Account (r:1 w:1) fn call() -> Weight { - (159_247_000 as Weight) + (140_912_000 as Weight) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } + // Storage: Contracts CodeStorage (r:1 w:1) + // Storage: Contracts PristineCode (r:0 w:1) + // Storage: Contracts OwnerInfoOf (r:0 w:1) + fn upload_code(c: u32, ) -> Weight { + (42_493_000 as Weight) + // Standard Error: 0 + .saturating_add((49_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Contracts OwnerInfoOf (r:1 w:1) + // Storage: Contracts CodeStorage (r:0 w:1) + // Storage: Contracts PristineCode (r:0 w:1) + fn remove_code() -> Weight { + (24_533_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_caller(r: u32, ) -> Weight { - (422_263_000 as Weight) - // Standard Error: 159_000 - .saturating_add((125_490_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (220_009_000 as Weight) + // Standard Error: 80_000 + .saturating_add((47_887_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - fn seal_address(r: u32, ) -> Weight { - (423_009_000 as Weight) - // Standard Error: 183_000 - .saturating_add((125_795_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + fn seal_is_contract(r: u32, ) -> Weight { + (71_779_000 as Weight) + // Standard Error: 900_000 + .saturating_add((371_278_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - fn seal_gas_left(r: u32, ) -> Weight { - (429_297_000 as Weight) - // Standard Error: 164_000 - .saturating_add((124_324_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + fn seal_code_hash(r: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 2_329_000 + .saturating_add((451_731_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) + fn seal_own_code_hash(r: u32, ) -> Weight { + (227_824_000 as Weight) + // Standard Error: 128_000 + .saturating_add((52_843_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } // Storage: System Account (r:1 w:0) - fn seal_balance(r: u32, ) -> Weight { - (442_330_000 as Weight) - // Standard Error: 187_000 - .saturating_add((354_665_000 as Weight).saturating_mul(r as Weight)) + // Storage: Contracts ContractInfoOf (r:1 w:1) + // Storage: Contracts CodeStorage (r:1 w:0) + // Storage: Timestamp Now (r:1 w:0) + fn seal_caller_is_origin(r: u32, ) -> Weight { + (213_057_000 as Weight) + // Standard Error: 43_000 + .saturating_add((21_023_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - fn seal_value_transferred(r: u32, ) -> Weight { - (411_893_000 as Weight) - // Standard Error: 178_000 - .saturating_add((125_971_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + fn seal_address(r: u32, ) -> Weight { + (219_066_000 as Weight) + // Standard Error: 117_000 + .saturating_add((48_056_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - fn seal_minimum_balance(r: u32, ) -> Weight { - (413_273_000 as Weight) - // Standard Error: 180_000 - .saturating_add((125_103_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + fn seal_gas_left(r: u32, ) -> Weight { + (218_844_000 as Weight) + // Standard Error: 101_000 + .saturating_add((47_325_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - fn seal_tombstone_deposit(r: u32, ) -> Weight { - (415_613_000 as Weight) - // Standard Error: 192_000 - .saturating_add((126_106_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + fn seal_balance(r: u32, ) -> Weight { + (219_234_000 as Weight) + // Standard Error: 171_000 + .saturating_add((142_534_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) + // Storage: Contracts ContractInfoOf (r:1 w:1) + // Storage: Contracts CodeStorage (r:1 w:0) + // Storage: Timestamp Now (r:1 w:0) + fn seal_value_transferred(r: u32, ) -> Weight { + (215_128_000 as Weight) + // Standard Error: 119_000 + .saturating_add((48_392_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: System Account (r:1 w:0) + // Storage: Contracts ContractInfoOf (r:1 w:1) + // Storage: Contracts CodeStorage (r:1 w:0) + // Storage: Timestamp Now (r:1 w:0) + fn seal_minimum_balance(r: u32, ) -> Weight { + (214_603_000 as Weight) + // Standard Error: 115_000 + .saturating_add((48_041_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_block_number(r: u32, ) -> Weight { - (414_718_000 as Weight) - // Standard Error: 170_000 - .saturating_add((124_962_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (214_091_000 as Weight) + // Standard Error: 126_000 + .saturating_add((48_067_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_now(r: u32, ) -> Weight { - (419_120_000 as Weight) - // Standard Error: 178_000 - .saturating_add((125_188_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (214_418_000 as Weight) + // Standard Error: 100_000 + .saturating_add((47_791_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) // Storage: TransactionPayment NextFeeMultiplier (r:1 w:0) fn seal_weight_to_fee(r: u32, ) -> Weight { - (419_125_000 as Weight) - // Standard Error: 216_000 - .saturating_add((290_592_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + (229_261_000 as Weight) + // Standard Error: 150_000 + .saturating_add((121_988_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_gas(r: u32, ) -> Weight { - (149_609_000 as Weight) - // Standard Error: 117_000 - .saturating_add((56_860_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (127_983_000 as Weight) + // Standard Error: 56_000 + .saturating_add((24_016_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_input(r: u32, ) -> Weight { - (423_570_000 as Weight) - // Standard Error: 151_000 - .saturating_add((106_985_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (216_634_000 as Weight) + // Standard Error: 114_000 + .saturating_add((46_864_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_input_per_kb(n: u32, ) -> Weight { - (566_496_000 as Weight) - // Standard Error: 6_000 - .saturating_add((38_091_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (285_180_000 as Weight) + // Standard Error: 4_000 + .saturating_add((11_899_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - fn seal_return(r: u32, ) -> Weight { - (406_811_000 as Weight) - // Standard Error: 1_833_000 - .saturating_add((6_551_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + fn seal_return(_r: u32, ) -> Weight { + (215_379_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_return_per_kb(n: u32, ) -> Weight { - (412_094_000 as Weight) - // Standard Error: 1_000 - .saturating_add((631_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (213_957_000 as Weight) + // Standard Error: 0 + .saturating_add((201_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) // Storage: Contracts DeletionQueue (r:1 w:1) - // Storage: System Account (r:2 w:2) + // Storage: Contracts OwnerInfoOf (r:1 w:1) fn seal_terminate(r: u32, ) -> Weight { - (415_716_000 as Weight) - // Standard Error: 1_608_000 - .saturating_add((72_648_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(r as Weight))) + (215_782_000 as Weight) + // Standard Error: 149_000 + .saturating_add((52_421_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(r as Weight))) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes((4 as Weight).saturating_mul(r as Weight))) + .saturating_add(RocksDbWeight::get().writes((5 as Weight).saturating_mul(r as Weight))) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) // Storage: RandomnessCollectiveFlip RandomMaterial (r:1 w:0) fn seal_random(r: u32, ) -> Weight { - (421_387_000 as Weight) - // Standard Error: 275_000 - .saturating_add((393_452_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + (217_910_000 as Weight) + // Standard Error: 149_000 + .saturating_add((157_525_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_deposit_event(r: u32, ) -> Weight { - (428_591_000 as Weight) - // Standard Error: 293_000 - .saturating_add((690_833_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (230_787_000 as Weight) + // Standard Error: 210_000 + .saturating_add((296_973_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) // Storage: System EventTopics (r:100 w:100) fn seal_deposit_event_per_topic_and_kb(t: u32, n: u32, ) -> Weight { - (1_245_676_000 as Weight) - // Standard Error: 2_636_000 - .saturating_add((484_691_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 519_000 - .saturating_add((165_836_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (539_238_000 as Weight) + // Standard Error: 1_701_000 + .saturating_add((294_348_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 335_000 + .saturating_add((82_116_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().reads((100 as Weight).saturating_mul(t as Weight))) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) .saturating_add(RocksDbWeight::get().writes((100 as Weight).saturating_mul(t as Weight))) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_debug_message(r: u32, ) -> Weight { - (162_162_000 as Weight) - // Standard Error: 127_000 - .saturating_add((72_828_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (135_081_000 as Weight) + // Standard Error: 94_000 + .saturating_add((39_247_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Skipped Metadata (r:0 w:0) fn seal_set_storage(r: u32, ) -> Weight { - (399_301_000 as Weight) - // Standard Error: 221_000 - .saturating_add((245_222_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (41_752_000 as Weight) + // Standard Error: 1_107_000 + .saturating_add((403_473_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) .saturating_add(RocksDbWeight::get().writes((100 as Weight).saturating_mul(r as Weight))) } - // Storage: Contracts ContractInfoOf (r:1 w:1) - // Storage: Contracts CodeStorage (r:1 w:0) - // Storage: Timestamp Now (r:1 w:0) - // Storage: unknown [0x7afa01283080ef247df84e0ba38ea5a587d25ce6633a6bfbba02068c14023441] (r:0 w:1) - fn seal_set_storage_per_kb(n: u32, ) -> Weight { - (623_011_000 as Weight) - // Standard Error: 246_000 - .saturating_add((72_051_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + // Storage: Skipped Metadata (r:0 w:0) + fn seal_set_storage_per_new_kb(n: u32, ) -> Weight { + (602_028_000 as Weight) + // Standard Error: 255_000 + .saturating_add((28_303_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(105 as Weight)) + .saturating_add(RocksDbWeight::get().writes(103 as Weight)) + } + // Storage: Skipped Metadata (r:0 w:0) + fn seal_set_storage_per_old_kb(n: u32, ) -> Weight { + (620_964_000 as Weight) + // Standard Error: 308_000 + .saturating_add((11_338_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(105 as Weight)) + .saturating_add(RocksDbWeight::get().writes(103 as Weight)) } // Storage: Skipped Metadata (r:0 w:0) fn seal_clear_storage(r: u32, ) -> Weight { - (445_102_000 as Weight) - // Standard Error: 247_000 - .saturating_add((224_384_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + (88_113_000 as Weight) + // Standard Error: 851_000 + .saturating_add((381_671_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) + .saturating_add(RocksDbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) .saturating_add(RocksDbWeight::get().writes((100 as Weight).saturating_mul(r as Weight))) } // Storage: Skipped Metadata (r:0 w:0) + fn seal_clear_storage_per_kb(n: u32, ) -> Weight { + (603_193_000 as Weight) + // Standard Error: 262_000 + .saturating_add((10_286_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(105 as Weight)) + .saturating_add(RocksDbWeight::get().writes(103 as Weight)) + } + // Storage: Skipped Metadata (r:0 w:0) fn seal_get_storage(r: u32, ) -> Weight { - (290_227_000 as Weight) - // Standard Error: 694_000 - .saturating_add((547_193_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (112_477_000 as Weight) + // Standard Error: 666_000 + .saturating_add((324_824_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } - // Storage: Contracts ContractInfoOf (r:1 w:1) - // Storage: Contracts CodeStorage (r:1 w:0) - // Storage: Timestamp Now (r:1 w:0) - // Storage: unknown [0x7afa01283080ef247df84e0ba38ea5a587d25ce6633a6bfbba02068c14023441] (r:1 w:0) + // Storage: Skipped Metadata (r:0 w:0) fn seal_get_storage_per_kb(n: u32, ) -> Weight { - (737_772_000 as Weight) - // Standard Error: 267_000 - .saturating_add((112_216_000 as Weight).saturating_mul(n as Weight)) + (564_781_000 as Weight) + // Standard Error: 403_000 + .saturating_add((63_824_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(104 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Skipped Metadata (r:0 w:0) + fn seal_contains_storage(r: u32, ) -> Weight { + (115_207_000 as Weight) + // Standard Error: 672_000 + .saturating_add((290_919_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Skipped Metadata (r:0 w:0) + fn seal_contains_storage_per_kb(n: u32, ) -> Weight { + (511_026_000 as Weight) + // Standard Error: 224_000 + .saturating_add((10_138_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(104 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: Skipped Metadata (r:0 w:0) + fn seal_take_storage(r: u32, ) -> Weight { + (79_113_000 as Weight) + // Standard Error: 904_000 + .saturating_add((417_022_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) + .saturating_add(RocksDbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes((100 as Weight).saturating_mul(r as Weight))) + } + // Storage: Skipped Metadata (r:0 w:0) + fn seal_take_storage_per_kb(n: u32, ) -> Weight { + (651_769_000 as Weight) + // Standard Error: 338_000 + .saturating_add((65_576_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(105 as Weight)) + .saturating_add(RocksDbWeight::get().writes(103 as Weight)) + } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - // Storage: System Account (r:101 w:101) fn seal_transfer(r: u32, ) -> Weight { - (383_402_000 as Weight) - // Standard Error: 2_184_000 - .saturating_add((4_335_681_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + (93_588_000 as Weight) + // Standard Error: 1_444_000 + .saturating_add((1_803_217_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) .saturating_add(RocksDbWeight::get().writes((100 as Weight).saturating_mul(r as Weight))) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_call(r: u32, ) -> Weight { (0 as Weight) - // Standard Error: 11_019_000 - .saturating_add((39_806_777_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + // Standard Error: 3_050_000 + .saturating_add((19_925_209_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().reads((100 as Weight).saturating_mul(r as Weight))) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) .saturating_add(RocksDbWeight::get().writes((100 as Weight).saturating_mul(r as Weight))) } + // Storage: System Account (r:1 w:0) + // Storage: Contracts ContractInfoOf (r:1 w:1) + // Storage: Contracts CodeStorage (r:1 w:0) + // Storage: Timestamp Now (r:1 w:0) + fn seal_delegate_call(r: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 7_377_000 + .saturating_add((19_978_301_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads((99 as Weight).saturating_mul(r as Weight))) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:101 w:101) // Storage: Contracts CodeStorage (r:2 w:0) // Storage: Timestamp Now (r:1 w:0) - // Storage: System Account (r:101 w:101) - fn seal_call_per_transfer_input_output_kb(t: u32, i: u32, o: u32, ) -> Weight { - (38_662_592_000 as Weight) - // Standard Error: 52_762_000 - .saturating_add((3_888_801_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 18_000 - .saturating_add((63_571_000 as Weight).saturating_mul(i as Weight)) - // Standard Error: 20_000 - .saturating_add((101_610_000 as Weight).saturating_mul(o as Weight)) - .saturating_add(RocksDbWeight::get().reads(104 as Weight)) + fn seal_call_per_transfer_clone_kb(t: u32, c: u32, ) -> Weight { + (11_124_804_000 as Weight) + // Standard Error: 21_475_000 + .saturating_add((1_635_442_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 9_000 + .saturating_add((11_981_000 as Weight).saturating_mul(c as Weight)) + .saturating_add(RocksDbWeight::get().reads(105 as Weight)) .saturating_add(RocksDbWeight::get().reads((101 as Weight).saturating_mul(t as Weight))) .saturating_add(RocksDbWeight::get().writes(101 as Weight)) .saturating_add(RocksDbWeight::get().writes((101 as Weight).saturating_mul(t as Weight))) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) - // Storage: Contracts AccountCounter (r:1 w:1) - // Storage: System Account (r:101 w:101) + // Storage: Contracts Nonce (r:1 w:1) + // Storage: Contracts OwnerInfoOf (r:100 w:100) fn seal_instantiate(r: u32, ) -> Weight { - (626_132_000 as Weight) - // Standard Error: 39_245_000 - .saturating_add((46_398_859_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(5 as Weight)) - .saturating_add(RocksDbWeight::get().reads((300 as Weight).saturating_mul(r as Weight))) + (0 as Weight) + // Standard Error: 47_682_000 + .saturating_add((27_883_754_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(6 as Weight)) + .saturating_add(RocksDbWeight::get().reads((400 as Weight).saturating_mul(r as Weight))) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) - .saturating_add(RocksDbWeight::get().writes((300 as Weight).saturating_mul(r as Weight))) + .saturating_add(RocksDbWeight::get().writes((400 as Weight).saturating_mul(r as Weight))) } + // Storage: System Account (r:101 w:101) // Storage: Contracts ContractInfoOf (r:101 w:101) // Storage: Contracts CodeStorage (r:2 w:1) // Storage: Timestamp Now (r:1 w:0) - // Storage: Contracts AccountCounter (r:1 w:1) - // Storage: System Account (r:101 w:101) - fn seal_instantiate_per_input_output_salt_kb(i: u32, o: u32, s: u32, ) -> Weight { - (46_649_369_000 as Weight) - // Standard Error: 26_000 - .saturating_add((63_469_000 as Weight).saturating_mul(i as Weight)) - // Standard Error: 26_000 - .saturating_add((100_694_000 as Weight).saturating_mul(o as Weight)) - // Standard Error: 26_000 - .saturating_add((201_705_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(206 as Weight)) - .saturating_add(RocksDbWeight::get().writes(204 as Weight)) + // Storage: Contracts Nonce (r:1 w:1) + // Storage: Contracts OwnerInfoOf (r:1 w:1) + fn seal_instantiate_per_transfer_salt_kb(t: u32, s: u32, ) -> Weight { + (14_824_308_000 as Weight) + // Standard Error: 39_823_000 + .saturating_add((880_630_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 18_000 + .saturating_add((156_232_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(207 as Weight)) + .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(t as Weight))) + .saturating_add(RocksDbWeight::get().writes(205 as Weight)) + .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(t as Weight))) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_sha2_256(r: u32, ) -> Weight { - (417_820_000 as Weight) - // Standard Error: 160_000 - .saturating_add((133_795_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (218_378_000 as Weight) + // Standard Error: 131_000 + .saturating_add((78_260_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_sha2_256_per_kb(n: u32, ) -> Weight { - (609_012_000 as Weight) - // Standard Error: 23_000 - .saturating_add((499_227_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (202_849_000 as Weight) + // Standard Error: 61_000 + .saturating_add((466_532_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_keccak_256(r: u32, ) -> Weight { - (419_043_000 as Weight) - // Standard Error: 177_000 - .saturating_add((140_704_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (220_258_000 as Weight) + // Standard Error: 147_000 + .saturating_add((90_363_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_keccak_256_per_kb(n: u32, ) -> Weight { - (564_451_000 as Weight) - // Standard Error: 19_000 - .saturating_add((346_948_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (232_371_000 as Weight) + // Standard Error: 23_000 + .saturating_add((307_036_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_blake2_256(r: u32, ) -> Weight { - (420_951_000 as Weight) - // Standard Error: 163_000 - .saturating_add((113_596_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (217_991_000 as Weight) + // Standard Error: 124_000 + .saturating_add((62_273_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_blake2_256_per_kb(n: u32, ) -> Weight { - (563_168_000 as Weight) - // Standard Error: 17_000 - .saturating_add((164_114_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (396_282_000 as Weight) + // Standard Error: 13_000 + .saturating_add((119_575_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_blake2_128(r: u32, ) -> Weight { - (418_794_000 as Weight) - // Standard Error: 167_000 - .saturating_add((113_205_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (217_578_000 as Weight) + // Standard Error: 104_000 + .saturating_add((62_189_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_hash_blake2_128_per_kb(n: u32, ) -> Weight { - (584_668_000 as Weight) + (358_167_000 as Weight) // Standard Error: 15_000 - .saturating_add((164_127_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add((119_692_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) // Storage: Contracts ContractInfoOf (r:1 w:1) // Storage: Contracts CodeStorage (r:1 w:0) // Storage: Timestamp Now (r:1 w:0) fn seal_ecdsa_recover(r: u32, ) -> Weight { - (435_443_000 as Weight) - // Standard Error: 1_408_000 - .saturating_add((15_624_877_000 as Weight).saturating_mul(r as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (292_884_000 as Weight) + // Standard Error: 683_000 + .saturating_add((3_824_902_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: System Account (r:1 w:0) + // Storage: Contracts ContractInfoOf (r:1 w:1) + // Storage: Contracts CodeStorage (r:1 w:0) + // Storage: Timestamp Now (r:1 w:0) + // Storage: Contracts OwnerInfoOf (r:36 w:36) + fn seal_set_code_hash(r: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 2_302_000 + .saturating_add((922_467_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads((99 as Weight).saturating_mul(r as Weight))) + .saturating_add(RocksDbWeight::get().writes((99 as Weight).saturating_mul(r as Weight))) + } fn instr_i64const(r: u32, ) -> Weight { - (45_937_000 as Weight) - // Standard Error: 10_000 - .saturating_add((1_108_000 as Weight).saturating_mul(r as Weight)) + (74_516_000 as Weight) + // Standard Error: 1_000 + .saturating_add((592_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64load(r: u32, ) -> Weight { - (44_001_000 as Weight) - // Standard Error: 11_000 - .saturating_add((2_412_000 as Weight).saturating_mul(r as Weight)) + (74_430_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_320_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64store(r: u32, ) -> Weight { - (43_157_000 as Weight) - // Standard Error: 12_000 - .saturating_add((2_677_000 as Weight).saturating_mul(r as Weight)) + (74_440_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_428_000 as Weight).saturating_mul(r as Weight)) } fn instr_select(r: u32, ) -> Weight { - (48_475_000 as Weight) - // Standard Error: 8_000 - .saturating_add((2_604_000 as Weight).saturating_mul(r as Weight)) + (74_151_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_782_000 as Weight).saturating_mul(r as Weight)) } fn instr_if(r: u32, ) -> Weight { - (50_649_000 as Weight) - // Standard Error: 12_000 - .saturating_add((2_553_000 as Weight).saturating_mul(r as Weight)) + (74_225_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_887_000 as Weight).saturating_mul(r as Weight)) } fn instr_br(r: u32, ) -> Weight { - (48_433_000 as Weight) - // Standard Error: 8_000 - .saturating_add((1_670_000 as Weight).saturating_mul(r as Weight)) + (73_987_000 as Weight) + // Standard Error: 1_000 + .saturating_add((898_000 as Weight).saturating_mul(r as Weight)) } fn instr_br_if(r: u32, ) -> Weight { - (49_244_000 as Weight) - // Standard Error: 16_000 - .saturating_add((1_946_000 as Weight).saturating_mul(r as Weight)) + (73_305_000 as Weight) + // Standard Error: 4_000 + .saturating_add((1_465_000 as Weight).saturating_mul(r as Weight)) } fn instr_br_table(r: u32, ) -> Weight { - (46_117_000 as Weight) - // Standard Error: 17_000 - .saturating_add((2_387_000 as Weight).saturating_mul(r as Weight)) + (73_037_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_605_000 as Weight).saturating_mul(r as Weight)) } - fn instr_br_table_per_entry(_e: u32, ) -> Weight { - (55_204_000 as Weight) + fn instr_br_table_per_entry(e: u32, ) -> Weight { + (76_434_000 as Weight) + // Standard Error: 0 + .saturating_add((4_000 as Weight).saturating_mul(e as Weight)) } fn instr_call(r: u32, ) -> Weight { - (43_651_000 as Weight) - // Standard Error: 26_000 - .saturating_add((19_163_000 as Weight).saturating_mul(r as Weight)) + (75_461_000 as Weight) + // Standard Error: 10_000 + .saturating_add((7_446_000 as Weight).saturating_mul(r as Weight)) } fn instr_call_indirect(r: u32, ) -> Weight { - (54_063_000 as Weight) - // Standard Error: 32_000 - .saturating_add((27_970_000 as Weight).saturating_mul(r as Weight)) + (87_222_000 as Weight) + // Standard Error: 15_000 + .saturating_add((9_406_000 as Weight).saturating_mul(r as Weight)) } fn instr_call_indirect_per_param(p: u32, ) -> Weight { - (88_527_000 as Weight) - // Standard Error: 6_000 - .saturating_add((958_000 as Weight).saturating_mul(p as Weight)) + (97_204_000 as Weight) + // Standard Error: 1_000 + .saturating_add((472_000 as Weight).saturating_mul(p as Weight)) } fn instr_local_get(r: u32, ) -> Weight { - (55_066_000 as Weight) - // Standard Error: 12_000 - .saturating_add((682_000 as Weight).saturating_mul(r as Weight)) + (75_299_000 as Weight) + // Standard Error: 1_000 + .saturating_add((601_000 as Weight).saturating_mul(r as Weight)) } fn instr_local_set(r: u32, ) -> Weight { - (55_298_000 as Weight) - // Standard Error: 13_000 - .saturating_add((778_000 as Weight).saturating_mul(r as Weight)) + (74_827_000 as Weight) + // Standard Error: 3_000 + .saturating_add((686_000 as Weight).saturating_mul(r as Weight)) } fn instr_local_tee(r: u32, ) -> Weight { - (56_302_000 as Weight) - // Standard Error: 11_000 - .saturating_add((1_079_000 as Weight).saturating_mul(r as Weight)) + (74_624_000 as Weight) + // Standard Error: 2_000 + .saturating_add((895_000 as Weight).saturating_mul(r as Weight)) } fn instr_global_get(r: u32, ) -> Weight { - (71_567_000 as Weight) - // Standard Error: 11_000 - .saturating_add((1_107_000 as Weight).saturating_mul(r as Weight)) + (77_435_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_201_000 as Weight).saturating_mul(r as Weight)) } fn instr_global_set(r: u32, ) -> Weight { - (71_186_000 as Weight) - // Standard Error: 12_000 - .saturating_add((1_151_000 as Weight).saturating_mul(r as Weight)) + (76_693_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_410_000 as Weight).saturating_mul(r as Weight)) } fn instr_memory_current(r: u32, ) -> Weight { - (46_240_000 as Weight) - // Standard Error: 10_000 - .saturating_add((1_044_000 as Weight).saturating_mul(r as Weight)) + (74_244_000 as Weight) + // Standard Error: 1_000 + .saturating_add((660_000 as Weight).saturating_mul(r as Weight)) } fn instr_memory_grow(r: u32, ) -> Weight { - (52_369_000 as Weight) - // Standard Error: 2_508_000 - .saturating_add((615_448_000 as Weight).saturating_mul(r as Weight)) + (73_527_000 as Weight) + // Standard Error: 931_000 + .saturating_add((184_946_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64clz(r: u32, ) -> Weight { - (47_623_000 as Weight) - // Standard Error: 9_000 - .saturating_add((1_583_000 as Weight).saturating_mul(r as Weight)) + (74_181_000 as Weight) + // Standard Error: 6_000 + .saturating_add((906_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64ctz(r: u32, ) -> Weight { - (47_670_000 as Weight) - // Standard Error: 9_000 - .saturating_add((1_583_000 as Weight).saturating_mul(r as Weight)) + (74_339_000 as Weight) + // Standard Error: 1_000 + .saturating_add((896_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64popcnt(r: u32, ) -> Weight { - (47_508_000 as Weight) - // Standard Error: 9_000 - .saturating_add((1_583_000 as Weight).saturating_mul(r as Weight)) + (74_444_000 as Weight) + // Standard Error: 3_000 + .saturating_add((889_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64eqz(r: u32, ) -> Weight { - (48_109_000 as Weight) - // Standard Error: 9_000 - .saturating_add((1_580_000 as Weight).saturating_mul(r as Weight)) + (74_572_000 as Weight) + // Standard Error: 1_000 + .saturating_add((908_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64extendsi32(r: u32, ) -> Weight { - (55_270_000 as Weight) - // Standard Error: 9_000 - .saturating_add((1_102_000 as Weight).saturating_mul(r as Weight)) + (74_349_000 as Weight) + // Standard Error: 2_000 + .saturating_add((881_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64extendui32(r: u32, ) -> Weight { - (55_093_000 as Weight) - // Standard Error: 9_000 - .saturating_add((1_108_000 as Weight).saturating_mul(r as Weight)) + (74_426_000 as Weight) + // Standard Error: 1_000 + .saturating_add((875_000 as Weight).saturating_mul(r as Weight)) } fn instr_i32wrapi64(r: u32, ) -> Weight { - (48_265_000 as Weight) - // Standard Error: 10_000 - .saturating_add((1_573_000 as Weight).saturating_mul(r as Weight)) + (74_172_000 as Weight) + // Standard Error: 2_000 + .saturating_add((906_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64eq(r: u32, ) -> Weight { - (48_733_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_088_000 as Weight).saturating_mul(r as Weight)) + (74_169_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_362_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64ne(r: u32, ) -> Weight { - (48_831_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_085_000 as Weight).saturating_mul(r as Weight)) + (74_205_000 as Weight) + // Standard Error: 4_000 + .saturating_add((1_360_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64lts(r: u32, ) -> Weight { - (49_147_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_056_000 as Weight).saturating_mul(r as Weight)) + (74_237_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_356_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64ltu(r: u32, ) -> Weight { - (49_596_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_049_000 as Weight).saturating_mul(r as Weight)) + (74_181_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_357_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64gts(r: u32, ) -> Weight { - (49_872_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_038_000 as Weight).saturating_mul(r as Weight)) + (74_038_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_360_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64gtu(r: u32, ) -> Weight { - (48_843_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_081_000 as Weight).saturating_mul(r as Weight)) + (73_881_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_372_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64les(r: u32, ) -> Weight { - (48_765_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_089_000 as Weight).saturating_mul(r as Weight)) + (73_969_000 as Weight) + // Standard Error: 0 + .saturating_add((1_361_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64leu(r: u32, ) -> Weight { - (48_720_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_083_000 as Weight).saturating_mul(r as Weight)) + (74_497_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_359_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64ges(r: u32, ) -> Weight { - (48_736_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_097_000 as Weight).saturating_mul(r as Weight)) + (74_275_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_354_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64geu(r: u32, ) -> Weight { - (48_772_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_093_000 as Weight).saturating_mul(r as Weight)) + (74_349_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_357_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64add(r: u32, ) -> Weight { - (48_827_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_082_000 as Weight).saturating_mul(r as Weight)) + (74_192_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_333_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64sub(r: u32, ) -> Weight { - (48_961_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_072_000 as Weight).saturating_mul(r as Weight)) + (74_271_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_340_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64mul(r: u32, ) -> Weight { - (49_069_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_067_000 as Weight).saturating_mul(r as Weight)) + (73_971_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_340_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64divs(r: u32, ) -> Weight { - (49_035_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_677_000 as Weight).saturating_mul(r as Weight)) + (74_546_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_995_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64divu(r: u32, ) -> Weight { - (48_842_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_449_000 as Weight).saturating_mul(r as Weight)) + (74_194_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_050_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64rems(r: u32, ) -> Weight { - (48_536_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_723_000 as Weight).saturating_mul(r as Weight)) + (74_106_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_997_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64remu(r: u32, ) -> Weight { - (48_851_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_432_000 as Weight).saturating_mul(r as Weight)) + (74_219_000 as Weight) + // Standard Error: 5_000 + .saturating_add((2_061_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64and(r: u32, ) -> Weight { - (48_624_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_093_000 as Weight).saturating_mul(r as Weight)) + (74_157_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_338_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64or(r: u32, ) -> Weight { - (49_348_000 as Weight) - // Standard Error: 8_000 - .saturating_add((2_073_000 as Weight).saturating_mul(r as Weight)) + (74_135_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_336_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64xor(r: u32, ) -> Weight { - (49_112_000 as Weight) - // Standard Error: 6_000 - .saturating_add((2_055_000 as Weight).saturating_mul(r as Weight)) + (74_038_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_345_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64shl(r: u32, ) -> Weight { - (49_654_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_051_000 as Weight).saturating_mul(r as Weight)) + (74_011_000 as Weight) + // Standard Error: 3_000 + .saturating_add((1_362_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64shrs(r: u32, ) -> Weight { - (48_848_000 as Weight) - // Standard Error: 8_000 - .saturating_add((2_089_000 as Weight).saturating_mul(r as Weight)) + (74_054_000 as Weight) + // Standard Error: 2_000 + .saturating_add((1_356_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64shru(r: u32, ) -> Weight { - (49_455_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_054_000 as Weight).saturating_mul(r as Weight)) + (73_900_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_358_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64rotl(r: u32, ) -> Weight { - (49_640_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_048_000 as Weight).saturating_mul(r as Weight)) + (73_948_000 as Weight) + // Standard Error: 0 + .saturating_add((1_357_000 as Weight).saturating_mul(r as Weight)) } fn instr_i64rotr(r: u32, ) -> Weight { - (49_498_000 as Weight) - // Standard Error: 7_000 - .saturating_add((2_068_000 as Weight).saturating_mul(r as Weight)) + (73_972_000 as Weight) + // Standard Error: 1_000 + .saturating_add((1_358_000 as Weight).saturating_mul(r as Weight)) } } diff --git a/frame/conviction-voting/Cargo.toml b/frame/conviction-voting/Cargo.toml new file mode 100644 index 000000000000..5877bf0c5da2 --- /dev/null +++ b/frame/conviction-voting/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "pallet-conviction-voting" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet for conviction voting in referenda" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +serde = { version = "1.0.136", optional = true, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ + "derive", +] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +assert_matches = "1.3.0" + +[dev-dependencies] +sp-core = { version = "6.0.0", path = "../../primitives/core" } +pallet-balances = { version = "4.0.0-dev", path = "../balances" } +pallet-scheduler = { version = "4.0.0-dev", path = "../scheduler" } + +[features] +default = ["std"] +std = [ + "serde", + "codec/std", + "scale-info/std", + "sp-std/std", + "sp-io/std", + "frame-benchmarking/std", + "frame-support/std", + "sp-runtime/std", + "frame-system/std", +] +runtime-benchmarks = [ + "frame-benchmarking", + "frame-system/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", +] +try-runtime = ["frame-support/try-runtime"] diff --git a/frame/conviction-voting/README.md b/frame/conviction-voting/README.md new file mode 100644 index 000000000000..5dc5d526d5c2 --- /dev/null +++ b/frame/conviction-voting/README.md @@ -0,0 +1,8 @@ +# Voting Pallet + +- [`assembly::Config`](https://docs.rs/pallet-assembly/latest/pallet_assembly/trait.Config.html) +- [`Call`](https://docs.rs/pallet-assembly/latest/pallet_assembly/enum.Call.html) + +## Overview + +Pallet for voting in referenda. diff --git a/frame/conviction-voting/src/benchmarking.rs b/frame/conviction-voting/src/benchmarking.rs new file mode 100644 index 000000000000..2beee4f3b49d --- /dev/null +++ b/frame/conviction-voting/src/benchmarking.rs @@ -0,0 +1,278 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! ConvictionVoting pallet benchmarking. + +use super::*; + +use assert_matches::assert_matches; +use frame_benchmarking::{account, benchmarks, whitelist_account}; +use frame_support::{ + dispatch::RawOrigin, + traits::{fungible, Currency, Get}, +}; +use sp_runtime::traits::Bounded; +use sp_std::collections::btree_map::BTreeMap; + +use crate::Pallet as ConvictionVoting; + +const SEED: u32 = 0; + +/// Fill all classes as much as possible up to `MaxVotes` and return the Class with the most votes +/// ongoing. +fn fill_voting() -> (ClassOf, BTreeMap, Vec>>) { + let mut r = BTreeMap::, Vec>>::new(); + for class in T::Polls::classes().into_iter() { + for _ in 0..T::MaxVotes::get() { + match T::Polls::create_ongoing(class.clone()) { + Ok(i) => r.entry(class.clone()).or_default().push(i), + Err(()) => break, + } + } + } + let c = r.iter().max_by_key(|(_, ref v)| v.len()).unwrap().0.clone(); + (c, r) +} + +fn funded_account(name: &'static str, index: u32) -> T::AccountId { + let caller: T::AccountId = account(name, index, SEED); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + caller +} + +fn account_vote(b: BalanceOf) -> AccountVote> { + let v = Vote { aye: true, conviction: Conviction::Locked1x }; + + AccountVote::Standard { vote: v, balance: b } +} + +benchmarks! { + where_clause { where T::MaxVotes: core::fmt::Debug } + + vote_new { + let caller = funded_account::("caller", 0); + whitelist_account!(caller); + let account_vote = account_vote::(100u32.into()); + + let (class, all_polls) = fill_voting::(); + let polls = &all_polls[&class]; + let r = polls.len() - 1; + // We need to create existing votes + for i in polls.iter().skip(1) { + ConvictionVoting::::vote(RawOrigin::Signed(caller.clone()).into(), *i, account_vote.clone())?; + } + let votes = match VotingFor::::get(&caller, &class) { + Voting::Casting(Casting { votes, .. }) => votes, + _ => return Err("Votes are not direct".into()), + }; + assert_eq!(votes.len(), r as usize, "Votes were not recorded."); + + let index = polls[0]; + }: vote(RawOrigin::Signed(caller.clone()), index, account_vote) + verify { + assert_matches!( + VotingFor::::get(&caller, &class), + Voting::Casting(Casting { votes, .. }) if votes.len() == (r + 1) as usize + ); + } + + vote_existing { + let caller = funded_account::("caller", 0); + whitelist_account!(caller); + let old_account_vote = account_vote::(100u32.into()); + + let (class, all_polls) = fill_voting::(); + let polls = &all_polls[&class]; + let r = polls.len(); + // We need to create existing votes + for i in polls.iter() { + ConvictionVoting::::vote(RawOrigin::Signed(caller.clone()).into(), *i, old_account_vote.clone())?; + } + let votes = match VotingFor::::get(&caller, &class) { + Voting::Casting(Casting { votes, .. }) => votes, + _ => return Err("Votes are not direct".into()), + }; + assert_eq!(votes.len(), r, "Votes were not recorded."); + + let new_account_vote = account_vote::(200u32.into()); + let index = polls[0]; + }: vote(RawOrigin::Signed(caller.clone()), index, new_account_vote) + verify { + assert_matches!( + VotingFor::::get(&caller, &class), + Voting::Casting(Casting { votes, .. }) if votes.len() == r as usize + ); + } + + remove_vote { + let caller = funded_account::("caller", 0); + whitelist_account!(caller); + let old_account_vote = account_vote::(100u32.into()); + + let (class, all_polls) = fill_voting::(); + let polls = &all_polls[&class]; + let r = polls.len(); + // We need to create existing votes + for i in polls.iter() { + ConvictionVoting::::vote(RawOrigin::Signed(caller.clone()).into(), *i, old_account_vote.clone())?; + } + let votes = match VotingFor::::get(&caller, &class) { + Voting::Casting(Casting { votes, .. }) => votes, + _ => return Err("Votes are not direct".into()), + }; + assert_eq!(votes.len(), r, "Votes were not recorded."); + + let index = polls[0]; + }: _(RawOrigin::Signed(caller.clone()), Some(class.clone()), index) + verify { + assert_matches!( + VotingFor::::get(&caller, &class), + Voting::Casting(Casting { votes, .. }) if votes.len() == (r - 1) as usize + ); + } + + remove_other_vote { + let caller = funded_account::("caller", 0); + let voter = funded_account::("caller", 0); + whitelist_account!(caller); + let old_account_vote = account_vote::(100u32.into()); + + let (class, all_polls) = fill_voting::(); + let polls = &all_polls[&class]; + let r = polls.len(); + // We need to create existing votes + for i in polls.iter() { + ConvictionVoting::::vote(RawOrigin::Signed(voter.clone()).into(), *i, old_account_vote.clone())?; + } + let votes = match VotingFor::::get(&caller, &class) { + Voting::Casting(Casting { votes, .. }) => votes, + _ => return Err("Votes are not direct".into()), + }; + assert_eq!(votes.len(), r, "Votes were not recorded."); + + let index = polls[0]; + assert!(T::Polls::end_ongoing(index, false).is_ok()); + }: _(RawOrigin::Signed(caller.clone()), voter.clone(), class.clone(), index) + verify { + assert_matches!( + VotingFor::::get(&voter, &class), + Voting::Casting(Casting { votes, .. }) if votes.len() == (r - 1) as usize + ); + } + + delegate { + let r in 0 .. T::MaxVotes::get().min(T::Polls::max_ongoing().1); + + let all_polls = fill_voting::().1; + let class = T::Polls::max_ongoing().0; + let polls = &all_polls[&class]; + let voter = funded_account::("voter", 0); + let caller = funded_account::("caller", 0); + whitelist_account!(caller); + + let delegated_balance: BalanceOf = 1000u32.into(); + let delegate_vote = account_vote::(delegated_balance); + + // We need to create existing delegations + for i in polls.iter().take(r as usize) { + ConvictionVoting::::vote(RawOrigin::Signed(voter.clone()).into(), *i, delegate_vote.clone())?; + } + assert_matches!( + VotingFor::::get(&voter, &class), + Voting::Casting(Casting { votes, .. }) if votes.len() == r as usize + ); + + }: _(RawOrigin::Signed(caller.clone()), class.clone(), voter.clone(), Conviction::Locked1x, delegated_balance) + verify { + assert_matches!(VotingFor::::get(&caller, &class), Voting::Delegating(_)); + } + + undelegate { + let r in 0 .. T::MaxVotes::get().min(T::Polls::max_ongoing().1); + + let all_polls = fill_voting::().1; + let class = T::Polls::max_ongoing().0; + let polls = &all_polls[&class]; + let voter = funded_account::("voter", 0); + let caller = funded_account::("caller", 0); + whitelist_account!(caller); + + let delegated_balance: BalanceOf = 1000u32.into(); + let delegate_vote = account_vote::(delegated_balance); + + ConvictionVoting::::delegate( + RawOrigin::Signed(caller.clone()).into(), + class.clone(), + voter.clone(), + Conviction::Locked1x, + delegated_balance, + )?; + + // We need to create delegations + for i in polls.iter().take(r as usize) { + ConvictionVoting::::vote(RawOrigin::Signed(voter.clone()).into(), *i, delegate_vote.clone())?; + } + assert_matches!( + VotingFor::::get(&voter, &class), + Voting::Casting(Casting { votes, .. }) if votes.len() == r as usize + ); + assert_matches!(VotingFor::::get(&caller, &class), Voting::Delegating(_)); + }: _(RawOrigin::Signed(caller.clone()), class.clone()) + verify { + assert_matches!(VotingFor::::get(&caller, &class), Voting::Casting(_)); + } + + unlock { + let caller = funded_account::("caller", 0); + whitelist_account!(caller); + let normal_account_vote = account_vote::(T::Currency::free_balance(&caller) - 100u32.into()); + let big_account_vote = account_vote::(T::Currency::free_balance(&caller)); + + // Fill everything up to the max by filling all classes with votes and voting on them all. + let (class, all_polls) = fill_voting::(); + assert!(all_polls.len() > 0); + for (class, polls) in all_polls.iter() { + assert!(polls.len() > 0); + for i in polls.iter() { + ConvictionVoting::::vote(RawOrigin::Signed(caller.clone()).into(), *i, normal_account_vote.clone())?; + } + } + + let orig_usable = >::reducible_balance(&caller, false); + let polls = &all_polls[&class]; + + // Vote big on the class with the most ongoing votes of them to bump the lock and make it + // hard to recompute when removed. + ConvictionVoting::::vote(RawOrigin::Signed(caller.clone()).into(), polls[0], big_account_vote.clone())?; + let now_usable = >::reducible_balance(&caller, false); + assert_eq!(orig_usable - now_usable, 100u32.into()); + + // Remove the vote + ConvictionVoting::::remove_vote(RawOrigin::Signed(caller.clone()).into(), Some(class.clone()), polls[0])?; + + // We can now unlock on `class` from 200 to 100... + }: _(RawOrigin::Signed(caller.clone()), class, caller.clone()) + verify { + assert_eq!(orig_usable, >::reducible_balance(&caller, false)); + } + + impl_benchmark_test_suite!( + ConvictionVoting, + crate::tests::new_test_ext(), + crate::tests::Test + ); +} diff --git a/frame/conviction-voting/src/conviction.rs b/frame/conviction-voting/src/conviction.rs new file mode 100644 index 000000000000..1feff35b19fc --- /dev/null +++ b/frame/conviction-voting/src/conviction.rs @@ -0,0 +1,131 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The conviction datatype. + +use codec::{Decode, Encode, MaxEncodedLen}; +use scale_info::TypeInfo; +use sp_runtime::{ + traits::{Bounded, CheckedDiv, CheckedMul, Zero}, + RuntimeDebug, +}; + +use crate::types::Delegations; + +/// A value denoting the strength of conviction of a vote. +#[derive( + Encode, + Decode, + Copy, + Clone, + Eq, + PartialEq, + Ord, + PartialOrd, + RuntimeDebug, + TypeInfo, + MaxEncodedLen, +)] +pub enum Conviction { + /// 0.1x votes, unlocked. + None, + /// 1x votes, locked for an enactment period following a successful vote. + Locked1x, + /// 2x votes, locked for 2x enactment periods following a successful vote. + Locked2x, + /// 3x votes, locked for 4x... + Locked3x, + /// 4x votes, locked for 8x... + Locked4x, + /// 5x votes, locked for 16x... + Locked5x, + /// 6x votes, locked for 32x... + Locked6x, +} + +impl Default for Conviction { + fn default() -> Self { + Conviction::None + } +} + +impl From for u8 { + fn from(c: Conviction) -> u8 { + match c { + Conviction::None => 0, + Conviction::Locked1x => 1, + Conviction::Locked2x => 2, + Conviction::Locked3x => 3, + Conviction::Locked4x => 4, + Conviction::Locked5x => 5, + Conviction::Locked6x => 6, + } + } +} + +impl TryFrom for Conviction { + type Error = (); + fn try_from(i: u8) -> Result { + Ok(match i { + 0 => Conviction::None, + 1 => Conviction::Locked1x, + 2 => Conviction::Locked2x, + 3 => Conviction::Locked3x, + 4 => Conviction::Locked4x, + 5 => Conviction::Locked5x, + 6 => Conviction::Locked6x, + _ => return Err(()), + }) + } +} + +impl Conviction { + /// The amount of time (in number of periods) that our conviction implies a successful voter's + /// balance should be locked for. + pub fn lock_periods(self) -> u32 { + match self { + Conviction::None => 0, + Conviction::Locked1x => 1, + Conviction::Locked2x => 2, + Conviction::Locked3x => 4, + Conviction::Locked4x => 8, + Conviction::Locked5x => 16, + Conviction::Locked6x => 32, + } + } + + /// The votes of a voter of the given `balance` with our conviction. + pub fn votes + Zero + Copy + CheckedMul + CheckedDiv + Bounded>( + self, + capital: B, + ) -> Delegations { + let votes = match self { + Conviction::None => capital.checked_div(&10u8.into()).unwrap_or_else(Zero::zero), + x => capital.checked_mul(&u8::from(x).into()).unwrap_or_else(B::max_value), + }; + Delegations { votes, capital } + } +} + +impl Bounded for Conviction { + fn min_value() -> Self { + Conviction::None + } + fn max_value() -> Self { + Conviction::Locked6x + } +} diff --git a/frame/conviction-voting/src/lib.rs b/frame/conviction-voting/src/lib.rs new file mode 100644 index 000000000000..af91e99fb379 --- /dev/null +++ b/frame/conviction-voting/src/lib.rs @@ -0,0 +1,649 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Voting Pallet +//! +//! - [`Config`] +//! - [`Call`] +//! +//! ## Overview +//! +//! Pallet for managing actual voting in polls. + +#![recursion_limit = "256"] +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_support::{ + dispatch::{DispatchError, DispatchResult}, + ensure, + traits::{ + fungible, Currency, Get, LockIdentifier, LockableCurrency, PollStatus, Polling, + ReservableCurrency, WithdrawReasons, + }, +}; +use sp_runtime::{ + traits::{AtLeast32BitUnsigned, Saturating, Zero}, + ArithmeticError, Perbill, +}; +use sp_std::prelude::*; + +mod conviction; +mod types; +mod vote; +pub mod weights; + +pub use self::{ + conviction::Conviction, + pallet::*, + types::{Delegations, Tally, UnvoteScope}, + vote::{AccountVote, Casting, Delegating, Vote, Voting}, + weights::WeightInfo, +}; + +#[cfg(test)] +mod tests; + +#[cfg(feature = "runtime-benchmarks")] +pub mod benchmarking; + +const CONVICTION_VOTING_ID: LockIdentifier = *b"pyconvot"; + +type BalanceOf = + <>::Currency as Currency<::AccountId>>::Balance; +type VotingOf = Voting< + BalanceOf, + ::AccountId, + ::BlockNumber, + PollIndexOf, + >::MaxVotes, +>; +#[allow(dead_code)] +type DelegatingOf = Delegating< + BalanceOf, + ::AccountId, + ::BlockNumber, +>; +pub type TallyOf = Tally, >::MaxTurnout>; +pub type VotesOf = BalanceOf; +type PollIndexOf = <>::Polls as Polling>>::Index; +#[cfg(feature = "runtime-benchmarks")] +type IndexOf = <>::Polls as Polling>>::Index; +type ClassOf = <>::Polls as Polling>>::Class; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config + Sized { + // System level stuff. + type Event: From> + IsType<::Event>; + /// Weight information for extrinsics in this pallet. + type WeightInfo: WeightInfo; + /// Currency type with which voting happens. + type Currency: ReservableCurrency + + LockableCurrency + + fungible::Inspect; + + /// The implementation of the logic which conducts polls. + type Polls: Polling< + TallyOf, + Votes = BalanceOf, + Moment = Self::BlockNumber, + >; + + /// The maximum amount of tokens which may be used for voting. May just be + /// `Currency::total_issuance`, but you might want to reduce this in order to account for + /// funds in the system which are unable to vote (e.g. parachain auction deposits). + type MaxTurnout: Get>; + + /// The maximum number of concurrent votes an account may have. + /// + /// Also used to compute weight, an overly large value can + /// lead to extrinsic with large weight estimation: see `delegate` for instance. + #[pallet::constant] + type MaxVotes: Get; + + /// The minimum period of vote locking. + /// + /// It should be no shorter than enactment period to ensure that in the case of an approval, + /// those successful voters are locked into the consequences that their votes entail. + #[pallet::constant] + type VoteLockingPeriod: Get; + } + + /// All voting for a particular voter in a particular voting class. We store the balance for the + /// number of votes that we have recorded. + #[pallet::storage] + pub type VotingFor, I: 'static = ()> = StorageDoubleMap< + _, + Twox64Concat, + T::AccountId, + Twox64Concat, + ClassOf, + VotingOf, + ValueQuery, + >; + + /// The voting classes which have a non-zero lock requirement and the lock amounts which they + /// require. The actual amount locked on behalf of this pallet should always be the maximum of + /// this list. + #[pallet::storage] + pub type ClassLocksFor, I: 'static = ()> = StorageMap< + _, + Twox64Concat, + T::AccountId, + Vec<(ClassOf, BalanceOf)>, + ValueQuery, + >; + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event, I: 'static = ()> { + /// An account has delegated their vote to another account. \[who, target\] + Delegated(T::AccountId, T::AccountId), + /// An \[account\] has cancelled a previous delegation operation. + Undelegated(T::AccountId), + } + + #[pallet::error] + pub enum Error { + /// Poll is not ongoing. + NotOngoing, + /// The given account did not vote on the poll. + NotVoter, + /// The actor has no permission to conduct the action. + NoPermission, + /// The actor has no permission to conduct the action right now but will do in the future. + NoPermissionYet, + /// The account is already delegating. + AlreadyDelegating, + /// The account currently has votes attached to it and the operation cannot succeed until + /// these are removed, either through `unvote` or `reap_vote`. + AlreadyVoting, + /// Too high a balance was provided that the account cannot afford. + InsufficientFunds, + /// The account is not currently delegating. + NotDelegating, + /// Delegation to oneself makes no sense. + Nonsense, + /// Maximum number of votes reached. + MaxVotesReached, + /// The class must be supplied since it is not easily determinable from the state. + ClassNeeded, + /// The class ID supplied is invalid. + BadClass, + } + + #[pallet::call] + impl, I: 'static> Pallet { + /// Vote in a poll. If `vote.is_aye()`, the vote is to enact the proposal; + /// otherwise it is a vote to keep the status quo. + /// + /// The dispatch origin of this call must be _Signed_. + /// + /// - `poll_index`: The index of the poll to vote for. + /// - `vote`: The vote configuration. + /// + /// Weight: `O(R)` where R is the number of polls the voter has voted on. + #[pallet::weight(T::WeightInfo::vote_new().max(T::WeightInfo::vote_existing()))] + pub fn vote( + origin: OriginFor, + #[pallet::compact] poll_index: PollIndexOf, + vote: AccountVote>, + ) -> DispatchResult { + let who = ensure_signed(origin)?; + Self::try_vote(&who, poll_index, vote) + } + + /// Delegate the voting power (with some given conviction) of the sending account for a + /// particular class of polls. + /// + /// The balance delegated is locked for as long as it's delegated, and thereafter for the + /// time appropriate for the conviction's lock period. + /// + /// The dispatch origin of this call must be _Signed_, and the signing account must either: + /// - be delegating already; or + /// - have no voting activity (if there is, then it will need to be removed/consolidated + /// through `reap_vote` or `unvote`). + /// + /// - `to`: The account whose voting the `target` account's voting power will follow. + /// - `class`: The class of polls to delegate. To delegate multiple classes, multiple calls + /// to this function are required. + /// - `conviction`: The conviction that will be attached to the delegated votes. When the + /// account is undelegated, the funds will be locked for the corresponding period. + /// - `balance`: The amount of the account's balance to be used in delegating. This must not + /// be more than the account's current balance. + /// + /// Emits `Delegated`. + /// + /// Weight: `O(R)` where R is the number of polls the voter delegating to has + /// voted on. Weight is initially charged as if maximum votes, but is refunded later. + // NOTE: weight must cover an incorrect voting of origin with max votes, this is ensure + // because a valid delegation cover decoding a direct voting with max votes. + #[pallet::weight(T::WeightInfo::delegate(T::MaxVotes::get()))] + pub fn delegate( + origin: OriginFor, + class: ClassOf, + to: T::AccountId, + conviction: Conviction, + balance: BalanceOf, + ) -> DispatchResultWithPostInfo { + let who = ensure_signed(origin)?; + let votes = Self::try_delegate(who, class, to, conviction, balance)?; + + Ok(Some(T::WeightInfo::delegate(votes)).into()) + } + + /// Undelegate the voting power of the sending account for a particular class of polls. + /// + /// Tokens may be unlocked following once an amount of time consistent with the lock period + /// of the conviction with which the delegation was issued. + /// + /// The dispatch origin of this call must be _Signed_ and the signing account must be + /// currently delegating. + /// + /// - `class`: The class of polls to remove the delegation from. + /// + /// Emits `Undelegated`. + /// + /// Weight: `O(R)` where R is the number of polls the voter delegating to has + /// voted on. Weight is initially charged as if maximum votes, but is refunded later. + // NOTE: weight must cover an incorrect voting of origin with max votes, this is ensure + // because a valid delegation cover decoding a direct voting with max votes. + #[pallet::weight(T::WeightInfo::undelegate(T::MaxVotes::get().into()))] + pub fn undelegate( + origin: OriginFor, + class: ClassOf, + ) -> DispatchResultWithPostInfo { + let who = ensure_signed(origin)?; + let votes = Self::try_undelegate(who, class)?; + Ok(Some(T::WeightInfo::undelegate(votes)).into()) + } + + /// Remove the lock caused prior voting/delegating which has expired within a particluar + /// class. + /// + /// The dispatch origin of this call must be _Signed_. + /// + /// - `class`: The class of polls to unlock. + /// - `target`: The account to remove the lock on. + /// + /// Weight: `O(R)` with R number of vote of target. + #[pallet::weight(T::WeightInfo::unlock())] + pub fn unlock( + origin: OriginFor, + class: ClassOf, + target: T::AccountId, + ) -> DispatchResult { + ensure_signed(origin)?; + Self::update_lock(&class, &target); + Ok(()) + } + + /// Remove a vote for a poll. + /// + /// If: + /// - the poll was cancelled, or + /// - the poll is ongoing, or + /// - the poll has ended such that + /// - the vote of the account was in opposition to the result; or + /// - there was no conviction to the account's vote; or + /// - the account made a split vote + /// ...then the vote is removed cleanly and a following call to `unlock` may result in more + /// funds being available. + /// + /// If, however, the poll has ended and: + /// - it finished corresponding to the vote of the account, and + /// - the account made a standard vote with conviction, and + /// - the lock period of the conviction is not over + /// ...then the lock will be aggregated into the overall account's lock, which may involve + /// *overlocking* (where the two locks are combined into a single lock that is the maximum + /// of both the amount locked and the time is it locked for). + /// + /// The dispatch origin of this call must be _Signed_, and the signer must have a vote + /// registered for poll `index`. + /// + /// - `index`: The index of poll of the vote to be removed. + /// - `class`: Optional parameter, if given it indicates the class of the poll. For polls + /// which have finished or are cancelled, this must be `Some`. + /// + /// Weight: `O(R + log R)` where R is the number of polls that `target` has voted on. + /// Weight is calculated for the maximum number of vote. + #[pallet::weight(T::WeightInfo::remove_vote())] + pub fn remove_vote( + origin: OriginFor, + class: Option>, + index: PollIndexOf, + ) -> DispatchResult { + let who = ensure_signed(origin)?; + Self::try_remove_vote(&who, index, class, UnvoteScope::Any) + } + + /// Remove a vote for a poll. + /// + /// If the `target` is equal to the signer, then this function is exactly equivalent to + /// `remove_vote`. If not equal to the signer, then the vote must have expired, + /// either because the poll was cancelled, because the voter lost the poll or + /// because the conviction period is over. + /// + /// The dispatch origin of this call must be _Signed_. + /// + /// - `target`: The account of the vote to be removed; this account must have voted for poll + /// `index`. + /// - `index`: The index of poll of the vote to be removed. + /// - `class`: The class of the poll. + /// + /// Weight: `O(R + log R)` where R is the number of polls that `target` has voted on. + /// Weight is calculated for the maximum number of vote. + #[pallet::weight(T::WeightInfo::remove_other_vote())] + pub fn remove_other_vote( + origin: OriginFor, + target: T::AccountId, + class: ClassOf, + index: PollIndexOf, + ) -> DispatchResult { + let who = ensure_signed(origin)?; + let scope = if target == who { UnvoteScope::Any } else { UnvoteScope::OnlyExpired }; + Self::try_remove_vote(&target, index, Some(class), scope)?; + Ok(()) + } + } +} + +impl, I: 'static> Pallet { + /// Actually enact a vote, if legit. + fn try_vote( + who: &T::AccountId, + poll_index: PollIndexOf, + vote: AccountVote>, + ) -> DispatchResult { + ensure!(vote.balance() <= T::Currency::free_balance(who), Error::::InsufficientFunds); + T::Polls::try_access_poll(poll_index, |poll_status| { + let (tally, class) = poll_status.ensure_ongoing().ok_or(Error::::NotOngoing)?; + VotingFor::::try_mutate(who, &class, |voting| { + if let Voting::Casting(Casting { ref mut votes, delegations, .. }) = voting { + match votes.binary_search_by_key(&poll_index, |i| i.0) { + Ok(i) => { + // Shouldn't be possible to fail, but we handle it gracefully. + tally.remove(votes[i].1).ok_or(ArithmeticError::Underflow)?; + if let Some(approve) = votes[i].1.as_standard() { + tally.reduce(approve, *delegations); + } + votes[i].1 = vote; + }, + Err(i) => { + votes + .try_insert(i, (poll_index, vote)) + .map_err(|()| Error::::MaxVotesReached)?; + }, + } + // Shouldn't be possible to fail, but we handle it gracefully. + tally.add(vote).ok_or(ArithmeticError::Overflow)?; + if let Some(approve) = vote.as_standard() { + tally.increase(approve, *delegations); + } + } else { + return Err(Error::::AlreadyDelegating.into()) + } + // Extend the lock to `balance` (rather than setting it) since we don't know what + // other votes are in place. + Self::extend_lock(who, &class, vote.balance()); + Ok(()) + }) + }) + } + + /// Remove the account's vote for the given poll if possible. This is possible when: + /// - The poll has not finished. + /// - The poll has finished and the voter lost their direction. + /// - The poll has finished and the voter's lock period is up. + /// + /// This will generally be combined with a call to `unlock`. + fn try_remove_vote( + who: &T::AccountId, + poll_index: PollIndexOf, + class_hint: Option>, + scope: UnvoteScope, + ) -> DispatchResult { + let class = class_hint + .or_else(|| Some(T::Polls::as_ongoing(poll_index)?.1)) + .ok_or(Error::::ClassNeeded)?; + VotingFor::::try_mutate(who, class, |voting| { + if let Voting::Casting(Casting { ref mut votes, delegations, ref mut prior }) = voting { + let i = votes + .binary_search_by_key(&poll_index, |i| i.0) + .map_err(|_| Error::::NotVoter)?; + let v = votes.remove(i); + + T::Polls::try_access_poll(poll_index, |poll_status| match poll_status { + PollStatus::Ongoing(tally, _) => { + ensure!(matches!(scope, UnvoteScope::Any), Error::::NoPermission); + // Shouldn't be possible to fail, but we handle it gracefully. + tally.remove(v.1).ok_or(ArithmeticError::Underflow)?; + if let Some(approve) = v.1.as_standard() { + tally.reduce(approve, *delegations); + } + Ok(()) + }, + PollStatus::Completed(end, approved) => { + if let Some((lock_periods, balance)) = v.1.locked_if(approved) { + let unlock_at = end.saturating_add( + T::VoteLockingPeriod::get().saturating_mul(lock_periods.into()), + ); + let now = frame_system::Pallet::::block_number(); + if now < unlock_at { + ensure!( + matches!(scope, UnvoteScope::Any), + Error::::NoPermissionYet + ); + prior.accumulate(unlock_at, balance) + } + } + Ok(()) + }, + PollStatus::None => Ok(()), // Poll was cancelled. + }) + } else { + Ok(()) + } + }) + } + + /// Return the number of votes for `who` + fn increase_upstream_delegation( + who: &T::AccountId, + class: &ClassOf, + amount: Delegations>, + ) -> u32 { + VotingFor::::mutate(who, class, |voting| match voting { + Voting::Delegating(Delegating { delegations, .. }) => { + // We don't support second level delegating, so we don't need to do anything more. + *delegations = delegations.saturating_add(amount); + 1 + }, + Voting::Casting(Casting { votes, delegations, .. }) => { + *delegations = delegations.saturating_add(amount); + for &(poll_index, account_vote) in votes.iter() { + if let AccountVote::Standard { vote, .. } = account_vote { + T::Polls::access_poll(poll_index, |poll_status| { + if let PollStatus::Ongoing(tally, _) = poll_status { + tally.increase(vote.aye, amount); + } + }); + } + } + votes.len() as u32 + }, + }) + } + + /// Return the number of votes for `who` + fn reduce_upstream_delegation( + who: &T::AccountId, + class: &ClassOf, + amount: Delegations>, + ) -> u32 { + VotingFor::::mutate(who, class, |voting| match voting { + Voting::Delegating(Delegating { delegations, .. }) => { + // We don't support second level delegating, so we don't need to do anything more. + *delegations = delegations.saturating_sub(amount); + 1 + }, + Voting::Casting(Casting { votes, delegations, .. }) => { + *delegations = delegations.saturating_sub(amount); + for &(poll_index, account_vote) in votes.iter() { + if let AccountVote::Standard { vote, .. } = account_vote { + T::Polls::access_poll(poll_index, |poll_status| { + if let PollStatus::Ongoing(tally, _) = poll_status { + tally.reduce(vote.aye, amount); + } + }); + } + } + votes.len() as u32 + }, + }) + } + + /// Attempt to delegate `balance` times `conviction` of voting power from `who` to `target`. + /// + /// Return the upstream number of votes. + fn try_delegate( + who: T::AccountId, + class: ClassOf, + target: T::AccountId, + conviction: Conviction, + balance: BalanceOf, + ) -> Result { + ensure!(who != target, Error::::Nonsense); + T::Polls::classes().binary_search(&class).map_err(|_| Error::::BadClass)?; + ensure!(balance <= T::Currency::free_balance(&who), Error::::InsufficientFunds); + let votes = + VotingFor::::try_mutate(&who, &class, |voting| -> Result { + let old = sp_std::mem::replace( + voting, + Voting::Delegating(Delegating { + balance, + target: target.clone(), + conviction, + delegations: Default::default(), + prior: Default::default(), + }), + ); + match old { + Voting::Delegating(Delegating { .. }) => Err(Error::::AlreadyDelegating)?, + Voting::Casting(Casting { votes, delegations, prior }) => { + // here we just ensure that we're currently idling with no votes recorded. + ensure!(votes.is_empty(), Error::::AlreadyVoting); + voting.set_common(delegations, prior); + }, + } + + let votes = + Self::increase_upstream_delegation(&target, &class, conviction.votes(balance)); + // Extend the lock to `balance` (rather than setting it) since we don't know what + // other votes are in place. + Self::extend_lock(&who, &class, balance); + Ok(votes) + })?; + Self::deposit_event(Event::::Delegated(who, target)); + Ok(votes) + } + + /// Attempt to end the current delegation. + /// + /// Return the number of votes of upstream. + fn try_undelegate(who: T::AccountId, class: ClassOf) -> Result { + let votes = + VotingFor::::try_mutate(&who, &class, |voting| -> Result { + match sp_std::mem::replace(voting, Voting::default()) { + Voting::Delegating(Delegating { + balance, + target, + conviction, + delegations, + mut prior, + }) => { + // remove any delegation votes to our current target. + let votes = Self::reduce_upstream_delegation( + &target, + &class, + conviction.votes(balance), + ); + let now = frame_system::Pallet::::block_number(); + let lock_periods = conviction.lock_periods().into(); + prior.accumulate( + now.saturating_add( + T::VoteLockingPeriod::get().saturating_mul(lock_periods), + ), + balance, + ); + voting.set_common(delegations, prior); + + Ok(votes) + }, + Voting::Casting(_) => Err(Error::::NotDelegating.into()), + } + })?; + Self::deposit_event(Event::::Undelegated(who)); + Ok(votes) + } + + fn extend_lock(who: &T::AccountId, class: &ClassOf, amount: BalanceOf) { + ClassLocksFor::::mutate(who, |locks| { + match locks.iter().position(|x| &x.0 == class) { + Some(i) => locks[i].1 = locks[i].1.max(amount), + None => locks.push((class.clone(), amount)), + } + }); + T::Currency::extend_lock(CONVICTION_VOTING_ID, who, amount, WithdrawReasons::TRANSFER); + } + + /// Rejig the lock on an account. It will never get more stringent (since that would indicate + /// a security hole) but may be reduced from what they are currently. + fn update_lock(class: &ClassOf, who: &T::AccountId) { + let class_lock_needed = VotingFor::::mutate(who, class, |voting| { + voting.rejig(frame_system::Pallet::::block_number()); + voting.locked_balance() + }); + let lock_needed = ClassLocksFor::::mutate(who, |locks| { + locks.retain(|x| &x.0 != class); + if !class_lock_needed.is_zero() { + locks.push((class.clone(), class_lock_needed)); + } + locks.iter().map(|x| x.1).max().unwrap_or(Zero::zero()) + }); + if lock_needed.is_zero() { + T::Currency::remove_lock(CONVICTION_VOTING_ID, who); + } else { + T::Currency::set_lock( + CONVICTION_VOTING_ID, + who, + lock_needed, + WithdrawReasons::TRANSFER, + ); + } + } +} diff --git a/frame/conviction-voting/src/tests.rs b/frame/conviction-voting/src/tests.rs new file mode 100644 index 000000000000..6a8bad5d8944 --- /dev/null +++ b/frame/conviction-voting/src/tests.rs @@ -0,0 +1,821 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The crate's tests. + +use std::collections::BTreeMap; + +use frame_support::{ + assert_noop, assert_ok, parameter_types, + traits::{ConstU32, ConstU64, Contains, Polling}, +}; +use sp_core::H256; +use sp_runtime::{ + testing::Header, + traits::{BlakeTwo256, IdentityLookup}, +}; + +use super::*; +use crate as pallet_conviction_voting; + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; + +frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, + Voting: pallet_conviction_voting::{Pallet, Call, Storage, Event}, + } +); + +// Test that a fitlered call can be dispatched. +pub struct BaseFilter; +impl Contains for BaseFilter { + fn contains(call: &Call) -> bool { + !matches!(call, &Call::Balances(pallet_balances::Call::set_balance { .. })) + } +} + +parameter_types! { + pub BlockWeights: frame_system::limits::BlockWeights = + frame_system::limits::BlockWeights::simple_max(1_000_000); +} +impl frame_system::Config for Test { + type BaseCallFilter = BaseFilter; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Call = Call; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = ConstU64<250>; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = ConstU32<16>; +} + +impl pallet_balances::Config for Test { + type MaxReserves = (); + type ReserveIdentifier = [u8; 8]; + type MaxLocks = ConstU32<10>; + type Balance = u64; + type Event = Event; + type DustRemoval = (); + type ExistentialDeposit = ConstU64<1>; + type AccountStore = System; + type WeightInfo = (); +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum TestPollState { + Ongoing(TallyOf, u8), + Completed(u64, bool), +} +use TestPollState::*; + +parameter_types! { + pub static Polls: BTreeMap = vec![ + (1, Completed(1, true)), + (2, Completed(2, false)), + (3, Ongoing(Tally::from_parts(0, 0, 0), 0)), + ].into_iter().collect(); +} + +pub struct TestPolls; +impl Polling> for TestPolls { + type Index = u8; + type Votes = u64; + type Moment = u64; + type Class = u8; + fn classes() -> Vec { + vec![0, 1, 2] + } + fn as_ongoing(index: u8) -> Option<(TallyOf, Self::Class)> { + Polls::get().remove(&index).and_then(|x| { + if let TestPollState::Ongoing(t, c) = x { + Some((t, c)) + } else { + None + } + }) + } + fn access_poll( + index: Self::Index, + f: impl FnOnce(PollStatus<&mut TallyOf, u64, u8>) -> R, + ) -> R { + let mut polls = Polls::get(); + let entry = polls.get_mut(&index); + let r = match entry { + Some(Ongoing(ref mut tally_mut_ref, class)) => + f(PollStatus::Ongoing(tally_mut_ref, *class)), + Some(Completed(when, succeeded)) => f(PollStatus::Completed(*when, *succeeded)), + None => f(PollStatus::None), + }; + Polls::set(polls); + r + } + fn try_access_poll( + index: Self::Index, + f: impl FnOnce(PollStatus<&mut TallyOf, u64, u8>) -> Result, + ) -> Result { + let mut polls = Polls::get(); + let entry = polls.get_mut(&index); + let r = match entry { + Some(Ongoing(ref mut tally_mut_ref, class)) => + f(PollStatus::Ongoing(tally_mut_ref, *class)), + Some(Completed(when, succeeded)) => f(PollStatus::Completed(*when, *succeeded)), + None => f(PollStatus::None), + }?; + Polls::set(polls); + Ok(r) + } + + #[cfg(feature = "runtime-benchmarks")] + fn create_ongoing(class: Self::Class) -> Result { + let mut polls = Polls::get(); + let i = polls.keys().rev().next().map_or(0, |x| x + 1); + polls.insert(i, Ongoing(Tally::default(), class)); + Polls::set(polls); + Ok(i) + } + + #[cfg(feature = "runtime-benchmarks")] + fn end_ongoing(index: Self::Index, approved: bool) -> Result<(), ()> { + let mut polls = Polls::get(); + match polls.get(&index) { + Some(Ongoing(..)) => {}, + _ => return Err(()), + } + let now = frame_system::Pallet::::block_number(); + polls.insert(index, Completed(now, approved)); + Polls::set(polls); + Ok(()) + } +} + +impl Config for Test { + type Event = Event; + type Currency = pallet_balances::Pallet; + type VoteLockingPeriod = ConstU64<3>; + type MaxVotes = ConstU32<3>; + type WeightInfo = (); + type MaxTurnout = frame_support::traits::TotalIssuanceOf; + type Polls = TestPolls; +} + +pub fn new_test_ext() -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + pallet_balances::GenesisConfig:: { + balances: vec![(1, 10), (2, 20), (3, 30), (4, 40), (5, 50), (6, 60)], + } + .assimilate_storage(&mut t) + .unwrap(); + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext +} + +#[test] +fn params_should_work() { + new_test_ext().execute_with(|| { + assert_eq!(Balances::free_balance(42), 0); + assert_eq!(Balances::total_issuance(), 210); + }); +} + +fn next_block() { + System::set_block_number(System::block_number() + 1); +} + +#[allow(dead_code)] +fn run_to(n: u64) { + while System::block_number() < n { + next_block(); + } +} + +fn aye(amount: u64, conviction: u8) -> AccountVote { + let vote = Vote { aye: true, conviction: conviction.try_into().unwrap() }; + AccountVote::Standard { vote, balance: amount } +} + +fn nay(amount: u64, conviction: u8) -> AccountVote { + let vote = Vote { aye: false, conviction: conviction.try_into().unwrap() }; + AccountVote::Standard { vote, balance: amount } +} + +fn tally(index: u8) -> TallyOf { + >>::as_ongoing(index).expect("No poll").0 +} + +fn class(index: u8) -> u8 { + >>::as_ongoing(index).expect("No poll").1 +} + +#[test] +#[ignore] +#[should_panic(expected = "No poll")] +fn unknown_poll_should_panic() { + let _ = tally(0); +} + +#[test] +#[ignore] +#[should_panic(expected = "No poll")] +fn completed_poll_should_panic() { + let _ = tally(1); +} + +#[test] +fn basic_stuff() { + new_test_ext().execute_with(|| { + assert_eq!(tally(3), Tally::from_parts(0, 0, 0)); + }); +} + +#[test] +fn basic_voting_works() { + new_test_ext().execute_with(|| { + assert_ok!(Voting::vote(Origin::signed(1), 3, aye(2, 5))); + assert_eq!(tally(3), Tally::from_parts(10, 0, 2)); + assert_ok!(Voting::vote(Origin::signed(1), 3, nay(2, 5))); + assert_eq!(tally(3), Tally::from_parts(0, 10, 2)); + assert_eq!(Balances::usable_balance(1), 8); + + assert_ok!(Voting::vote(Origin::signed(1), 3, aye(5, 1))); + assert_eq!(tally(3), Tally::from_parts(5, 0, 5)); + assert_ok!(Voting::vote(Origin::signed(1), 3, nay(5, 1))); + assert_eq!(tally(3), Tally::from_parts(0, 5, 5)); + assert_eq!(Balances::usable_balance(1), 5); + + assert_ok!(Voting::vote(Origin::signed(1), 3, aye(10, 0))); + assert_eq!(tally(3), Tally::from_parts(1, 0, 10)); + assert_ok!(Voting::vote(Origin::signed(1), 3, nay(10, 0))); + assert_eq!(tally(3), Tally::from_parts(0, 1, 10)); + assert_eq!(Balances::usable_balance(1), 0); + + assert_ok!(Voting::remove_vote(Origin::signed(1), None, 3)); + assert_eq!(tally(3), Tally::from_parts(0, 0, 0)); + + assert_ok!(Voting::unlock(Origin::signed(1), class(3), 1)); + assert_eq!(Balances::usable_balance(1), 10); + }); +} + +#[test] +fn voting_balance_gets_locked() { + new_test_ext().execute_with(|| { + assert_ok!(Voting::vote(Origin::signed(1), 3, aye(2, 5))); + assert_eq!(tally(3), Tally::from_parts(10, 0, 2)); + assert_ok!(Voting::vote(Origin::signed(1), 3, nay(2, 5))); + assert_eq!(tally(3), Tally::from_parts(0, 10, 2)); + assert_eq!(Balances::usable_balance(1), 8); + + assert_ok!(Voting::vote(Origin::signed(1), 3, aye(5, 1))); + assert_eq!(tally(3), Tally::from_parts(5, 0, 5)); + assert_ok!(Voting::vote(Origin::signed(1), 3, nay(5, 1))); + assert_eq!(tally(3), Tally::from_parts(0, 5, 5)); + assert_eq!(Balances::usable_balance(1), 5); + + assert_ok!(Voting::vote(Origin::signed(1), 3, aye(10, 0))); + assert_eq!(tally(3), Tally::from_parts(1, 0, 10)); + assert_ok!(Voting::vote(Origin::signed(1), 3, nay(10, 0))); + assert_eq!(tally(3), Tally::from_parts(0, 1, 10)); + assert_eq!(Balances::usable_balance(1), 0); + + assert_ok!(Voting::remove_vote(Origin::signed(1), None, 3)); + assert_eq!(tally(3), Tally::from_parts(0, 0, 0)); + + assert_ok!(Voting::unlock(Origin::signed(1), class(3), 1)); + assert_eq!(Balances::usable_balance(1), 10); + }); +} + +#[test] +fn successful_but_zero_conviction_vote_balance_can_be_unlocked() { + new_test_ext().execute_with(|| { + assert_ok!(Voting::vote(Origin::signed(1), 3, aye(1, 1))); + assert_ok!(Voting::vote(Origin::signed(2), 3, nay(20, 0))); + let c = class(3); + Polls::set(vec![(3, Completed(3, false))].into_iter().collect()); + assert_ok!(Voting::remove_vote(Origin::signed(2), Some(c), 3)); + assert_ok!(Voting::unlock(Origin::signed(2), c, 2)); + assert_eq!(Balances::usable_balance(2), 20); + }); +} + +#[test] +fn unsuccessful_conviction_vote_balance_can_be_unlocked() { + new_test_ext().execute_with(|| { + assert_ok!(Voting::vote(Origin::signed(1), 3, aye(1, 1))); + assert_ok!(Voting::vote(Origin::signed(2), 3, nay(20, 0))); + let c = class(3); + Polls::set(vec![(3, Completed(3, false))].into_iter().collect()); + assert_ok!(Voting::remove_vote(Origin::signed(1), Some(c), 3)); + assert_ok!(Voting::unlock(Origin::signed(1), c, 1)); + assert_eq!(Balances::usable_balance(1), 10); + }); +} + +#[test] +fn successful_conviction_vote_balance_stays_locked_for_correct_time() { + new_test_ext().execute_with(|| { + for i in 1..=5 { + assert_ok!(Voting::vote(Origin::signed(i), 3, aye(10, i as u8))); + } + let c = class(3); + Polls::set(vec![(3, Completed(3, true))].into_iter().collect()); + for i in 1..=5 { + assert_ok!(Voting::remove_vote(Origin::signed(i), Some(c), 3)); + } + for block in 1..=(3 + 5 * 3) { + run_to(block); + for i in 1..=5 { + assert_ok!(Voting::unlock(Origin::signed(i), c, i)); + let expired = block >= (3 << (i - 1)) + 3; + assert_eq!(Balances::usable_balance(i), i * 10 - if expired { 0 } else { 10 }); + } + } + }); +} + +#[test] +fn classwise_delegation_works() { + new_test_ext().execute_with(|| { + Polls::set( + vec![ + (0, Ongoing(Tally::default(), 0)), + (1, Ongoing(Tally::default(), 1)), + (2, Ongoing(Tally::default(), 2)), + (3, Ongoing(Tally::default(), 2)), + ] + .into_iter() + .collect(), + ); + assert_ok!(Voting::delegate(Origin::signed(1), 0, 2, Conviction::Locked1x, 5)); + assert_ok!(Voting::delegate(Origin::signed(1), 1, 3, Conviction::Locked1x, 5)); + assert_ok!(Voting::delegate(Origin::signed(1), 2, 4, Conviction::Locked1x, 5)); + assert_eq!(Balances::usable_balance(1), 5); + + assert_ok!(Voting::vote(Origin::signed(2), 0, aye(10, 0))); + assert_ok!(Voting::vote(Origin::signed(2), 1, nay(10, 0))); + assert_ok!(Voting::vote(Origin::signed(2), 2, nay(10, 0))); + assert_ok!(Voting::vote(Origin::signed(3), 0, nay(10, 0))); + assert_ok!(Voting::vote(Origin::signed(3), 1, aye(10, 0))); + assert_ok!(Voting::vote(Origin::signed(3), 2, nay(10, 0))); + assert_ok!(Voting::vote(Origin::signed(4), 0, nay(10, 0))); + assert_ok!(Voting::vote(Origin::signed(4), 1, nay(10, 0))); + assert_ok!(Voting::vote(Origin::signed(4), 2, aye(10, 0))); + // 4 hasn't voted yet + + assert_eq!( + Polls::get(), + vec![ + (0, Ongoing(Tally::from_parts(6, 2, 35), 0)), + (1, Ongoing(Tally::from_parts(6, 2, 35), 1)), + (2, Ongoing(Tally::from_parts(6, 2, 35), 2)), + (3, Ongoing(Tally::from_parts(0, 0, 0), 2)), + ] + .into_iter() + .collect() + ); + + // 4 votes nay to 3. + assert_ok!(Voting::vote(Origin::signed(4), 3, nay(10, 0))); + assert_eq!( + Polls::get(), + vec![ + (0, Ongoing(Tally::from_parts(6, 2, 35), 0)), + (1, Ongoing(Tally::from_parts(6, 2, 35), 1)), + (2, Ongoing(Tally::from_parts(6, 2, 35), 2)), + (3, Ongoing(Tally::from_parts(0, 6, 15), 2)), + ] + .into_iter() + .collect() + ); + + // Redelegate for class 2 to account 3. + assert_ok!(Voting::undelegate(Origin::signed(1), 2)); + assert_ok!(Voting::delegate(Origin::signed(1), 2, 3, Conviction::Locked1x, 5)); + assert_eq!( + Polls::get(), + vec![ + (0, Ongoing(Tally::from_parts(6, 2, 35), 0)), + (1, Ongoing(Tally::from_parts(6, 2, 35), 1)), + (2, Ongoing(Tally::from_parts(1, 7, 35), 2)), + (3, Ongoing(Tally::from_parts(0, 1, 10), 2)), + ] + .into_iter() + .collect() + ); + + // Redelegating with a lower lock does not forget previous lock and updates correctly. + assert_ok!(Voting::undelegate(Origin::signed(1), 0)); + assert_ok!(Voting::undelegate(Origin::signed(1), 1)); + assert_ok!(Voting::undelegate(Origin::signed(1), 2)); + assert_ok!(Voting::delegate(Origin::signed(1), 0, 2, Conviction::Locked1x, 3)); + assert_ok!(Voting::delegate(Origin::signed(1), 1, 3, Conviction::Locked1x, 3)); + assert_ok!(Voting::delegate(Origin::signed(1), 2, 4, Conviction::Locked1x, 3)); + assert_eq!( + Polls::get(), + vec![ + (0, Ongoing(Tally::from_parts(4, 2, 33), 0)), + (1, Ongoing(Tally::from_parts(4, 2, 33), 1)), + (2, Ongoing(Tally::from_parts(4, 2, 33), 2)), + (3, Ongoing(Tally::from_parts(0, 4, 13), 2)), + ] + .into_iter() + .collect() + ); + assert_eq!(Balances::usable_balance(1), 5); + + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 1, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 2, 1)); + // unlock does nothing since the delegation already took place. + assert_eq!(Balances::usable_balance(1), 5); + + // Redelegating with higher amount extends previous lock. + assert_ok!(Voting::undelegate(Origin::signed(1), 0)); + assert_ok!(Voting::delegate(Origin::signed(1), 0, 2, Conviction::Locked1x, 6)); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 4); + assert_ok!(Voting::undelegate(Origin::signed(1), 1)); + assert_ok!(Voting::delegate(Origin::signed(1), 1, 3, Conviction::Locked1x, 7)); + assert_ok!(Voting::unlock(Origin::signed(1), 1, 1)); + assert_eq!(Balances::usable_balance(1), 3); + assert_ok!(Voting::undelegate(Origin::signed(1), 2)); + assert_ok!(Voting::delegate(Origin::signed(1), 2, 4, Conviction::Locked1x, 8)); + assert_ok!(Voting::unlock(Origin::signed(1), 2, 1)); + assert_eq!(Balances::usable_balance(1), 2); + assert_eq!( + Polls::get(), + vec![ + (0, Ongoing(Tally::from_parts(7, 2, 36), 0)), + (1, Ongoing(Tally::from_parts(8, 2, 37), 1)), + (2, Ongoing(Tally::from_parts(9, 2, 38), 2)), + (3, Ongoing(Tally::from_parts(0, 9, 18), 2)), + ] + .into_iter() + .collect() + ); + }); +} + +#[test] +fn redelegation_after_vote_ending_should_keep_lock() { + new_test_ext().execute_with(|| { + Polls::set(vec![(0, Ongoing(Tally::default(), 0))].into_iter().collect()); + assert_ok!(Voting::delegate(Origin::signed(1), 0, 2, Conviction::Locked1x, 5)); + assert_ok!(Voting::vote(Origin::signed(2), 0, aye(10, 1))); + Polls::set(vec![(0, Completed(1, true))].into_iter().collect()); + assert_eq!(Balances::usable_balance(1), 5); + assert_ok!(Voting::undelegate(Origin::signed(1), 0)); + assert_ok!(Voting::delegate(Origin::signed(1), 0, 3, Conviction::Locked1x, 3)); + assert_eq!(Balances::usable_balance(1), 5); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 5); + }); +} + +#[test] +fn lock_amalgamation_valid_with_multiple_removed_votes() { + new_test_ext().execute_with(|| { + Polls::set( + vec![ + (0, Ongoing(Tally::default(), 0)), + (1, Ongoing(Tally::default(), 0)), + (2, Ongoing(Tally::default(), 0)), + ] + .into_iter() + .collect(), + ); + assert_ok!(Voting::vote(Origin::signed(1), 0, aye(5, 1))); + assert_ok!(Voting::vote(Origin::signed(1), 1, aye(10, 1))); + assert_ok!(Voting::vote(Origin::signed(1), 2, aye(5, 2))); + assert_eq!(Balances::usable_balance(1), 0); + + Polls::set( + vec![(0, Completed(1, true)), (1, Completed(1, true)), (2, Completed(1, true))] + .into_iter() + .collect(), + ); + assert_ok!(Voting::remove_vote(Origin::signed(1), Some(0), 0)); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 0); + + assert_ok!(Voting::remove_vote(Origin::signed(1), Some(0), 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 0); + + assert_ok!(Voting::remove_vote(Origin::signed(1), Some(0), 2)); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 0); + + run_to(3); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 0); + + run_to(6); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert!(Balances::usable_balance(1) <= 5); + + run_to(7); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 10); + }); +} + +#[test] +fn lock_amalgamation_valid_with_multiple_delegations() { + new_test_ext().execute_with(|| { + assert_ok!(Voting::delegate(Origin::signed(1), 0, 2, Conviction::Locked1x, 5)); + assert_ok!(Voting::undelegate(Origin::signed(1), 0)); + assert_ok!(Voting::delegate(Origin::signed(1), 0, 2, Conviction::Locked1x, 10)); + assert_ok!(Voting::undelegate(Origin::signed(1), 0)); + assert_ok!(Voting::delegate(Origin::signed(1), 0, 2, Conviction::Locked2x, 5)); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 0); + assert_ok!(Voting::undelegate(Origin::signed(1), 0)); + + run_to(3); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 0); + + run_to(6); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert!(Balances::usable_balance(1) <= 5); + + run_to(7); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 10); + }); +} + +#[test] +fn lock_amalgamation_valid_with_move_roundtrip_to_delegation() { + new_test_ext().execute_with(|| { + Polls::set(vec![(0, Ongoing(Tally::default(), 0))].into_iter().collect()); + assert_ok!(Voting::vote(Origin::signed(1), 0, aye(5, 1))); + Polls::set(vec![(0, Completed(1, true))].into_iter().collect()); + assert_ok!(Voting::remove_vote(Origin::signed(1), Some(0), 0)); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 5); + + assert_ok!(Voting::delegate(Origin::signed(1), 0, 2, Conviction::Locked1x, 10)); + assert_ok!(Voting::undelegate(Origin::signed(1), 0)); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 0); + + Polls::set(vec![(1, Ongoing(Tally::default(), 0))].into_iter().collect()); + assert_ok!(Voting::vote(Origin::signed(1), 1, aye(5, 2))); + Polls::set(vec![(1, Completed(1, true))].into_iter().collect()); + assert_ok!(Voting::remove_vote(Origin::signed(1), Some(0), 1)); + + run_to(3); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 0); + + run_to(6); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert!(Balances::usable_balance(1) <= 5); + + run_to(7); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 10); + }); +} + +#[test] +fn lock_amalgamation_valid_with_move_roundtrip_to_casting() { + new_test_ext().execute_with(|| { + assert_ok!(Voting::delegate(Origin::signed(1), 0, 2, Conviction::Locked1x, 5)); + assert_ok!(Voting::undelegate(Origin::signed(1), 0)); + + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 5); + + Polls::set(vec![(0, Ongoing(Tally::default(), 0))].into_iter().collect()); + assert_ok!(Voting::vote(Origin::signed(1), 0, aye(10, 1))); + Polls::set(vec![(0, Completed(1, true))].into_iter().collect()); + assert_ok!(Voting::remove_vote(Origin::signed(1), Some(0), 0)); + + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 0); + + assert_ok!(Voting::delegate(Origin::signed(1), 0, 2, Conviction::Locked2x, 10)); + assert_ok!(Voting::undelegate(Origin::signed(1), 0)); + + run_to(3); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 0); + + run_to(6); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert!(Balances::usable_balance(1) <= 5); + + run_to(7); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_eq!(Balances::usable_balance(1), 10); + }); +} + +#[test] +fn lock_aggregation_over_different_classes_with_delegation_works() { + new_test_ext().execute_with(|| { + assert_ok!(Voting::delegate(Origin::signed(1), 0, 2, Conviction::Locked1x, 5)); + assert_ok!(Voting::delegate(Origin::signed(1), 1, 2, Conviction::Locked2x, 5)); + assert_ok!(Voting::delegate(Origin::signed(1), 2, 2, Conviction::Locked1x, 10)); + + assert_ok!(Voting::undelegate(Origin::signed(1), 0)); + assert_ok!(Voting::undelegate(Origin::signed(1), 1)); + assert_ok!(Voting::undelegate(Origin::signed(1), 2)); + + run_to(3); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 1, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 2, 1)); + assert_eq!(Balances::usable_balance(1), 0); + + run_to(6); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 1, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 2, 1)); + assert_eq!(Balances::usable_balance(1), 5); + + run_to(7); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 1, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 2, 1)); + assert_eq!(Balances::usable_balance(1), 10); + }); +} + +#[test] +fn lock_aggregation_over_different_classes_with_casting_works() { + new_test_ext().execute_with(|| { + Polls::set( + vec![ + (0, Ongoing(Tally::default(), 0)), + (1, Ongoing(Tally::default(), 1)), + (2, Ongoing(Tally::default(), 2)), + ] + .into_iter() + .collect(), + ); + assert_ok!(Voting::vote(Origin::signed(1), 0, aye(5, 1))); + assert_ok!(Voting::vote(Origin::signed(1), 1, aye(10, 1))); + assert_ok!(Voting::vote(Origin::signed(1), 2, aye(5, 2))); + Polls::set( + vec![(0, Completed(1, true)), (1, Completed(1, true)), (2, Completed(1, true))] + .into_iter() + .collect(), + ); + assert_ok!(Voting::remove_vote(Origin::signed(1), Some(0), 0)); + assert_ok!(Voting::remove_vote(Origin::signed(1), Some(1), 1)); + assert_ok!(Voting::remove_vote(Origin::signed(1), Some(2), 2)); + + run_to(3); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 1, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 2, 1)); + assert_eq!(Balances::usable_balance(1), 0); + + run_to(6); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 1, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 2, 1)); + assert_eq!(Balances::usable_balance(1), 5); + + run_to(7); + assert_ok!(Voting::unlock(Origin::signed(1), 0, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 1, 1)); + assert_ok!(Voting::unlock(Origin::signed(1), 2, 1)); + assert_eq!(Balances::usable_balance(1), 10); + }); +} + +#[test] +fn errors_with_vote_work() { + new_test_ext().execute_with(|| { + assert_noop!(Voting::vote(Origin::signed(1), 0, aye(10, 0)), Error::::NotOngoing); + assert_noop!(Voting::vote(Origin::signed(1), 1, aye(10, 0)), Error::::NotOngoing); + assert_noop!(Voting::vote(Origin::signed(1), 2, aye(10, 0)), Error::::NotOngoing); + assert_noop!( + Voting::vote(Origin::signed(1), 3, aye(11, 0)), + Error::::InsufficientFunds + ); + + assert_ok!(Voting::delegate(Origin::signed(1), 0, 2, Conviction::None, 10)); + assert_noop!( + Voting::vote(Origin::signed(1), 3, aye(10, 0)), + Error::::AlreadyDelegating + ); + + assert_ok!(Voting::undelegate(Origin::signed(1), 0)); + Polls::set( + vec![ + (0, Ongoing(Tally::default(), 0)), + (1, Ongoing(Tally::default(), 0)), + (2, Ongoing(Tally::default(), 0)), + (3, Ongoing(Tally::default(), 0)), + ] + .into_iter() + .collect(), + ); + assert_ok!(Voting::vote(Origin::signed(1), 0, aye(10, 0))); + assert_ok!(Voting::vote(Origin::signed(1), 1, aye(10, 0))); + assert_ok!(Voting::vote(Origin::signed(1), 2, aye(10, 0))); + assert_noop!( + Voting::vote(Origin::signed(1), 3, aye(10, 0)), + Error::::MaxVotesReached + ); + }); +} + +#[test] +fn errors_with_delegating_work() { + new_test_ext().execute_with(|| { + assert_noop!( + Voting::delegate(Origin::signed(1), 0, 2, Conviction::None, 11), + Error::::InsufficientFunds + ); + assert_noop!( + Voting::delegate(Origin::signed(1), 3, 2, Conviction::None, 10), + Error::::BadClass + ); + + assert_ok!(Voting::vote(Origin::signed(1), 3, aye(10, 0))); + assert_noop!( + Voting::delegate(Origin::signed(1), 0, 2, Conviction::None, 10), + Error::::AlreadyVoting + ); + + assert_noop!(Voting::undelegate(Origin::signed(1), 0), Error::::NotDelegating); + }); +} + +#[test] +fn remove_other_vote_works() { + new_test_ext().execute_with(|| { + assert_noop!( + Voting::remove_other_vote(Origin::signed(2), 1, 0, 3), + Error::::NotVoter + ); + assert_ok!(Voting::vote(Origin::signed(1), 3, aye(10, 2))); + assert_noop!( + Voting::remove_other_vote(Origin::signed(2), 1, 0, 3), + Error::::NoPermission + ); + Polls::set(vec![(3, Completed(1, true))].into_iter().collect()); + run_to(6); + assert_noop!( + Voting::remove_other_vote(Origin::signed(2), 1, 0, 3), + Error::::NoPermissionYet + ); + run_to(7); + assert_ok!(Voting::remove_other_vote(Origin::signed(2), 1, 0, 3)); + }); +} + +#[test] +fn errors_with_remove_vote_work() { + new_test_ext().execute_with(|| { + assert_noop!(Voting::remove_vote(Origin::signed(1), Some(0), 3), Error::::NotVoter); + assert_ok!(Voting::vote(Origin::signed(1), 3, aye(10, 2))); + Polls::set(vec![(3, Completed(1, true))].into_iter().collect()); + assert_noop!(Voting::remove_vote(Origin::signed(1), None, 3), Error::::ClassNeeded); + }); +} diff --git a/frame/conviction-voting/src/types.rs b/frame/conviction-voting/src/types.rs new file mode 100644 index 000000000000..e2b5844ddd5d --- /dev/null +++ b/frame/conviction-voting/src/types.rs @@ -0,0 +1,237 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Miscellaneous additional datatypes. + +use sp_std::marker::PhantomData; + +use codec::{Codec, Decode, Encode, MaxEncodedLen}; +use frame_support::{ + traits::VoteTally, CloneNoBound, DefaultNoBound, EqNoBound, PartialEqNoBound, + RuntimeDebugNoBound, +}; +use scale_info::TypeInfo; +use sp_runtime::{ + traits::{Saturating, Zero}, + RuntimeDebug, +}; + +use super::*; +use crate::{AccountVote, Conviction, Vote}; + +/// Info regarding an ongoing referendum. +#[derive( + CloneNoBound, + DefaultNoBound, + PartialEqNoBound, + EqNoBound, + RuntimeDebugNoBound, + TypeInfo, + Encode, + Decode, + MaxEncodedLen, +)] +#[scale_info(skip_type_params(Total))] +pub struct Tally< + Votes: Clone + Default + PartialEq + Eq + sp_std::fmt::Debug + TypeInfo + Codec, + Total, +> { + /// The number of aye votes, expressed in terms of post-conviction lock-vote. + pub ayes: Votes, + /// The number of nay votes, expressed in terms of post-conviction lock-vote. + pub nays: Votes, + /// The amount of funds currently expressing its opinion. Pre-conviction. + pub turnout: Votes, + /// Dummy. + dummy: PhantomData, +} + +impl< + Votes: Clone + + Default + + PartialEq + + Eq + + sp_std::fmt::Debug + + Copy + + AtLeast32BitUnsigned + + TypeInfo + + Codec, + Total: Get, + > VoteTally for Tally +{ + fn ayes(&self) -> Votes { + self.ayes + } + + fn turnout(&self) -> Perbill { + Perbill::from_rational(self.turnout, Total::get()) + } + + fn approval(&self) -> Perbill { + Perbill::from_rational(self.ayes, self.ayes.saturating_add(self.nays)) + } + + #[cfg(feature = "runtime-benchmarks")] + fn unanimity() -> Self { + Self { ayes: Total::get(), nays: Zero::zero(), turnout: Total::get(), dummy: PhantomData } + } + + #[cfg(feature = "runtime-benchmarks")] + fn from_requirements(turnout: Perbill, approval: Perbill) -> Self { + let turnout = turnout.mul_ceil(Total::get()); + let ayes = approval.mul_ceil(turnout); + Self { ayes, nays: turnout - ayes, turnout, dummy: PhantomData } + } +} + +impl< + Votes: Clone + + Default + + PartialEq + + Eq + + sp_std::fmt::Debug + + Copy + + AtLeast32BitUnsigned + + TypeInfo + + Codec, + Total: Get, + > Tally +{ + /// Create a new tally. + pub fn new(vote: Vote, balance: Votes) -> Self { + let Delegations { votes, capital } = vote.conviction.votes(balance); + Self { + ayes: if vote.aye { votes } else { Zero::zero() }, + nays: if vote.aye { Zero::zero() } else { votes }, + turnout: capital, + dummy: PhantomData, + } + } + + pub fn from_parts(ayes: Votes, nays: Votes, turnout: Votes) -> Self { + Self { ayes, nays, turnout, dummy: PhantomData } + } + + /// Add an account's vote into the tally. + pub fn add(&mut self, vote: AccountVote) -> Option<()> { + match vote { + AccountVote::Standard { vote, balance } => { + let Delegations { votes, capital } = vote.conviction.votes(balance); + self.turnout = self.turnout.checked_add(&capital)?; + match vote.aye { + true => self.ayes = self.ayes.checked_add(&votes)?, + false => self.nays = self.nays.checked_add(&votes)?, + } + }, + AccountVote::Split { aye, nay } => { + let aye = Conviction::None.votes(aye); + let nay = Conviction::None.votes(nay); + self.turnout = self.turnout.checked_add(&aye.capital)?.checked_add(&nay.capital)?; + self.ayes = self.ayes.checked_add(&aye.votes)?; + self.nays = self.nays.checked_add(&nay.votes)?; + }, + } + Some(()) + } + + /// Remove an account's vote from the tally. + pub fn remove(&mut self, vote: AccountVote) -> Option<()> { + match vote { + AccountVote::Standard { vote, balance } => { + let Delegations { votes, capital } = vote.conviction.votes(balance); + self.turnout = self.turnout.checked_sub(&capital)?; + match vote.aye { + true => self.ayes = self.ayes.checked_sub(&votes)?, + false => self.nays = self.nays.checked_sub(&votes)?, + } + }, + AccountVote::Split { aye, nay } => { + let aye = Conviction::None.votes(aye); + let nay = Conviction::None.votes(nay); + self.turnout = self.turnout.checked_sub(&aye.capital)?.checked_sub(&nay.capital)?; + self.ayes = self.ayes.checked_sub(&aye.votes)?; + self.nays = self.nays.checked_sub(&nay.votes)?; + }, + } + Some(()) + } + + /// Increment some amount of votes. + pub fn increase(&mut self, approve: bool, delegations: Delegations) { + self.turnout = self.turnout.saturating_add(delegations.capital); + match approve { + true => self.ayes = self.ayes.saturating_add(delegations.votes), + false => self.nays = self.nays.saturating_add(delegations.votes), + } + } + + /// Decrement some amount of votes. + pub fn reduce(&mut self, approve: bool, delegations: Delegations) { + self.turnout = self.turnout.saturating_sub(delegations.capital); + match approve { + true => self.ayes = self.ayes.saturating_sub(delegations.votes), + false => self.nays = self.nays.saturating_sub(delegations.votes), + } + } +} + +/// Amount of votes and capital placed in delegation for an account. +#[derive( + Encode, Decode, Default, Copy, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen, +)] +pub struct Delegations { + /// The number of votes (this is post-conviction). + pub votes: Balance, + /// The amount of raw capital, used for the turnout. + pub capital: Balance, +} + +impl Saturating for Delegations { + fn saturating_add(self, o: Self) -> Self { + Self { + votes: self.votes.saturating_add(o.votes), + capital: self.capital.saturating_add(o.capital), + } + } + + fn saturating_sub(self, o: Self) -> Self { + Self { + votes: self.votes.saturating_sub(o.votes), + capital: self.capital.saturating_sub(o.capital), + } + } + + fn saturating_mul(self, o: Self) -> Self { + Self { + votes: self.votes.saturating_mul(o.votes), + capital: self.capital.saturating_mul(o.capital), + } + } + + fn saturating_pow(self, exp: usize) -> Self { + Self { votes: self.votes.saturating_pow(exp), capital: self.capital.saturating_pow(exp) } + } +} + +/// Whether an `unvote` operation is able to make actions that are not strictly always in the +/// interest of an account. +pub enum UnvoteScope { + /// Permitted to do everything. + Any, + /// Permitted to do only the changes that do not need the owner's permission. + OnlyExpired, +} diff --git a/frame/conviction-voting/src/vote.rs b/frame/conviction-voting/src/vote.rs new file mode 100644 index 000000000000..e608a6dcfd01 --- /dev/null +++ b/frame/conviction-voting/src/vote.rs @@ -0,0 +1,254 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The vote datatype. + +use crate::{Conviction, Delegations}; +use codec::{Decode, Encode, EncodeLike, Input, MaxEncodedLen, Output}; +use frame_support::{pallet_prelude::Get, BoundedVec}; +use scale_info::TypeInfo; +use sp_runtime::{ + traits::{Saturating, Zero}, + RuntimeDebug, +}; +use sp_std::prelude::*; + +/// A number of lock periods, plus a vote, one way or the other. +#[derive(Copy, Clone, Eq, PartialEq, Default, RuntimeDebug, MaxEncodedLen)] +pub struct Vote { + pub aye: bool, + pub conviction: Conviction, +} + +impl Encode for Vote { + fn encode_to(&self, output: &mut T) { + output.push_byte(u8::from(self.conviction) | if self.aye { 0b1000_0000 } else { 0 }); + } +} + +impl EncodeLike for Vote {} + +impl Decode for Vote { + fn decode(input: &mut I) -> Result { + let b = input.read_byte()?; + Ok(Vote { + aye: (b & 0b1000_0000) == 0b1000_0000, + conviction: Conviction::try_from(b & 0b0111_1111) + .map_err(|_| codec::Error::from("Invalid conviction"))?, + }) + } +} + +impl TypeInfo for Vote { + type Identity = Self; + + fn type_info() -> scale_info::Type { + scale_info::Type::builder() + .path(scale_info::Path::new("Vote", module_path!())) + .composite( + scale_info::build::Fields::unnamed() + .field(|f| f.ty::().docs(&["Raw vote byte, encodes aye + conviction"])), + ) + } +} + +/// A vote for a referendum of a particular account. +#[derive(Encode, Decode, Copy, Clone, Eq, PartialEq, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub enum AccountVote { + /// A standard vote, one-way (approve or reject) with a given amount of conviction. + Standard { vote: Vote, balance: Balance }, + /// A split vote with balances given for both ways, and with no conviction, useful for + /// parachains when voting. + Split { aye: Balance, nay: Balance }, +} + +impl AccountVote { + /// Returns `Some` of the lock periods that the account is locked for, assuming that the + /// referendum passed iff `approved` is `true`. + pub fn locked_if(self, approved: bool) -> Option<(u32, Balance)> { + // winning side: can only be removed after the lock period ends. + match self { + AccountVote::Standard { vote: Vote { conviction: Conviction::None, .. }, .. } => None, + AccountVote::Standard { vote, balance } if vote.aye == approved => + Some((vote.conviction.lock_periods(), balance)), + _ => None, + } + } + + /// The total balance involved in this vote. + pub fn balance(self) -> Balance { + match self { + AccountVote::Standard { balance, .. } => balance, + AccountVote::Split { aye, nay } => aye.saturating_add(nay), + } + } + + /// Returns `Some` with whether the vote is an aye vote if it is standard, otherwise `None` if + /// it is split. + pub fn as_standard(self) -> Option { + match self { + AccountVote::Standard { vote, .. } => Some(vote.aye), + _ => None, + } + } +} + +/// A "prior" lock, i.e. a lock for some now-forgotten reason. +#[derive( + Encode, + Decode, + Default, + Copy, + Clone, + Eq, + PartialEq, + Ord, + PartialOrd, + RuntimeDebug, + TypeInfo, + MaxEncodedLen, +)] +pub struct PriorLock(BlockNumber, Balance); + +impl PriorLock { + /// Accumulates an additional lock. + pub fn accumulate(&mut self, until: BlockNumber, amount: Balance) { + self.0 = self.0.max(until); + self.1 = self.1.max(amount); + } + + pub fn locked(&self) -> Balance { + self.1 + } + + pub fn rejig(&mut self, now: BlockNumber) { + if now >= self.0 { + self.0 = Zero::zero(); + self.1 = Zero::zero(); + } + } +} + +/// Information concerning the delegation of some voting power. +#[derive(Encode, Decode, Clone, Eq, PartialEq, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub struct Delegating { + /// The amount of balance delegated. + pub balance: Balance, + /// The account to which the voting power is delegated. + pub target: AccountId, + /// The conviction with which the voting power is delegated. When this gets undelegated, the + /// relevant lock begins. + pub conviction: Conviction, + /// The total amount of delegations that this account has received, post-conviction-weighting. + pub delegations: Delegations, + /// Any pre-existing locks from past voting/delegating activity. + pub prior: PriorLock, +} + +/// Information concerning the direct vote-casting of some voting power. +#[derive(Encode, Decode, Clone, Eq, PartialEq, RuntimeDebug, TypeInfo, MaxEncodedLen)] +#[scale_info(skip_type_params(MaxVotes))] +pub struct Casting +where + MaxVotes: Get, +{ + /// The current votes of the account. + pub votes: BoundedVec<(PollIndex, AccountVote), MaxVotes>, + /// The total amount of delegations that this account has received, post-conviction-weighting. + pub delegations: Delegations, + /// Any pre-existing locks from past voting/delegating activity. + pub prior: PriorLock, +} + +/// An indicator for what an account is doing; it can either be delegating or voting. +#[derive(Encode, Decode, Clone, Eq, PartialEq, RuntimeDebug, TypeInfo, MaxEncodedLen)] +#[scale_info(skip_type_params(MaxVotes))] +pub enum Voting +where + MaxVotes: Get, +{ + /// The account is voting directly. + Casting(Casting), + /// The account is delegating `balance` of its balance to a `target` account with `conviction`. + Delegating(Delegating), +} + +impl Default + for Voting +where + MaxVotes: Get, +{ + fn default() -> Self { + Voting::Casting(Casting { + votes: Default::default(), + delegations: Default::default(), + prior: PriorLock(Zero::zero(), Default::default()), + }) + } +} + +impl AsMut> + for Voting +where + MaxVotes: Get, +{ + fn as_mut(&mut self) -> &mut PriorLock { + match self { + Voting::Casting(Casting { prior, .. }) => prior, + Voting::Delegating(Delegating { prior, .. }) => prior, + } + } +} + +impl< + Balance: Saturating + Ord + Zero + Copy, + BlockNumber: Ord + Copy + Zero, + AccountId, + PollIndex, + MaxVotes, + > Voting +where + MaxVotes: Get, +{ + pub fn rejig(&mut self, now: BlockNumber) { + AsMut::>::as_mut(self).rejig(now); + } + + /// The amount of this account's balance that much currently be locked due to voting. + pub fn locked_balance(&self) -> Balance { + match self { + Voting::Casting(Casting { votes, prior, .. }) => + votes.iter().map(|i| i.1.balance()).fold(prior.locked(), |a, i| a.max(i)), + Voting::Delegating(Delegating { balance, prior, .. }) => *balance.max(&prior.locked()), + } + } + + pub fn set_common( + &mut self, + delegations: Delegations, + prior: PriorLock, + ) { + let (d, p) = match self { + Voting::Casting(Casting { ref mut delegations, ref mut prior, .. }) => + (delegations, prior), + Voting::Delegating(Delegating { ref mut delegations, ref mut prior, .. }) => + (delegations, prior), + }; + *d = delegations; + *p = prior; + } +} diff --git a/frame/conviction-voting/src/weights.rs b/frame/conviction-voting/src/weights.rs new file mode 100644 index 000000000000..da15aac0b47c --- /dev/null +++ b/frame/conviction-voting/src/weights.rs @@ -0,0 +1,201 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for pallet_conviction_voting +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2022-01-09, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 + +// Executed Command: +// target/release/substrate +// benchmark +// --chain=dev +// --steps=50 +// --repeat=20 +// --pallet=pallet_conviction_voting +// --extrinsic=* +// --execution=wasm +// --wasm-execution=compiled +// --heap-pages=4096 +// --output=./frame/conviction-voting/src/weights.rs +// --template=./.maintain/frame-weight-template.hbs + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use sp_std::marker::PhantomData; + +/// Weight functions needed for pallet_conviction_voting. +pub trait WeightInfo { + fn vote_new() -> Weight; + fn vote_existing() -> Weight; + fn remove_vote() -> Weight; + fn remove_other_vote() -> Weight; + fn delegate(r: u32, ) -> Weight; + fn undelegate(r: u32, ) -> Weight; + fn unlock() -> Weight; +} + +/// Weights for pallet_conviction_voting using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: ConvictionVoting VotingFor (r:1 w:1) + // Storage: ConvictionVoting ClassLocksFor (r:1 w:1) + // Storage: Balances Locks (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn vote_new() -> Weight { + (159_647_000 as Weight) + .saturating_add(T::DbWeight::get().reads(6 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: ConvictionVoting VotingFor (r:1 w:1) + // Storage: ConvictionVoting ClassLocksFor (r:1 w:1) + // Storage: Balances Locks (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn vote_existing() -> Weight { + (339_851_000 as Weight) + .saturating_add(T::DbWeight::get().reads(6 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) + } + // Storage: ConvictionVoting VotingFor (r:1 w:1) + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn remove_vote() -> Weight { + (317_673_000 as Weight) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + // Storage: ConvictionVoting VotingFor (r:1 w:1) + // Storage: Referenda ReferendumInfoFor (r:1 w:0) + fn remove_other_vote() -> Weight { + (52_222_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: ConvictionVoting VotingFor (r:2 w:2) + // Storage: ConvictionVoting ClassLocksFor (r:1 w:1) + // Storage: Balances Locks (r:1 w:1) + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn delegate(r: u32, ) -> Weight { + (61_553_000 as Weight) + // Standard Error: 123_000 + .saturating_add((33_092_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(r as Weight))) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(r as Weight))) + } + // Storage: ConvictionVoting VotingFor (r:2 w:2) + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn undelegate(r: u32, ) -> Weight { + (42_037_000 as Weight) + // Standard Error: 582_000 + .saturating_add((32_296_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(r as Weight))) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(r as Weight))) + } + // Storage: ConvictionVoting VotingFor (r:1 w:1) + // Storage: ConvictionVoting ClassLocksFor (r:1 w:1) + // Storage: Balances Locks (r:1 w:1) + fn unlock() -> Weight { + (69_017_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } +} + +// For backwards compatibility and tests +impl WeightInfo for () { + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: ConvictionVoting VotingFor (r:1 w:1) + // Storage: ConvictionVoting ClassLocksFor (r:1 w:1) + // Storage: Balances Locks (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn vote_new() -> Weight { + (159_647_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(6 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: ConvictionVoting VotingFor (r:1 w:1) + // Storage: ConvictionVoting ClassLocksFor (r:1 w:1) + // Storage: Balances Locks (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn vote_existing() -> Weight { + (339_851_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(6 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + } + // Storage: ConvictionVoting VotingFor (r:1 w:1) + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn remove_vote() -> Weight { + (317_673_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + // Storage: ConvictionVoting VotingFor (r:1 w:1) + // Storage: Referenda ReferendumInfoFor (r:1 w:0) + fn remove_other_vote() -> Weight { + (52_222_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: ConvictionVoting VotingFor (r:2 w:2) + // Storage: ConvictionVoting ClassLocksFor (r:1 w:1) + // Storage: Balances Locks (r:1 w:1) + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn delegate(r: u32, ) -> Weight { + (61_553_000 as Weight) + // Standard Error: 123_000 + .saturating_add((33_092_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(r as Weight))) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(r as Weight))) + } + // Storage: ConvictionVoting VotingFor (r:2 w:2) + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn undelegate(r: u32, ) -> Weight { + (42_037_000 as Weight) + // Standard Error: 582_000 + .saturating_add((32_296_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(r as Weight))) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(r as Weight))) + } + // Storage: ConvictionVoting VotingFor (r:1 w:1) + // Storage: ConvictionVoting ClassLocksFor (r:1 w:1) + // Storage: Balances Locks (r:1 w:1) + fn unlock() -> Weight { + (69_017_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } +} diff --git a/frame/democracy/Cargo.toml b/frame/democracy/Cargo.toml index 94719553e28a..49b21d0eecdc 100644 --- a/frame/democracy/Cargo.toml +++ b/frame/democracy/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-democracy" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for democracy" readme = "README.md" @@ -13,20 +13,20 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -serde = { version = "1.0.126", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +serde = { version = "1.0.136", optional = true, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-scheduler = { version = "4.0.0-dev", path = "../scheduler" } @@ -44,7 +44,7 @@ std = [ "frame-system/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-system/runtime-benchmarks", "frame-support/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/frame/democracy/src/benchmarking.rs b/frame/democracy/src/benchmarking.rs index a00e6f4686fd..bce830b385a9 100644 --- a/frame/democracy/src/benchmarking.rs +++ b/frame/democracy/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,7 @@ use super::*; -use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite, whitelist_account}; +use frame_benchmarking::{account, benchmarks, whitelist_account}; use frame_support::{ assert_noop, assert_ok, codec::Decode, @@ -70,7 +70,7 @@ fn add_referendum(n: u32) -> Result { let referendum_index: ReferendumIndex = ReferendumCount::::get() - 1; T::Scheduler::schedule_named( (DEMOCRACY_ID, referendum_index).encode(), - DispatchTime::At(1u32.into()), + DispatchTime::At(2u32.into()), None, 63, frame_system::RawOrigin::Root.into(), @@ -239,9 +239,14 @@ benchmarks! { let origin = T::ExternalOrigin::successful_origin(); let proposal_hash = T::Hashing::hash_of(&0); // Add proposal to blacklist with block number 0 + + let addresses = (0..v) + .into_iter() + .map(|i| account::("blacklist", i, SEED)) + .collect::>(); Blacklist::::insert( proposal_hash, - (T::BlockNumber::zero(), vec![T::AccountId::default(); v as usize]) + (T::BlockNumber::zero(), addresses), ); }: _(origin, proposal_hash) verify { @@ -292,7 +297,7 @@ benchmarks! { let mut vetoers: Vec = Vec::new(); for i in 0 .. v { - vetoers.push(account("vetoer", i, SEED)); + vetoers.push(account::("vetoer", i, SEED)); } vetoers.sort(); Blacklist::::insert(proposal_hash, (T::BlockNumber::zero(), vetoers)); @@ -422,7 +427,39 @@ benchmarks! { assert_eq!(Democracy::::referendum_count(), r, "referenda not created"); assert_eq!(Democracy::::lowest_unbaked(), 0, "invalid referenda init"); - }: { Democracy::::on_initialize(0u32.into()) } + }: { Democracy::::on_initialize(1u32.into()) } + verify { + // All should be on going + for i in 0 .. r { + if let Some(value) = ReferendumInfoOf::::get(i) { + match value { + ReferendumInfo::Finished { .. } => return Err("Referendum has been finished".into()), + ReferendumInfo::Ongoing(_) => (), + } + } + } + } + + on_initialize_base_with_launch_period { + let r in 1 .. MAX_REFERENDUMS; + + for i in 0..r { + add_referendum::(i)?; + } + + for (key, mut info) in ReferendumInfoOf::::iter() { + if let ReferendumInfo::Ongoing(ref mut status) = info { + status.end += 100u32.into(); + } + ReferendumInfoOf::::insert(key, info); + } + + assert_eq!(Democracy::::referendum_count(), r, "referenda not created"); + assert_eq!(Democracy::::lowest_unbaked(), 0, "invalid referenda init"); + + let block_number = T::LaunchPeriod::get(); + + }: { Democracy::::on_initialize(block_number) } verify { // All should be on going for i in 0 .. r { @@ -742,7 +779,7 @@ benchmarks! { }: enact_proposal(RawOrigin::Root, proposal_hash, 0) verify { // Fails due to mismatched origin - assert_last_event::(Event::::Executed(0, Err(BadOrigin.into())).into()); + assert_last_event::(Event::::Executed { ref_index: 0, result: Err(BadOrigin.into()) }.into()); } #[extra] @@ -770,6 +807,10 @@ benchmarks! { Err(Error::::PreimageInvalid.into()) ); } -} -impl_benchmark_test_suite!(Democracy, crate::tests::new_test_ext(), crate::tests::Test); + impl_benchmark_test_suite!( + Democracy, + crate::tests::new_test_ext(), + crate::tests::Test + ); +} diff --git a/frame/democracy/src/conviction.rs b/frame/democracy/src/conviction.rs index b4f24c93bb40..57d631e8c1f4 100644 --- a/frame/democracy/src/conviction.rs +++ b/frame/democracy/src/conviction.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,7 +24,7 @@ use sp_runtime::{ traits::{Bounded, CheckedDiv, CheckedMul, Zero}, RuntimeDebug, }; -use sp_std::{convert::TryFrom, result::Result}; +use sp_std::{prelude::*, result::Result}; /// A value denoting the strength of conviction of a vote. #[derive(Encode, Decode, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, RuntimeDebug, TypeInfo)] diff --git a/frame/democracy/src/lib.rs b/frame/democracy/src/lib.rs index 8bc6921c4f8a..8588f1876d7e 100644 --- a/frame/democracy/src/lib.rs +++ b/frame/democracy/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -149,13 +149,14 @@ //! - `cancel_queued` - Cancels a proposal that is queued for enactment. //! - `clear_public_proposal` - Removes all public proposals. -#![recursion_limit = "128"] +#![recursion_limit = "256"] #![cfg_attr(not(feature = "std"), no_std)] use codec::{Decode, Encode, Input}; use frame_support::{ ensure, traits::{ + defensive_prelude::*, schedule::{DispatchTime, Named as ScheduleNamed}, BalanceStatus, Currency, Get, LockIdentifier, LockableCurrency, OnUnbalanced, ReservableCurrency, WithdrawReasons, @@ -240,19 +241,13 @@ enum Releases { #[frame_support::pallet] pub mod pallet { - use super::*; - use frame_support::{ - dispatch::DispatchResultWithPostInfo, - pallet_prelude::*, - traits::EnsureOrigin, - weights::{DispatchClass, Pays}, - Parameter, - }; - use frame_system::{ensure_root, ensure_signed, pallet_prelude::*}; - use sp_runtime::DispatchResult; + use super::{DispatchResult, *}; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); #[pallet::config] @@ -444,14 +439,6 @@ pub mod pallet { ValueQuery, >; - /// Accounts for which there are locks in action which may be removed at some point in the - /// future. The value is the block number at which the lock expires and may be removed. - /// - /// TWOX-NOTE: OK ― `AccountId` is a secure hash. - #[pallet::storage] - #[pallet::getter(fn locks)] - pub type Locks = StorageMap<_, Twox64Concat, T::AccountId, T::BlockNumber>; - /// True if the last referendum tabled was submitted externally. False if it was a public /// proposal. // TODO: There should be any number of tabling origins, not just public and "external" @@ -507,45 +494,49 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// A motion has been proposed by a public account. \[proposal_index, deposit\] - Proposed(PropIndex, BalanceOf), - /// A public proposal has been tabled for referendum vote. \[proposal_index, deposit, - /// depositors\] - Tabled(PropIndex, BalanceOf, Vec), + /// A motion has been proposed by a public account. + Proposed { proposal_index: PropIndex, deposit: BalanceOf }, + /// A public proposal has been tabled for referendum vote. + Tabled { proposal_index: PropIndex, deposit: BalanceOf, depositors: Vec }, /// An external proposal has been tabled. ExternalTabled, - /// A referendum has begun. \[ref_index, threshold\] - Started(ReferendumIndex, VoteThreshold), - /// A proposal has been approved by referendum. \[ref_index\] - Passed(ReferendumIndex), - /// A proposal has been rejected by referendum. \[ref_index\] - NotPassed(ReferendumIndex), - /// A referendum has been cancelled. \[ref_index\] - Cancelled(ReferendumIndex), - /// A proposal has been enacted. \[ref_index, result\] - Executed(ReferendumIndex, DispatchResult), - /// An account has delegated their vote to another account. \[who, target\] - Delegated(T::AccountId, T::AccountId), - /// An \[account\] has cancelled a previous delegation operation. - Undelegated(T::AccountId), - /// An external proposal has been vetoed. \[who, proposal_hash, until\] - Vetoed(T::AccountId, T::Hash, T::BlockNumber), - /// A proposal's preimage was noted, and the deposit taken. \[proposal_hash, who, deposit\] - PreimageNoted(T::Hash, T::AccountId, BalanceOf), + /// A referendum has begun. + Started { ref_index: ReferendumIndex, threshold: VoteThreshold }, + /// A proposal has been approved by referendum. + Passed { ref_index: ReferendumIndex }, + /// A proposal has been rejected by referendum. + NotPassed { ref_index: ReferendumIndex }, + /// A referendum has been cancelled. + Cancelled { ref_index: ReferendumIndex }, + /// A proposal has been enacted. + Executed { ref_index: ReferendumIndex, result: DispatchResult }, + /// An account has delegated their vote to another account. + Delegated { who: T::AccountId, target: T::AccountId }, + /// An account has cancelled a previous delegation operation. + Undelegated { account: T::AccountId }, + /// An external proposal has been vetoed. + Vetoed { who: T::AccountId, proposal_hash: T::Hash, until: T::BlockNumber }, + /// A proposal's preimage was noted, and the deposit taken. + PreimageNoted { proposal_hash: T::Hash, who: T::AccountId, deposit: BalanceOf }, /// A proposal preimage was removed and used (the deposit was returned). - /// \[proposal_hash, provider, deposit\] - PreimageUsed(T::Hash, T::AccountId, BalanceOf), + PreimageUsed { proposal_hash: T::Hash, provider: T::AccountId, deposit: BalanceOf }, /// A proposal could not be executed because its preimage was invalid. - /// \[proposal_hash, ref_index\] - PreimageInvalid(T::Hash, ReferendumIndex), + PreimageInvalid { proposal_hash: T::Hash, ref_index: ReferendumIndex }, /// A proposal could not be executed because its preimage was missing. - /// \[proposal_hash, ref_index\] - PreimageMissing(T::Hash, ReferendumIndex), + PreimageMissing { proposal_hash: T::Hash, ref_index: ReferendumIndex }, /// A registered preimage was removed and the deposit collected by the reaper. - /// \[proposal_hash, provider, deposit, reaper\] - PreimageReaped(T::Hash, T::AccountId, BalanceOf, T::AccountId), - /// A proposal \[hash\] has been blacklisted permanently. - Blacklisted(T::Hash), + PreimageReaped { + proposal_hash: T::Hash, + provider: T::AccountId, + deposit: BalanceOf, + reaper: T::AccountId, + }, + /// A proposal_hash has been blacklisted permanently. + Blacklisted { proposal_hash: T::Hash }, + /// An account has voted in a referendum + Voted { voter: T::AccountId, ref_index: ReferendumIndex, vote: AccountVote> }, + /// An account has secconded a proposal + Seconded { seconder: T::AccountId, prop_index: PropIndex }, } #[pallet::error] @@ -613,10 +604,7 @@ pub mod pallet { impl Hooks> for Pallet { /// Weight: see `begin_block` fn on_initialize(n: T::BlockNumber) -> Weight { - Self::begin_block(n).unwrap_or_else(|e| { - sp_runtime::print(e); - 0 - }) + Self::begin_block(n) } } @@ -660,7 +648,7 @@ pub mod pallet { >::append((index, proposal_hash, who)); - Self::deposit_event(Event::::Proposed(index, value)); + Self::deposit_event(Event::::Proposed { proposal_index: index, deposit: value }); Ok(()) } @@ -687,8 +675,9 @@ pub mod pallet { ensure!(seconds <= seconds_upper_bound, Error::::WrongUpperBound); let mut deposit = Self::deposit_of(proposal).ok_or(Error::::ProposalMissing)?; T::Currency::reserve(&who, deposit.1)?; - deposit.0.push(who); + deposit.0.push(who.clone()); >::insert(proposal, deposit); + Self::deposit_event(Event::::Seconded { seconder: who, prop_index: proposal }); Ok(()) } @@ -853,7 +842,12 @@ pub mod pallet { >::kill(); let now = >::block_number(); - Self::inject_referendum(now + voting_period, proposal_hash, threshold, delay); + Self::inject_referendum( + now.saturating_add(voting_period), + proposal_hash, + threshold, + delay, + ); Ok(()) } @@ -882,10 +876,11 @@ pub mod pallet { existing_vetoers.binary_search(&who).err().ok_or(Error::::AlreadyVetoed)?; existing_vetoers.insert(insert_position, who.clone()); - let until = >::block_number() + T::CooloffPeriod::get(); + let until = + >::block_number().saturating_add(T::CooloffPeriod::get()); >::insert(&proposal_hash, (until, existing_vetoers)); - Self::deposit_event(Event::::Vetoed(who, proposal_hash, until)); + Self::deposit_event(Event::::Vetoed { who, proposal_hash, until }); >::kill(); Ok(()) } @@ -1100,14 +1095,22 @@ pub mod pallet { let now = >::block_number(); let (voting, enactment) = (T::VotingPeriod::get(), T::EnactmentPeriod::get()); let additional = if who == provider { Zero::zero() } else { enactment }; - ensure!(now >= since + voting + additional, Error::::TooEarly); + ensure!( + now >= since.saturating_add(voting).saturating_add(additional), + Error::::TooEarly + ); ensure!(expiry.map_or(true, |e| now > e), Error::::Imminent); let res = T::Currency::repatriate_reserved(&provider, &who, deposit, BalanceStatus::Free); debug_assert!(res.is_ok()); >::remove(&proposal_hash); - Self::deposit_event(Event::::PreimageReaped(proposal_hash, provider, deposit, who)); + Self::deposit_event(Event::::PreimageReaped { + proposal_hash, + provider, + deposit, + reaper: who, + }); Ok(()) } @@ -1252,7 +1255,7 @@ pub mod pallet { } } - Self::deposit_event(Event::::Blacklisted(proposal_hash)); + Self::deposit_event(Event::::Blacklisted { proposal_hash }); Ok(()) } @@ -1288,7 +1291,7 @@ impl Pallet { /// Get the amount locked in support of `proposal`; `None` if proposal isn't a valid proposal /// index. pub fn backing_for(proposal: PropIndex) -> Option> { - Self::deposit_of(proposal).map(|(l, d)| d * (l.len() as u32).into()) + Self::deposit_of(proposal).map(|(l, d)| d.saturating_mul((l.len() as u32).into())) } /// Get all referenda ready for tally at block `n`. @@ -1324,7 +1327,7 @@ impl Pallet { delay: T::BlockNumber, ) -> ReferendumIndex { >::inject_referendum( - >::block_number() + T::VotingPeriod::get(), + >::block_number().saturating_add(T::VotingPeriod::get()), proposal_hash, threshold, delay, @@ -1333,7 +1336,7 @@ impl Pallet { /// Remove a referendum. pub fn internal_cancel_referendum(ref_index: ReferendumIndex) { - Self::deposit_event(Event::::Cancelled(ref_index)); + Self::deposit_event(Event::::Cancelled { ref_index }); ReferendumInfoOf::::remove(ref_index); } @@ -1383,6 +1386,7 @@ impl Pallet { votes.insert(i, (ref_index, vote)); }, } + Self::deposit_event(Event::::Voted { voter: who.clone(), ref_index, vote }); // Shouldn't be possible to fail, but we handle it gracefully. status.tally.add(vote).ok_or(ArithmeticError::Overflow)?; if let Some(approve) = vote.as_standard() { @@ -1429,7 +1433,9 @@ impl Pallet { }, Some(ReferendumInfo::Finished { end, approved }) => { if let Some((lock_periods, balance)) = votes[i].1.locked_if(approved) { - let unlock_at = end + T::VoteLockingPeriod::get() * lock_periods.into(); + let unlock_at = end.saturating_add( + T::VoteLockingPeriod::get().saturating_mul(lock_periods.into()), + ); let now = frame_system::Pallet::::block_number(); if now < unlock_at { ensure!( @@ -1518,9 +1524,16 @@ impl Pallet { }; sp_std::mem::swap(&mut old, voting); match old { - Voting::Delegating { balance, target, conviction, delegations, prior, .. } => { + Voting::Delegating { + balance, target, conviction, delegations, mut prior, .. + } => { // remove any delegation votes to our current target. Self::reduce_upstream_delegation(&target, conviction.votes(balance)); + let now = frame_system::Pallet::::block_number(); + let lock_periods = conviction.lock_periods().into(); + let unlock_block = now + .saturating_add(T::VoteLockingPeriod::get().saturating_mul(lock_periods)); + prior.accumulate(unlock_block, balance); voting.set_common(delegations, prior); }, Voting::Direct { votes, delegations, prior } => { @@ -1535,7 +1548,7 @@ impl Pallet { T::Currency::extend_lock(DEMOCRACY_ID, &who, balance, WithdrawReasons::TRANSFER); Ok(votes) })?; - Self::deposit_event(Event::::Delegated(who, target)); + Self::deposit_event(Event::::Delegated { who, target }); Ok(votes) } @@ -1553,7 +1566,9 @@ impl Pallet { Self::reduce_upstream_delegation(&target, conviction.votes(balance)); let now = frame_system::Pallet::::block_number(); let lock_periods = conviction.lock_periods().into(); - prior.accumulate(now + T::VoteLockingPeriod::get() * lock_periods, balance); + let unlock_block = now + .saturating_add(T::VoteLockingPeriod::get().saturating_mul(lock_periods)); + prior.accumulate(unlock_block, balance); voting.set_common(delegations, prior); Ok(votes) @@ -1561,7 +1576,7 @@ impl Pallet { Voting::Direct { .. } => Err(Error::::NotDelegating.into()), } })?; - Self::deposit_event(Event::::Undelegated(who)); + Self::deposit_event(Event::::Undelegated { account: who }); Ok(votes) } @@ -1592,7 +1607,7 @@ impl Pallet { ReferendumStatus { end, proposal_hash, threshold, delay, tally: Default::default() }; let item = ReferendumInfo::Ongoing(status); >::insert(ref_index, item); - Self::deposit_event(Event::::Started(ref_index, threshold)); + Self::deposit_event(Event::::Started { ref_index, threshold }); ref_index } @@ -1612,7 +1627,7 @@ impl Pallet { LastTabledWasExternal::::put(true); Self::deposit_event(Event::::ExternalTabled); Self::inject_referendum( - now + T::VotingPeriod::get(), + now.saturating_add(T::VotingPeriod::get()), proposal, threshold, T::EnactmentPeriod::get(), @@ -1628,7 +1643,7 @@ impl Pallet { let mut public_props = Self::public_props(); if let Some((winner_index, _)) = public_props.iter().enumerate().max_by_key( // defensive only: All current public proposals have an amount locked - |x| Self::backing_for((x.1).0).unwrap_or_else(Zero::zero), + |x| Self::backing_for((x.1).0).defensive_unwrap_or_else(Zero::zero), ) { let (prop_index, proposal, _) = public_props.swap_remove(winner_index); >::put(public_props); @@ -1638,9 +1653,13 @@ impl Pallet { for d in &depositors { T::Currency::unreserve(d, deposit); } - Self::deposit_event(Event::::Tabled(prop_index, deposit, depositors)); + Self::deposit_event(Event::::Tabled { + proposal_index: prop_index, + deposit, + depositors, + }); Self::inject_referendum( - now + T::VotingPeriod::get(), + now.saturating_add(T::VotingPeriod::get()), proposal, VoteThreshold::SuperMajorityApprove, T::EnactmentPeriod::get(), @@ -1658,22 +1677,25 @@ impl Pallet { if let Ok(proposal) = T::Proposal::decode(&mut &data[..]) { let err_amount = T::Currency::unreserve(&provider, deposit); debug_assert!(err_amount.is_zero()); - Self::deposit_event(Event::::PreimageUsed(proposal_hash, provider, deposit)); + Self::deposit_event(Event::::PreimageUsed { proposal_hash, provider, deposit }); let res = proposal .dispatch(frame_system::RawOrigin::Root.into()) .map(|_| ()) .map_err(|e| e.error); - Self::deposit_event(Event::::Executed(index, res)); + Self::deposit_event(Event::::Executed { ref_index: index, result: res }); Ok(()) } else { T::Slash::on_unbalanced(T::Currency::slash_reserved(&provider, deposit).0); - Self::deposit_event(Event::::PreimageInvalid(proposal_hash, index)); + Self::deposit_event(Event::::PreimageInvalid { + proposal_hash, + ref_index: index, + }); Err(Error::::PreimageInvalid.into()) } } else { - Self::deposit_event(Event::::PreimageMissing(proposal_hash, index)); + Self::deposit_event(Event::::PreimageMissing { proposal_hash, ref_index: index }); Err(Error::::PreimageMissing.into()) } } @@ -1682,16 +1704,16 @@ impl Pallet { now: T::BlockNumber, index: ReferendumIndex, status: ReferendumStatus>, - ) -> Result { + ) -> bool { let total_issuance = T::Currency::total_issuance(); let approved = status.threshold.approved(status.tally, total_issuance); if approved { - Self::deposit_event(Event::::Passed(index)); + Self::deposit_event(Event::::Passed { ref_index: index }); if status.delay.is_zero() { let _ = Self::do_enact_proposal(status.proposal_hash, index); } else { - let when = now + status.delay; + let when = now.saturating_add(status.delay); // Note that we need the preimage now. Preimages::::mutate_exists( &status.proposal_hash, @@ -1716,47 +1738,71 @@ impl Pallet { } } } else { - Self::deposit_event(Event::::NotPassed(index)); + Self::deposit_event(Event::::NotPassed { ref_index: index }); } - Ok(approved) + approved } /// Current era is ending; we should finish up any proposals. /// /// /// # - /// If a referendum is launched or maturing, this will take full block weight. Otherwise: + /// If a referendum is launched or maturing, this will take full block weight if queue is not + /// empty. Otherwise: /// - Complexity: `O(R)` where `R` is the number of unbaked referenda. /// - Db reads: `LastTabledWasExternal`, `NextExternal`, `PublicProps`, `account`, /// `ReferendumCount`, `LowestUnbaked` /// - Db writes: `PublicProps`, `account`, `ReferendumCount`, `DepositOf`, `ReferendumInfoOf` /// - Db reads per R: `DepositOf`, `ReferendumInfoOf` /// # - fn begin_block(now: T::BlockNumber) -> Result { + fn begin_block(now: T::BlockNumber) -> Weight { let max_block_weight = T::BlockWeights::get().max_block; let mut weight = 0; + let next = Self::lowest_unbaked(); + let last = Self::referendum_count(); + let r = last.saturating_sub(next); + // pick out another public referendum if it's time. if (now % T::LaunchPeriod::get()).is_zero() { - // Errors come from the queue being empty. we don't really care about that, and even if - // we did, there is nothing we can do here. - let _ = Self::launch_next(now); - weight = max_block_weight; + // Errors come from the queue being empty. If the queue is not empty, it will take + // full block weight. + if Self::launch_next(now).is_ok() { + weight = max_block_weight; + } else { + weight = + weight.saturating_add(T::WeightInfo::on_initialize_base_with_launch_period(r)); + } + } else { + weight = weight.saturating_add(T::WeightInfo::on_initialize_base(r)); } - let next = Self::lowest_unbaked(); - let last = Self::referendum_count(); - let r = last.saturating_sub(next); - weight = weight.saturating_add(T::WeightInfo::on_initialize_base(r)); // tally up votes for any expiring referenda. for (index, info) in Self::maturing_referenda_at_inner(now, next..last).into_iter() { - let approved = Self::bake_referendum(now, index, info)?; + let approved = Self::bake_referendum(now, index, info); ReferendumInfoOf::::insert(index, ReferendumInfo::Finished { end: now, approved }); weight = max_block_weight; } - Ok(weight) + // Notes: + // * We don't consider the lowest unbaked to be the last maturing in case some refendum have + // longer voting period than others. + // * The iteration here shouldn't trigger any storage read that are not in cache, due to + // `maturing_referenda_at_inner` having already read them. + // * We shouldn't iterate more than `LaunchPeriod/VotingPeriod + 1` times because the number + // of unbaked referendum is bounded by this number. In case those number have changed in a + // runtime upgrade the formula should be adjusted but the bound should still be sensible. + >::mutate(|ref_index| { + while *ref_index < last && + Self::referendum_info(*ref_index) + .map_or(true, |info| matches!(info, ReferendumInfo::Finished { .. })) + { + *ref_index += 1 + } + }); + + weight } /// Reads the length of account in DepositOf without getting the complete value in the runtime. @@ -1849,7 +1895,7 @@ impl Pallet { }; >::insert(proposal_hash, a); - Self::deposit_event(Event::::PreimageNoted(proposal_hash, who, deposit)); + Self::deposit_event(Event::::PreimageNoted { proposal_hash, who, deposit }); Ok(()) } @@ -1875,7 +1921,7 @@ impl Pallet { }; >::insert(proposal_hash, a); - Self::deposit_event(Event::::PreimageNoted(proposal_hash, who, free)); + Self::deposit_event(Event::::PreimageNoted { proposal_hash, who, deposit: free }); Ok(()) } diff --git a/frame/democracy/src/tests.rs b/frame/democracy/src/tests.rs index 75104db51b97..0fe83a07610d 100644 --- a/frame/democracy/src/tests.rs +++ b/frame/democracy/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,9 @@ use crate as pallet_democracy; use codec::Encode; use frame_support::{ assert_noop, assert_ok, ord_parameter_types, parameter_types, - traits::{Contains, GenesisBuild, OnInitialize, SortedMembers}, + traits::{ + ConstU32, ConstU64, Contains, EqualPrivilegeOnly, GenesisBuild, OnInitialize, SortedMembers, + }, weights::Weight, }; use frame_system::{EnsureRoot, EnsureSignedBy}; @@ -50,8 +52,6 @@ const NAY: Vote = Vote { aye: false, conviction: Conviction::None }; const BIG_AYE: Vote = Vote { aye: true, conviction: Conviction::Locked1x }; const BIG_NAY: Vote = Vote { aye: false, conviction: Conviction::Locked1x }; -const MAX_PROPOSALS: u32 = 100; - type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; type Block = frame_system::mocking::MockBlock; @@ -63,7 +63,7 @@ frame_support::construct_runtime!( { System: frame_system::{Pallet, Call, Config, Storage, Event}, Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, - Scheduler: pallet_scheduler::{Pallet, Call, Storage, Config, Event}, + Scheduler: pallet_scheduler::{Pallet, Call, Storage, Event}, Democracy: pallet_democracy::{Pallet, Call, Storage, Config, Event}, } ); @@ -77,7 +77,6 @@ impl Contains for BaseFilter { } parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1_000_000); } @@ -96,7 +95,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -105,6 +104,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } parameter_types! { pub MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * BlockWeights::get().max_block; @@ -118,32 +118,23 @@ impl pallet_scheduler::Config for Test { type ScheduleOrigin = EnsureRoot; type MaxScheduledPerBlock = (); type WeightInfo = (); + type OriginPrivilegeCmp = EqualPrivilegeOnly; + type PreimageProvider = (); + type NoPreimagePostponement = (); } -parameter_types! { - pub const ExistentialDeposit: u64 = 1; - pub const MaxLocks: u32 = 10; -} + impl pallet_balances::Config for Test { type MaxReserves = (); type ReserveIdentifier = [u8; 8]; - type MaxLocks = MaxLocks; + type MaxLocks = ConstU32<10>; type Balance = u64; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } parameter_types! { - pub const LaunchPeriod: u64 = 2; - pub const VotingPeriod: u64 = 2; - pub const FastTrackVotingPeriod: u64 = 2; - pub const MinimumDeposit: u64 = 1; - pub const EnactmentPeriod: u64 = 2; - pub const VoteLockingPeriod: u64 = 3; - pub const CooloffPeriod: u64 = 2; - pub const MaxVotes: u32 = 100; - pub const MaxProposals: u32 = MAX_PROPOSALS; pub static PreimageByteDeposit: u64 = 0; pub static InstantAllowed: bool = false; } @@ -168,12 +159,12 @@ impl Config for Test { type Proposal = Call; type Event = Event; type Currency = pallet_balances::Pallet; - type EnactmentPeriod = EnactmentPeriod; - type LaunchPeriod = LaunchPeriod; - type VotingPeriod = VotingPeriod; - type VoteLockingPeriod = VoteLockingPeriod; - type FastTrackVotingPeriod = FastTrackVotingPeriod; - type MinimumDeposit = MinimumDeposit; + type EnactmentPeriod = ConstU64<2>; + type LaunchPeriod = ConstU64<2>; + type VotingPeriod = ConstU64<2>; + type VoteLockingPeriod = ConstU64<3>; + type FastTrackVotingPeriod = ConstU64<2>; + type MinimumDeposit = ConstU64<1>; type ExternalOrigin = EnsureSignedBy; type ExternalMajorityOrigin = EnsureSignedBy; type ExternalDefaultOrigin = EnsureSignedBy; @@ -182,17 +173,17 @@ impl Config for Test { type BlacklistOrigin = EnsureRoot; type CancelProposalOrigin = EnsureRoot; type VetoOrigin = EnsureSignedBy; - type CooloffPeriod = CooloffPeriod; + type CooloffPeriod = ConstU64<2>; type PreimageByteDeposit = PreimageByteDeposit; type Slash = (); type InstantOrigin = EnsureSignedBy; type InstantAllowed = InstantAllowed; type Scheduler = Scheduler; - type MaxVotes = MaxVotes; + type MaxVotes = ConstU32<100>; type OperationalPreimageOrigin = EnsureSignedBy; type PalletsOrigin = OriginCaller; type WeightInfo = (); - type MaxProposals = MaxProposals; + type MaxProposals = ConstU32<100>; } pub fn new_test_ext() -> sp_io::TestExternalities { @@ -264,7 +255,7 @@ fn propose_set_balance_and_note(who: u64, value: u64, delay: u64) -> DispatchRes fn next_block() { System::set_block_number(System::block_number() + 1); Scheduler::on_initialize(System::block_number()); - assert!(Democracy::begin_block(System::block_number()).is_ok()); + Democracy::begin_block(System::block_number()); } fn fast_forward_to(n: u64) { diff --git a/frame/democracy/src/tests/cancellation.rs b/frame/democracy/src/tests/cancellation.rs index c2bd725ce934..9035e17c5c80 100644 --- a/frame/democracy/src/tests/cancellation.rs +++ b/frame/democracy/src/tests/cancellation.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,10 +30,14 @@ fn cancel_referendum_should_work() { ); assert_ok!(Democracy::vote(Origin::signed(1), r, aye(1))); assert_ok!(Democracy::cancel_referendum(Origin::root(), r.into())); + assert_eq!(Democracy::lowest_unbaked(), 0); next_block(); + next_block(); + assert_eq!(Democracy::lowest_unbaked(), 1); + assert_eq!(Democracy::lowest_unbaked(), Democracy::referendum_count()); assert_eq!(Balances::free_balance(42), 0); }); } diff --git a/frame/democracy/src/tests/decoders.rs b/frame/democracy/src/tests/decoders.rs index 3c1729c4355c..1fbb88060549 100644 --- a/frame/democracy/src/tests/decoders.rs +++ b/frame/democracy/src/tests/decoders.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/democracy/src/tests/delegation.rs b/frame/democracy/src/tests/delegation.rs index d3afa1c13f90..3551ca8f9112 100644 --- a/frame/democracy/src/tests/delegation.rs +++ b/frame/democracy/src/tests/delegation.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -158,7 +158,7 @@ fn conviction_should_be_honored_in_delegation() { // If transactor voted, delegated vote is overwritten. new_test_ext().execute_with(|| { let r = begin_referendum(); - // Delegate, undelegate and vote. + // Delegate and vote. assert_ok!(Democracy::delegate(Origin::signed(2), 1, Conviction::Locked6x, 20)); assert_ok!(Democracy::vote(Origin::signed(1), r, aye(1))); // Delegated vote is huge. @@ -177,3 +177,42 @@ fn split_vote_delegation_should_be_ignored() { assert_eq!(tally(r), Tally { ayes: 1, nays: 0, turnout: 10 }); }); } + +#[test] +fn redelegation_keeps_lock() { + // If transactor voted, delegated vote is overwritten. + new_test_ext().execute_with(|| { + let r = begin_referendum(); + // Delegate and vote. + assert_ok!(Democracy::delegate(Origin::signed(2), 1, Conviction::Locked6x, 20)); + assert_ok!(Democracy::vote(Origin::signed(1), r, aye(1))); + // Delegated vote is huge. + assert_eq!(tally(r), Tally { ayes: 121, nays: 0, turnout: 30 }); + + let mut prior_lock = vote::PriorLock::default(); + + // Locked balance of delegator exists + assert_eq!(VotingOf::::get(2).locked_balance(), 20); + assert_eq!(VotingOf::::get(2).prior(), &prior_lock); + + // Delegate someone else at a lower conviction and amount + assert_ok!(Democracy::delegate(Origin::signed(2), 3, Conviction::None, 10)); + + // 6x prior should appear w/ locked balance. + prior_lock.accumulate(98, 20); + assert_eq!(VotingOf::::get(2).prior(), &prior_lock); + assert_eq!(VotingOf::::get(2).locked_balance(), 20); + // Unlock shouldn't work + assert_ok!(Democracy::unlock(Origin::signed(2), 2)); + assert_eq!(VotingOf::::get(2).prior(), &prior_lock); + assert_eq!(VotingOf::::get(2).locked_balance(), 20); + + fast_forward_to(100); + + // Now unlock can remove the prior lock and reduce the locked amount. + assert_eq!(VotingOf::::get(2).prior(), &prior_lock); + assert_ok!(Democracy::unlock(Origin::signed(2), 2)); + assert_eq!(VotingOf::::get(2).prior(), &vote::PriorLock::default()); + assert_eq!(VotingOf::::get(2).locked_balance(), 10); + }); +} diff --git a/frame/democracy/src/tests/external_proposing.rs b/frame/democracy/src/tests/external_proposing.rs index 7442964584fa..5d4a9f2a7cbf 100644 --- a/frame/democracy/src/tests/external_proposing.rs +++ b/frame/democracy/src/tests/external_proposing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/democracy/src/tests/fast_tracking.rs b/frame/democracy/src/tests/fast_tracking.rs index 9b2f2760bde1..7a15bb8de4da 100644 --- a/frame/democracy/src/tests/fast_tracking.rs +++ b/frame/democracy/src/tests/fast_tracking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/democracy/src/tests/lock_voting.rs b/frame/democracy/src/tests/lock_voting.rs index 8b80b39c14aa..071873436731 100644 --- a/frame/democracy/src/tests/lock_voting.rs +++ b/frame/democracy/src/tests/lock_voting.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,6 @@ //! The tests for functionality concerning locking and lock-voting. use super::*; -use std::convert::TryFrom; fn aye(x: u8, balance: u64) -> AccountVote { AccountVote::Standard { diff --git a/frame/democracy/src/tests/preimage.rs b/frame/democracy/src/tests/preimage.rs index 6d478fcaa68c..21303c8eddae 100644 --- a/frame/democracy/src/tests/preimage.rs +++ b/frame/democracy/src/tests/preimage.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/democracy/src/tests/public_proposals.rs b/frame/democracy/src/tests/public_proposals.rs index 34713c3e1572..aadc92d24f41 100644 --- a/frame/democracy/src/tests/public_proposals.rs +++ b/frame/democracy/src/tests/public_proposals.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/democracy/src/tests/scheduling.rs b/frame/democracy/src/tests/scheduling.rs index 06b492bc6093..d28f24d76bb5 100644 --- a/frame/democracy/src/tests/scheduling.rs +++ b/frame/democracy/src/tests/scheduling.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,8 +30,10 @@ fn simple_passing_should_work() { ); assert_ok!(Democracy::vote(Origin::signed(1), r, aye(1))); assert_eq!(tally(r), Tally { ayes: 1, nays: 0, turnout: 10 }); + assert_eq!(Democracy::lowest_unbaked(), 0); next_block(); next_block(); + assert_eq!(Democracy::lowest_unbaked(), 1); assert_eq!(Balances::free_balance(42), 2); }); } @@ -110,3 +112,45 @@ fn delayed_enactment_should_work() { assert_eq!(Balances::free_balance(42), 2); }); } + +#[test] +fn lowest_unbaked_should_be_sensible() { + new_test_ext().execute_with(|| { + let r1 = Democracy::inject_referendum( + 3, + set_balance_proposal_hash_and_note(1), + VoteThreshold::SuperMajorityApprove, + 0, + ); + let r2 = Democracy::inject_referendum( + 2, + set_balance_proposal_hash_and_note(2), + VoteThreshold::SuperMajorityApprove, + 0, + ); + let r3 = Democracy::inject_referendum( + 10, + set_balance_proposal_hash_and_note(3), + VoteThreshold::SuperMajorityApprove, + 0, + ); + assert_ok!(Democracy::vote(Origin::signed(1), r1, aye(1))); + assert_ok!(Democracy::vote(Origin::signed(1), r2, aye(1))); + // r3 is canceled + assert_ok!(Democracy::cancel_referendum(Origin::root(), r3.into())); + assert_eq!(Democracy::lowest_unbaked(), 0); + + next_block(); + + // r2 is approved + assert_eq!(Balances::free_balance(42), 2); + assert_eq!(Democracy::lowest_unbaked(), 0); + + next_block(); + + // r1 is approved + assert_eq!(Balances::free_balance(42), 1); + assert_eq!(Democracy::lowest_unbaked(), 3); + assert_eq!(Democracy::lowest_unbaked(), Democracy::referendum_count()); + }); +} diff --git a/frame/democracy/src/tests/voting.rs b/frame/democracy/src/tests/voting.rs index e035c2d46c1b..d4fceaf0ee48 100644 --- a/frame/democracy/src/tests/voting.rs +++ b/frame/democracy/src/tests/voting.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/democracy/src/types.rs b/frame/democracy/src/types.rs index 2eb004ba61bc..52ab8a40eb3e 100644 --- a/frame/democracy/src/types.rs +++ b/frame/democracy/src/types.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/democracy/src/vote.rs b/frame/democracy/src/vote.rs index 03ca020ca094..c74623d4dfeb 100644 --- a/frame/democracy/src/vote.rs +++ b/frame/democracy/src/vote.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,7 +24,7 @@ use sp_runtime::{ traits::{Saturating, Zero}, RuntimeDebug, }; -use sp_std::{convert::TryFrom, prelude::*, result::Result}; +use sp_std::prelude::*; /// A number of lock periods, plus a vote, one way or the other. #[derive(Copy, Clone, Eq, PartialEq, Default, RuntimeDebug)] @@ -183,7 +183,7 @@ impl votes.iter().map(|i| i.1.balance()).fold(prior.locked(), |a, i| a.max(i)), - Voting::Delegating { balance, .. } => *balance, + Voting::Delegating { balance, prior, .. } => *balance.max(&prior.locked()), } } @@ -199,4 +199,11 @@ impl &PriorLock { + match self { + Voting::Direct { prior, .. } => prior, + Voting::Delegating { prior, .. } => prior, + } + } } diff --git a/frame/democracy/src/vote_threshold.rs b/frame/democracy/src/vote_threshold.rs index ad8bce290ed4..443d6b116619 100644 --- a/frame/democracy/src/vote_threshold.rs +++ b/frame/democracy/src/vote_threshold.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/democracy/src/weights.rs b/frame/democracy/src/weights.rs index e3f22f4fc0ab..19bf602c5d29 100644 --- a/frame/democracy/src/weights.rs +++ b/frame/democracy/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_democracy //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-30, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/democracy/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -60,6 +61,7 @@ pub trait WeightInfo { fn cancel_referendum() -> Weight; fn cancel_queued(r: u32, ) -> Weight; fn on_initialize_base(r: u32, ) -> Weight; + fn on_initialize_base_with_launch_period(r: u32, ) -> Weight; fn delegate(r: u32, ) -> Weight; fn undelegate(r: u32, ) -> Weight; fn clear_public_proposals() -> Weight; @@ -80,15 +82,15 @@ impl WeightInfo for SubstrateWeight { // Storage: Democracy Blacklist (r:1 w:0) // Storage: Democracy DepositOf (r:0 w:1) fn propose() -> Weight { - (65_665_000 as Weight) + (42_560_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Democracy DepositOf (r:1 w:1) fn second(s: u32, ) -> Weight { - (40_003_000 as Weight) + (25_603_000 as Weight) // Standard Error: 1_000 - .saturating_add((180_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((167_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -96,9 +98,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Democracy VotingOf (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vote_new(r: u32, ) -> Weight { - (45_465_000 as Weight) - // Standard Error: 1_000 - .saturating_add((220_000 as Weight).saturating_mul(r as Weight)) + (34_248_000 as Weight) + // Standard Error: 2_000 + .saturating_add((135_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -106,16 +108,16 @@ impl WeightInfo for SubstrateWeight { // Storage: Democracy VotingOf (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vote_existing(r: u32, ) -> Weight { - (45_112_000 as Weight) - // Standard Error: 1_000 - .saturating_add((222_000 as Weight).saturating_mul(r as Weight)) + (33_927_000 as Weight) + // Standard Error: 2_000 + .saturating_add((138_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Democracy ReferendumInfoOf (r:1 w:1) // Storage: Democracy Cancellations (r:1 w:1) fn emergency_cancel() -> Weight { - (26_651_000 as Weight) + (14_714_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -126,45 +128,45 @@ impl WeightInfo for SubstrateWeight { // Storage: Democracy DepositOf (r:1 w:1) // Storage: System Account (r:1 w:1) fn blacklist(p: u32, ) -> Weight { - (77_737_000 as Weight) - // Standard Error: 4_000 - .saturating_add((512_000 as Weight).saturating_mul(p as Weight)) + (49_423_000 as Weight) + // Standard Error: 3_000 + .saturating_add((275_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) } // Storage: Democracy NextExternal (r:1 w:1) // Storage: Democracy Blacklist (r:1 w:0) fn external_propose(v: u32, ) -> Weight { - (13_126_000 as Weight) + (7_321_000 as Weight) // Standard Error: 0 - .saturating_add((89_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((75_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Democracy NextExternal (r:0 w:1) fn external_propose_majority() -> Weight { - (2_923_000 as Weight) + (1_325_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Democracy NextExternal (r:0 w:1) fn external_propose_default() -> Weight { - (2_889_000 as Weight) + (1_283_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Democracy NextExternal (r:1 w:1) // Storage: Democracy ReferendumCount (r:1 w:1) // Storage: Democracy ReferendumInfoOf (r:0 w:1) fn fast_track() -> Weight { - (27_598_000 as Weight) + (15_355_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Democracy NextExternal (r:1 w:1) // Storage: Democracy Blacklist (r:1 w:1) fn veto_external(v: u32, ) -> Weight { - (28_416_000 as Weight) + (16_597_000 as Weight) // Standard Error: 0 - .saturating_add((132_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((90_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -172,46 +174,58 @@ impl WeightInfo for SubstrateWeight { // Storage: Democracy DepositOf (r:1 w:1) // Storage: System Account (r:1 w:1) fn cancel_proposal(p: u32, ) -> Weight { - (52_836_000 as Weight) + (37_695_000 as Weight) // Standard Error: 2_000 - .saturating_add((478_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((258_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Democracy ReferendumInfoOf (r:0 w:1) fn cancel_referendum() -> Weight { - (16_891_000 as Weight) + (9_456_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Scheduler Lookup (r:1 w:1) // Storage: Scheduler Agenda (r:1 w:1) fn cancel_queued(r: u32, ) -> Weight { - (30_504_000 as Weight) - // Standard Error: 2_000 - .saturating_add((1_480_000 as Weight).saturating_mul(r as Weight)) + (19_915_000 as Weight) + // Standard Error: 1_000 + .saturating_add((609_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } + // Storage: Democracy LowestUnbaked (r:1 w:1) + // Storage: Democracy ReferendumCount (r:1 w:0) + // Storage: Democracy ReferendumInfoOf (r:1 w:0) + fn on_initialize_base(r: u32, ) -> Weight { + (3_787_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_852_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(r as Weight))) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Democracy LowestUnbaked (r:1 w:1) + // Storage: Democracy ReferendumCount (r:1 w:0) // Storage: Democracy LastTabledWasExternal (r:1 w:0) // Storage: Democracy NextExternal (r:1 w:0) // Storage: Democracy PublicProps (r:1 w:0) - // Storage: Democracy LowestUnbaked (r:1 w:0) - // Storage: Democracy ReferendumCount (r:1 w:0) // Storage: Democracy ReferendumInfoOf (r:1 w:0) - fn on_initialize_base(r: u32, ) -> Weight { - (6_259_000 as Weight) - // Standard Error: 4_000 - .saturating_add((5_032_000 as Weight).saturating_mul(r as Weight)) + fn on_initialize_base_with_launch_period(r: u32, ) -> Weight { + (7_890_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_856_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(r as Weight))) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Democracy VotingOf (r:3 w:3) // Storage: Democracy ReferendumInfoOf (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn delegate(r: u32, ) -> Weight { - (51_719_000 as Weight) - // Standard Error: 5_000 - .saturating_add((7_210_000 as Weight).saturating_mul(r as Weight)) + (31_190_000 as Weight) + // Standard Error: 3_000 + .saturating_add((3_753_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(r as Weight))) .saturating_add(T::DbWeight::get().writes(4 as Weight)) @@ -220,9 +234,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Democracy VotingOf (r:2 w:2) // Storage: Democracy ReferendumInfoOf (r:1 w:1) fn undelegate(r: u32, ) -> Weight { - (23_203_000 as Weight) - // Standard Error: 5_000 - .saturating_add((7_206_000 as Weight).saturating_mul(r as Weight)) + (15_121_000 as Weight) + // Standard Error: 3_000 + .saturating_add((3_852_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(r as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) @@ -230,31 +244,31 @@ impl WeightInfo for SubstrateWeight { } // Storage: Democracy PublicProps (r:0 w:1) fn clear_public_proposals() -> Weight { - (3_127_000 as Weight) + (1_583_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Democracy Preimages (r:1 w:1) fn note_preimage(b: u32, ) -> Weight { - (44_130_000 as Weight) + (22_681_000 as Weight) // Standard Error: 0 - .saturating_add((3_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(b as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Democracy Preimages (r:1 w:1) fn note_imminent_preimage(b: u32, ) -> Weight { - (28_756_000 as Weight) + (15_599_000 as Weight) // Standard Error: 0 - .saturating_add((3_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(b as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Democracy Preimages (r:1 w:1) // Storage: System Account (r:1 w:0) fn reap_preimage(b: u32, ) -> Weight { - (39_922_000 as Weight) + (23_663_000 as Weight) // Standard Error: 0 - .saturating_add((2_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((1_000 as Weight).saturating_mul(b as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -262,9 +276,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn unlock_remove(r: u32, ) -> Weight { - (38_621_000 as Weight) + (21_928_000 as Weight) // Standard Error: 1_000 - .saturating_add((110_000 as Weight).saturating_mul(r as Weight)) + .saturating_add((61_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -272,27 +286,27 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn unlock_set(r: u32, ) -> Weight { - (36_631_000 as Weight) + (21_548_000 as Weight) // Standard Error: 1_000 - .saturating_add((214_000 as Weight).saturating_mul(r as Weight)) + .saturating_add((123_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Democracy ReferendumInfoOf (r:1 w:1) // Storage: Democracy VotingOf (r:1 w:1) fn remove_vote(r: u32, ) -> Weight { - (21_025_000 as Weight) + (11_797_000 as Weight) // Standard Error: 1_000 - .saturating_add((195_000 as Weight).saturating_mul(r as Weight)) + .saturating_add((117_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Democracy ReferendumInfoOf (r:1 w:1) // Storage: Democracy VotingOf (r:1 w:1) fn remove_other_vote(r: u32, ) -> Weight { - (20_628_000 as Weight) + (11_930_000 as Weight) // Standard Error: 1_000 - .saturating_add((214_000 as Weight).saturating_mul(r as Weight)) + .saturating_add((122_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -305,15 +319,15 @@ impl WeightInfo for () { // Storage: Democracy Blacklist (r:1 w:0) // Storage: Democracy DepositOf (r:0 w:1) fn propose() -> Weight { - (65_665_000 as Weight) + (42_560_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Democracy DepositOf (r:1 w:1) fn second(s: u32, ) -> Weight { - (40_003_000 as Weight) + (25_603_000 as Weight) // Standard Error: 1_000 - .saturating_add((180_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((167_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -321,9 +335,9 @@ impl WeightInfo for () { // Storage: Democracy VotingOf (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vote_new(r: u32, ) -> Weight { - (45_465_000 as Weight) - // Standard Error: 1_000 - .saturating_add((220_000 as Weight).saturating_mul(r as Weight)) + (34_248_000 as Weight) + // Standard Error: 2_000 + .saturating_add((135_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -331,16 +345,16 @@ impl WeightInfo for () { // Storage: Democracy VotingOf (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vote_existing(r: u32, ) -> Weight { - (45_112_000 as Weight) - // Standard Error: 1_000 - .saturating_add((222_000 as Weight).saturating_mul(r as Weight)) + (33_927_000 as Weight) + // Standard Error: 2_000 + .saturating_add((138_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Democracy ReferendumInfoOf (r:1 w:1) // Storage: Democracy Cancellations (r:1 w:1) fn emergency_cancel() -> Weight { - (26_651_000 as Weight) + (14_714_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -351,45 +365,45 @@ impl WeightInfo for () { // Storage: Democracy DepositOf (r:1 w:1) // Storage: System Account (r:1 w:1) fn blacklist(p: u32, ) -> Weight { - (77_737_000 as Weight) - // Standard Error: 4_000 - .saturating_add((512_000 as Weight).saturating_mul(p as Weight)) + (49_423_000 as Weight) + // Standard Error: 3_000 + .saturating_add((275_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } // Storage: Democracy NextExternal (r:1 w:1) // Storage: Democracy Blacklist (r:1 w:0) fn external_propose(v: u32, ) -> Weight { - (13_126_000 as Weight) + (7_321_000 as Weight) // Standard Error: 0 - .saturating_add((89_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((75_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Democracy NextExternal (r:0 w:1) fn external_propose_majority() -> Weight { - (2_923_000 as Weight) + (1_325_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Democracy NextExternal (r:0 w:1) fn external_propose_default() -> Weight { - (2_889_000 as Weight) + (1_283_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Democracy NextExternal (r:1 w:1) // Storage: Democracy ReferendumCount (r:1 w:1) // Storage: Democracy ReferendumInfoOf (r:0 w:1) fn fast_track() -> Weight { - (27_598_000 as Weight) + (15_355_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Democracy NextExternal (r:1 w:1) // Storage: Democracy Blacklist (r:1 w:1) fn veto_external(v: u32, ) -> Weight { - (28_416_000 as Weight) + (16_597_000 as Weight) // Standard Error: 0 - .saturating_add((132_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((90_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -397,46 +411,58 @@ impl WeightInfo for () { // Storage: Democracy DepositOf (r:1 w:1) // Storage: System Account (r:1 w:1) fn cancel_proposal(p: u32, ) -> Weight { - (52_836_000 as Weight) + (37_695_000 as Weight) // Standard Error: 2_000 - .saturating_add((478_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((258_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Democracy ReferendumInfoOf (r:0 w:1) fn cancel_referendum() -> Weight { - (16_891_000 as Weight) + (9_456_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Scheduler Lookup (r:1 w:1) // Storage: Scheduler Agenda (r:1 w:1) fn cancel_queued(r: u32, ) -> Weight { - (30_504_000 as Weight) - // Standard Error: 2_000 - .saturating_add((1_480_000 as Weight).saturating_mul(r as Weight)) + (19_915_000 as Weight) + // Standard Error: 1_000 + .saturating_add((609_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } + // Storage: Democracy LowestUnbaked (r:1 w:1) + // Storage: Democracy ReferendumCount (r:1 w:0) + // Storage: Democracy ReferendumInfoOf (r:1 w:0) + fn on_initialize_base(r: u32, ) -> Weight { + (3_787_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_852_000 as Weight).saturating_mul(r as Weight)) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(r as Weight))) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Democracy LowestUnbaked (r:1 w:1) + // Storage: Democracy ReferendumCount (r:1 w:0) // Storage: Democracy LastTabledWasExternal (r:1 w:0) // Storage: Democracy NextExternal (r:1 w:0) // Storage: Democracy PublicProps (r:1 w:0) - // Storage: Democracy LowestUnbaked (r:1 w:0) - // Storage: Democracy ReferendumCount (r:1 w:0) // Storage: Democracy ReferendumInfoOf (r:1 w:0) - fn on_initialize_base(r: u32, ) -> Weight { - (6_259_000 as Weight) - // Standard Error: 4_000 - .saturating_add((5_032_000 as Weight).saturating_mul(r as Weight)) + fn on_initialize_base_with_launch_period(r: u32, ) -> Weight { + (7_890_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_856_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(r as Weight))) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Democracy VotingOf (r:3 w:3) // Storage: Democracy ReferendumInfoOf (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn delegate(r: u32, ) -> Weight { - (51_719_000 as Weight) - // Standard Error: 5_000 - .saturating_add((7_210_000 as Weight).saturating_mul(r as Weight)) + (31_190_000 as Weight) + // Standard Error: 3_000 + .saturating_add((3_753_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(r as Weight))) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) @@ -445,9 +471,9 @@ impl WeightInfo for () { // Storage: Democracy VotingOf (r:2 w:2) // Storage: Democracy ReferendumInfoOf (r:1 w:1) fn undelegate(r: u32, ) -> Weight { - (23_203_000 as Weight) - // Standard Error: 5_000 - .saturating_add((7_206_000 as Weight).saturating_mul(r as Weight)) + (15_121_000 as Weight) + // Standard Error: 3_000 + .saturating_add((3_852_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(r as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) @@ -455,31 +481,31 @@ impl WeightInfo for () { } // Storage: Democracy PublicProps (r:0 w:1) fn clear_public_proposals() -> Weight { - (3_127_000 as Weight) + (1_583_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Democracy Preimages (r:1 w:1) fn note_preimage(b: u32, ) -> Weight { - (44_130_000 as Weight) + (22_681_000 as Weight) // Standard Error: 0 - .saturating_add((3_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(b as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Democracy Preimages (r:1 w:1) fn note_imminent_preimage(b: u32, ) -> Weight { - (28_756_000 as Weight) + (15_599_000 as Weight) // Standard Error: 0 - .saturating_add((3_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(b as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Democracy Preimages (r:1 w:1) // Storage: System Account (r:1 w:0) fn reap_preimage(b: u32, ) -> Weight { - (39_922_000 as Weight) + (23_663_000 as Weight) // Standard Error: 0 - .saturating_add((2_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((1_000 as Weight).saturating_mul(b as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -487,9 +513,9 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn unlock_remove(r: u32, ) -> Weight { - (38_621_000 as Weight) + (21_928_000 as Weight) // Standard Error: 1_000 - .saturating_add((110_000 as Weight).saturating_mul(r as Weight)) + .saturating_add((61_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -497,27 +523,27 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn unlock_set(r: u32, ) -> Weight { - (36_631_000 as Weight) + (21_548_000 as Weight) // Standard Error: 1_000 - .saturating_add((214_000 as Weight).saturating_mul(r as Weight)) + .saturating_add((123_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Democracy ReferendumInfoOf (r:1 w:1) // Storage: Democracy VotingOf (r:1 w:1) fn remove_vote(r: u32, ) -> Weight { - (21_025_000 as Weight) + (11_797_000 as Weight) // Standard Error: 1_000 - .saturating_add((195_000 as Weight).saturating_mul(r as Weight)) + .saturating_add((117_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Democracy ReferendumInfoOf (r:1 w:1) // Storage: Democracy VotingOf (r:1 w:1) fn remove_other_vote(r: u32, ) -> Weight { - (20_628_000 as Weight) + (11_930_000 as Weight) // Standard Error: 1_000 - .saturating_add((214_000 as Weight).saturating_mul(r as Weight)) + .saturating_add((122_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } diff --git a/frame/election-provider-multi-phase/Cargo.toml b/frame/election-provider-multi-phase/Cargo.toml index b2d50321e8cd..25f98d965d86 100644 --- a/frame/election-provider-multi-phase/Cargo.toml +++ b/frame/election-provider-multi-phase/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-election-provider-multi-phase" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "PALLET two phase election providers" readme = "README.md" @@ -14,21 +14,21 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] static_assertions = "1.1.0" -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } log = { version = "0.4.14", default-features = false } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-npos-elections = { version = "4.0.0-dev", default-features = false, path = "../../primitives/npos-elections" } -sp-arithmetic = { version = "4.0.0-dev", default-features = false, path = "../../primitives/arithmetic" } +sp-arithmetic = { version = "5.0.0", default-features = false, path = "../../primitives/arithmetic" } frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } # Optional imports for benchmarking @@ -37,18 +37,16 @@ rand = { version = "0.7.3", default-features = false, optional = true, features "alloc", "small_rng", ] } -strum = { optional = true, version = "0.21.0" } -strum_macros = { optional = true, version = "0.21.1" } +strum = { optional = true, default-features = false, version = "0.23.0", features = ["derive"] } [dev-dependencies] -parking_lot = "0.11.0" +parking_lot = "0.12.0" rand = { version = "0.7.3" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } sp-npos-elections = { version = "4.0.0-dev", default-features = false, path = "../../primitives/npos-elections" } -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } -frame-election-provider-support = { version = "4.0.0-dev", features = [ -], path = "../election-provider-support" } +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } +frame-election-provider-support = { version = "4.0.0-dev", path = "../election-provider-support" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } @@ -70,12 +68,15 @@ std = [ "sp-arithmetic/std", "frame-election-provider-support/std", "log/std", + + "frame-benchmarking/std", + "rand/std", + "strum/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-election-provider-support/runtime-benchmarks", "rand", "strum", - "strum_macros", ] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/election-provider-multi-phase/src/benchmarking.rs b/frame/election-provider-multi-phase/src/benchmarking.rs index b8d7bc45c448..923e9e2d984c 100644 --- a/frame/election-provider-multi-phase/src/benchmarking.rs +++ b/frame/election-provider-multi-phase/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,17 +19,16 @@ use super::*; use crate::{unsigned::IndexAssignmentOf, Pallet as MultiPhase}; -use frame_benchmarking::{account, impl_benchmark_test_suite}; -use frame_support::{assert_ok, traits::Hooks}; +use frame_benchmarking::account; +use frame_support::{ + assert_ok, + traits::{Hooks, TryCollect}, + BoundedVec, +}; use frame_system::RawOrigin; use rand::{prelude::SliceRandom, rngs::SmallRng, SeedableRng}; use sp_arithmetic::{per_things::Percent, traits::One}; -use sp_npos_elections::IndexAssignment; use sp_runtime::InnerOf; -use sp_std::{ - boxed::Box, - convert::{TryFrom, TryInto}, -}; const SEED: u32 = 999; @@ -73,11 +72,12 @@ fn solution_with_size( let active_voters = (0..active_voters_count) .map(|i| { // chose a random subset of winners. - let winner_votes = winners + let winner_votes: BoundedVec<_, _> = winners .as_slice() .choose_multiple(&mut rng, >::LIMIT) .cloned() - .collect::>(); + .try_collect() + .expect(">::LIMIT is the correct bound; qed."); let voter = frame_benchmarking::account::("Voter", i, SEED); (voter, stake, winner_votes) }) @@ -91,10 +91,11 @@ fn solution_with_size( .collect::>(); let rest_voters = (active_voters_count..size.voters) .map(|i| { - let votes = (&non_winners) + let votes: BoundedVec<_, _> = (&non_winners) .choose_multiple(&mut rng, >::LIMIT) .cloned() - .collect::>(); + .try_collect() + .expect(">::LIMIT is the correct bound; qed."); let voter = frame_benchmarking::account::("Voter", i, SEED); (voter, stake, votes) }) @@ -146,7 +147,10 @@ fn solution_with_size( let score = solution.clone().score(stake_of, voter_at, target_at).unwrap(); let round = >::round(); - assert!(score[0] > 0, "score is zero, this probably means that the stakes are not set."); + assert!( + score.minimal_stake > 0, + "score is zero, this probably means that the stakes are not set." + ); Ok(RawSolution { solution, score, round }) } @@ -156,7 +160,7 @@ fn set_up_data_provider(v: u32, t: u32) { info, "setting up with voters = {} [degree = {}], targets = {}", v, - T::DataProvider::MAXIMUM_VOTES_PER_VOTER, + ::MaxVotesPerVoter::get(), t ); @@ -169,14 +173,16 @@ fn set_up_data_provider(v: u32, t: u32) { }) .collect::>(); // we should always have enough voters to fill. - assert!(targets.len() > T::DataProvider::MAXIMUM_VOTES_PER_VOTER as usize); - targets.truncate(T::DataProvider::MAXIMUM_VOTES_PER_VOTER as usize); + assert!( + targets.len() > ::MaxVotesPerVoter::get() as usize + ); + targets.truncate(::MaxVotesPerVoter::get() as usize); // fill voters. (0..v).for_each(|i| { let voter = frame_benchmarking::account::("Voter", i, SEED); let weight = T::Currency::minimum_balance().saturated_into::() * 1000; - T::DataProvider::add_voter(voter, weight, targets.clone()); + T::DataProvider::add_voter(voter, weight, targets.clone().try_into().unwrap()); }); } @@ -213,7 +219,11 @@ frame_benchmarking::benchmarks! { let receiver = account("receiver", 0, SEED); let initial_balance = T::Currency::minimum_balance() * 10u32.into(); T::Currency::make_free_balance_be(&receiver, initial_balance); - let ready: ReadySolution = Default::default(); + let ready = ReadySolution { + supports: vec![], + score: Default::default(), + compute: Default::default() + }; let deposit: BalanceOf = 10u32.into(); let reward: BalanceOf = 20u32.into(); @@ -250,8 +260,8 @@ frame_benchmarking::benchmarks! { // we don't directly need the data-provider to be populated, but it is just easy to use it. set_up_data_provider::(v, t); - let targets = T::DataProvider::targets(None)?; - let voters = T::DataProvider::voters(None)?; + let targets = T::DataProvider::electable_targets(None)?; + let voters = T::DataProvider::electing_voters(None)?; let desired_targets = T::DataProvider::desired_targets()?; assert!(>::snapshot().is_none()); }: { @@ -289,7 +299,7 @@ frame_benchmarking::benchmarks! { assert!(>::get().is_some()); assert!(>::get().is_some()); }: { - assert_ok!( as ElectionProvider>::elect()); + assert_ok!( as ElectionProvider>::elect()); } verify { assert!(>::queued_solution().is_none()); assert!(>::get().is_none()); @@ -299,12 +309,10 @@ frame_benchmarking::benchmarks! { } submit { - let c in 1 .. (T::SignedMaxSubmissions::get() - 1); - // the solution will be worse than all of them meaning the score need to be checked against // ~ log2(c) let solution = RawSolution { - score: [(10_000_000u128 - 1).into(), 0, 0], + score: ElectionScore { minimal_stake: 10_000_000u128 - 1, ..Default::default() }, ..Default::default() }; @@ -313,22 +321,34 @@ frame_benchmarking::benchmarks! { >::put(1); let mut signed_submissions = SignedSubmissions::::get(); - for i in 0..c { + + // Insert `max - 1` submissions because the call to `submit` will insert another + // submission and the score is worse then the previous scores. + for i in 0..(T::SignedMaxSubmissions::get() - 1) { let raw_solution = RawSolution { - score: [(10_000_000 + i).into(), 0, 0], + score: ElectionScore { minimal_stake: 10_000_000u128 + (i as u128), ..Default::default() }, ..Default::default() }; - let signed_submission = SignedSubmission { raw_solution, ..Default::default() }; + let signed_submission = SignedSubmission { + raw_solution, + who: account("submitters", i, SEED), + deposit: Default::default(), + reward: Default::default(), + }; signed_submissions.insert(signed_submission); } signed_submissions.put(); let caller = frame_benchmarking::whitelisted_caller(); - T::Currency::make_free_balance_be(&caller, T::Currency::minimum_balance() * 10u32.into()); + let deposit = MultiPhase::::deposit_for( + &solution, + MultiPhase::::snapshot_metadata().unwrap_or_default(), + ); + T::Currency::make_free_balance_be(&caller, T::Currency::minimum_balance() * 1000u32.into() + deposit); - }: _(RawOrigin::Signed(caller), Box::new(solution), c) + }: _(RawOrigin::Signed(caller), Box::new(solution)) verify { - assert!(>::signed_submissions().len() as u32 == c + 1); + assert!(>::signed_submissions().len() as u32 == T::SignedMaxSubmissions::get()); } submit_unsigned { @@ -443,6 +463,7 @@ frame_benchmarking::benchmarks! { T::BenchmarkingConfig::DESIRED_TARGETS[1]; // Subtract this percentage from the actual encoded size let f in 0 .. 95; + use frame_election_provider_support::IndexAssignment; // Compute a random solution, then work backwards to get the lists of voters, targets, and // assignments @@ -497,10 +518,10 @@ frame_benchmarking::benchmarks! { log!(trace, "actual encoded size = {}", encoding.len()); assert!(encoding.len() <= desired_size); } -} -impl_benchmark_test_suite!( - MultiPhase, - crate::mock::ExtBuilder::default().build_offchainify(10).0, - crate::mock::Runtime, -); + impl_benchmark_test_suite!( + MultiPhase, + crate::mock::ExtBuilder::default().build_offchainify(10).0, + crate::mock::Runtime, + ); +} diff --git a/frame/election-provider-multi-phase/src/helpers.rs b/frame/election-provider-multi-phase/src/helpers.rs index 72b1b23f27f3..48da194cc65d 100644 --- a/frame/election-provider-multi-phase/src/helpers.rs +++ b/frame/election-provider-multi-phase/src/helpers.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,8 +17,8 @@ //! Some helper functions/macros for this crate. -use super::{Config, SolutionTargetIndexOf, SolutionVoterIndexOf, VoteWeight}; -use sp_std::{collections::btree_map::BTreeMap, convert::TryInto, prelude::*}; +use crate::{unsigned::VoterOf, Config, SolutionTargetIndexOf, SolutionVoterIndexOf, VoteWeight}; +use sp_std::{collections::btree_map::BTreeMap, prelude::*}; #[macro_export] macro_rules! log { @@ -34,7 +34,7 @@ macro_rules! log { /// /// This can be used to efficiently build index getter closures. pub fn generate_voter_cache( - snapshot: &Vec<(T::AccountId, VoteWeight, Vec)>, + snapshot: &Vec>, ) -> BTreeMap { let mut cache: BTreeMap = BTreeMap::new(); snapshot.iter().enumerate().for_each(|(i, (x, _, _))| { @@ -97,7 +97,7 @@ pub fn voter_index_fn_usize( /// Not meant to be used in production. #[cfg(test)] pub fn voter_index_fn_linear( - snapshot: &Vec<(T::AccountId, VoteWeight, Vec)>, + snapshot: &Vec>, ) -> impl Fn(&T::AccountId) -> Option> + '_ { move |who| { snapshot @@ -148,7 +148,7 @@ pub fn target_index_fn_linear( /// Create a function that can map a voter index ([`SolutionVoterIndexOf`]) to the actual voter /// account using a linearly indexible snapshot. pub fn voter_at_fn( - snapshot: &Vec<(T::AccountId, VoteWeight, Vec)>, + snapshot: &Vec>, ) -> impl Fn(SolutionVoterIndexOf) -> Option + '_ { move |i| { as TryInto>::try_into(i) @@ -174,7 +174,7 @@ pub fn target_at_fn( /// This is not optimized and uses a linear search. #[cfg(test)] pub fn stake_of_fn_linear( - snapshot: &Vec<(T::AccountId, VoteWeight, Vec)>, + snapshot: &Vec>, ) -> impl Fn(&T::AccountId) -> VoteWeight + '_ { move |who| { snapshot @@ -192,7 +192,7 @@ pub fn stake_of_fn_linear( /// The cache need must be derived from the same snapshot. Zero is returned if a voter is /// non-existent. pub fn stake_of_fn<'a, T: Config>( - snapshot: &'a Vec<(T::AccountId, VoteWeight, Vec)>, + snapshot: &'a Vec>, cache: &'a BTreeMap, ) -> impl Fn(&T::AccountId) -> VoteWeight + 'a { move |who| { diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 6b0329afc0d7..e67a5cab8d64 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -67,11 +67,11 @@ //! origin can not bail out in any way, if their solution is queued. //! //! Upon the end of the signed phase, the solutions are examined from best to worse (i.e. `pop()`ed -//! until drained). Each solution undergoes an expensive `Pallet::feasibility_check`, which -//! ensures the score claimed by this score was correct, and it is valid based on the election data -//! (i.e. votes and candidates). At each step, if the current best solution passes the feasibility -//! check, it is considered to be the best one. The sender of the origin is rewarded, and the rest -//! of the queued solutions get their deposit back and are discarded, without being checked. +//! until drained). Each solution undergoes an expensive `Pallet::feasibility_check`, which ensures +//! the score claimed by this score was correct, and it is valid based on the election data (i.e. +//! votes and targets). At each step, if the current best solution passes the feasibility check, +//! it is considered to be the best one. The sender of the origin is rewarded, and the rest of the +//! queued solutions get their deposit back and are discarded, without being checked. //! //! The following example covers all of the cases at the end of the signed phase: //! @@ -103,7 +103,7 @@ //! //! Validators will only submit solutions if the one that they have computed is sufficiently better //! than the best queued one (see [`pallet::Config::SolutionImprovementThreshold`]) and will limit -//! the weigh of the solution to [`pallet::Config::MinerMaxWeight`]. +//! the weight of the solution to [`pallet::Config::MinerMaxWeight`]. //! //! The unsigned phase can be made passive depending on how the previous signed phase went, by //! setting the first inner value of [`Phase`] to `false`. For now, the signed phase is always @@ -121,21 +121,39 @@ //! //! If, for any of the below reasons: //! -//! 1. No signed or unsigned solution submitted & Fallback is `None` or failed -//! 2. Internal error +//! 1. No **signed** or **unsigned** solution submitted, and no successful [`Config::Fallback`] is +//! provided +//! 2. Any other unforeseen internal error //! //! A call to `T::ElectionProvider::elect` is made, and `Ok(_)` cannot be returned, then the pallet //! proceeds to the [`Phase::Emergency`]. During this phase, any solution can be submitted from -//! [`Config::ForceOrigin`], without any checking. Once submitted, the forced solution is kept in -//! [`QueuedSolution`] until the next call to `T::ElectionProvider::elect`, where it is returned and -//! [`Phase`] goes back to `Off`. +//! [`Config::ForceOrigin`], without any checking, via [`Pallet::set_emergency_election_result`] +//! transaction. Hence, `[`Config::ForceOrigin`]` should only be set to a trusted origin, such as +//! the council or root. Once submitted, the forced solution is kept in [`QueuedSolution`] until the +//! next call to `T::ElectionProvider::elect`, where it is returned and [`Phase`] goes back to +//! `Off`. //! //! This implies that the user of this pallet (i.e. a staking pallet) should re-try calling -//! `T::ElectionProvider::elect` in case of error until `OK(_)` is returned. +//! `T::ElectionProvider::elect` in case of error, until `OK(_)` is returned. +//! +//! To generate an emergency solution, one must only provide one argument: [`Supports`]. This is +//! essentially a collection of elected winners for the election, and voters who support them. The +//! supports can be generated by any means. In the simplest case, it could be manual. For example, +//! in the case of massive network failure or misbehavior, [`Config::ForceOrigin`] might decide to +//! select only a small number of emergency winners (which would greatly restrict the next validator +//! set, if this pallet is used with `pallet-staking`). If the failure is for other technical +//! reasons, then a simple and safe way to generate supports is using the staking-miner binary +//! provided in the Polkadot repository. This binary has a subcommand named `emergency-solution` +//! which is capable of connecting to a live network, and generating appropriate `supports` using a +//! standard algorithm, and outputting the `supports` in hex format, ready for submission. Note that +//! while this binary lives in the Polkadot repository, this particular subcommand of it can work +//! against any substrate-based chain. +//! +//! See the `staking-miner` documentation in the Polkadot repository for more information. //! //! ## Feasible Solution (correct solution) //! -//! All submissions must undergo a feasibility check. Signed solutions are checked on by one at the +//! All submissions must undergo a feasibility check. Signed solutions are checked one by one at the //! end of the signed phase, and the unsigned solutions are checked on the spot. A feasible solution //! is as follows: //! @@ -146,16 +164,16 @@ //! //! ## Accuracy //! -//! The accuracy of the election is configured via -//! [`SolutionAccuracyOf`] which is the accuracy that the submitted solutions must adhere to. +//! The accuracy of the election is configured via [`SolutionAccuracyOf`] which is the accuracy that +//! the submitted solutions must adhere to. //! //! Note that the accuracy is of great importance. The offchain solution should be as small as //! possible, reducing solutions size/weight. //! //! ## Error types //! -//! This pallet provides a verbose error system to ease future debugging and debugging. The -//! overall hierarchy of errors is as follows: +//! This pallet provides a verbose error system to ease future debugging and debugging. The overall +//! hierarchy of errors is as follows: //! //! 1. [`pallet::Error`]: These are the errors that can be returned in the dispatchables of the //! pallet, either signed or unsigned. Since decomposition with nested enums is not possible @@ -165,14 +183,17 @@ //! are helpful for logging and are thus nested as: //! - [`ElectionError::Miner`]: wraps a [`unsigned::MinerError`]. //! - [`ElectionError::Feasibility`]: wraps a [`FeasibilityError`]. -//! - [`ElectionError::OnChainFallback`]: wraps a -//! [`frame_election_provider_support::onchain::Error`]. +//! - [`ElectionError::Fallback`]: wraps a fallback error. +//! - [`ElectionError::DataProvider`]: wraps a static str. //! //! Note that there could be an overlap between these sub-errors. For example, A //! `SnapshotUnavailable` can happen in both miner and feasibility check phase. //! //! ## Future Plans //! +//! **Emergency-phase recovery script**: This script should be taken out of staking-miner in +//! polkadot and ideally live in `substrate/utils/frame/elections`. +//! //! **Challenge Phase**. We plan on adding a third phase to the pallet, called the challenge phase. //! This is a phase in which no further solutions are processed, and the current best solution might //! be challenged by anyone (signed or unsigned). The main plan here is to enforce the solution to @@ -209,7 +230,9 @@ #![cfg_attr(not(feature = "std"), no_std)] use codec::{Decode, Encode}; -use frame_election_provider_support::{ElectionDataProvider, ElectionProvider}; +use frame_election_provider_support::{ + ElectionDataProvider, ElectionProvider, InstantElectionProvider, NposSolution, +}; use frame_support::{ dispatch::DispatchResultWithPostInfo, ensure, @@ -219,22 +242,20 @@ use frame_support::{ use frame_system::{ensure_none, offchain::SendTransactionTypes}; use scale_info::TypeInfo; use sp_arithmetic::{ - traits::{CheckedAdd, Saturating, Zero}, + traits::{Bounded, CheckedAdd, Saturating, Zero}, UpperOf, }; use sp_npos_elections::{ - assignment_ratio_to_staked_normalized, ElectionScore, EvaluateSupport, NposSolution, Supports, - VoteWeight, + assignment_ratio_to_staked_normalized, ElectionScore, EvaluateSupport, Supports, VoteWeight, }; use sp_runtime::{ - traits::Bounded, transaction_validity::{ InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity, TransactionValidityError, ValidTransaction, }, - DispatchError, PerThing, Perbill, RuntimeDebug, SaturatedConversion, + DispatchError, ModuleError, PerThing, Perbill, RuntimeDebug, SaturatedConversion, }; -use sp_std::{convert::TryInto, prelude::*}; +use sp_std::prelude::*; #[cfg(feature = "runtime-benchmarks")] mod benchmarking; @@ -248,6 +269,7 @@ const LOG_TARGET: &'static str = "runtime::election-provider"; pub mod signed; pub mod unsigned; pub mod weights; +use unsigned::VoterOf; pub use weights::WeightInfo; pub use signed::{ @@ -265,10 +287,7 @@ pub type SolutionTargetIndexOf = as NposSolution>::TargetIndex /// The accuracy of the election, when submitted from offchain. Derived from [`SolutionOf`]. pub type SolutionAccuracyOf = as NposSolution>::Accuracy; /// The fallback election type. -pub type FallbackErrorOf = <::Fallback as ElectionProvider< - ::AccountId, - ::BlockNumber, ->>::Error; +pub type FallbackErrorOf = <::Fallback as ElectionProvider>::Error; /// Configuration for the benchmarks of the pallet. pub trait BenchmarkingConfig { @@ -288,20 +307,12 @@ pub trait BenchmarkingConfig { const MAXIMUM_TARGETS: u32; } -impl BenchmarkingConfig for () { - const VOTERS: [u32; 2] = [4000, 6000]; - const TARGETS: [u32; 2] = [1000, 1600]; - const ACTIVE_VOTERS: [u32; 2] = [1000, 3000]; - const DESIRED_TARGETS: [u32; 2] = [400, 800]; - const SNAPSHOT_MAXIMUM_VOTERS: u32 = 10_000; - const MINER_MAXIMUM_VOTERS: u32 = 10_000; - const MAXIMUM_TARGETS: u32 = 2_000; -} - /// A fallback implementation that transitions the pallet to the emergency phase. pub struct NoFallback(sp_std::marker::PhantomData); -impl ElectionProvider for NoFallback { +impl ElectionProvider for NoFallback { + type AccountId = T::AccountId; + type BlockNumber = T::BlockNumber; type DataProvider = T::DataProvider; type Error = &'static str; @@ -311,6 +322,12 @@ impl ElectionProvider for NoFallback } } +impl InstantElectionProvider for NoFallback { + fn elect_with_bounds(_: usize, _: usize) -> Result, Self::Error> { + Err("NoFallback.") + } +} + /// Current phase of the pallet. #[derive(PartialEq, Eq, Clone, Copy, Encode, Decode, Debug, TypeInfo)] pub enum Phase { @@ -438,11 +455,13 @@ pub struct ReadySolution { /// /// These are stored together because they are often accessed together. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, Default, TypeInfo)] -pub struct RoundSnapshot { +#[codec(mel_bound())] +#[scale_info(skip_type_params(T))] +pub struct RoundSnapshot { /// All of the voters. - pub voters: Vec<(A, VoteWeight, Vec)>, + pub voters: Vec>, /// All of the targets. - pub targets: Vec, + pub targets: Vec, } /// Encodes the length of a solution or a snapshot. @@ -464,7 +483,7 @@ pub struct SolutionOrSnapshotSize { /// /// Note that this is different from [`pallet::Error`]. #[derive(frame_support::DebugNoBound)] -#[cfg_attr(feature = "runtime-benchmarks", derive(strum_macros::IntoStaticStr))] +#[cfg_attr(feature = "runtime-benchmarks", derive(strum::IntoStaticStr))] pub enum ElectionError { /// An error happened in the feasibility check sub-system. Feasibility(FeasibilityError), @@ -509,7 +528,7 @@ impl From> for ElectionError { /// Errors that can happen in the feasibility check. #[derive(Debug, Eq, PartialEq)] -#[cfg_attr(feature = "runtime-benchmarks", derive(strum_macros::IntoStaticStr))] +#[cfg_attr(feature = "runtime-benchmarks", derive(strum::IntoStaticStr))] pub enum FeasibilityError { /// Wrong number of winners presented. WrongWinnerCount, @@ -542,7 +561,7 @@ pub use pallet::*; #[frame_support::pallet] pub mod pallet { use super::*; - use frame_election_provider_support::NposSolver; + use frame_election_provider_support::{InstantElectionProvider, NposSolver}; use frame_support::{pallet_prelude::*, traits::EstimateCallFee}; use frame_system::pallet_prelude::*; @@ -620,14 +639,15 @@ pub mod pallet { #[pallet::constant] type SignedDepositWeight: Get>; - /// The maximum number of voters to put in the snapshot. At the moment, snapshots are only - /// over a single block, but once multi-block elections are introduced they will take place - /// over multiple blocks. - /// - /// Also, note the data type: If the voters are represented by a `u32` in `type - /// CompactSolution`, the same `u32` is used here to ensure bounds are respected. + /// The maximum number of electing voters to put in the snapshot. At the moment, snapshots + /// are only over a single block, but once multi-block elections are introduced they will + /// take place over multiple blocks. + #[pallet::constant] + type MaxElectingVoters: Get>; + + /// The maximum number of electable targets to put in the snapshot. #[pallet::constant] - type VoterSnapshotPerBlock: Get>; + type MaxElectableTargets: Get>; /// Handler for the slashed deposits. type SlashHandler: OnUnbalanced>; @@ -643,7 +663,10 @@ pub mod pallet { type MinerMaxLength: Get; /// Something that will provide the election data. - type DataProvider: ElectionDataProvider; + type DataProvider: ElectionDataProvider< + AccountId = Self::AccountId, + BlockNumber = Self::BlockNumber, + >; /// The solution type. type Solution: codec::Codec @@ -656,10 +679,20 @@ pub mod pallet { + NposSolution + TypeInfo; - /// Configuration for the fallback - type Fallback: ElectionProvider< - Self::AccountId, - Self::BlockNumber, + /// Configuration for the fallback. + type Fallback: InstantElectionProvider< + AccountId = Self::AccountId, + BlockNumber = Self::BlockNumber, + DataProvider = Self::DataProvider, + >; + + /// Configuration of the governance-only fallback. + /// + /// As a side-note, it is recommend for test-nets to use `type ElectionProvider = + /// BoundedExecution<_>` if the test-net is not expected to have thousands of nominators. + type GovernanceFallback: InstantElectionProvider< + AccountId = Self::AccountId, + BlockNumber = Self::BlockNumber, DataProvider = Self::DataProvider, >; @@ -751,7 +784,7 @@ pub mod pallet { Self::on_initialize_open_unsigned(enabled, now); T::WeightInfo::on_initialize_open_unsigned() } - } + }, _ => T::WeightInfo::on_initialize_nothing(), } } @@ -781,7 +814,7 @@ pub mod pallet { fn integrity_test() { use sp_std::mem::size_of; // The index type of both voters and targets need to be smaller than that of usize (very - // unlikely to be the case, but anyhow). + // unlikely to be the case, but anyhow).. assert!(size_of::>() <= size_of::()); assert!(size_of::>() <= size_of::()); @@ -807,7 +840,7 @@ pub mod pallet { // NOTE that this pallet does not really need to enforce this in runtime. The // solution cannot represent any voters more than `LIMIT` anyhow. assert_eq!( - >::MAXIMUM_VOTES_PER_VOTER, + ::MaxVotesPerVoter::get(), as NposSolution>::LIMIT as u32, ); } @@ -865,10 +898,10 @@ pub mod pallet { log!(info, "queued unsigned solution with score {:?}", ready.score); let ejected_a_solution = >::exists(); >::put(ready); - Self::deposit_event(Event::SolutionStored( - ElectionCompute::Unsigned, - ejected_a_solution, - )); + Self::deposit_event(Event::SolutionStored { + election_compute: ElectionCompute::Unsigned, + prev_ejected: ejected_a_solution, + }); Ok(None.into()) } @@ -907,8 +940,11 @@ pub mod pallet { // Note: we don't `rotate_round` at this point; the next call to // `ElectionProvider::elect` will succeed and take care of that. - let solution = - ReadySolution { supports, score: [0, 0, 0], compute: ElectionCompute::Emergency }; + let solution = ReadySolution { + supports, + score: Default::default(), + compute: ElectionCompute::Emergency, + }; >::put(solution); Ok(()) @@ -923,25 +959,13 @@ pub mod pallet { /// /// A deposit is reserved and recorded for the solution. Based on the outcome, the solution /// might be rewarded, slashed, or get all or a part of the deposit back. - /// - /// # - /// Queue size must be provided as witness data. - /// # - #[pallet::weight(T::WeightInfo::submit(*num_signed_submissions))] + #[pallet::weight(T::WeightInfo::submit())] pub fn submit( origin: OriginFor, raw_solution: Box>>, - num_signed_submissions: u32, ) -> DispatchResult { let who = ensure_signed(origin)?; - // ensure witness data is correct. - ensure!( - num_signed_submissions >= - >::decode_len().unwrap_or_default() as u32, - Error::::SignedInvalidWitness, - ); - // ensure solution is timely. ensure!(Self::current_phase().is_signed(), Error::::PreDispatchEarlySubmission); @@ -960,8 +984,7 @@ pub mod pallet { // create the submission let deposit = Self::deposit_for(&raw_solution, size); let reward = { - let call = - Call::submit { raw_solution: raw_solution.clone(), num_signed_submissions }; + let call = Call::submit { raw_solution: raw_solution.clone() }; let call_fee = T::EstimateCallFee::estimate_call_fee(&call, None.into()); T::SignedRewardBase::get().saturating_add(call_fee) }; @@ -991,7 +1014,45 @@ pub mod pallet { } signed_submissions.put(); - Self::deposit_event(Event::SolutionStored(ElectionCompute::Signed, ejected_a_solution)); + Self::deposit_event(Event::SolutionStored { + election_compute: ElectionCompute::Signed, + prev_ejected: ejected_a_solution, + }); + Ok(()) + } + + /// Trigger the governance fallback. + /// + /// This can only be called when [`Phase::Emergency`] is enabled, as an alternative to + /// calling [`Call::set_emergency_election_result`]. + #[pallet::weight(T::DbWeight::get().reads_writes(1, 1))] + pub fn governance_fallback( + origin: OriginFor, + maybe_max_voters: Option, + maybe_max_targets: Option, + ) -> DispatchResult { + T::ForceOrigin::ensure_origin(origin)?; + ensure!(Self::current_phase().is_emergency(), >::CallNotAllowed); + + let maybe_max_voters = maybe_max_voters.map(|x| x as usize); + let maybe_max_targets = maybe_max_targets.map(|x| x as usize); + + let supports = T::GovernanceFallback::elect_with_bounds( + maybe_max_voters.unwrap_or(Bounded::max_value()), + maybe_max_targets.unwrap_or(Bounded::max_value()), + ) + .map_err(|e| { + log!(error, "GovernanceFallback failed: {:?}", e); + Error::::FallbackFailed + })?; + + let solution = ReadySolution { + supports, + score: Default::default(), + compute: ElectionCompute::Fallback, + }; + + >::put(solution); Ok(()) } } @@ -1005,18 +1066,18 @@ pub mod pallet { /// solution is unsigned, this means that it has also been processed. /// /// The `bool` is `true` when a previous solution was ejected to make room for this one. - SolutionStored(ElectionCompute, bool), + SolutionStored { election_compute: ElectionCompute, prev_ejected: bool }, /// The election has been finalized, with `Some` of the given computation, or else if the /// election failed, `None`. - ElectionFinalized(Option), + ElectionFinalized { election_compute: Option }, /// An account has been rewarded for their signed submission being finalized. - Rewarded(::AccountId, BalanceOf), + Rewarded { account: ::AccountId, value: BalanceOf }, /// An account has been slashed for submitting an invalid signed submission. - Slashed(::AccountId, BalanceOf), + Slashed { account: ::AccountId, value: BalanceOf }, /// The signed phase of the given round has started. - SignedPhaseStarted(u32), + SignedPhaseStarted { round: u32 }, /// The unsigned phase of the given round has started. - UnsignedPhaseStarted(u32), + UnsignedPhaseStarted { round: u32 }, } /// Error of the pallet that can be returned in response to dispatches. @@ -1044,6 +1105,8 @@ pub mod pallet { InvalidSubmissionIndex, /// The call is not allowed at this point. CallNotAllowed, + /// The fallback failed + FallbackFailed, } #[pallet::validate_unsigned] @@ -1065,10 +1128,10 @@ pub mod pallet { .map_err(dispatch_error_to_invalid)?; ValidTransaction::with_tag_prefix("OffchainElection") - // The higher the score[0], the better a solution is. + // The higher the score.minimal_stake, the better a solution is. .priority( T::MinerTxPriority::get() - .saturating_add(raw_solution.score[0].saturated_into()), + .saturating_add(raw_solution.score.minimal_stake.saturated_into()), ) // Used to deduplicate unsigned solutions: each validator should produce one // solution per round at most, and solutions are not propagate. @@ -1124,7 +1187,7 @@ pub mod pallet { /// This is created at the beginning of the signed phase and cleared upon calling `elect`. #[pallet::storage] #[pallet::getter(fn snapshot)] - pub type Snapshot = StorageValue<_, RoundSnapshot>; + pub type Snapshot = StorageValue<_, RoundSnapshot>; /// Desired number of targets to elect for this round. /// @@ -1154,7 +1217,7 @@ pub mod pallet { /// capacity, it will simply saturate. We can't just iterate over `SignedSubmissionsMap`, /// because iteration is slow. Instead, we store the value here. #[pallet::storage] - pub(crate) type SignedSubmissionNextIndex = StorageValue<_, u32, ValueQuery>; + pub type SignedSubmissionNextIndex = StorageValue<_, u32, ValueQuery>; /// A sorted, bounded set of `(score, index)`, where each `index` points to a value in /// `SignedSubmissions`. @@ -1163,7 +1226,7 @@ pub mod pallet { /// can be quite large, so we're willing to pay the cost of multiple database accesses to access /// them one at a time instead of reading and decoding all of them at once. #[pallet::storage] - pub(crate) type SignedSubmissionIndices = + pub type SignedSubmissionIndices = StorageValue<_, SubmissionIndicesOf, ValueQuery>; /// Unchecked, signed solutions. @@ -1174,8 +1237,8 @@ pub mod pallet { /// Twox note: the key of the map is an auto-incrementing index which users cannot inspect or /// affect; we shouldn't need a cryptographically secure hasher. #[pallet::storage] - pub(crate) type SignedSubmissionsMap = - StorageMap<_, Twox64Concat, u32, SignedSubmissionOf, ValueQuery>; + pub type SignedSubmissionsMap = + StorageMap<_, Twox64Concat, u32, SignedSubmissionOf, OptionQuery>; // `SignedSubmissions` items end here. @@ -1189,6 +1252,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(PhantomData); } @@ -1201,8 +1265,12 @@ impl Pallet { match current_phase { Phase::Unsigned((true, opened)) if opened == now => { // Mine a new solution, cache it, and attempt to submit it - let initial_output = Self::ensure_offchain_repeat_frequency(now) - .and_then(|_| Self::mine_check_save_submit()); + let initial_output = Self::ensure_offchain_repeat_frequency(now).and_then(|_| { + // This is executed at the beginning of each round. Any cache is now invalid. + // Clear it. + unsigned::kill_ocw_solution::(); + Self::mine_check_save_submit() + }); log!(debug, "initial offchain thread output: {:?}", initial_output); }, Phase::Unsigned((true, opened)) if opened < now => { @@ -1214,35 +1282,21 @@ impl Pallet { }, _ => {}, } - - // After election finalization, clear OCW solution storage. - // - // We can read the events here because offchain worker doesn't affect PoV. - if >::read_events_no_consensus() - .into_iter() - .filter_map(|event_record| { - let local_event = ::Event::from(event_record.event); - local_event.try_into().ok() - }) - .any(|event| matches!(event, Event::ElectionFinalized(_))) - { - unsigned::kill_ocw_solution::(); - } } - /// Logic for [`::on_initialize`] when signed phase is being opened. + /// Logic for `::on_initialize` when signed phase is being opened. pub fn on_initialize_open_signed() { log!(info, "Starting signed phase round {}.", Self::round()); >::put(Phase::Signed); - Self::deposit_event(Event::SignedPhaseStarted(Self::round())); + Self::deposit_event(Event::SignedPhaseStarted { round: Self::round() }); } - /// Logic for [`>::on_initialize`] when unsigned phase is being opened. + /// Logic for `>::on_initialize` when unsigned phase is being opened. pub fn on_initialize_open_unsigned(enabled: bool, now: T::BlockNumber) { let round = Self::round(); log!(info, "Starting unsigned phase round {} enabled {}.", round, enabled); >::put(Phase::Unsigned((enabled, now))); - Self::deposit_event(Event::UnsignedPhaseStarted(round)); + Self::deposit_event(Event::UnsignedPhaseStarted { round }); } /// Parts of [`create_snapshot`] that happen inside of this pallet. @@ -1250,7 +1304,7 @@ impl Pallet { /// Extracted for easier weight calculation. fn create_snapshot_internal( targets: Vec, - voters: Vec>, + voters: Vec>, desired_targets: u32, ) { let metadata = @@ -1263,7 +1317,7 @@ impl Pallet { // instead of using storage APIs, we do a manual encoding into a fixed-size buffer. // `encoded_size` encodes it without storing it anywhere, this should not cause any // allocation. - let snapshot = RoundSnapshot { voters, targets }; + let snapshot = RoundSnapshot:: { voters, targets }; let size = snapshot.encoded_size(); log!(debug, "snapshot pre-calculated size {:?}", size); let mut buffer = Vec::with_capacity(size); @@ -1281,16 +1335,15 @@ impl Pallet { /// /// Extracted for easier weight calculation. fn create_snapshot_external( - ) -> Result<(Vec, Vec>, u32), ElectionError> { - let target_limit = >::max_value().saturated_into::(); - // for now we have just a single block snapshot. - let voter_limit = T::VoterSnapshotPerBlock::get().saturated_into::(); - - let targets = - T::DataProvider::targets(Some(target_limit)).map_err(ElectionError::DataProvider)?; - let voters = - T::DataProvider::voters(Some(voter_limit)).map_err(ElectionError::DataProvider)?; - let desired_targets = + ) -> Result<(Vec, Vec>, u32), ElectionError> { + let target_limit = T::MaxElectableTargets::get().saturated_into::(); + let voter_limit = T::MaxElectingVoters::get().saturated_into::(); + + let targets = T::DataProvider::electable_targets(Some(target_limit)) + .map_err(ElectionError::DataProvider)?; + let voters = T::DataProvider::electing_voters(Some(voter_limit)) + .map_err(ElectionError::DataProvider)?; + let mut desired_targets = T::DataProvider::desired_targets().map_err(ElectionError::DataProvider)?; // Defensive-only. @@ -1299,6 +1352,22 @@ impl Pallet { return Err(ElectionError::DataProvider("Snapshot too big for submission.")) } + // If `desired_targets` > `targets.len()`, cap `desired_targets` to that level and emit a + // warning + let max_len = targets + .len() + .try_into() + .map_err(|_| ElectionError::DataProvider("Failed to convert usize"))?; + if desired_targets > max_len { + log!( + warn, + "desired_targets: {} > targets.len(): {}, capping desired_targets", + desired_targets, + max_len + ); + desired_targets = max_len; + } + Ok((targets, voters, desired_targets)) } @@ -1357,16 +1426,13 @@ impl Pallet { let desired_targets = Self::desired_targets().ok_or(FeasibilityError::SnapshotUnavailable)?; - // NOTE: this is a bit of duplicate, but we keep it around for veracity. The unsigned path - // already checked this in `unsigned_per_dispatch_checks`. The signed path *could* check it - // upon arrival, thus we would then remove it here. Given overlay it is cheap anyhow ensure!(winners.len() as u32 == desired_targets, FeasibilityError::WrongWinnerCount); // Ensure that the solution's score can pass absolute min-score. let submitted_score = raw_solution.score.clone(); ensure!( Self::minimum_untrusted_score().map_or(true, |min_score| { - sp_npos_elections::is_score_better(submitted_score, min_score, Perbill::zero()) + submitted_score.strict_threshold_better(min_score, Perbill::zero()) }), FeasibilityError::UntrustedScoreTooLow ); @@ -1462,14 +1528,14 @@ impl Pallet { |ReadySolution { supports, compute, .. }| Ok((supports, compute)), ) .map(|(supports, compute)| { - Self::deposit_event(Event::ElectionFinalized(Some(compute))); + Self::deposit_event(Event::ElectionFinalized { election_compute: Some(compute) }); if Self::round() != 1 { log!(info, "Finalized election round with compute {:?}.", compute); } supports }) .map_err(|err| { - Self::deposit_event(Event::ElectionFinalized(None)); + Self::deposit_event(Event::ElectionFinalized { election_compute: None }); if Self::round() != 1 { log!(warn, "Failed to finalize election round. reason {:?}", err); } @@ -1488,7 +1554,9 @@ impl Pallet { } } -impl ElectionProvider for Pallet { +impl ElectionProvider for Pallet { + type AccountId = T::AccountId; + type BlockNumber = T::BlockNumber; type Error = ElectionError; type DataProvider = T::DataProvider; @@ -1513,7 +1581,7 @@ impl ElectionProvider for Pallet { /// number. pub fn dispatch_error_to_invalid(error: DispatchError) -> InvalidTransaction { let error_number = match error { - DispatchError::Module { error, .. } => error, + DispatchError::Module(ModuleError { error, .. }) => error[0], _ => 0, }; InvalidTransaction::Custom(error_number) @@ -1530,7 +1598,7 @@ mod feasibility_check { raw_solution, roll_to, EpochLength, ExtBuilder, MultiPhase, Runtime, SignedPhase, TargetIndex, UnsignedPhase, VoterIndex, }; - use frame_support::assert_noop; + use frame_support::{assert_noop, assert_ok}; const COMPUTE: ElectionCompute = ElectionCompute::OnChain; @@ -1567,7 +1635,7 @@ mod feasibility_check { } #[test] - fn desired_targets() { + fn desired_targets_gets_capped() { ExtBuilder::default().desired_targets(8).build_and_execute(|| { roll_to(::get() - ::get() - ::get()); assert!(MultiPhase::current_phase().is_signed()); @@ -1575,8 +1643,30 @@ mod feasibility_check { let raw = raw_solution(); assert_eq!(raw.solution.unique_targets().len(), 4); - assert_eq!(MultiPhase::desired_targets().unwrap(), 8); + // desired_targets is capped to the number of targets which is 4 + assert_eq!(MultiPhase::desired_targets().unwrap(), 4); + // It should succeed + assert_ok!(MultiPhase::feasibility_check(raw, COMPUTE)); + }) + } + + #[test] + fn less_than_desired_targets_fails() { + ExtBuilder::default().desired_targets(8).build_and_execute(|| { + roll_to(::get() - ::get() - ::get()); + assert!(MultiPhase::current_phase().is_signed()); + + let mut raw = raw_solution(); + + assert_eq!(raw.solution.unique_targets().len(), 4); + // desired_targets is capped to the number of targets which is 4 + assert_eq!(MultiPhase::desired_targets().unwrap(), 4); + + // Force the number of winners to be bigger to fail + raw.solution.votes1[0].1 = 4; + + // It should succeed assert_noop!( MultiPhase::feasibility_check(raw, COMPUTE), FeasibilityError::WrongWinnerCount, @@ -1684,7 +1774,7 @@ mod feasibility_check { assert_eq!(MultiPhase::snapshot().unwrap().voters.len(), 8); // Simply faff with the score. - solution.score[0] += 1; + solution.score.minimal_stake += 1; assert_noop!( MultiPhase::feasibility_check(solution, COMPUTE), @@ -1726,7 +1816,7 @@ mod tests { roll_to(15); assert_eq!(MultiPhase::current_phase(), Phase::Signed); - assert_eq!(multi_phase_events(), vec![Event::SignedPhaseStarted(1)]); + assert_eq!(multi_phase_events(), vec![Event::SignedPhaseStarted { round: 1 }]); assert!(MultiPhase::snapshot().is_some()); assert_eq!(MultiPhase::round(), 1); @@ -1739,7 +1829,10 @@ mod tests { assert_eq!(MultiPhase::current_phase(), Phase::Unsigned((true, 25))); assert_eq!( multi_phase_events(), - vec![Event::SignedPhaseStarted(1), Event::UnsignedPhaseStarted(1)], + vec![ + Event::SignedPhaseStarted { round: 1 }, + Event::UnsignedPhaseStarted { round: 1 } + ], ); assert!(MultiPhase::snapshot().is_some()); @@ -1850,7 +1943,7 @@ mod tests { assert_eq!(MultiPhase::current_phase(), Phase::Off); roll_to(15); - assert_eq!(multi_phase_events(), vec![Event::SignedPhaseStarted(1)]); + assert_eq!(multi_phase_events(), vec![Event::SignedPhaseStarted { round: 1 }]); assert_eq!(MultiPhase::current_phase(), Phase::Signed); assert_eq!(MultiPhase::round(), 1); @@ -1862,8 +1955,8 @@ mod tests { assert_eq!( multi_phase_events(), vec![ - Event::SignedPhaseStarted(1), - Event::ElectionFinalized(Some(ElectionCompute::Fallback)) + Event::SignedPhaseStarted { round: 1 }, + Event::ElectionFinalized { election_compute: Some(ElectionCompute::Fallback) } ], ); // All storage items must be cleared. @@ -1885,18 +1978,17 @@ mod tests { assert_eq!(MultiPhase::current_phase(), Phase::Off); roll_to(15); - assert_eq!(multi_phase_events(), vec![Event::SignedPhaseStarted(1)]); + assert_eq!(multi_phase_events(), vec![Event::SignedPhaseStarted { round: 1 }]); assert_eq!(MultiPhase::current_phase(), Phase::Signed); assert_eq!(MultiPhase::round(), 1); // fill the queue with signed submissions for s in 0..SignedMaxSubmissions::get() { - let solution = RawSolution { score: [(5 + s).into(), 0, 0], ..Default::default() }; - assert_ok!(MultiPhase::submit( - crate::mock::Origin::signed(99), - Box::new(solution), - MultiPhase::signed_submissions().len() as u32 - )); + let solution = RawSolution { + score: ElectionScore { minimal_stake: (5 + s).into(), ..Default::default() }, + ..Default::default() + }; + assert_ok!(MultiPhase::submit(crate::mock::Origin::signed(99), Box::new(solution))); } // an unexpected call to elect. @@ -1995,7 +2087,7 @@ mod tests { // we have 8 voters in total. assert_eq!(crate::mock::Voters::get().len(), 8); // but we want to take 2. - crate::mock::VoterSnapshotPerBlock::set(2); + crate::mock::MaxElectingVoters::set(2); // Signed phase opens just fine. roll_to(15); @@ -2018,13 +2110,19 @@ mod tests { crate::mock::Balancing::set(Some((2, 0))); let (solution, _) = MultiPhase::mine_solution::<::Solver>().unwrap(); - // Default solution has a score of [50, 100, 5000]. - assert_eq!(solution.score, [50, 100, 5000]); + // Default solution's score. + assert!(matches!(solution.score, ElectionScore { minimal_stake: 50, .. })); - >::put([49, 0, 0]); + >::put(ElectionScore { + minimal_stake: 49, + ..Default::default() + }); assert_ok!(MultiPhase::feasibility_check(solution.clone(), ElectionCompute::Signed)); - >::put([51, 0, 0]); + >::put(ElectionScore { + minimal_stake: 51, + ..Default::default() + }); assert_noop!( MultiPhase::feasibility_check(solution, ElectionCompute::Signed), FeasibilityError::UntrustedScoreTooLow, diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index 1a65316be1f1..d6f040363dba 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,16 @@ use super::*; use crate as multi_phase; use frame_election_provider_support::{ - data_provider, onchain, ElectionDataProvider, SequentialPhragmen, + data_provider, onchain, ElectionDataProvider, NposSolution, SequentialPhragmen, }; pub use frame_support::{assert_noop, assert_ok}; -use frame_support::{parameter_types, traits::Hooks, weights::Weight}; -use multi_phase::unsigned::{IndexAssignmentOf, Voter}; +use frame_support::{ + bounded_vec, parameter_types, + traits::{ConstU32, Hooks}, + weights::Weight, + BoundedVec, +}; +use multi_phase::unsigned::{IndexAssignmentOf, VoterOf}; use parking_lot::RwLock; use sp_core::{ offchain::{ @@ -33,14 +38,14 @@ use sp_core::{ }; use sp_npos_elections::{ assignment_ratio_to_staked_normalized, seq_phragmen, to_supports, ElectionResult, - EvaluateSupport, ExtendedBalance, NposSolution, + EvaluateSupport, ExtendedBalance, }; use sp_runtime::{ testing::Header, traits::{BlakeTwo256, IdentityLookup}, PerU16, }; -use std::{convert::TryFrom, sync::Arc}; +use std::sync::Arc; pub type Block = sp_runtime::generic::Block; pub type UncheckedExtrinsic = sp_runtime::generic::UncheckedExtrinsic; @@ -63,9 +68,14 @@ pub(crate) type BlockNumber = u64; pub(crate) type VoterIndex = u32; pub(crate) type TargetIndex = u16; -sp_npos_elections::generate_solution_type!( +frame_election_provider_support::generate_solution_type!( #[compact] - pub struct TestNposSolution::(16) + pub struct TestNposSolution::< + VoterIndex = VoterIndex, + TargetIndex = TargetIndex, + Accuracy = PerU16, + MaxVoters = ConstU32::<2_000> + >(16) ); /// All events of this pallet. @@ -96,7 +106,7 @@ pub fn roll_to_with_ocw(n: BlockNumber) { } pub struct TrimHelpers { - pub voters: Vec>, + pub voters: Vec>, pub assignments: Vec>, pub encoded_size_of: Box]) -> Result>, @@ -127,13 +137,8 @@ pub fn trim_helpers() -> TrimHelpers { let desired_targets = MultiPhase::desired_targets().unwrap(); - let ElectionResult { mut assignments, .. } = seq_phragmen::<_, SolutionAccuracyOf>( - desired_targets as usize, - targets.clone(), - voters.clone(), - None, - ) - .unwrap(); + let ElectionResult::<_, SolutionAccuracyOf> { mut assignments, .. } = + seq_phragmen(desired_targets as usize, targets.clone(), voters.clone(), None).unwrap(); // sort by decreasing order of stake assignments.sort_unstable_by_key(|assignment| { @@ -159,14 +164,8 @@ pub fn raw_solution() -> RawSolution> { let RoundSnapshot { voters, targets } = MultiPhase::snapshot().unwrap(); let desired_targets = MultiPhase::desired_targets().unwrap(); - let ElectionResult { winners: _, assignments } = - seq_phragmen::<_, SolutionAccuracyOf>( - desired_targets as usize, - targets.clone(), - voters.clone(), - None, - ) - .unwrap(); + let ElectionResult::<_, SolutionAccuracyOf> { winners: _, assignments } = + seq_phragmen(desired_targets as usize, targets.clone(), voters.clone(), None).unwrap(); // closures let cache = helpers::generate_voter_cache::(&voters); @@ -218,6 +217,7 @@ impl frame_system::Config for Runtime { type OnKilledAccount = (); type SystemWeightInfo = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } const NORMAL_DISPATCH_RATIO: Perbill = Perbill::from_percent(75); @@ -241,16 +241,16 @@ impl pallet_balances::Config for Runtime { parameter_types! { pub static Targets: Vec = vec![10, 20, 30, 40]; - pub static Voters: Vec<(AccountId, VoteWeight, Vec)> = vec![ - (1, 10, vec![10, 20]), - (2, 10, vec![30, 40]), - (3, 10, vec![40]), - (4, 10, vec![10, 20, 30, 40]), + pub static Voters: Vec> = vec![ + (1, 10, bounded_vec![10, 20]), + (2, 10, bounded_vec![30, 40]), + (3, 10, bounded_vec![40]), + (4, 10, bounded_vec![10, 20, 30, 40]), // self votes. - (10, 10, vec![10]), - (20, 20, vec![20]), - (30, 30, vec![30]), - (40, 40, vec![40]), + (10, 10, bounded_vec![10]), + (20, 20, bounded_vec![20]), + (30, 30, bounded_vec![30]), + (40, 40, bounded_vec![40]), ]; pub static DesiredTargets: u32 = 2; @@ -268,28 +268,45 @@ parameter_types! { pub static MinerMaxWeight: Weight = BlockWeights::get().max_block; pub static MinerMaxLength: u32 = 256; pub static MockWeightInfo: bool = false; - pub static VoterSnapshotPerBlock: VoterIndex = u32::max_value(); + pub static MaxElectingVoters: VoterIndex = u32::max_value(); + pub static MaxElectableTargets: TargetIndex = TargetIndex::max_value(); pub static EpochLength: u64 = 30; - pub static OnChianFallback: bool = true; + pub static OnChainFallback: bool = true; } -impl onchain::Config for Runtime { - type Accuracy = sp_runtime::Perbill; +pub struct OnChainSeqPhragmen; +impl onchain::ExecutionConfig for OnChainSeqPhragmen { + type System = Runtime; + type Solver = SequentialPhragmen, Balancing>; type DataProvider = StakingMock; } pub struct MockFallback; -impl ElectionProvider for MockFallback { +impl ElectionProvider for MockFallback { + type AccountId = AccountId; + type BlockNumber = u64; type Error = &'static str; type DataProvider = StakingMock; fn elect() -> Result, Self::Error> { - if OnChianFallback::get() { - onchain::OnChainSequentialPhragmen::::elect() - .map_err(|_| "OnChainSequentialPhragmen failed") + Self::elect_with_bounds(Bounded::max_value(), Bounded::max_value()) + } +} + +impl InstantElectionProvider for MockFallback { + fn elect_with_bounds( + max_voters: usize, + max_targets: usize, + ) -> Result, Self::Error> { + if OnChainFallback::get() { + onchain::UnboundedExecution::::elect_with_bounds( + max_voters, + max_targets, + ) + .map_err(|_| "UnboundedExecution failed") } else { - super::NoFallback::::elect() + super::NoFallback::::elect_with_bounds(max_voters, max_targets) } } } @@ -346,11 +363,11 @@ impl multi_phase::weights::WeightInfo for DualMockWeightInfo { <() as multi_phase::weights::WeightInfo>::finalize_signed_phase_reject_solution() } } - fn submit(c: u32) -> Weight { + fn submit() -> Weight { if MockWeightInfo::get() { Zero::zero() } else { - <() as multi_phase::weights::WeightInfo>::submit(c) + <() as multi_phase::weights::WeightInfo>::submit() } } fn submit_unsigned(v: u32, t: u32, a: u32, d: u32) -> Weight { @@ -377,6 +394,19 @@ parameter_types! { pub static Balancing: Option<(usize, ExtendedBalance)> = Some((0, 0)); } +pub struct TestBenchmarkingConfig; +impl BenchmarkingConfig for TestBenchmarkingConfig { + const VOTERS: [u32; 2] = [400, 600]; + const ACTIVE_VOTERS: [u32; 2] = [100, 300]; + const TARGETS: [u32; 2] = [200, 400]; + const DESIRED_TARGETS: [u32; 2] = [100, 180]; + + const SNAPSHOT_MAXIMUM_VOTERS: u32 = 1000; + const MINER_MAXIMUM_VOTERS: u32 = 1000; + + const MAXIMUM_TARGETS: u32 = 200; +} + impl crate::Config for Runtime { type Event = Event; type Currency = Balances; @@ -398,11 +428,13 @@ impl crate::Config for Runtime { type RewardHandler = (); type DataProvider = StakingMock; type WeightInfo = DualMockWeightInfo; - type BenchmarkingConfig = (); + type BenchmarkingConfig = TestBenchmarkingConfig; type Fallback = MockFallback; + type GovernanceFallback = NoFallback; type ForceOrigin = frame_system::EnsureRoot; type Solution = TestNposSolution; - type VoterSnapshotPerBlock = VoterSnapshotPerBlock; + type MaxElectingVoters = MaxElectingVoters; + type MaxElectableTargets = MaxElectableTargets; type Solver = SequentialPhragmen, Balancing>; } @@ -416,13 +448,20 @@ where pub type Extrinsic = sp_runtime::testing::TestXt; +parameter_types! { + pub MaxNominations: u32 = ::LIMIT as u32; +} + #[derive(Default)] pub struct ExtBuilder {} pub struct StakingMock; -impl ElectionDataProvider for StakingMock { - const MAXIMUM_VOTES_PER_VOTER: u32 = ::LIMIT as u32; - fn targets(maybe_max_len: Option) -> data_provider::Result> { +impl ElectionDataProvider for StakingMock { + type AccountId = AccountId; + type BlockNumber = u64; + type MaxVotesPerVoter = MaxNominations; + + fn electable_targets(maybe_max_len: Option) -> data_provider::Result> { let targets = Targets::get(); if maybe_max_len.map_or(false, |max_len| targets.len() > max_len) { @@ -432,9 +471,9 @@ impl ElectionDataProvider for StakingMock { Ok(targets) } - fn voters( + fn electing_voters( maybe_max_len: Option, - ) -> data_provider::Result)>> { + ) -> data_provider::Result>> { let mut voters = Voters::get(); if let Some(max_len) = maybe_max_len { voters.truncate(max_len) @@ -453,7 +492,7 @@ impl ElectionDataProvider for StakingMock { #[cfg(feature = "runtime-benchmarks")] fn put_snapshot( - voters: Vec<(AccountId, VoteWeight, Vec)>, + voters: Vec>, targets: Vec, _target_stake: Option, ) { @@ -468,7 +507,11 @@ impl ElectionDataProvider for StakingMock { } #[cfg(feature = "runtime-benchmarks")] - fn add_voter(voter: AccountId, weight: VoteWeight, targets: Vec) { + fn add_voter( + voter: AccountId, + weight: VoteWeight, + targets: frame_support::BoundedVec, + ) { let mut current = Voters::get(); current.push((voter, weight, targets)); Voters::set(current); @@ -483,7 +526,7 @@ impl ElectionDataProvider for StakingMock { // to be on-par with staking, we add a self vote as well. the stake is really not that // important. let mut current = Voters::get(); - current.push((target, ExistentialDeposit::get() as u64, vec![target])); + current.push((target, ExistentialDeposit::get() as u64, bounded_vec![target])); Voters::set(current); } } @@ -503,7 +546,7 @@ impl ExtBuilder { self } pub fn onchain_fallback(self, onchain: bool) -> Self { - ::set(onchain); + ::set(onchain); self } pub fn miner_weight(self, weight: Weight) -> Self { @@ -518,7 +561,12 @@ impl ExtBuilder { ::set(t); self } - pub fn add_voter(self, who: AccountId, stake: Balance, targets: Vec) -> Self { + pub fn add_voter( + self, + who: AccountId, + stake: Balance, + targets: BoundedVec, + ) -> Self { VOTERS.with(|v| v.borrow_mut().push((who, stake, targets))); self } diff --git a/frame/election-provider-multi-phase/src/signed.rs b/frame/election-provider-multi-phase/src/signed.rs index 61215059c53a..82b40e327603 100644 --- a/frame/election-provider-multi-phase/src/signed.rs +++ b/frame/election-provider-multi-phase/src/signed.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,12 +23,13 @@ use crate::{ SolutionOrSnapshotSize, Weight, WeightInfo, }; use codec::{Decode, Encode, HasCompact}; +use frame_election_provider_support::NposSolution; use frame_support::{ storage::bounded_btree_map::BoundedBTreeMap, - traits::{Currency, Get, OnUnbalanced, ReservableCurrency}, + traits::{defensive_prelude::*, Currency, Get, OnUnbalanced, ReservableCurrency}, }; use sp_arithmetic::traits::SaturatedConversion; -use sp_npos_elections::{is_score_better, ElectionScore, NposSolution}; +use sp_npos_elections::ElectionScore; use sp_runtime::{ traits::{Saturating, Zero}, RuntimeDebug, @@ -42,7 +43,7 @@ use sp_std::{ /// A raw, unchecked signed submission. /// /// This is just a wrapper around [`RawSolution`] and some additional info. -#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, Default, scale_info::TypeInfo)] +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, scale_info::TypeInfo)] pub struct SignedSubmission { /// Who submitted this solution. pub who: AccountId, @@ -167,17 +168,17 @@ impl SignedSubmissions { } /// Get the submission at a particular index. - fn get_submission(&self, idx: u32) -> Option> { - if self.deletion_overlay.contains(&idx) { + fn get_submission(&self, index: u32) -> Option> { + if self.deletion_overlay.contains(&index) { // Note: can't actually remove the item from the insertion overlay (if present) // because we don't want to use `&mut self` here. There may be some kind of // `RefCell` optimization possible here in the future. None } else { self.insertion_overlay - .get(&idx) + .get(&index) .cloned() - .or_else(|| SignedSubmissionsMap::::try_get(idx).ok()) + .or_else(|| SignedSubmissionsMap::::get(index)) } } @@ -200,18 +201,18 @@ impl SignedSubmissions { remove_score: ElectionScore, insert: Option<(ElectionScore, u32)>, ) -> Option> { - let remove_idx = self.indices.remove(&remove_score)?; + let remove_index = self.indices.remove(&remove_score)?; if let Some((insert_score, insert_idx)) = insert { self.indices .try_insert(insert_score, insert_idx) .expect("just removed an item, we must be under capacity; qed"); } - self.insertion_overlay.remove(&remove_idx).or_else(|| { - (!self.deletion_overlay.contains(&remove_idx)) + self.insertion_overlay.remove(&remove_index).or_else(|| { + (!self.deletion_overlay.contains(&remove_index)) .then(|| { - self.deletion_overlay.insert(remove_idx); - SignedSubmissionsMap::::try_get(remove_idx).ok() + self.deletion_overlay.insert(remove_index); + SignedSubmissionsMap::::get(remove_index) }) .flatten() }) @@ -293,7 +294,7 @@ impl SignedSubmissions { let threshold = T::SolutionImprovementThreshold::get(); // if we haven't improved on the weakest score, don't change anything. - if !is_score_better(insert_score, weakest_score, threshold) { + if !insert_score.strict_threshold_better(weakest_score, threshold) { return InsertResult::NotInserted } @@ -365,7 +366,7 @@ impl Pallet { let active_voters = raw_solution.solution.voter_count() as u32; let feasibility_weight = { // defensive only: at the end of signed phase, snapshot will exits. - let desired_targets = Self::desired_targets().unwrap_or_default(); + let desired_targets = Self::desired_targets().defensive_unwrap_or_default(); T::WeightInfo::feasibility_check(voters, targets, active_voters, desired_targets) }; // the feasibility check itself has some weight @@ -429,7 +430,7 @@ impl Pallet { >::put(ready_solution); // emit reward event - Self::deposit_event(crate::Event::Rewarded(who.clone(), reward)); + Self::deposit_event(crate::Event::Rewarded { account: who.clone(), value: reward }); // unreserve deposit. let _remaining = T::Currency::unreserve(who, deposit); @@ -446,7 +447,7 @@ impl Pallet { /// /// Infallible pub fn finalize_signed_phase_reject_solution(who: &T::AccountId, deposit: BalanceOf) { - Self::deposit_event(crate::Event::Slashed(who.clone(), deposit)); + Self::deposit_event(crate::Event::Slashed { account: who.clone(), value: deposit }); let (negative_imbalance, _remaining) = T::Currency::slash_reserved(who, deposit); debug_assert!(_remaining.is_zero()); T::SlashHandler::on_unbalanced(negative_imbalance); @@ -500,18 +501,7 @@ mod tests { }, Error, Phase, }; - use frame_support::{assert_noop, assert_ok, assert_storage_noop, dispatch::DispatchResult}; - - fn submit_with_witness( - origin: Origin, - solution: RawSolution>, - ) -> DispatchResult { - MultiPhase::submit( - origin, - Box::new(solution), - MultiPhase::signed_submissions().len() as u32, - ) - } + use frame_support::{assert_noop, assert_ok, assert_storage_noop}; #[test] fn cannot_submit_too_early() { @@ -524,31 +514,12 @@ mod tests { let solution = raw_solution(); assert_noop!( - submit_with_witness(Origin::signed(10), solution), + MultiPhase::submit(Origin::signed(10), Box::new(solution)), Error::::PreDispatchEarlySubmission, ); }) } - #[test] - fn wrong_witness_fails() { - ExtBuilder::default().build_and_execute(|| { - roll_to(15); - assert!(MultiPhase::current_phase().is_signed()); - - let solution = raw_solution(); - // submit this once correctly - assert_ok!(submit_with_witness(Origin::signed(99), solution.clone())); - assert_eq!(MultiPhase::signed_submissions().len(), 1); - - // now try and cheat by passing a lower queue length - assert_noop!( - MultiPhase::submit(Origin::signed(99), Box::new(solution), 0), - Error::::SignedInvalidWitness, - ); - }) - } - #[test] fn should_pay_deposit() { ExtBuilder::default().build_and_execute(|| { @@ -558,7 +529,7 @@ mod tests { let solution = raw_solution(); assert_eq!(balances(&99), (100, 0)); - assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); assert_eq!(balances(&99), (95, 5)); assert_eq!(MultiPhase::signed_submissions().iter().next().unwrap().deposit, 5); @@ -574,7 +545,7 @@ mod tests { let solution = raw_solution(); assert_eq!(balances(&99), (100, 0)); - assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); assert_eq!(balances(&99), (95, 5)); assert!(MultiPhase::finalize_signed_phase()); @@ -592,9 +563,9 @@ mod tests { assert_eq!(balances(&99), (100, 0)); // make the solution invalid. - solution.score[0] += 1; + solution.score.minimal_stake += 1; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); assert_eq!(balances(&99), (95, 5)); // no good solution was stored. @@ -615,11 +586,11 @@ mod tests { assert_eq!(balances(&999), (100, 0)); // submit as correct. - assert_ok!(submit_with_witness(Origin::signed(99), solution.clone())); + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution.clone()))); // make the solution invalid and weaker. - solution.score[0] -= 1; - assert_ok!(submit_with_witness(Origin::signed(999), solution)); + solution.score.minimal_stake -= 1; + assert_ok!(MultiPhase::submit(Origin::signed(999), Box::new(solution))); assert_eq!(balances(&99), (95, 5)); assert_eq!(balances(&999), (95, 5)); @@ -641,15 +612,21 @@ mod tests { for s in 0..SignedMaxSubmissions::get() { // score is always getting better - let solution = RawSolution { score: [(5 + s).into(), 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); + let solution = RawSolution { + score: ElectionScore { minimal_stake: (5 + s).into(), ..Default::default() }, + ..Default::default() + }; + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); } // weaker. - let solution = RawSolution { score: [4, 0, 0], ..Default::default() }; + let solution = RawSolution { + score: ElectionScore { minimal_stake: 4, ..Default::default() }, + ..Default::default() + }; assert_noop!( - submit_with_witness(Origin::signed(99), solution), + MultiPhase::submit(Origin::signed(99), Box::new(solution)), Error::::SignedQueueFull, ); }) @@ -663,27 +640,33 @@ mod tests { for s in 0..SignedMaxSubmissions::get() { // score is always getting better - let solution = RawSolution { score: [(5 + s).into(), 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); + let solution = RawSolution { + score: ElectionScore { minimal_stake: (5 + s).into(), ..Default::default() }, + ..Default::default() + }; + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); } assert_eq!( MultiPhase::signed_submissions() .iter() - .map(|s| s.raw_solution.score[0]) + .map(|s| s.raw_solution.score.minimal_stake) .collect::>(), vec![5, 6, 7, 8, 9] ); // better. - let solution = RawSolution { score: [20, 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); + let solution = RawSolution { + score: ElectionScore { minimal_stake: 20, ..Default::default() }, + ..Default::default() + }; + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); // the one with score 5 was rejected, the new one inserted. assert_eq!( MultiPhase::signed_submissions() .iter() - .map(|s| s.raw_solution.score[0]) + .map(|s| s.raw_solution.score.minimal_stake) .collect::>(), vec![6, 7, 8, 9, 20] ); @@ -698,30 +681,39 @@ mod tests { for s in 1..SignedMaxSubmissions::get() { // score is always getting better - let solution = RawSolution { score: [(5 + s).into(), 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); + let solution = RawSolution { + score: ElectionScore { minimal_stake: (5 + s).into(), ..Default::default() }, + ..Default::default() + }; + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); } - let solution = RawSolution { score: [4, 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); + let solution = RawSolution { + score: ElectionScore { minimal_stake: 4, ..Default::default() }, + ..Default::default() + }; + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); assert_eq!( MultiPhase::signed_submissions() .iter() - .map(|s| s.raw_solution.score[0]) + .map(|s| s.raw_solution.score.minimal_stake) .collect::>(), vec![4, 6, 7, 8, 9], ); // better. - let solution = RawSolution { score: [5, 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); + let solution = RawSolution { + score: ElectionScore { minimal_stake: 5, ..Default::default() }, + ..Default::default() + }; + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); // the one with score 5 was rejected, the new one inserted. assert_eq!( MultiPhase::signed_submissions() .iter() - .map(|s| s.raw_solution.score[0]) + .map(|s| s.raw_solution.score.minimal_stake) .collect::>(), vec![5, 6, 7, 8, 9], ); @@ -736,16 +728,22 @@ mod tests { for s in 0..SignedMaxSubmissions::get() { // score is always getting better - let solution = RawSolution { score: [(5 + s).into(), 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); + let solution = RawSolution { + score: ElectionScore { minimal_stake: (5 + s).into(), ..Default::default() }, + ..Default::default() + }; + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); } assert_eq!(balances(&99).1, 2 * 5); assert_eq!(balances(&999).1, 0); // better. - let solution = RawSolution { score: [20, 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(999), solution)); + let solution = RawSolution { + score: ElectionScore { minimal_stake: 20, ..Default::default() }, + ..Default::default() + }; + assert_ok!(MultiPhase::submit(Origin::signed(999), Box::new(solution))); // got one bond back. assert_eq!(balances(&99).1, 2 * 4); @@ -760,21 +758,27 @@ mod tests { assert!(MultiPhase::current_phase().is_signed()); for i in 0..SignedMaxSubmissions::get() { - let solution = RawSolution { score: [(5 + i).into(), 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); + let solution = RawSolution { + score: ElectionScore { minimal_stake: (5 + i).into(), ..Default::default() }, + ..Default::default() + }; + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); } assert_eq!( MultiPhase::signed_submissions() .iter() - .map(|s| s.raw_solution.score[0]) + .map(|s| s.raw_solution.score.minimal_stake) .collect::>(), vec![5, 6, 7] ); // 5 is not accepted. This will only cause processing with no benefit. - let solution = RawSolution { score: [5, 0, 0], ..Default::default() }; + let solution = RawSolution { + score: ElectionScore { minimal_stake: 5, ..Default::default() }, + ..Default::default() + }; assert_noop!( - submit_with_witness(Origin::signed(99), solution), + MultiPhase::submit(Origin::signed(99), Box::new(solution)), Error::::SignedQueueFull, ); }) @@ -796,18 +800,18 @@ mod tests { let solution = raw_solution(); // submit a correct one. - assert_ok!(submit_with_witness(Origin::signed(99), solution.clone())); + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution.clone()))); // make the solution invalidly better and submit. This ought to be slashed. let mut solution_999 = solution.clone(); - solution_999.score[0] += 1; - assert_ok!(submit_with_witness(Origin::signed(999), solution_999)); + solution_999.score.minimal_stake += 1; + assert_ok!(MultiPhase::submit(Origin::signed(999), Box::new(solution_999))); // make the solution invalidly worse and submit. This ought to be suppressed and // returned. let mut solution_9999 = solution.clone(); - solution_9999.score[0] -= 1; - assert_ok!(submit_with_witness(Origin::signed(9999), solution_9999)); + solution_9999.score.minimal_stake -= 1; + assert_ok!(MultiPhase::submit(Origin::signed(9999), Box::new(solution_9999))); assert_eq!( MultiPhase::signed_submissions().iter().map(|x| x.who).collect::>(), @@ -848,14 +852,14 @@ mod tests { assert_eq!(raw.solution.voter_count(), 5); assert_eq!(::SignedMaxWeight::get(), 40); - assert_ok!(submit_with_witness(Origin::signed(99), raw.clone())); + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(raw.clone()))); ::set(30); // note: resubmitting the same solution is technically okay as long as the queue has // space. assert_noop!( - submit_with_witness(Origin::signed(99), raw), + MultiPhase::submit(Origin::signed(99), Box::new(raw)), Error::::SignedTooMuchWeight, ); }) @@ -871,7 +875,7 @@ mod tests { assert_eq!(balances(&123), (0, 0)); assert_noop!( - submit_with_witness(Origin::signed(123), solution), + MultiPhase::submit(Origin::signed(123), Box::new(solution)), Error::::SignedCannotPayDeposit, ); @@ -889,19 +893,25 @@ mod tests { for s in 0..SignedMaxSubmissions::get() { // score is always getting better - let solution = RawSolution { score: [(5 + s).into(), 0, 0], ..Default::default() }; - assert_ok!(submit_with_witness(Origin::signed(99), solution)); + let solution = RawSolution { + score: ElectionScore { minimal_stake: (5 + s).into(), ..Default::default() }, + ..Default::default() + }; + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); } // this solution has a higher score than any in the queue let solution = RawSolution { - score: [(5 + SignedMaxSubmissions::get()).into(), 0, 0], + score: ElectionScore { + minimal_stake: (5 + SignedMaxSubmissions::get()).into(), + ..Default::default() + }, ..Default::default() }; assert_eq!(balances(&123), (0, 0)); assert_noop!( - submit_with_witness(Origin::signed(123), solution), + MultiPhase::submit(Origin::signed(123), Box::new(solution)), Error::::SignedCannotPayDeposit, ); @@ -930,7 +940,7 @@ mod tests { let solution = raw_solution(); // submit a correct one. - assert_ok!(submit_with_witness(Origin::signed(99), solution.clone())); + assert_ok!(MultiPhase::submit(Origin::signed(99), Box::new(solution))); // _some_ good solution was stored. assert!(MultiPhase::finalize_signed_phase()); diff --git a/frame/election-provider-multi-phase/src/unsigned.rs b/frame/election-provider-multi-phase/src/unsigned.rs index 31ad502ac076..d210852bac19 100644 --- a/frame/election-provider-multi-phase/src/unsigned.rs +++ b/frame/election-provider-multi-phase/src/unsigned.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,19 +23,17 @@ use crate::{ WeightInfo, }; use codec::Encode; -use frame_election_provider_support::{NposSolver, PerThing128}; +use frame_election_provider_support::{NposSolution, NposSolver, PerThing128}; use frame_support::{dispatch::DispatchResult, ensure, traits::Get}; use frame_system::offchain::SubmitTransaction; -use sp_arithmetic::Perbill; use sp_npos_elections::{ - assignment_ratio_to_staked_normalized, assignment_staked_to_ratio_normalized, is_score_better, - ElectionResult, NposSolution, + assignment_ratio_to_staked_normalized, assignment_staked_to_ratio_normalized, ElectionResult, }; use sp_runtime::{ offchain::storage::{MutateStorageError, StorageValueRef}, DispatchError, SaturatedConversion, }; -use sp_std::{boxed::Box, cmp::Ordering, convert::TryFrom, vec::Vec}; +use sp_std::{cmp::Ordering, prelude::*}; /// Storage key used to store the last block number at which offchain worker ran. pub(crate) const OFFCHAIN_LAST_BLOCK: &[u8] = b"parity/multi-phase-unsigned-election"; @@ -47,19 +45,15 @@ pub(crate) const OFFCHAIN_CACHED_CALL: &[u8] = b"parity/multi-phase-unsigned-ele /// A voter's fundamental data: their ID, their stake, and the list of candidates for whom they /// voted. -pub type Voter = ( - ::AccountId, - sp_npos_elections::VoteWeight, - Vec<::AccountId>, -); +pub type VoterOf = frame_election_provider_support::VoterOf<::DataProvider>; /// The relative distribution of a voter's stake among the winning targets. pub type Assignment = sp_npos_elections::Assignment<::AccountId, SolutionAccuracyOf>; -/// The [`IndexAssignment`][sp_npos_elections::IndexAssignment] type specialized for a particular -/// runtime `T`. -pub type IndexAssignmentOf = sp_npos_elections::IndexAssignmentOf>; +/// The [`IndexAssignment`][frame_election_provider_support::IndexAssignment] type specialized for a +/// particular runtime `T`. +pub type IndexAssignmentOf = frame_election_provider_support::IndexAssignmentOf>; /// Error type of the pallet's [`crate::Config::Solver`]. pub type SolverErrorOf = <::Solver as NposSolver>::Error; @@ -544,7 +538,7 @@ impl Pallet { // Time to finish. We might have reduced less than expected due to rounding error. Increase // one last time if we have any room left, the reduce until we are sure we are below limit. - while voters + 1 <= max_voters && weight_with(voters + 1) < max_weight { + while voters < max_voters && weight_with(voters + 1) < max_weight { voters += 1; } while voters.checked_sub(1).is_some() && weight_with(voters) > max_weight { @@ -628,11 +622,9 @@ impl Pallet { // ensure score is being improved. Panic henceforth. ensure!( - Self::queued_solution().map_or(true, |q: ReadySolution<_>| is_score_better::( - raw_solution.score, - q.score, - T::SolutionImprovementThreshold::get() - )), + Self::queued_solution().map_or(true, |q: ReadySolution<_>| raw_solution + .score + .strict_threshold_better(q.score, T::SolutionImprovementThreshold::get())), Error::::PreDispatchWeakSubmission, ); @@ -669,7 +661,7 @@ mod max_weight { fn finalize_signed_phase_reject_solution() -> Weight { unreachable!() } - fn submit(c: u32) -> Weight { + fn submit() -> Weight { unreachable!() } fn submit_unsigned(v: u32, t: u32, a: u32, d: u32) -> Weight { @@ -749,12 +741,15 @@ mod tests { }; use codec::Decode; use frame_benchmarking::Zero; - use frame_support::{assert_noop, assert_ok, dispatch::Dispatchable, traits::OffchainWorker}; - use sp_npos_elections::IndexAssignment; + use frame_election_provider_support::IndexAssignment; + use frame_support::{ + assert_noop, assert_ok, bounded_vec, dispatch::Dispatchable, traits::OffchainWorker, + }; + use sp_npos_elections::ElectionScore; use sp_runtime::{ offchain::storage_lock::{BlockAndTime, StorageLock}, traits::ValidateUnsigned, - PerU16, + ModuleError, PerU16, Perbill, }; type Assignment = crate::unsigned::Assignment; @@ -762,8 +757,10 @@ mod tests { #[test] fn validate_unsigned_retracts_wrong_phase() { ExtBuilder::default().desired_targets(0).build_and_execute(|| { - let solution = - RawSolution:: { score: [5, 0, 0], ..Default::default() }; + let solution = RawSolution:: { + score: ElectionScore { minimal_stake: 5, ..Default::default() }, + ..Default::default() + }; let call = Call::submit_unsigned { raw_solution: Box::new(solution.clone()), witness: witness(), @@ -835,8 +832,10 @@ mod tests { roll_to(25); assert!(MultiPhase::current_phase().is_unsigned()); - let solution = - RawSolution:: { score: [5, 0, 0], ..Default::default() }; + let solution = RawSolution:: { + score: ElectionScore { minimal_stake: 5, ..Default::default() }, + ..Default::default() + }; let call = Call::submit_unsigned { raw_solution: Box::new(solution.clone()), witness: witness(), @@ -851,7 +850,10 @@ mod tests { assert!(::pre_dispatch(&call).is_ok()); // set a better score - let ready = ReadySolution { score: [10, 0, 0], ..Default::default() }; + let ready = ReadySolution { + score: ElectionScore { minimal_stake: 10, ..Default::default() }, + ..Default::default() + }; >::put(ready); // won't work anymore. @@ -876,7 +878,10 @@ mod tests { roll_to(25); assert!(MultiPhase::current_phase().is_unsigned()); - let raw = RawSolution:: { score: [5, 0, 0], ..Default::default() }; + let raw = RawSolution:: { + score: ElectionScore { minimal_stake: 5, ..Default::default() }, + ..Default::default() + }; let call = Call::submit_unsigned { raw_solution: Box::new(raw.clone()), witness: witness() }; assert_eq!(raw.solution.unique_targets().len(), 0); @@ -902,8 +907,10 @@ mod tests { roll_to(25); assert!(MultiPhase::current_phase().is_unsigned()); - let solution = - RawSolution:: { score: [5, 0, 0], ..Default::default() }; + let solution = RawSolution:: { + score: ElectionScore { minimal_stake: 5, ..Default::default() }, + ..Default::default() + }; let call = Call::submit_unsigned { raw_solution: Box::new(solution.clone()), witness: witness(), @@ -924,16 +931,18 @@ mod tests { #[test] #[should_panic(expected = "Invalid unsigned submission must produce invalid block and \ deprive validator from their authoring reward.: \ - Module { index: 2, error: 1, message: \ - Some(\"PreDispatchWrongWinnerCount\") }")] + Module(ModuleError { index: 2, error: [1, 0, 0, 0], message: \ + Some(\"PreDispatchWrongWinnerCount\") })")] fn unfeasible_solution_panics() { ExtBuilder::default().build_and_execute(|| { roll_to(25); assert!(MultiPhase::current_phase().is_unsigned()); // This is in itself an invalid BS solution. - let solution = - RawSolution:: { score: [5, 0, 0], ..Default::default() }; + let solution = RawSolution:: { + score: ElectionScore { minimal_stake: 5, ..Default::default() }, + ..Default::default() + }; let call = Call::submit_unsigned { raw_solution: Box::new(solution.clone()), witness: witness(), @@ -952,8 +961,10 @@ mod tests { assert!(MultiPhase::current_phase().is_unsigned()); // This solution is unfeasible as well, but we won't even get there. - let solution = - RawSolution:: { score: [5, 0, 0], ..Default::default() }; + let solution = RawSolution:: { + score: ElectionScore { minimal_stake: 5, ..Default::default() }, + ..Default::default() + }; let mut correct_witness = witness(); correct_witness.voters += 1; @@ -1033,13 +1044,18 @@ mod tests { roll_to(25); assert!(MultiPhase::current_phase().is_unsigned()); + // Force the number of winners to be bigger to fail + let (mut solution, _) = + MultiPhase::mine_solution::<::Solver>().unwrap(); + solution.solution.votes1[0].1 = 4; + assert_eq!( - MultiPhase::mine_check_save_submit().unwrap_err(), - MinerError::PreDispatchChecksFailed(DispatchError::Module { + MultiPhase::basic_checks(&solution, "mined").unwrap_err(), + MinerError::PreDispatchChecksFailed(DispatchError::Module(ModuleError { index: 2, - error: 1, + error: [1, 0, 0, 0], message: Some("PreDispatchWrongWinnerCount"), - }), + })), ); }) } @@ -1048,8 +1064,8 @@ mod tests { fn unsigned_per_dispatch_checks_can_only_submit_threshold_better() { ExtBuilder::default() .desired_targets(1) - .add_voter(7, 2, vec![10]) - .add_voter(8, 5, vec![10]) + .add_voter(7, 2, bounded_vec![10]) + .add_voter(8, 5, bounded_vec![10]) .solution_improvement_threshold(Perbill::from_percent(50)) .build_and_execute(|| { roll_to(25); @@ -1072,7 +1088,7 @@ mod tests { Box::new(solution), witness )); - assert_eq!(MultiPhase::queued_solution().unwrap().score[0], 10); + assert_eq!(MultiPhase::queued_solution().unwrap().score.minimal_stake, 10); // trial 1: a solution who's score is only 2, i.e. 20% better in the first element. let result = ElectionResult { @@ -1088,7 +1104,7 @@ mod tests { }; let (solution, _) = MultiPhase::prepare_election_result(result).unwrap(); // 12 is not 50% more than 10 - assert_eq!(solution.score[0], 12); + assert_eq!(solution.score.minimal_stake, 12); assert_noop!( MultiPhase::unsigned_pre_dispatch_checks(&solution), Error::::PreDispatchWeakSubmission, @@ -1109,7 +1125,7 @@ mod tests { ], }; let (solution, witness) = MultiPhase::prepare_election_result(result).unwrap(); - assert_eq!(solution.score[0], 17); + assert_eq!(solution.score.minimal_stake, 17); // and it is fine assert_ok!(MultiPhase::unsigned_pre_dispatch_checks(&solution)); @@ -1241,35 +1257,62 @@ mod tests { } #[test] - fn ocw_clears_cache_after_election() { - let (mut ext, _pool) = ExtBuilder::default().build_offchainify(0); + fn ocw_clears_cache_on_unsigned_phase_open() { + let (mut ext, pool) = ExtBuilder::default().build_offchainify(0); ext.execute_with(|| { - roll_to(25); - assert_eq!(MultiPhase::current_phase(), Phase::Unsigned((true, 25))); + const BLOCK: u64 = 25; + let block_plus = |delta: u64| BLOCK + delta; + let offchain_repeat = ::OffchainRepeat::get(); - // we must clear the offchain storage to ensure the offchain execution check doesn't get - // in the way. - let mut storage = StorageValueRef::persistent(&OFFCHAIN_LAST_BLOCK); - storage.clear(); + roll_to(BLOCK); + // we are on the first block of the unsigned phase + assert_eq!(MultiPhase::current_phase(), Phase::Unsigned((true, BLOCK))); assert!( !ocw_solution_exists::(), "no solution should be present before we mine one", ); - // creates and cache a solution - MultiPhase::offchain_worker(25); + // create and cache a solution on the first block of the unsigned phase + MultiPhase::offchain_worker(BLOCK); assert!( ocw_solution_exists::(), "a solution must be cached after running the worker", ); - // after an election, the solution must be cleared + // record the submitted tx, + let tx_cache_1 = pool.read().transactions[0].clone(); + // and assume it has been processed. + pool.try_write().unwrap().transactions.clear(); + + // after an election, the solution is not cleared // we don't actually care about the result of the election - roll_to(26); let _ = MultiPhase::do_elect(); - MultiPhase::offchain_worker(26); - assert!(!ocw_solution_exists::(), "elections must clear the ocw cache"); + MultiPhase::offchain_worker(block_plus(1)); + assert!(ocw_solution_exists::(), "elections does not clear the ocw cache"); + + // submit a solution with the offchain worker after the repeat interval + MultiPhase::offchain_worker(block_plus(offchain_repeat + 1)); + + // record the submitted tx, + let tx_cache_2 = pool.read().transactions[0].clone(); + // and assume it has been processed. + pool.try_write().unwrap().transactions.clear(); + + // the OCW submitted the same solution twice since the cache was not cleared. + assert_eq!(tx_cache_1, tx_cache_2); + + let current_block = block_plus(offchain_repeat * 2 + 2); + // force the unsigned phase to start on the current block. + CurrentPhase::::set(Phase::Unsigned((true, current_block))); + + // clear the cache and create a solution since we are on the first block of the unsigned + // phase. + MultiPhase::offchain_worker(current_block); + let tx_cache_3 = pool.read().transactions[0].clone(); + + // the submitted solution changes because the cache was cleared. + assert_eq!(tx_cache_1, tx_cache_3); }) } diff --git a/frame/election-provider-multi-phase/src/weights.rs b/frame/election-provider-multi-phase/src/weights.rs index 4d49f60fabfc..54c519681922 100644 --- a/frame/election-provider-multi-phase/src/weights.rs +++ b/frame/election-provider-multi-phase/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_election_provider_multi_phase //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-09-22, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-03-03, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -35,7 +35,6 @@ // --output=./frame/election-provider-multi-phase/src/weights.rs // --template=./.maintain/frame-weight-template.hbs - #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] #![allow(unused_imports)] @@ -52,7 +51,7 @@ pub trait WeightInfo { fn finalize_signed_phase_reject_solution() -> Weight; fn create_snapshot_internal(v: u32, t: u32, ) -> Weight; fn elect_queued(a: u32, d: u32, ) -> Weight; - fn submit(c: u32, ) -> Weight; + fn submit() -> Weight; fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight; fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight; } @@ -69,33 +68,33 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking ForceEra (r:1 w:0) // Storage: ElectionProviderMultiPhase CurrentPhase (r:1 w:0) fn on_initialize_nothing() -> Weight { - (22_784_000 as Weight) + (13_342_000 as Weight) .saturating_add(T::DbWeight::get().reads(8 as Weight)) } // Storage: ElectionProviderMultiPhase Round (r:1 w:0) // Storage: ElectionProviderMultiPhase CurrentPhase (r:0 w:1) fn on_initialize_open_signed() -> Weight { - (32_763_000 as Weight) + (13_503_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: ElectionProviderMultiPhase Round (r:1 w:0) // Storage: ElectionProviderMultiPhase CurrentPhase (r:0 w:1) fn on_initialize_open_unsigned() -> Weight { - (29_117_000 as Weight) + (13_688_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: System Account (r:1 w:1) // Storage: ElectionProviderMultiPhase QueuedSolution (r:0 w:1) fn finalize_signed_phase_accept_solution() -> Weight { - (48_996_000 as Weight) + (29_124_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: System Account (r:1 w:1) fn finalize_signed_phase_reject_solution() -> Weight { - (32_508_000 as Weight) + (21_950_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -103,11 +102,11 @@ impl WeightInfo for SubstrateWeight { // Storage: ElectionProviderMultiPhase DesiredTargets (r:0 w:1) // Storage: ElectionProviderMultiPhase Snapshot (r:0 w:1) fn create_snapshot_internal(v: u32, t: u32, ) -> Weight { - (96_001_000 as Weight) + (17_274_000 as Weight) // Standard Error: 1_000 - .saturating_add((307_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 2_000 - .saturating_add((133_000 as Weight).saturating_mul(t as Weight)) + .saturating_add((191_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 3_000 + .saturating_add((53_000 as Weight).saturating_mul(t as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: ElectionProviderMultiPhase SignedSubmissionIndices (r:1 w:1) @@ -120,24 +119,22 @@ impl WeightInfo for SubstrateWeight { // Storage: ElectionProviderMultiPhase Snapshot (r:0 w:1) // Storage: ElectionProviderMultiPhase CurrentPhase (r:0 w:1) fn elect_queued(a: u32, d: u32, ) -> Weight { - (100_505_000 as Weight) + (145_826_000 as Weight) + // Standard Error: 4_000 + .saturating_add((604_000 as Weight).saturating_mul(a as Weight)) // Standard Error: 6_000 - .saturating_add((1_665_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 10_000 - .saturating_add((443_000 as Weight).saturating_mul(d as Weight)) + .saturating_add((72_000 as Weight).saturating_mul(d as Weight)) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(8 as Weight)) } - // Storage: ElectionProviderMultiPhase SignedSubmissionIndices (r:1 w:1) // Storage: ElectionProviderMultiPhase CurrentPhase (r:1 w:0) // Storage: ElectionProviderMultiPhase SnapshotMetadata (r:1 w:0) // Storage: TransactionPayment NextFeeMultiplier (r:1 w:0) + // Storage: ElectionProviderMultiPhase SignedSubmissionIndices (r:1 w:1) // Storage: ElectionProviderMultiPhase SignedSubmissionNextIndex (r:1 w:1) // Storage: ElectionProviderMultiPhase SignedSubmissionsMap (r:0 w:1) - fn submit(c: u32, ) -> Weight { - (74_088_000 as Weight) - // Standard Error: 59_000 - .saturating_add((187_000 as Weight).saturating_mul(c as Weight)) + fn submit() -> Weight { + (41_579_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -150,14 +147,14 @@ impl WeightInfo for SubstrateWeight { // Storage: ElectionProviderMultiPhase Snapshot (r:1 w:0) fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 5_000 - .saturating_add((1_970_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 10_000 - .saturating_add((173_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 3_000 + .saturating_add((882_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 7_000 + .saturating_add((144_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 12_000 + .saturating_add((6_534_000 as Weight).saturating_mul(a as Weight)) // Standard Error: 18_000 - .saturating_add((9_783_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 27_000 - .saturating_add((2_224_000 as Weight).saturating_mul(d as Weight)) + .saturating_add((1_312_000 as Weight).saturating_mul(d as Weight)) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -165,16 +162,14 @@ impl WeightInfo for SubstrateWeight { // Storage: ElectionProviderMultiPhase DesiredTargets (r:1 w:0) // Storage: ElectionProviderMultiPhase MinimumUntrustedScore (r:1 w:0) // Storage: ElectionProviderMultiPhase Snapshot (r:1 w:0) - fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + fn feasibility_check(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) // Standard Error: 3_000 - .saturating_add((1_910_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 7_000 - .saturating_add((111_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 13_000 - .saturating_add((7_741_000 as Weight).saturating_mul(a as Weight)) + .saturating_add((835_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 12_000 + .saturating_add((5_395_000 as Weight).saturating_mul(a as Weight)) // Standard Error: 19_000 - .saturating_add((1_844_000 as Weight).saturating_mul(d as Weight)) + .saturating_add((1_243_000 as Weight).saturating_mul(d as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) } } @@ -190,33 +185,33 @@ impl WeightInfo for () { // Storage: Staking ForceEra (r:1 w:0) // Storage: ElectionProviderMultiPhase CurrentPhase (r:1 w:0) fn on_initialize_nothing() -> Weight { - (22_784_000 as Weight) + (13_342_000 as Weight) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) } // Storage: ElectionProviderMultiPhase Round (r:1 w:0) // Storage: ElectionProviderMultiPhase CurrentPhase (r:0 w:1) fn on_initialize_open_signed() -> Weight { - (32_763_000 as Weight) + (13_503_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: ElectionProviderMultiPhase Round (r:1 w:0) // Storage: ElectionProviderMultiPhase CurrentPhase (r:0 w:1) fn on_initialize_open_unsigned() -> Weight { - (29_117_000 as Weight) + (13_688_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: System Account (r:1 w:1) // Storage: ElectionProviderMultiPhase QueuedSolution (r:0 w:1) fn finalize_signed_phase_accept_solution() -> Weight { - (48_996_000 as Weight) + (29_124_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: System Account (r:1 w:1) fn finalize_signed_phase_reject_solution() -> Weight { - (32_508_000 as Weight) + (21_950_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -224,11 +219,11 @@ impl WeightInfo for () { // Storage: ElectionProviderMultiPhase DesiredTargets (r:0 w:1) // Storage: ElectionProviderMultiPhase Snapshot (r:0 w:1) fn create_snapshot_internal(v: u32, t: u32, ) -> Weight { - (96_001_000 as Weight) + (17_274_000 as Weight) // Standard Error: 1_000 - .saturating_add((307_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 2_000 - .saturating_add((133_000 as Weight).saturating_mul(t as Weight)) + .saturating_add((191_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 3_000 + .saturating_add((53_000 as Weight).saturating_mul(t as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: ElectionProviderMultiPhase SignedSubmissionIndices (r:1 w:1) @@ -241,24 +236,22 @@ impl WeightInfo for () { // Storage: ElectionProviderMultiPhase Snapshot (r:0 w:1) // Storage: ElectionProviderMultiPhase CurrentPhase (r:0 w:1) fn elect_queued(a: u32, d: u32, ) -> Weight { - (100_505_000 as Weight) + (145_826_000 as Weight) + // Standard Error: 4_000 + .saturating_add((604_000 as Weight).saturating_mul(a as Weight)) // Standard Error: 6_000 - .saturating_add((1_665_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 10_000 - .saturating_add((443_000 as Weight).saturating_mul(d as Weight)) + .saturating_add((72_000 as Weight).saturating_mul(d as Weight)) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } - // Storage: ElectionProviderMultiPhase SignedSubmissionIndices (r:1 w:1) // Storage: ElectionProviderMultiPhase CurrentPhase (r:1 w:0) // Storage: ElectionProviderMultiPhase SnapshotMetadata (r:1 w:0) // Storage: TransactionPayment NextFeeMultiplier (r:1 w:0) + // Storage: ElectionProviderMultiPhase SignedSubmissionIndices (r:1 w:1) // Storage: ElectionProviderMultiPhase SignedSubmissionNextIndex (r:1 w:1) // Storage: ElectionProviderMultiPhase SignedSubmissionsMap (r:0 w:1) - fn submit(c: u32, ) -> Weight { - (74_088_000 as Weight) - // Standard Error: 59_000 - .saturating_add((187_000 as Weight).saturating_mul(c as Weight)) + fn submit() -> Weight { + (41_579_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -271,14 +264,14 @@ impl WeightInfo for () { // Storage: ElectionProviderMultiPhase Snapshot (r:1 w:0) fn submit_unsigned(v: u32, t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 5_000 - .saturating_add((1_970_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 10_000 - .saturating_add((173_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 3_000 + .saturating_add((882_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 7_000 + .saturating_add((144_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 12_000 + .saturating_add((6_534_000 as Weight).saturating_mul(a as Weight)) // Standard Error: 18_000 - .saturating_add((9_783_000 as Weight).saturating_mul(a as Weight)) - // Standard Error: 27_000 - .saturating_add((2_224_000 as Weight).saturating_mul(d as Weight)) + .saturating_add((1_312_000 as Weight).saturating_mul(d as Weight)) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -286,16 +279,14 @@ impl WeightInfo for () { // Storage: ElectionProviderMultiPhase DesiredTargets (r:1 w:0) // Storage: ElectionProviderMultiPhase MinimumUntrustedScore (r:1 w:0) // Storage: ElectionProviderMultiPhase Snapshot (r:1 w:0) - fn feasibility_check(v: u32, t: u32, a: u32, d: u32, ) -> Weight { + fn feasibility_check(v: u32, _t: u32, a: u32, d: u32, ) -> Weight { (0 as Weight) // Standard Error: 3_000 - .saturating_add((1_910_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 7_000 - .saturating_add((111_000 as Weight).saturating_mul(t as Weight)) - // Standard Error: 13_000 - .saturating_add((7_741_000 as Weight).saturating_mul(a as Weight)) + .saturating_add((835_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 12_000 + .saturating_add((5_395_000 as Weight).saturating_mul(a as Weight)) // Standard Error: 19_000 - .saturating_add((1_844_000 as Weight).saturating_mul(d as Weight)) + .saturating_add((1_243_000 as Weight).saturating_mul(d as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) } } diff --git a/frame/election-provider-support/Cargo.toml b/frame/election-provider-support/Cargo.toml index dfe2b1102433..be0c05e46df3 100644 --- a/frame/election-provider-support/Cargo.toml +++ b/frame/election-provider-support/Cargo.toml @@ -2,9 +2,9 @@ name = "frame-election-provider-support" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "election provider supporting traits" readme = "README.md" @@ -13,19 +13,21 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-arithmetic = { version = "4.0.0-dev", default-features = false, path = "../../primitives/arithmetic" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-arithmetic = { version = "5.0.0", default-features = false, path = "../../primitives/arithmetic" } sp-npos-elections = { version = "4.0.0-dev", default-features = false, path = "../../primitives/npos-elections" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +frame-election-provider-solution-type = { version = "4.0.0-dev", path = "solution-type" } [dev-dependencies] +rand = "0.7.3" sp-npos-elections = { version = "4.0.0-dev", path = "../../primitives/npos-elections" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } [features] default = ["std"] diff --git a/frame/election-provider-support/solution-type/Cargo.toml b/frame/election-provider-support/solution-type/Cargo.toml new file mode 100644 index 000000000000..ca3038c9145c --- /dev/null +++ b/frame/election-provider-support/solution-type/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "frame-election-provider-solution-type" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "NPoS Solution Type" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0.82", features = ["full", "visit"] } +quote = "1.0" +proc-macro2 = "1.0.36" +proc-macro-crate = "1.1.3" + +[dev-dependencies] +parity-scale-codec = "3.0.0" +scale-info = "2.0.1" +sp-arithmetic = { version = "5.0.0", path = "../../../primitives/arithmetic" } +# used by generate_solution_type: +frame-election-provider-support = { version = "4.0.0-dev", path = ".." } +frame-support = { version = "4.0.0-dev", path = "../../support" } +trybuild = "1.0.53" diff --git a/frame/election-provider-support/solution-type/fuzzer/Cargo.toml b/frame/election-provider-support/solution-type/fuzzer/Cargo.toml new file mode 100644 index 000000000000..e2bae3f72a4f --- /dev/null +++ b/frame/election-provider-support/solution-type/fuzzer/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "frame-election-solution-type-fuzzer" +version = "2.0.0-alpha.5" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "Fuzzer for phragmén solution type implementation." +publish = false + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +clap = { version = "3.0", features = ["derive"] } +honggfuzz = "0.5" +rand = { version = "0.8", features = ["std", "small_rng"] } + +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +frame-election-provider-solution-type = { version = "4.0.0-dev", path = ".." } +frame-election-provider-support = { version = "4.0.0-dev", path = "../.." } +sp-arithmetic = { version = "5.0.0", path = "../../../../primitives/arithmetic" } +sp-runtime = { version = "6.0.0", path = "../../../../primitives/runtime" } +# used by generate_solution_type: +sp-npos-elections = { version = "4.0.0-dev", default-features = false, path = "../../../../primitives/npos-elections" } +frame-support = { version = "4.0.0-dev", path = "../../../support" } + +[[bin]] +name = "compact" +path = "src/compact.rs" diff --git a/primitives/npos-elections/fuzzer/src/compact.rs b/frame/election-provider-support/solution-type/fuzzer/src/compact.rs similarity index 86% rename from primitives/npos-elections/fuzzer/src/compact.rs rename to frame/election-provider-support/solution-type/fuzzer/src/compact.rs index 4e78c94b8257..e7ef440ff219 100644 --- a/primitives/npos-elections/fuzzer/src/compact.rs +++ b/frame/election-provider-support/solution-type/fuzzer/src/compact.rs @@ -1,5 +1,6 @@ +use frame_election_provider_solution_type::generate_solution_type; use honggfuzz::fuzz; -use sp_npos_elections::{generate_solution_type, sp_arithmetic::Percent}; +use sp_arithmetic::Percent; use sp_runtime::codec::{Encode, Error}; fn main() { @@ -7,11 +8,12 @@ fn main() { VoterIndex = u32, TargetIndex = u32, Accuracy = Percent, + MaxVoters = frame_support::traits::ConstU32::<100_000>, >(16)); loop { fuzz!(|fuzzer_data: &[u8]| { let result_decoded: Result = - ::decode(&mut &fuzzer_data[..]); + ::decode(&mut &*fuzzer_data); // Ignore errors as not every random sequence of bytes can be decoded as // InnerTestSolutionCompact if let Ok(decoded) = result_decoded { diff --git a/primitives/npos-elections/solution-type/src/codec.rs b/frame/election-provider-support/solution-type/src/codec.rs similarity index 70% rename from primitives/npos-elections/solution-type/src/codec.rs rename to frame/election-provider-support/solution-type/src/codec.rs index 2dac076fcde4..05d753d3d845 100644 --- a/primitives/npos-elections/solution-type/src/codec.rs +++ b/frame/election-provider-support/solution-type/src/codec.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -51,14 +51,14 @@ fn decode_impl( quote! { let #name = < - _npos::sp_std::prelude::Vec<(_npos::codec::Compact<#voter_type>, _npos::codec::Compact<#target_type>)> + _feps::sp_std::prelude::Vec<(_feps::codec::Compact<#voter_type>, _feps::codec::Compact<#target_type>)> as - _npos::codec::Decode + _feps::codec::Decode >::decode(value)?; let #name = #name .into_iter() .map(|(v, t)| (v.0, t.0)) - .collect::<_npos::sp_std::prelude::Vec<_>>(); + .collect::<_feps::sp_std::prelude::Vec<_>>(); } }; @@ -73,12 +73,12 @@ fn decode_impl( quote! { let #name = < - _npos::sp_std::prelude::Vec<( - _npos::codec::Compact<#voter_type>, - [(_npos::codec::Compact<#target_type>, _npos::codec::Compact<#weight_type>); #c-1], - _npos::codec::Compact<#target_type>, + _feps::sp_std::prelude::Vec<( + _feps::codec::Compact<#voter_type>, + [(_feps::codec::Compact<#target_type>, _feps::codec::Compact<#weight_type>); #c-1], + _feps::codec::Compact<#target_type>, )> - as _npos::codec::Decode + as _feps::codec::Decode >::decode(value)?; let #name = #name .into_iter() @@ -87,7 +87,7 @@ fn decode_impl( [ #inner_impl ], t_last.0, )) - .collect::<_npos::sp_std::prelude::Vec<_>>(); + .collect::<_feps::sp_std::prelude::Vec<_>>(); } }) .collect::(); @@ -100,8 +100,8 @@ fn decode_impl( .collect::(); quote!( - impl _npos::codec::Decode for #ident { - fn decode(value: &mut I) -> Result { + impl _feps::codec::Decode for #ident { + fn decode(value: &mut I) -> Result { #decode_impl_single #decode_impl_rest @@ -123,10 +123,10 @@ fn encode_impl(ident: &syn::Ident, count: usize) -> TokenStream2 { let #name = self.#name .iter() .map(|(v, t)| ( - _npos::codec::Compact(v.clone()), - _npos::codec::Compact(t.clone()), + _feps::codec::Compact(v.clone()), + _feps::codec::Compact(t.clone()), )) - .collect::<_npos::sp_std::prelude::Vec<_>>(); + .collect::<_feps::sp_std::prelude::Vec<_>>(); #name.encode_to(&mut r); } }; @@ -139,8 +139,8 @@ fn encode_impl(ident: &syn::Ident, count: usize) -> TokenStream2 { let inners_solution_array = (0..c - 1) .map(|i| { quote! {( - _npos::codec::Compact(inner[#i].0.clone()), - _npos::codec::Compact(inner[#i].1.clone()), + _feps::codec::Compact(inner[#i].0.clone()), + _feps::codec::Compact(inner[#i].1.clone()), ),} }) .collect::(); @@ -149,19 +149,19 @@ fn encode_impl(ident: &syn::Ident, count: usize) -> TokenStream2 { let #name = self.#name .iter() .map(|(v, inner, t_last)| ( - _npos::codec::Compact(v.clone()), + _feps::codec::Compact(v.clone()), [ #inners_solution_array ], - _npos::codec::Compact(t_last.clone()), + _feps::codec::Compact(t_last.clone()), )) - .collect::<_npos::sp_std::prelude::Vec<_>>(); + .collect::<_feps::sp_std::prelude::Vec<_>>(); #name.encode_to(&mut r); } }) .collect::(); quote!( - impl _npos::codec::Encode for #ident { - fn encode(&self) -> _npos::sp_std::prelude::Vec { + impl _feps::codec::Encode for #ident { + fn encode(&self) -> _feps::sp_std::prelude::Vec { let mut r = vec![]; #encode_impl_single #encode_impl_rest @@ -182,8 +182,8 @@ fn scale_info_impl( let name = format!("{}", vote_field(1)); quote! { .field(|f| - f.ty::<_npos::sp_std::prelude::Vec< - (_npos::codec::Compact<#voter_type>, _npos::codec::Compact<#target_type>) + f.ty::<_feps::sp_std::prelude::Vec< + (_feps::codec::Compact<#voter_type>, _feps::codec::Compact<#target_type>) >>() .name(#name) ) @@ -194,10 +194,10 @@ fn scale_info_impl( let name = format!("{}", vote_field(2)); quote! { .field(|f| - f.ty::<_npos::sp_std::prelude::Vec<( - _npos::codec::Compact<#voter_type>, - (_npos::codec::Compact<#target_type>, _npos::codec::Compact<#weight_type>), - _npos::codec::Compact<#target_type> + f.ty::<_feps::sp_std::prelude::Vec<( + _feps::codec::Compact<#voter_type>, + (_feps::codec::Compact<#target_type>, _feps::codec::Compact<#weight_type>), + _feps::codec::Compact<#target_type> )>>() .name(#name) ) @@ -209,13 +209,13 @@ fn scale_info_impl( let name = format!("{}", vote_field(c)); quote! { .field(|f| - f.ty::<_npos::sp_std::prelude::Vec<( - _npos::codec::Compact<#voter_type>, + f.ty::<_feps::sp_std::prelude::Vec<( + _feps::codec::Compact<#voter_type>, [ - (_npos::codec::Compact<#target_type>, _npos::codec::Compact<#weight_type>); + (_feps::codec::Compact<#target_type>, _feps::codec::Compact<#weight_type>); #c - 1 ], - _npos::codec::Compact<#target_type> + _feps::codec::Compact<#target_type> )>>() .name(#name) ) @@ -224,14 +224,14 @@ fn scale_info_impl( .collect::(); quote!( - impl _npos::scale_info::TypeInfo for #ident { + impl _feps::scale_info::TypeInfo for #ident { type Identity = Self; - fn type_info() -> _npos::scale_info::Type<_npos::scale_info::form::MetaForm> { - _npos::scale_info::Type::builder() - .path(_npos::scale_info::Path::new(stringify!(#ident), module_path!())) + fn type_info() -> _feps::scale_info::Type<_feps::scale_info::form::MetaForm> { + _feps::scale_info::Type::builder() + .path(_feps::scale_info::Path::new(stringify!(#ident), module_path!())) .composite( - _npos::scale_info::build::Fields::named() + _feps::scale_info::build::Fields::named() #scale_info_impl_single #scale_info_impl_double #scale_info_impl_rest diff --git a/primitives/npos-elections/solution-type/src/from_assignment_helpers.rs b/frame/election-provider-support/solution-type/src/from_assignment_helpers.rs similarity index 96% rename from primitives/npos-elections/solution-type/src/from_assignment_helpers.rs rename to frame/election-provider-support/solution-type/src/from_assignment_helpers.rs index dc194baa6d9e..e0613afc0d39 100644 --- a/primitives/npos-elections/solution-type/src/from_assignment_helpers.rs +++ b/frame/election-provider-support/solution-type/src/from_assignment_helpers.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/npos-elections/solution-type/src/index_assignment.rs b/frame/election-provider-support/solution-type/src/index_assignment.rs similarity index 96% rename from primitives/npos-elections/solution-type/src/index_assignment.rs rename to frame/election-provider-support/solution-type/src/index_assignment.rs index d38dc3ec309d..ccdcb4ba25cd 100644 --- a/primitives/npos-elections/solution-type/src/index_assignment.rs +++ b/frame/election-provider-support/solution-type/src/index_assignment.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/npos-elections/solution-type/src/lib.rs b/frame/election-provider-support/solution-type/src/lib.rs similarity index 78% rename from primitives/npos-elections/solution-type/src/lib.rs rename to frame/election-provider-support/solution-type/src/lib.rs index 9b0ec56fc74d..57d939377b62 100644 --- a/primitives/npos-elections/solution-type/src/lib.rs +++ b/frame/election-provider-support/solution-type/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -49,6 +49,12 @@ pub(crate) fn syn_err(message: &'static str) -> syn::Error { /// compact encoding. /// - The accuracy of the ratios. This must be one of the `PerThing` types defined in /// `sp-arithmetic`. +/// - The maximum number of voters. This must be of type `Get`. Check +/// for more details. This is used to bound the struct, by leveraging the fact that `votes1.len() +/// < votes2.len() < ... < votesn.len()` (the details of the struct is explained further below). +/// We know that `sum_i votes_i.len() <= MaxVoters`, and we know that the maximum size of the +/// struct would be achieved if all voters fall in the last bucket. One can also check the tests +/// and more specifically `max_encoded_len_exact` for a concrete example. /// /// Moreover, the maximum number of edges per voter (distribution per assignment) also need to be /// specified. Attempting to convert from/to an assignment with more distributions will fail. @@ -57,12 +63,14 @@ pub(crate) fn syn_err(message: &'static str) -> syn::Error { /// type, `u8` target type and `Perbill` accuracy with maximum of 4 edges per voter. /// /// ``` -/// # use sp_npos_elections_solution_type::generate_solution_type; +/// # use frame_election_provider_solution_type::generate_solution_type; /// # use sp_arithmetic::per_things::Perbill; +/// # use frame_support::traits::ConstU32; /// generate_solution_type!(pub struct TestSolution::< /// VoterIndex = u16, /// TargetIndex = u8, /// Accuracy = Perbill, +/// MaxVoters = ConstU32::<10>, /// >(4)); /// ``` /// @@ -88,7 +96,7 @@ pub(crate) fn syn_err(message: &'static str) -> syn::Error { /// ``` /// /// The given struct provides function to convert from/to `Assignment` as part of -/// [`sp_npos_elections::Solution`] trait: +/// `frame_election_provider_support::NposSolution` trait: /// /// - `fn from_assignment<..>(..)` /// - `fn into_assignment<..>(..)` @@ -100,12 +108,18 @@ pub(crate) fn syn_err(message: &'static str) -> syn::Error { /// for numbers will be used, similar to how `parity-scale-codec`'s `Compact` works. /// /// ``` -/// # use sp_npos_elections_solution_type::generate_solution_type; -/// # use sp_npos_elections::NposSolution; +/// # use frame_election_provider_solution_type::generate_solution_type; +/// # use frame_election_provider_support::NposSolution; /// # use sp_arithmetic::per_things::Perbill; +/// # use frame_support::traits::ConstU32; /// generate_solution_type!( /// #[compact] -/// pub struct TestSolutionCompact::(8) +/// pub struct TestSolutionCompact::< +/// VoterIndex = u16, +/// TargetIndex = u8, +/// Accuracy = Perbill, +/// MaxVoters = ConstU32::<10>, +/// >(8) /// ); /// ``` #[proc_macro] @@ -129,6 +143,7 @@ struct SolutionDef { voter_type: syn::Type, target_type: syn::Type, weight_type: syn::Type, + max_voters: syn::Type, count: usize, compact_encoding: bool, } @@ -167,11 +182,11 @@ impl Parse for SolutionDef { let _ = ::parse(input)?; let generics: syn::AngleBracketedGenericArguments = input.parse()?; - if generics.args.len() != 3 { - return Err(syn_err("Must provide 3 generic args.")) + if generics.args.len() != 4 { + return Err(syn_err("Must provide 4 generic args.")) } - let expected_types = ["VoterIndex", "TargetIndex", "Accuracy"]; + let expected_types = ["VoterIndex", "TargetIndex", "Accuracy", "MaxVoters"]; let mut types: Vec = generics .args @@ -197,6 +212,7 @@ impl Parse for SolutionDef { }) .collect::>()?; + let max_voters = types.pop().expect("Vector of length 4 can be popped; qed"); let weight_type = types.pop().expect("Vector of length 3 can be popped; qed"); let target_type = types.pop().expect("Vector of length 2 can be popped; qed"); let voter_type = types.pop().expect("Vector of length 1 can be popped; qed"); @@ -205,7 +221,16 @@ impl Parse for SolutionDef { let count_expr: syn::ExprParen = input.parse()?; let count = parse_parenthesized_number::(count_expr)?; - Ok(Self { vis, ident, voter_type, target_type, weight_type, count, compact_encoding }) + Ok(Self { + vis, + ident, + voter_type, + target_type, + weight_type, + max_voters, + count, + compact_encoding, + }) } } @@ -226,11 +251,11 @@ where } fn imports() -> Result { - match crate_name("sp-npos-elections") { - Ok(FoundCrate::Itself) => Ok(quote! { use crate as _npos; }), - Ok(FoundCrate::Name(sp_npos_elections)) => { - let ident = syn::Ident::new(&sp_npos_elections, Span::call_site()); - Ok(quote!( extern crate #ident as _npos; )) + match crate_name("frame-election-provider-support") { + Ok(FoundCrate::Itself) => Ok(quote! { use crate as _feps; }), + Ok(FoundCrate::Name(frame_election_provider_support)) => { + let ident = syn::Ident::new(&frame_election_provider_support, Span::call_site()); + Ok(quote!( extern crate #ident as _feps; )) }, Err(e) => Err(syn::Error::new(Span::call_site(), e)), } diff --git a/primitives/npos-elections/solution-type/src/single_page.rs b/frame/election-provider-support/solution-type/src/single_page.rs similarity index 74% rename from primitives/npos-elections/solution-type/src/single_page.rs rename to frame/election-provider-support/solution-type/src/single_page.rs index 33017d558331..a20f0542984d 100644 --- a/primitives/npos-elections/solution-type/src/single_page.rs +++ b/frame/election-provider-support/solution-type/src/single_page.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,6 +28,7 @@ pub(crate) fn generate(def: crate::SolutionDef) -> Result { voter_type, target_type, weight_type, + max_voters, compact_encoding, } = def; @@ -39,7 +40,7 @@ pub(crate) fn generate(def: crate::SolutionDef) -> Result { let name = vote_field(1); // NOTE: we use the visibility of the struct for the fields as well.. could be made better. quote!( - #vis #name: _npos::sp_std::prelude::Vec<(#voter_type, #target_type)>, + #vis #name: _feps::sp_std::prelude::Vec<(#voter_type, #target_type)>, ) }; @@ -48,7 +49,7 @@ pub(crate) fn generate(def: crate::SolutionDef) -> Result { let field_name = vote_field(c); let array_len = c - 1; quote!( - #vis #field_name: _npos::sp_std::prelude::Vec<( + #vis #field_name: _feps::sp_std::prelude::Vec<( #voter_type, [(#target_type, #weight_type); #array_len], #target_type @@ -83,9 +84,9 @@ pub(crate) fn generate(def: crate::SolutionDef) -> Result { Eq, Clone, Debug, - _npos::codec::Encode, - _npos::codec::Decode, - _npos::scale_info::TypeInfo, + _feps::codec::Encode, + _feps::codec::Decode, + _feps::scale_info::TypeInfo, )]) }; @@ -101,8 +102,8 @@ pub(crate) fn generate(def: crate::SolutionDef) -> Result { #derives_and_maybe_compact_encoding #vis struct #ident { #single #rest } - use _npos::__OrInvalidIndex; - impl _npos::NposSolution for #ident { + use _feps::__OrInvalidIndex; + impl _feps::NposSolution for #ident { const LIMIT: usize = #count; type VoterIndex = #voter_type; type TargetIndex = #target_type; @@ -114,34 +115,40 @@ pub(crate) fn generate(def: crate::SolutionDef) -> Result { } fn from_assignment( - assignments: &[_npos::Assignment], + assignments: &[_feps::Assignment], voter_index: FV, target_index: FT, - ) -> Result + ) -> Result where - A: _npos::IdentifierT, + A: _feps::IdentifierT, for<'r> FV: Fn(&'r A) -> Option, for<'r> FT: Fn(&'r A) -> Option, { + // Make sure that the voter bound is binding. + // `assignments.len()` actually represents the number of voters + if assignments.len() as u32 > <#max_voters as _feps::Get>::get() { + return Err(_feps::Error::TooManyVoters); + } let mut #struct_name: #ident = Default::default(); - for _npos::Assignment { who, distribution } in assignments { + for _feps::Assignment { who, distribution } in assignments { match distribution.len() { 0 => continue, #from_impl _ => { - return Err(_npos::Error::SolutionTargetOverflow); + return Err(_feps::Error::SolutionTargetOverflow); } } }; + Ok(#struct_name) } - fn into_assignment( + fn into_assignment( self, voter_at: impl Fn(Self::VoterIndex) -> Option, target_at: impl Fn(Self::TargetIndex) -> Option, - ) -> Result<_npos::sp_std::prelude::Vec<_npos::Assignment>, _npos::Error> { - let mut #assignment_name: _npos::sp_std::prelude::Vec<_npos::Assignment> = Default::default(); + ) -> Result<_feps::sp_std::prelude::Vec<_feps::Assignment>, _feps::Error> { + let mut #assignment_name: _feps::sp_std::prelude::Vec<_feps::Assignment> = Default::default(); #into_impl Ok(#assignment_name) } @@ -158,10 +165,10 @@ pub(crate) fn generate(def: crate::SolutionDef) -> Result { all_edges } - fn unique_targets(&self) -> _npos::sp_std::prelude::Vec { + fn unique_targets(&self) -> _feps::sp_std::prelude::Vec { // NOTE: this implementation returns the targets sorted, but we don't use it yet per // se, nor is the API enforcing it. - use _npos::sp_std::collections::btree_set::BTreeSet; + use _feps::sp_std::collections::btree_set::BTreeSet; let mut all_targets: BTreeSet = BTreeSet::new(); let mut maybe_insert_target = |t: Self::TargetIndex| { all_targets.insert(t); @@ -173,22 +180,43 @@ pub(crate) fn generate(def: crate::SolutionDef) -> Result { } } - type __IndexAssignment = _npos::IndexAssignment< - <#ident as _npos::NposSolution>::VoterIndex, - <#ident as _npos::NposSolution>::TargetIndex, - <#ident as _npos::NposSolution>::Accuracy, + type __IndexAssignment = _feps::IndexAssignment< + <#ident as _feps::NposSolution>::VoterIndex, + <#ident as _feps::NposSolution>::TargetIndex, + <#ident as _feps::NposSolution>::Accuracy, >; - impl<'a> _npos::sp_std::convert::TryFrom<&'a [__IndexAssignment]> for #ident { - type Error = _npos::Error; + impl _feps::codec::MaxEncodedLen for #ident { + fn max_encoded_len() -> usize { + use frame_support::traits::Get; + use _feps::codec::Encode; + let s: u32 = #max_voters::get(); + let max_element_size = + // the first voter.. + #voter_type::max_encoded_len() + // #count - 1 tuples.. + .saturating_add( + (#count - 1).saturating_mul( + #target_type::max_encoded_len().saturating_add(#weight_type::max_encoded_len()))) + // and the last target. + .saturating_add(#target_type::max_encoded_len()); + // The assumption is that it contains #count-1 empty elements + // and then last element with full size + #count + .saturating_mul(_feps::codec::Compact(0u32).encoded_size()) + .saturating_add((s as usize).saturating_mul(max_element_size)) + } + } + impl<'a> _feps::sp_std::convert::TryFrom<&'a [__IndexAssignment]> for #ident { + type Error = _feps::Error; fn try_from(index_assignments: &'a [__IndexAssignment]) -> Result { let mut #struct_name = #ident::default(); - for _npos::IndexAssignment { who, distribution } in index_assignments { + for _feps::IndexAssignment { who, distribution } in index_assignments { match distribution.len() { 0 => {} #from_index_impl _ => { - return Err(_npos::Error::SolutionTargetOverflow); + return Err(_feps::Error::SolutionTargetOverflow); } } }; @@ -310,7 +338,7 @@ pub(crate) fn into_impl( let name = vote_field(1); quote!( for (voter_index, target_index) in self.#name { - #assignments.push(_npos::Assignment { + #assignments.push(_feps::Assignment { who: voter_at(voter_index).or_invalid_index()?, distribution: vec![ (target_at(target_index).or_invalid_index()?, #per_thing::one()) @@ -329,25 +357,25 @@ pub(crate) fn into_impl( let mut inners_parsed = inners .iter() .map(|(ref t_idx, p)| { - sum = _npos::sp_arithmetic::traits::Saturating::saturating_add(sum, *p); + sum = _feps::sp_arithmetic::traits::Saturating::saturating_add(sum, *p); let target = target_at(*t_idx).or_invalid_index()?; Ok((target, *p)) }) - .collect::, _npos::Error>>()?; + .collect::, _feps::Error>>()?; if sum >= #per_thing::one() { - return Err(_npos::Error::SolutionWeightOverflow); + return Err(_feps::Error::SolutionWeightOverflow); } // defensive only. Since Percent doesn't have `Sub`. - let p_last = _npos::sp_arithmetic::traits::Saturating::saturating_sub( + let p_last = _feps::sp_arithmetic::traits::Saturating::saturating_sub( #per_thing::one(), sum, ); inners_parsed.push((target_at(t_last_idx).or_invalid_index()?, p_last)); - #assignments.push(_npos::Assignment { + #assignments.push(_feps::Assignment { who: voter_at(voter_index).or_invalid_index()?, distribution: inners_parsed, }); diff --git a/primitives/npos-elections/solution-type/tests/ui/fail/missing_accuracy.rs b/frame/election-provider-support/solution-type/tests/ui/fail/missing_accuracy.rs similarity index 55% rename from primitives/npos-elections/solution-type/tests/ui/fail/missing_accuracy.rs rename to frame/election-provider-support/solution-type/tests/ui/fail/missing_accuracy.rs index b74b857e4581..52ae9623fd38 100644 --- a/primitives/npos-elections/solution-type/tests/ui/fail/missing_accuracy.rs +++ b/frame/election-provider-support/solution-type/tests/ui/fail/missing_accuracy.rs @@ -1,9 +1,10 @@ -use sp_npos_elections_solution_type::generate_solution_type; +use frame_election_provider_solution_type::generate_solution_type; generate_solution_type!(pub struct TestSolution::< VoterIndex = u16, TargetIndex = u8, Perbill, + MaxVoters = ConstU32::<10>, >(8)); fn main() {} diff --git a/primitives/npos-elections/solution-type/tests/ui/fail/missing_accuracy.stderr b/frame/election-provider-support/solution-type/tests/ui/fail/missing_accuracy.stderr similarity index 100% rename from primitives/npos-elections/solution-type/tests/ui/fail/missing_accuracy.stderr rename to frame/election-provider-support/solution-type/tests/ui/fail/missing_accuracy.stderr diff --git a/frame/election-provider-support/solution-type/tests/ui/fail/missing_size_bound.rs b/frame/election-provider-support/solution-type/tests/ui/fail/missing_size_bound.rs new file mode 100644 index 000000000000..fe8ac04cc8d6 --- /dev/null +++ b/frame/election-provider-support/solution-type/tests/ui/fail/missing_size_bound.rs @@ -0,0 +1,10 @@ +use frame_election_provider_solution_type::generate_solution_type; + +generate_solution_type!(pub struct TestSolution::< + VoterIndex = u16, + TargetIndex = u8, + Accuracy = Perbill, + ConstU32::<10>, +>(8)); + +fn main() {} diff --git a/frame/election-provider-support/solution-type/tests/ui/fail/missing_size_bound.stderr b/frame/election-provider-support/solution-type/tests/ui/fail/missing_size_bound.stderr new file mode 100644 index 000000000000..c685ab816d39 --- /dev/null +++ b/frame/election-provider-support/solution-type/tests/ui/fail/missing_size_bound.stderr @@ -0,0 +1,5 @@ +error: Expected binding: `MaxVoters = ...` + --> tests/ui/fail/missing_size_bound.rs:7:2 + | +7 | ConstU32::<10>, + | ^^^^^^^^^^^^^^ diff --git a/primitives/npos-elections/solution-type/tests/ui/fail/missing_target.rs b/frame/election-provider-support/solution-type/tests/ui/fail/missing_target.rs similarity index 55% rename from primitives/npos-elections/solution-type/tests/ui/fail/missing_target.rs rename to frame/election-provider-support/solution-type/tests/ui/fail/missing_target.rs index 4c9cd51a3209..b457c4abada6 100644 --- a/primitives/npos-elections/solution-type/tests/ui/fail/missing_target.rs +++ b/frame/election-provider-support/solution-type/tests/ui/fail/missing_target.rs @@ -1,9 +1,10 @@ -use sp_npos_elections_solution_type::generate_solution_type; +use frame_election_provider_solution_type::generate_solution_type; generate_solution_type!(pub struct TestSolution::< VoterIndex = u16, u8, Accuracy = Perbill, + MaxVoters = ConstU32::<10>, >(8)); fn main() {} diff --git a/primitives/npos-elections/solution-type/tests/ui/fail/missing_target.stderr b/frame/election-provider-support/solution-type/tests/ui/fail/missing_target.stderr similarity index 100% rename from primitives/npos-elections/solution-type/tests/ui/fail/missing_target.stderr rename to frame/election-provider-support/solution-type/tests/ui/fail/missing_target.stderr diff --git a/primitives/npos-elections/solution-type/tests/ui/fail/missing_voter.rs b/frame/election-provider-support/solution-type/tests/ui/fail/missing_voter.rs similarity index 55% rename from primitives/npos-elections/solution-type/tests/ui/fail/missing_voter.rs rename to frame/election-provider-support/solution-type/tests/ui/fail/missing_voter.rs index b87037f77f1e..3d12e3e6b5ec 100644 --- a/primitives/npos-elections/solution-type/tests/ui/fail/missing_voter.rs +++ b/frame/election-provider-support/solution-type/tests/ui/fail/missing_voter.rs @@ -1,9 +1,10 @@ -use sp_npos_elections_solution_type::generate_solution_type; +use frame_election_provider_solution_type::generate_solution_type; generate_solution_type!(pub struct TestSolution::< u16, TargetIndex = u8, Accuracy = Perbill, + MaxVoters = ConstU32::<10>, >(8)); fn main() {} diff --git a/primitives/npos-elections/solution-type/tests/ui/fail/missing_voter.stderr b/frame/election-provider-support/solution-type/tests/ui/fail/missing_voter.stderr similarity index 100% rename from primitives/npos-elections/solution-type/tests/ui/fail/missing_voter.stderr rename to frame/election-provider-support/solution-type/tests/ui/fail/missing_voter.stderr diff --git a/frame/election-provider-support/solution-type/tests/ui/fail/no_annotations.rs b/frame/election-provider-support/solution-type/tests/ui/fail/no_annotations.rs new file mode 100644 index 000000000000..9aab15e7ec9a --- /dev/null +++ b/frame/election-provider-support/solution-type/tests/ui/fail/no_annotations.rs @@ -0,0 +1,10 @@ +use frame_election_provider_solution_type::generate_solution_type; + +generate_solution_type!(pub struct TestSolution::< + u16, + u8, + Perbill, + MaxVoters = ConstU32::<10>, +>(8)); + +fn main() {} diff --git a/primitives/npos-elections/solution-type/tests/ui/fail/no_annotations.stderr b/frame/election-provider-support/solution-type/tests/ui/fail/no_annotations.stderr similarity index 100% rename from primitives/npos-elections/solution-type/tests/ui/fail/no_annotations.stderr rename to frame/election-provider-support/solution-type/tests/ui/fail/no_annotations.stderr diff --git a/primitives/npos-elections/solution-type/tests/ui/fail/swap_voter_target.rs b/frame/election-provider-support/solution-type/tests/ui/fail/swap_voter_target.rs similarity index 57% rename from primitives/npos-elections/solution-type/tests/ui/fail/swap_voter_target.rs rename to frame/election-provider-support/solution-type/tests/ui/fail/swap_voter_target.rs index 443202d11b39..4275aae045a6 100644 --- a/primitives/npos-elections/solution-type/tests/ui/fail/swap_voter_target.rs +++ b/frame/election-provider-support/solution-type/tests/ui/fail/swap_voter_target.rs @@ -1,9 +1,10 @@ -use sp_npos_elections_solution_type::generate_solution_type; +use frame_election_provider_solution_type::generate_solution_type; generate_solution_type!(pub struct TestSolution::< TargetIndex = u16, VoterIndex = u8, Accuracy = Perbill, + MaxVoters = ConstU32::<10>, >(8)); fn main() {} diff --git a/primitives/npos-elections/solution-type/tests/ui/fail/swap_voter_target.stderr b/frame/election-provider-support/solution-type/tests/ui/fail/swap_voter_target.stderr similarity index 100% rename from primitives/npos-elections/solution-type/tests/ui/fail/swap_voter_target.stderr rename to frame/election-provider-support/solution-type/tests/ui/fail/swap_voter_target.stderr diff --git a/primitives/npos-elections/solution-type/tests/ui/fail/wrong_attribute.rs b/frame/election-provider-support/solution-type/tests/ui/fail/wrong_attribute.rs similarity index 60% rename from primitives/npos-elections/solution-type/tests/ui/fail/wrong_attribute.rs rename to frame/election-provider-support/solution-type/tests/ui/fail/wrong_attribute.rs index 3008277e36b7..a51cc724ad15 100644 --- a/primitives/npos-elections/solution-type/tests/ui/fail/wrong_attribute.rs +++ b/frame/election-provider-support/solution-type/tests/ui/fail/wrong_attribute.rs @@ -1,10 +1,11 @@ -use sp_npos_elections_solution_type::generate_solution_type; +use frame_election_provider_solution_type::generate_solution_type; generate_solution_type!( #[pages(1)] pub struct TestSolution::< VoterIndex = u8, TargetIndex = u16, Accuracy = Perbill, + MaxVoters = ConstU32::<10>, >(8) ); diff --git a/primitives/npos-elections/solution-type/tests/ui/fail/wrong_attribute.stderr b/frame/election-provider-support/solution-type/tests/ui/fail/wrong_attribute.stderr similarity index 100% rename from primitives/npos-elections/solution-type/tests/ui/fail/wrong_attribute.stderr rename to frame/election-provider-support/solution-type/tests/ui/fail/wrong_attribute.stderr diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index cb36e025c3be..19735cf6035a 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -80,6 +80,7 @@ //! ```rust //! # use frame_election_provider_support::{*, data_provider}; //! # use sp_npos_elections::{Support, Assignment}; +//! # use frame_support::traits::ConstU32; //! //! type AccountId = u64; //! type Balance = u64; @@ -90,25 +91,28 @@ //! //! pub trait Config: Sized { //! type ElectionProvider: ElectionProvider< -//! AccountId, -//! BlockNumber, -//! DataProvider = Module, +//! AccountId = AccountId, +//! BlockNumber = BlockNumber, +//! DataProvider = Pallet, //! >; //! } //! -//! pub struct Module(std::marker::PhantomData); +//! pub struct Pallet(std::marker::PhantomData); +//! +//! impl ElectionDataProvider for Pallet { +//! type AccountId = AccountId; +//! type BlockNumber = BlockNumber; +//! type MaxVotesPerVoter = ConstU32<1>; //! -//! impl ElectionDataProvider for Module { -//! const MAXIMUM_VOTES_PER_VOTER: u32 = 1; //! fn desired_targets() -> data_provider::Result { //! Ok(1) //! } -//! fn voters(maybe_max_len: Option) -//! -> data_provider::Result)>> +//! fn electing_voters(maybe_max_len: Option) +//! -> data_provider::Result>> //! { //! Ok(Default::default()) //! } -//! fn targets(maybe_max_len: Option) -> data_provider::Result> { +//! fn electable_targets(maybe_max_len: Option) -> data_provider::Result> { //! Ok(vec![10, 20, 30]) //! } //! fn next_election_prediction(now: BlockNumber) -> BlockNumber { @@ -124,15 +128,17 @@ //! pub struct GenericElectionProvider(std::marker::PhantomData); //! //! pub trait Config { -//! type DataProvider: ElectionDataProvider; +//! type DataProvider: ElectionDataProvider; //! } //! -//! impl ElectionProvider for GenericElectionProvider { +//! impl ElectionProvider for GenericElectionProvider { +//! type AccountId = AccountId; +//! type BlockNumber = BlockNumber; //! type Error = &'static str; //! type DataProvider = T::DataProvider; //! //! fn elect() -> Result, Self::Error> { -//! Self::DataProvider::targets(None) +//! Self::DataProvider::electable_targets(None) //! .map_err(|_| "failed to elect") //! .map(|t| vec![(t[0], Support::default())]) //! } @@ -146,7 +152,7 @@ //! //! struct Runtime; //! impl generic_election_provider::Config for Runtime { -//! type DataProvider = data_provider_mod::Module; +//! type DataProvider = data_provider_mod::Pallet; //! } //! //! impl data_provider_mod::Config for Runtime { @@ -161,15 +167,93 @@ #![cfg_attr(not(feature = "std"), no_std)] pub mod onchain; -use frame_support::traits::Get; +pub mod traits; +#[cfg(feature = "std")] +use codec::{Decode, Encode}; +use frame_support::{BoundedVec, RuntimeDebug}; +use sp_runtime::traits::Bounded; use sp_std::{fmt::Debug, prelude::*}; +/// Re-export the solution generation macro. +pub use frame_election_provider_solution_type::generate_solution_type; +pub use frame_support::traits::Get; /// Re-export some type as they are used in the interface. pub use sp_arithmetic::PerThing; pub use sp_npos_elections::{ - Assignment, ElectionResult, ExtendedBalance, IdentifierT, PerThing128, Support, Supports, - VoteWeight, + Assignment, ElectionResult, Error, ExtendedBalance, IdentifierT, PerThing128, Support, + Supports, VoteWeight, }; +pub use traits::NposSolution; + +// re-export for the solution macro, with the dependencies of the macro. +#[doc(hidden)] +pub use codec; +#[doc(hidden)] +pub use scale_info; +#[doc(hidden)] +pub use sp_arithmetic; +#[doc(hidden)] +pub use sp_std; + +#[cfg(test)] +mod mock; +#[cfg(test)] +mod tests; +// Simple Extension trait to easily convert `None` from index closures to `Err`. +// +// This is only generated and re-exported for the solution code to use. +#[doc(hidden)] +pub trait __OrInvalidIndex { + fn or_invalid_index(self) -> Result; +} + +impl __OrInvalidIndex for Option { + fn or_invalid_index(self) -> Result { + self.ok_or(Error::SolutionInvalidIndex) + } +} + +/// The [`IndexAssignment`] type is an intermediate between the assignments list +/// ([`&[Assignment]`][Assignment]) and `SolutionOf`. +/// +/// The voter and target identifiers have already been replaced with appropriate indices, +/// making it fast to repeatedly encode into a `SolutionOf`. This property turns out +/// to be important when trimming for solution length. +#[derive(RuntimeDebug, Clone, Default)] +#[cfg_attr(feature = "std", derive(PartialEq, Eq, Encode, Decode))] +pub struct IndexAssignment { + /// Index of the voter among the voters list. + pub who: VoterIndex, + /// The distribution of the voter's stake among winning targets. + /// + /// Targets are identified by their index in the canonical list. + pub distribution: Vec<(TargetIndex, P)>, +} + +impl IndexAssignment { + pub fn new( + assignment: &Assignment, + voter_index: impl Fn(&AccountId) -> Option, + target_index: impl Fn(&AccountId) -> Option, + ) -> Result { + Ok(Self { + who: voter_index(&assignment.who).or_invalid_index()?, + distribution: assignment + .distribution + .iter() + .map(|(target, proportion)| Some((target_index(target)?, proportion.clone()))) + .collect::>>() + .or_invalid_index()?, + }) + } +} + +/// A type alias for [`IndexAssignment`] made from [`NposSolution`]. +pub type IndexAssignmentOf = IndexAssignment< + ::VoterIndex, + ::TargetIndex, + ::Accuracy, +>; /// Types that are used by the data provider trait. pub mod data_provider { @@ -178,20 +262,29 @@ pub mod data_provider { } /// Something that can provide the data to an [`ElectionProvider`]. -pub trait ElectionDataProvider { +pub trait ElectionDataProvider { + /// The account identifier type. + type AccountId; + + /// The block number type. + type BlockNumber; + /// Maximum number of votes per voter that this data provider is providing. - const MAXIMUM_VOTES_PER_VOTER: u32; + type MaxVotesPerVoter: Get; - /// All possible targets for the election, i.e. the candidates. + /// All possible targets for the election, i.e. the targets that could become elected, thus + /// "electable". /// /// If `maybe_max_len` is `Some(v)` then the resulting vector MUST NOT be longer than `v` items /// long. /// /// This should be implemented as a self-weighing function. The implementor should register its /// appropriate weight at the end of execution with the system pallet directly. - fn targets(maybe_max_len: Option) -> data_provider::Result>; + fn electable_targets( + maybe_max_len: Option, + ) -> data_provider::Result>; - /// All possible voters for the election. + /// All the voters that participate in the election, thus "electing". /// /// Note that if a notion of self-vote exists, it should be represented here. /// @@ -200,14 +293,18 @@ pub trait ElectionDataProvider { /// /// This should be implemented as a self-weighing function. The implementor should register its /// appropriate weight at the end of execution with the system pallet directly. - fn voters( - maybe_max_len: Option, - ) -> data_provider::Result)>>; + fn electing_voters(maybe_max_len: Option) -> data_provider::Result>>; /// The number of targets to elect. /// /// This should be implemented as a self-weighing function. The implementor should register its /// appropriate weight at the end of execution with the system pallet directly. + /// + /// A sensible implementation should use the minimum between this value and + /// [`Self::targets().len()`], since desiring a winner set larger than candidates is not + /// feasible. + /// + /// This is documented further in issue: fn desired_targets() -> data_provider::Result; /// Provide a best effort prediction about when the next election is about to happen. @@ -216,14 +313,14 @@ pub trait ElectionDataProvider { /// [`ElectionProvider::elect`]. /// /// This is only useful for stateful election providers. - fn next_election_prediction(now: BlockNumber) -> BlockNumber; + fn next_election_prediction(now: Self::BlockNumber) -> Self::BlockNumber; /// Utility function only to be used in benchmarking scenarios, to be implemented optionally, /// else a noop. #[cfg(any(feature = "runtime-benchmarks", test))] fn put_snapshot( - _voters: Vec<(AccountId, VoteWeight, Vec)>, - _targets: Vec, + _voters: Vec>, + _targets: Vec, _target_stake: Option, ) { } @@ -233,67 +330,95 @@ pub trait ElectionDataProvider { /// /// Same as `put_snapshot`, but can add a single voter one by one. #[cfg(any(feature = "runtime-benchmarks", test))] - fn add_voter(_voter: AccountId, _weight: VoteWeight, _targets: Vec) {} + fn add_voter( + _voter: Self::AccountId, + _weight: VoteWeight, + _targets: BoundedVec, + ) { + } /// Utility function only to be used in benchmarking scenarios, to be implemented optionally, /// else a noop. /// /// Same as `put_snapshot`, but can add a single voter one by one. #[cfg(any(feature = "runtime-benchmarks", test))] - fn add_target(_target: AccountId) {} + fn add_target(_target: Self::AccountId) {} /// Clear all voters and targets. #[cfg(any(feature = "runtime-benchmarks", test))] fn clear() {} } -#[cfg(feature = "std")] -impl ElectionDataProvider for () { - const MAXIMUM_VOTES_PER_VOTER: u32 = 0; - fn targets(_maybe_max_len: Option) -> data_provider::Result> { - Ok(Default::default()) - } - fn voters( - _maybe_max_len: Option, - ) -> data_provider::Result)>> { - Ok(Default::default()) - } - fn desired_targets() -> data_provider::Result { - Ok(Default::default()) - } - fn next_election_prediction(now: BlockNumber) -> BlockNumber { - now - } -} - /// Something that can compute the result of an election and pass it back to the caller. /// /// This trait only provides an interface to _request_ an election, i.e. /// [`ElectionProvider::elect`]. That data required for the election need to be passed to the /// implemented of this trait through [`ElectionProvider::DataProvider`]. -pub trait ElectionProvider { +pub trait ElectionProvider { + /// The account identifier type. + type AccountId; + + /// The block number type. + type BlockNumber; + /// The error type that is returned by the provider. type Error: Debug; /// The data provider of the election. - type DataProvider: ElectionDataProvider; + type DataProvider: ElectionDataProvider< + AccountId = Self::AccountId, + BlockNumber = Self::BlockNumber, + >; - /// Elect a new set of winners. + /// Elect a new set of winners, without specifying any bounds on the amount of data fetched from + /// [`Self::DataProvider`]. An implementation could nonetheless impose its own custom limits. /// - /// The result is returned in a target major format, namely as vector of supports. + /// The result is returned in a target major format, namely as *vector of supports*. /// /// This should be implemented as a self-weighing function. The implementor should register its /// appropriate weight at the end of execution with the system pallet directly. - fn elect() -> Result, Self::Error>; + fn elect() -> Result, Self::Error>; } +/// A sub-trait of the [`ElectionProvider`] for cases where we need to be sure an election needs to +/// happen instantly, not asynchronously. +/// +/// The same `DataProvider` is assumed to be used. +/// +/// Consequently, allows for control over the amount of data that is being fetched from the +/// [`ElectionProvider::DataProvider`]. +pub trait InstantElectionProvider: ElectionProvider { + /// Elect a new set of winners, but unlike [`ElectionProvider::elect`] which cannot enforce + /// bounds, this trait method can enforce bounds on the amount of data provided by the + /// `DataProvider`. + /// + /// An implementing type, if itself bounded, should choose the minimum of the two bounds to + /// choose the final value of `max_voters` and `max_targets`. In other words, an implementation + /// should guarantee that `max_voter` and `max_targets` provided to this method are absolutely + /// respected. + fn elect_with_bounds( + max_voters: usize, + max_targets: usize, + ) -> Result, Self::Error>; +} + +/// An election provider to be used only for testing. +#[cfg(feature = "std")] +pub struct NoElection(sp_std::marker::PhantomData); + #[cfg(feature = "std")] -impl ElectionProvider for () { +impl ElectionProvider + for NoElection<(AccountId, BlockNumber, DataProvider)> +where + DataProvider: ElectionDataProvider, +{ + type AccountId = AccountId; + type BlockNumber = BlockNumber; type Error = &'static str; - type DataProvider = (); + type DataProvider = DataProvider; fn elect() -> Result, Self::Error> { - Err("<() as ElectionProvider> cannot do anything.") + Err(" cannot do anything.") } } @@ -302,19 +427,25 @@ impl ElectionProvider for () { /// This is generic over `AccountId` and it can represent a validator, a nominator, or any other /// entity. /// -/// To simplify the trait, the `VoteWeight` is hardcoded as the weight of each entity. The weights -/// are ascending, the higher, the better. In the long term, if this trait ends up having use cases -/// outside of the election context, it is easy enough to make it generic over the `VoteWeight`. +/// The scores (see [`Self::Score`]) are ascending, the higher, the better. /// /// Something that implements this trait will do a best-effort sort over ids, and thus can be /// used on the implementing side of [`ElectionDataProvider`]. pub trait SortedListProvider { /// The list's error type. - type Error; + type Error: sp_std::fmt::Debug; + + /// The type used by the list to compare nodes for ordering. + type Score: Bounded; /// An iterator over the list, which can have `take` called on it. fn iter() -> Box>; + /// Returns an iterator over the list, starting right after from the given voter. + /// + /// May return an error if `start` is invalid. + fn iter_from(start: &AccountId) -> Result>, Self::Error>; + /// The current count of ids in the list. fn count() -> u32; @@ -322,10 +453,10 @@ pub trait SortedListProvider { fn contains(id: &AccountId) -> bool; /// Hook for inserting a new id. - fn on_insert(id: AccountId, weight: VoteWeight) -> Result<(), Self::Error>; + fn on_insert(id: AccountId, score: Self::Score) -> Result<(), Self::Error>; /// Hook for updating a single id. - fn on_update(id: &AccountId, weight: VoteWeight); + fn on_update(id: &AccountId, score: Self::Score); /// Hook for removing am id from the list. fn on_remove(id: &AccountId); @@ -333,15 +464,23 @@ pub trait SortedListProvider { /// Regenerate this list from scratch. Returns the count of items inserted. /// /// This should typically only be used at a runtime upgrade. - fn regenerate( + /// + /// ## WARNING + /// + /// This function should be called with care, regenerate will remove the current list write the + /// new list, which can lead to too many storage accesses, exhausting the block weight. + fn unsafe_regenerate( all: impl IntoIterator, - weight_of: Box VoteWeight>, + score_of: Box Self::Score>, ) -> u32; - /// Remove `maybe_count` number of items from the list. Returns the number of items actually - /// removed. WARNING: removes all items if `maybe_count` is `None`, which should never be done - /// in production settings because it can lead to an unbounded amount of storage accesses. - fn clear(maybe_count: Option) -> u32; + /// Remove all items from the list. + /// + /// ## WARNING + /// + /// This function should never be called in production settings because it can lead to an + /// unbounded amount of storage accesses. + fn unsafe_clear(); /// Sanity check internal state of list. Only meant for debug compilation. fn sanity_check() -> Result<(), &'static str>; @@ -349,21 +488,23 @@ pub trait SortedListProvider { /// If `who` changes by the returned amount they are guaranteed to have a worst case change /// in their list position. #[cfg(feature = "runtime-benchmarks")] - fn weight_update_worst_case(_who: &AccountId, _is_increase: bool) -> VoteWeight { - VoteWeight::MAX + fn score_update_worst_case(_who: &AccountId, _is_increase: bool) -> Self::Score { + Self::Score::max_value() } } /// Something that can provide the `VoteWeight` of an account. Similar to [`ElectionProvider`] and /// [`ElectionDataProvider`], this should typically be implementing by whoever is supposed to *use* /// `SortedListProvider`. -pub trait VoteWeightProvider { - /// Get the current `VoteWeight` of `who`. - fn vote_weight(who: &AccountId) -> VoteWeight; +pub trait ScoreProvider { + type Score; + + /// Get the current `Score` of `who`. + fn score(who: &AccountId) -> Self::Score; /// For tests and benchmarks, set the `VoteWeight`. #[cfg(any(feature = "runtime-benchmarks", test))] - fn set_vote_weight_of(_: &AccountId, _: VoteWeight) {} + fn set_score_of(_: &AccountId, _: Self::Score) {} } /// Something that can compute the result to an NPoS solution. @@ -380,11 +521,11 @@ pub trait NposSolver { fn solve( to_elect: usize, targets: Vec, - voters: Vec<(Self::AccountId, VoteWeight, Vec)>, + voters: Vec<(Self::AccountId, VoteWeight, impl IntoIterator)>, ) -> Result, Self::Error>; } -/// A wrapper for [`sp_npos_elections::seq_phragmen`] that implements [`super::NposSolver`]. See the +/// A wrapper for [`sp_npos_elections::seq_phragmen`] that implements [`NposSolver`]. See the /// documentation of [`sp_npos_elections::seq_phragmen`] for more info. pub struct SequentialPhragmen( sp_std::marker::PhantomData<(AccountId, Accuracy, Balancing)>, @@ -402,14 +543,14 @@ impl< fn solve( winners: usize, targets: Vec, - voters: Vec<(Self::AccountId, VoteWeight, Vec)>, + voters: Vec<(Self::AccountId, VoteWeight, impl IntoIterator)>, ) -> Result, Self::Error> { sp_npos_elections::seq_phragmen(winners, targets, voters, Balancing::get()) } } -/// A wrapper for [`sp_npos_elections::phragmms`] that implements [`NposSolver`]. See the -/// documentation of [`sp_npos_elections::phragmms`] for more info. +/// A wrapper for [`sp_npos_elections::phragmms()`] that implements [`NposSolver`]. See the +/// documentation of [`sp_npos_elections::phragmms()`] for more info. pub struct PhragMMS( sp_std::marker::PhantomData<(AccountId, Accuracy, Balancing)>, ); @@ -426,8 +567,15 @@ impl< fn solve( winners: usize, targets: Vec, - voters: Vec<(Self::AccountId, VoteWeight, Vec)>, + voters: Vec<(Self::AccountId, VoteWeight, impl IntoIterator)>, ) -> Result, Self::Error> { sp_npos_elections::phragmms(winners, targets, voters, Balancing::get()) } } + +/// A voter, at the level of abstraction of this crate. +pub type Voter = (AccountId, VoteWeight, BoundedVec); + +/// Same as [`Voter`], but parameterized by an [`ElectionDataProvider`]. +pub type VoterOf = + Voter<::AccountId, ::MaxVotesPerVoter>; diff --git a/frame/election-provider-support/src/mock.rs b/frame/election-provider-support/src/mock.rs new file mode 100644 index 000000000000..7c834f06f3cd --- /dev/null +++ b/frame/election-provider-support/src/mock.rs @@ -0,0 +1,184 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Mock file for solution-type. + +#![cfg(test)] + +use std::{ + collections::{HashMap, HashSet}, + hash::Hash, +}; + +use rand::{seq::SliceRandom, Rng}; + +pub type AccountId = u64; + +/// The candidate mask allows easy disambiguation between voters and candidates: accounts +/// for which this bit is set are candidates, and without it, are voters. +pub const CANDIDATE_MASK: AccountId = 1 << ((std::mem::size_of::() * 8) - 1); + +pub type TestAccuracy = sp_runtime::Perbill; + +pub fn p(p: u8) -> TestAccuracy { + TestAccuracy::from_percent(p.into()) +} + +pub type MockAssignment = crate::Assignment; +pub type Voter = (AccountId, crate::VoteWeight, Vec); + +crate::generate_solution_type! { + pub struct TestSolution::< + VoterIndex = u32, + TargetIndex = u16, + Accuracy = TestAccuracy, + MaxVoters = frame_support::traits::ConstU32::<2_500>, + >(16) +} + +/// Generate voter and assignment lists. Makes no attempt to be realistic about winner or assignment +/// fairness. +/// +/// Maintains these invariants: +/// +/// - candidate ids have `CANDIDATE_MASK` bit set +/// - voter ids do not have `CANDIDATE_MASK` bit set +/// - assignments have the same ordering as voters +/// - `assignments.distribution.iter().map(|(_, frac)| frac).sum() == One::one()` +/// - a coherent set of winners is chosen. +/// - the winner set is a subset of the candidate set. +/// - `assignments.distribution.iter().all(|(who, _)| winners.contains(who))` +pub fn generate_random_votes( + candidate_count: usize, + voter_count: usize, + mut rng: impl Rng, +) -> (Vec, Vec, Vec) { + // cache for fast generation of unique candidate and voter ids + let mut used_ids = HashSet::with_capacity(candidate_count + voter_count); + + // candidates are easy: just a completely random set of IDs + let mut candidates: Vec = Vec::with_capacity(candidate_count); + while candidates.len() < candidate_count { + let mut new = || rng.gen::() | CANDIDATE_MASK; + let mut id = new(); + // insert returns `false` when the value was already present + while !used_ids.insert(id) { + id = new(); + } + candidates.push(id); + } + + // voters are random ids, random weights, random selection from the candidates + let mut voters = Vec::with_capacity(voter_count); + while voters.len() < voter_count { + let mut new = || rng.gen::() & !CANDIDATE_MASK; + let mut id = new(); + // insert returns `false` when the value was already present + while !used_ids.insert(id) { + id = new(); + } + + let vote_weight = rng.gen(); + + // it's not interesting if a voter chooses 0 or all candidates, so rule those cases out. + // also, let's not generate any cases which result in a compact overflow. + let n_candidates_chosen = + rng.gen_range(1, candidates.len().min(::LIMIT)); + + let mut chosen_candidates = Vec::with_capacity(n_candidates_chosen); + chosen_candidates.extend(candidates.choose_multiple(&mut rng, n_candidates_chosen)); + voters.push((id, vote_weight, chosen_candidates)); + } + + // always generate a sensible number of winners: elections are uninteresting if nobody wins, + // or everybody wins + let num_winners = rng.gen_range(1, candidate_count); + let mut winners: HashSet = HashSet::with_capacity(num_winners); + winners.extend(candidates.choose_multiple(&mut rng, num_winners)); + assert_eq!(winners.len(), num_winners); + + let mut assignments = Vec::with_capacity(voters.len()); + for (voter_id, _, votes) in voters.iter() { + let chosen_winners = votes.iter().filter(|vote| winners.contains(vote)).cloned(); + let num_chosen_winners = chosen_winners.clone().count(); + + // distribute the available stake randomly + let stake_distribution = if num_chosen_winners == 0 { + continue + } else { + let mut available_stake = 1000; + let mut stake_distribution = Vec::with_capacity(num_chosen_winners); + for _ in 0..num_chosen_winners - 1 { + let stake = rng.gen_range(0, available_stake).min(1); + stake_distribution.push(TestAccuracy::from_perthousand(stake)); + available_stake -= stake; + } + stake_distribution.push(TestAccuracy::from_perthousand(available_stake)); + stake_distribution.shuffle(&mut rng); + stake_distribution + }; + + assignments.push(MockAssignment { + who: *voter_id, + distribution: chosen_winners.zip(stake_distribution).collect(), + }); + } + + (voters, assignments, candidates) +} + +fn generate_cache(voters: Voters) -> HashMap +where + Voters: Iterator, + Item: Hash + Eq + Copy, +{ + let mut cache = HashMap::new(); + for (idx, voter_id) in voters.enumerate() { + cache.insert(voter_id, idx); + } + cache +} + +/// Create a function that returns the index of a voter in the voters list. +pub fn make_voter_fn(voters: &[Voter]) -> impl Fn(&AccountId) -> Option +where + usize: TryInto, +{ + let cache = generate_cache(voters.iter().map(|(id, _, _)| *id)); + move |who| { + if cache.get(who).is_none() { + println!("WARNING: voter {} will raise InvalidIndex", who); + } + cache.get(who).cloned().and_then(|i| i.try_into().ok()) + } +} + +/// Create a function that returns the index of a candidate in the candidates list. +pub fn make_target_fn( + candidates: &[AccountId], +) -> impl Fn(&AccountId) -> Option +where + usize: TryInto, +{ + let cache = generate_cache(candidates.iter().cloned()); + move |who| { + if cache.get(who).is_none() { + println!("WARNING: target {} will raise InvalidIndex", who); + } + cache.get(who).cloned().and_then(|i| i.try_into().ok()) + } +} diff --git a/frame/election-provider-support/src/onchain.rs b/frame/election-provider-support/src/onchain.rs index fb1ccfdfe256..57fd931a467d 100644 --- a/frame/election-provider-support/src/onchain.rs +++ b/frame/election-provider-support/src/onchain.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,9 +15,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! An implementation of [`ElectionProvider`] that does an on-chain sequential phragmen. +//! An implementation of [`ElectionProvider`] that uses an `NposSolver` to do the election. -use crate::{ElectionDataProvider, ElectionProvider}; +use crate::{ElectionDataProvider, ElectionProvider, InstantElectionProvider, NposSolver}; use frame_support::{traits::Get, weights::DispatchClass}; use sp_npos_elections::*; use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData, prelude::*}; @@ -41,60 +41,141 @@ impl From for Error { /// /// This will accept voting data on the fly and produce the results immediately. /// +/// Finally, the [`ElectionProvider`] implementation of this type does not impose any limits on the +/// number of voters and targets that are fetched. This could potentially make this unsuitable for +/// execution onchain. One could, however, impose bounds on it by using for example +/// `BoundedExecution` which will the bounds provided in the configuration. +/// +/// On the other hand, the [`InstantElectionProvider`] implementation does limit these inputs, +/// either via using `BoundedExecution` and imposing the bounds there, or dynamically via calling +/// `elect_with_bounds` providing these bounds. If you use `elect_with_bounds` along with +/// `InstantElectionProvider`, the bound that would be used is the minimum of the 2 bounds. +/// +/// It is advisable to use the former ([`ElectionProvider::elect`]) only at genesis, or for testing, +/// the latter [`InstantElectionProvider::elect_with_bounds`] for onchain operations, with +/// thoughtful bounds. +/// +/// Please use `BoundedExecution` at all times except at genesis or for testing, with thoughtful +/// bounds in order to bound the potential execution time. Limit the use `UnboundedExecution` at +/// genesis or for testing, as it does not bound the inputs. However, this can be used with +/// `[InstantElectionProvider::elect_with_bounds`] that dynamically imposes limits. +pub struct BoundedExecution(PhantomData); + +/// An unbounded variant of [`BoundedExecution`]. +/// /// ### Warning /// /// This can be very expensive to run frequently on-chain. Use with care. Moreover, this /// implementation ignores the additional data of the election data provider and gives no insight on /// how much weight was consumed. -/// -/// Finally, this implementation does not impose any limits on the number of voters and targets that -/// are provided. -pub struct OnChainSequentialPhragmen(PhantomData); +pub struct UnboundedExecution(PhantomData); -/// Configuration trait of [`OnChainSequentialPhragmen`]. -/// -/// Note that this is similar to a pallet traits, but [`OnChainSequentialPhragmen`] is not a pallet. -/// -/// WARNING: the user of this pallet must ensure that the `Accuracy` type will work nicely with the -/// normalization operation done inside `seq_phragmen`. See -/// [`sp_npos_elections::assignment::try_normalize`] for more info. -pub trait Config: frame_system::Config { - /// The accuracy used to compute the election: - type Accuracy: PerThing128; +/// Configuration trait of [`UnboundedExecution`]. +pub trait ExecutionConfig { + /// Something that implements the system pallet configs. This is to enable to register extra + /// weight. + type System: frame_system::Config; + /// `NposSolver` that should be used, an example would be `PhragMMS`. + type Solver: NposSolver< + AccountId = ::AccountId, + Error = sp_npos_elections::Error, + >; /// Something that provides the data for election. - type DataProvider: ElectionDataProvider; + type DataProvider: ElectionDataProvider< + AccountId = ::AccountId, + BlockNumber = ::BlockNumber, + >; +} + +/// Configuration trait of [`BoundedExecution`]. +pub trait BoundedExecutionConfig: ExecutionConfig { + /// Bounds the number of voters. + type VotersBound: Get; + /// Bounds the number of targets. + type TargetsBound: Get; +} + +fn elect_with( + maybe_max_voters: Option, + maybe_max_targets: Option, +) -> Result::AccountId>, Error> { + let voters = T::DataProvider::electing_voters(maybe_max_voters).map_err(Error::DataProvider)?; + let targets = + T::DataProvider::electable_targets(maybe_max_targets).map_err(Error::DataProvider)?; + let desired_targets = T::DataProvider::desired_targets().map_err(Error::DataProvider)?; + + let stake_map: BTreeMap<_, _> = voters + .iter() + .map(|(validator, vote_weight, _)| (validator.clone(), *vote_weight)) + .collect(); + + let stake_of = |w: &::AccountId| -> VoteWeight { + stake_map.get(w).cloned().unwrap_or_default() + }; + + let ElectionResult { winners: _, assignments } = + T::Solver::solve(desired_targets as usize, targets, voters).map_err(Error::from)?; + + let staked = assignment_ratio_to_staked_normalized(assignments, &stake_of)?; + + let weight = ::BlockWeights::get().max_block; + frame_system::Pallet::::register_extra_weight_unchecked( + weight, + DispatchClass::Mandatory, + ); + + Ok(to_supports(&staked)) } -impl ElectionProvider for OnChainSequentialPhragmen { +impl ElectionProvider for UnboundedExecution { + type AccountId = ::AccountId; + type BlockNumber = ::BlockNumber; type Error = Error; type DataProvider = T::DataProvider; - fn elect() -> Result, Self::Error> { - let voters = Self::DataProvider::voters(None).map_err(Error::DataProvider)?; - let targets = Self::DataProvider::targets(None).map_err(Error::DataProvider)?; - let desired_targets = Self::DataProvider::desired_targets().map_err(Error::DataProvider)?; - - let stake_map: BTreeMap = voters - .iter() - .map(|(validator, vote_weight, _)| (validator.clone(), *vote_weight)) - .collect(); + fn elect() -> Result::AccountId>, Self::Error> { + // This should not be called if not in `std` mode (and therefore neither in genesis nor in + // testing) + if cfg!(not(feature = "std")) { + frame_support::log::error!( + "Please use `InstantElectionProvider` instead to provide bounds on election if not in \ + genesis or testing mode" + ); + } - let stake_of = - |w: &T::AccountId| -> VoteWeight { stake_map.get(w).cloned().unwrap_or_default() }; + elect_with::(None, None) + } +} - let ElectionResult { winners: _, assignments } = - seq_phragmen::<_, T::Accuracy>(desired_targets as usize, targets, voters, None) - .map_err(Error::from)?; +impl InstantElectionProvider for UnboundedExecution { + fn elect_with_bounds( + max_voters: usize, + max_targets: usize, + ) -> Result, Self::Error> { + elect_with::(Some(max_voters), Some(max_targets)) + } +} - let staked = assignment_ratio_to_staked_normalized(assignments, &stake_of)?; +impl ElectionProvider for BoundedExecution { + type AccountId = ::AccountId; + type BlockNumber = ::BlockNumber; + type Error = Error; + type DataProvider = T::DataProvider; - let weight = T::BlockWeights::get().max_block; - frame_system::Pallet::::register_extra_weight_unchecked( - weight, - DispatchClass::Mandatory, - ); + fn elect() -> Result::AccountId>, Self::Error> { + elect_with::(Some(T::VotersBound::get() as usize), Some(T::TargetsBound::get() as usize)) + } +} - Ok(to_supports(&staked)) +impl InstantElectionProvider for BoundedExecution { + fn elect_with_bounds( + max_voters: usize, + max_targets: usize, + ) -> Result, Self::Error> { + elect_with::( + Some(max_voters.min(T::VotersBound::get() as usize)), + Some(max_targets.min(T::TargetsBound::get() as usize)), + ) } } @@ -103,7 +184,6 @@ mod tests { use super::*; use sp_npos_elections::Support; use sp_runtime::Perbill; - type AccountId = u64; type BlockNumber = u64; @@ -145,29 +225,37 @@ mod tests { type OnKilledAccount = (); type SystemWeightInfo = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } - impl Config for Runtime { - type Accuracy = Perbill; + impl ExecutionConfig for Runtime { + type System = Self; + type Solver = crate::SequentialPhragmen; type DataProvider = mock_data_provider::DataProvider; } - type OnChainPhragmen = OnChainSequentialPhragmen; + type OnChainPhragmen = UnboundedExecution; mod mock_data_provider { + use frame_support::{bounded_vec, traits::ConstU32}; + use super::*; - use crate::data_provider; + use crate::{data_provider, VoterOf}; pub struct DataProvider; - impl ElectionDataProvider for DataProvider { - const MAXIMUM_VOTES_PER_VOTER: u32 = 2; - fn voters( - _: Option, - ) -> data_provider::Result)>> { - Ok(vec![(1, 10, vec![10, 20]), (2, 20, vec![30, 20]), (3, 30, vec![10, 30])]) + impl ElectionDataProvider for DataProvider { + type AccountId = AccountId; + type BlockNumber = BlockNumber; + type MaxVotesPerVoter = ConstU32<2>; + fn electing_voters(_: Option) -> data_provider::Result>> { + Ok(vec![ + (1, 10, bounded_vec![10, 20]), + (2, 20, bounded_vec![30, 20]), + (3, 30, bounded_vec![10, 30]), + ]) } - fn targets(_: Option) -> data_provider::Result> { + fn electable_targets(_: Option) -> data_provider::Result> { Ok(vec![10, 20, 30]) } diff --git a/frame/election-provider-support/src/tests.rs b/frame/election-provider-support/src/tests.rs new file mode 100644 index 000000000000..1ccff79f3efd --- /dev/null +++ b/frame/election-provider-support/src/tests.rs @@ -0,0 +1,454 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Tests for solution-type. + +#![cfg(test)] + +use crate::{mock::*, IndexAssignment, NposSolution}; +use frame_support::traits::ConstU32; +use rand::SeedableRng; + +mod solution_type { + use super::*; + use codec::{Decode, Encode, MaxEncodedLen}; + // these need to come from the same dev-dependency `frame-election-provider-support`, not from + // the crate. + use crate::{generate_solution_type, Assignment, Error as NposError, NposSolution}; + use sp_std::fmt::Debug; + + #[allow(dead_code)] + mod __private { + // This is just to make sure that the solution can be generated in a scope without any + // imports. + use crate::generate_solution_type; + generate_solution_type!( + #[compact] + struct InnerTestSolutionIsolated::< + VoterIndex = u32, + TargetIndex = u8, + Accuracy = sp_runtime::Percent, + MaxVoters = crate::tests::ConstU32::<20>, + >(12) + ); + } + + #[test] + fn solution_struct_works_with_and_without_compact() { + // we use u32 size to make sure compact is smaller. + let without_compact = { + generate_solution_type!( + pub struct InnerTestSolution::< + VoterIndex = u32, + TargetIndex = u32, + Accuracy = TestAccuracy, + MaxVoters = ConstU32::<20>, + >(16) + ); + let solution = InnerTestSolution { + votes1: vec![(2, 20), (4, 40)], + votes2: vec![(1, [(10, p(80))], 11), (5, [(50, p(85))], 51)], + ..Default::default() + }; + + solution.encode().len() + }; + + let with_compact = { + generate_solution_type!( + #[compact] + pub struct InnerTestSolutionCompact::< + VoterIndex = u32, + TargetIndex = u32, + Accuracy = TestAccuracy, + MaxVoters = ConstU32::<20>, + >(16) + ); + let compact = InnerTestSolutionCompact { + votes1: vec![(2, 20), (4, 40)], + votes2: vec![(1, [(10, p(80))], 11), (5, [(50, p(85))], 51)], + ..Default::default() + }; + + compact.encode().len() + }; + + assert!(with_compact < without_compact); + } + + #[test] + fn from_assignment_fail_too_many_voters() { + let rng = rand::rngs::SmallRng::seed_from_u64(0); + + // This will produce 24 voters.. + let (voters, assignments, candidates) = generate_random_votes(10, 25, rng); + let voter_index = make_voter_fn(&voters); + let target_index = make_target_fn(&candidates); + + // Limit the voters to 20.. + generate_solution_type!( + pub struct InnerTestSolution::< + VoterIndex = u32, + TargetIndex = u16, + Accuracy = TestAccuracy, + MaxVoters = frame_support::traits::ConstU32::<20>, + >(16) + ); + + // 24 > 20, so this should fail. + assert_eq!( + InnerTestSolution::from_assignment(&assignments, &voter_index, &target_index) + .unwrap_err(), + NposError::TooManyVoters, + ); + } + + #[test] + fn max_encoded_len_too_small() { + generate_solution_type!( + pub struct InnerTestSolution::< + VoterIndex = u32, + TargetIndex = u32, + Accuracy = TestAccuracy, + MaxVoters = ConstU32::<1>, + >(3) + ); + let solution = InnerTestSolution { + votes1: vec![(2, 20), (4, 40)], + votes2: vec![(1, [(10, p(80))], 11), (5, [(50, p(85))], 51)], + ..Default::default() + }; + + // We actually have 4 voters, but the bound is 1 voter, so the implemented bound is too + // small. + assert!(solution.encode().len() > InnerTestSolution::max_encoded_len()); + } + + #[test] + fn max_encoded_len_upper_bound() { + generate_solution_type!( + pub struct InnerTestSolution::< + VoterIndex = u32, + TargetIndex = u32, + Accuracy = TestAccuracy, + MaxVoters = ConstU32::<4>, + >(3) + ); + let solution = InnerTestSolution { + votes1: vec![(2, 20), (4, 40)], + votes2: vec![(1, [(10, p(80))], 11), (5, [(50, p(85))], 51)], + ..Default::default() + }; + + // We actually have 4 voters, and the bound is 4 voters, so the implemented bound should be + // larger than the encoded len. + assert!(solution.encode().len() < InnerTestSolution::max_encoded_len()); + } + + #[test] + fn max_encoded_len_exact() { + generate_solution_type!( + pub struct InnerTestSolution::< + VoterIndex = u32, + TargetIndex = u32, + Accuracy = TestAccuracy, + MaxVoters = ConstU32::<4>, + >(3) + ); + let solution = InnerTestSolution { + votes1: vec![], + votes2: vec![], + votes3: vec![ + (1, [(10, p(50)), (11, p(20))], 12), + (2, [(20, p(50)), (21, p(20))], 22), + (3, [(30, p(50)), (31, p(20))], 32), + (4, [(40, p(50)), (41, p(20))], 42), + ], + }; + + // We have 4 voters, the bound is 4 voters, and all the voters voted for 3 targets, which is + // the max number of targets. This should represent the upper bound that `max_encoded_len` + // represents. + assert_eq!(solution.encode().len(), InnerTestSolution::max_encoded_len()); + } + + #[test] + fn solution_struct_is_codec() { + let solution = TestSolution { + votes1: vec![(2, 20), (4, 40)], + votes2: vec![(1, [(10, p(80))], 11), (5, [(50, p(85))], 51)], + ..Default::default() + }; + + let encoded = solution.encode(); + + assert_eq!(solution, Decode::decode(&mut &encoded[..]).unwrap()); + assert_eq!(solution.voter_count(), 4); + assert_eq!(solution.edge_count(), 2 + 4); + assert_eq!(solution.unique_targets(), vec![10, 11, 20, 40, 50, 51]); + } + + #[test] + fn remove_voter_works() { + let mut solution = TestSolution { + votes1: vec![(0, 2), (1, 6)], + votes2: vec![(2, [(0, p(80))], 1), (3, [(7, p(85))], 8)], + votes3: vec![(4, [(3, p(50)), (4, p(25))], 5)], + ..Default::default() + }; + + assert!(!solution.remove_voter(11)); + assert!(solution.remove_voter(2)); + assert_eq!( + solution, + TestSolution { + votes1: vec![(0, 2), (1, 6)], + votes2: vec![(3, [(7, p(85))], 8)], + votes3: vec![(4, [(3, p(50)), (4, p(25))], 5,)], + ..Default::default() + }, + ); + + assert!(solution.remove_voter(4)); + assert_eq!( + solution, + TestSolution { + votes1: vec![(0, 2), (1, 6)], + votes2: vec![(3, [(7, p(85))], 8)], + ..Default::default() + }, + ); + + assert!(solution.remove_voter(1)); + assert_eq!( + solution, + TestSolution { + votes1: vec![(0, 2)], + votes2: vec![(3, [(7, p(85))], 8),], + ..Default::default() + }, + ); + } + + #[test] + fn from_and_into_assignment_works() { + let voters = vec![2 as AccountId, 4, 1, 5, 3]; + let targets = vec![ + 10 as AccountId, + 11, + 20, // 2 + 30, + 31, // 4 + 32, + 40, // 6 + 50, + 51, // 8 + ]; + + let assignments = vec![ + Assignment { who: 2 as AccountId, distribution: vec![(20u64, p(100))] }, + Assignment { who: 4, distribution: vec![(40, p(100))] }, + Assignment { who: 1, distribution: vec![(10, p(80)), (11, p(20))] }, + Assignment { who: 5, distribution: vec![(50, p(85)), (51, p(15))] }, + Assignment { who: 3, distribution: vec![(30, p(50)), (31, p(25)), (32, p(25))] }, + ]; + + let voter_index = |a: &AccountId| -> Option { + voters.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() + }; + let target_index = |a: &AccountId| -> Option { + targets.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() + }; + + let solution = + TestSolution::from_assignment(&assignments, voter_index, target_index).unwrap(); + + // basically number of assignments that it is encoding. + assert_eq!(solution.voter_count(), assignments.len()); + assert_eq!( + solution.edge_count(), + assignments.iter().fold(0, |a, b| a + b.distribution.len()), + ); + + assert_eq!( + solution, + TestSolution { + votes1: vec![(0, 2), (1, 6)], + votes2: vec![(2, [(0, p(80))], 1), (3, [(7, p(85))], 8)], + votes3: vec![(4, [(3, p(50)), (4, p(25))], 5)], + ..Default::default() + } + ); + + assert_eq!(solution.unique_targets(), vec![0, 1, 2, 3, 4, 5, 6, 7, 8]); + + let voter_at = |a: u32| -> Option { + voters.get(>::try_into(a).unwrap()).cloned() + }; + let target_at = |a: u16| -> Option { + targets.get(>::try_into(a).unwrap()).cloned() + }; + + assert_eq!(solution.into_assignment(voter_at, target_at).unwrap(), assignments); + } + + #[test] + fn unique_targets_len_edge_count_works() { + // we don't really care about voters here so all duplicates. This is not invalid per se. + let solution = TestSolution { + votes1: vec![(99, 1), (99, 2)], + votes2: vec![(99, [(3, p(10))], 7), (99, [(4, p(10))], 8)], + votes3: vec![(99, [(11, p(10)), (12, p(10))], 13)], + // ensure the last one is also counted. + votes16: vec![( + 99, + [ + (66, p(10)), + (66, p(10)), + (66, p(10)), + (66, p(10)), + (66, p(10)), + (66, p(10)), + (66, p(10)), + (66, p(10)), + (66, p(10)), + (66, p(10)), + (66, p(10)), + (66, p(10)), + (66, p(10)), + (66, p(10)), + (66, p(10)), + ], + 67, + )], + ..Default::default() + }; + + assert_eq!(solution.unique_targets(), vec![1, 2, 3, 4, 7, 8, 11, 12, 13, 66, 67]); + assert_eq!(solution.edge_count(), 2 + (2 * 2) + 3 + 16); + assert_eq!(solution.voter_count(), 6); + + // this one has some duplicates. + let solution = TestSolution { + votes1: vec![(99, 1), (99, 1)], + votes2: vec![(99, [(3, p(10))], 7), (99, [(4, p(10))], 8)], + votes3: vec![(99, [(11, p(10)), (11, p(10))], 13)], + ..Default::default() + }; + + assert_eq!(solution.unique_targets(), vec![1, 3, 4, 7, 8, 11, 13]); + assert_eq!(solution.edge_count(), 2 + (2 * 2) + 3); + assert_eq!(solution.voter_count(), 5); + } + + #[test] + fn solution_into_assignment_must_report_overflow() { + // in votes2 + let solution = TestSolution { + votes1: Default::default(), + votes2: vec![(0, [(1, p(100))], 2)], + ..Default::default() + }; + + let voter_at = |a: u32| -> Option { Some(a as AccountId) }; + let target_at = |a: u16| -> Option { Some(a as AccountId) }; + + assert_eq!( + solution.into_assignment(&voter_at, &target_at).unwrap_err(), + NposError::SolutionWeightOverflow, + ); + + // in votes3 onwards + let solution = TestSolution { + votes1: Default::default(), + votes2: Default::default(), + votes3: vec![(0, [(1, p(70)), (2, p(80))], 3)], + ..Default::default() + }; + + assert_eq!( + solution.into_assignment(&voter_at, &target_at).unwrap_err(), + NposError::SolutionWeightOverflow, + ); + } + + #[test] + fn target_count_overflow_is_detected() { + let voter_index = |a: &AccountId| -> Option { Some(*a as u32) }; + let target_index = |a: &AccountId| -> Option { Some(*a as u16) }; + + let assignments = vec![Assignment { + who: 1 as AccountId, + distribution: (10..27).map(|i| (i as AccountId, p(i as u8))).collect::>(), + }]; + + let solution = TestSolution::from_assignment(&assignments, voter_index, target_index); + assert_eq!(solution.unwrap_err(), NposError::SolutionTargetOverflow); + } + + #[test] + fn zero_target_count_is_ignored() { + let voters = vec![1 as AccountId, 2]; + let targets = vec![10 as AccountId, 11]; + + let assignments = vec![ + Assignment { who: 1 as AccountId, distribution: vec![(10, p(50)), (11, p(50))] }, + Assignment { who: 2, distribution: vec![] }, + ]; + + let voter_index = |a: &AccountId| -> Option { + voters.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() + }; + let target_index = |a: &AccountId| -> Option { + targets.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() + }; + + let solution = + TestSolution::from_assignment(&assignments, voter_index, target_index).unwrap(); + + assert_eq!( + solution, + TestSolution { + votes1: Default::default(), + votes2: vec![(0, [(0, p(50))], 1)], + ..Default::default() + } + ); + } +} + +#[test] +fn index_assignments_generate_same_solution_as_plain_assignments() { + let rng = rand::rngs::SmallRng::seed_from_u64(0); + + let (voters, assignments, candidates) = generate_random_votes(1000, 2500, rng); + let voter_index = make_voter_fn(&voters); + let target_index = make_target_fn(&candidates); + + let solution = + TestSolution::from_assignment(&assignments, &voter_index, &target_index).unwrap(); + + let index_assignments = assignments + .into_iter() + .map(|assignment| IndexAssignment::new(&assignment, &voter_index, &target_index)) + .collect::, _>>() + .unwrap(); + + let index_compact = index_assignments.as_slice().try_into().unwrap(); + + assert_eq!(solution, index_compact); +} diff --git a/frame/election-provider-support/src/traits.rs b/frame/election-provider-support/src/traits.rs new file mode 100644 index 000000000000..ed812e2e0f2c --- /dev/null +++ b/frame/election-provider-support/src/traits.rs @@ -0,0 +1,125 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Traits for the election operations. + +use crate::{Assignment, IdentifierT, IndexAssignmentOf, PerThing128, VoteWeight}; +use codec::Encode; +use scale_info::TypeInfo; +use sp_arithmetic::traits::{Bounded, UniqueSaturatedInto}; +use sp_npos_elections::{ElectionScore, Error, EvaluateSupport}; +use sp_std::{fmt::Debug, prelude::*}; + +/// An opaque index-based, NPoS solution type. +pub trait NposSolution +where + Self: Sized + for<'a> TryFrom<&'a [IndexAssignmentOf], Error = Error>, +{ + /// The maximum number of votes that are allowed. + const LIMIT: usize; + + /// The voter type. Needs to be an index (convert to usize). + type VoterIndex: UniqueSaturatedInto + + TryInto + + TryFrom + + Debug + + Copy + + Clone + + Bounded + + Encode + + TypeInfo; + + /// The target type. Needs to be an index (convert to usize). + type TargetIndex: UniqueSaturatedInto + + TryInto + + TryFrom + + Debug + + Copy + + Clone + + Bounded + + Encode + + TypeInfo; + + /// The weight/accuracy type of each vote. + type Accuracy: PerThing128; + + /// Get the length of all the voters that this type is encoding. + /// + /// This is basically the same as the number of assignments, or number of active voters. + fn voter_count(&self) -> usize; + + /// Get the total count of edges. + /// + /// This is effectively in the range of {[`Self::voter_count`], [`Self::voter_count`] * + /// [`Self::LIMIT`]}. + fn edge_count(&self) -> usize; + + /// Get the number of unique targets in the whole struct. + /// + /// Once presented with a list of winners, this set and the set of winners must be + /// equal. + fn unique_targets(&self) -> Vec; + + /// Get the average edge count. + fn average_edge_count(&self) -> usize { + self.edge_count().checked_div(self.voter_count()).unwrap_or(0) + } + + /// Compute the score of this solution type. + fn score( + self, + stake_of: FS, + voter_at: impl Fn(Self::VoterIndex) -> Option, + target_at: impl Fn(Self::TargetIndex) -> Option, + ) -> Result + where + for<'r> FS: Fn(&'r A) -> VoteWeight, + A: IdentifierT, + { + let ratio = self.into_assignment(voter_at, target_at)?; + let staked = + sp_npos_elections::helpers::assignment_ratio_to_staked_normalized(ratio, stake_of)?; + let supports = sp_npos_elections::to_supports(&staked); + Ok(supports.evaluate()) + } + + /// Remove a certain voter. + /// + /// This will only search until the first instance of `to_remove`, and return true. If + /// no instance is found (no-op), then it returns false. + /// + /// In other words, if this return true, exactly **one** element must have been removed self. + fn remove_voter(&mut self, to_remove: Self::VoterIndex) -> bool; + + /// Build self from a list of assignments. + fn from_assignment( + assignments: &[Assignment], + voter_index: FV, + target_index: FT, + ) -> Result + where + A: IdentifierT, + for<'r> FV: Fn(&'r A) -> Option, + for<'r> FT: Fn(&'r A) -> Option; + + /// Convert self into a `Vec>` + fn into_assignment( + self, + voter_at: impl Fn(Self::VoterIndex) -> Option, + target_at: impl Fn(Self::TargetIndex) -> Option, + ) -> Result>, Error>; +} diff --git a/frame/elections-phragmen/Cargo.toml b/frame/elections-phragmen/Cargo.toml index f2771a9f7278..f772a82d73d8 100644 --- a/frame/elections-phragmen/Cargo.toml +++ b/frame/elections-phragmen/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-elections-phragmen" version = "5.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet based on seq-Phragmén election method." readme = "README.md" @@ -13,23 +13,23 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-npos-elections = { version = "4.0.0-dev", default-features = false, path = "../../primitives/npos-elections" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } log = { version = "0.4.14", default-features = false } [dev-dependencies] pallet-balances = { version = "4.0.0-dev", path = "../balances" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } substrate-test-utils = { version = "4.0.0-dev", path = "../../test-utils" } [features] @@ -47,7 +47,7 @@ std = [ "log/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", ] diff --git a/frame/elections-phragmen/src/benchmarking.rs b/frame/elections-phragmen/src/benchmarking.rs index 6e3ce0234c4f..05e9df60c7fb 100644 --- a/frame/elections-phragmen/src/benchmarking.rs +++ b/frame/elections-phragmen/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,9 +21,7 @@ use super::*; -use frame_benchmarking::{ - account, benchmarks, impl_benchmark_test_suite, whitelist, BenchmarkError, BenchmarkResult, -}; +use frame_benchmarking::{account, benchmarks, whitelist, BenchmarkError, BenchmarkResult}; use frame_support::{ dispatch::{DispatchResultWithPostInfo, UnfilteredDispatchable}, traits::OnInitialize, @@ -41,7 +39,9 @@ type Lookup = <::Lookup as StaticLookup>::Source; /// grab new account with infinite balance. fn endowed_account(name: &'static str, index: u32) -> T::AccountId { let account: T::AccountId = account(name, index, 0); - let amount = default_stake::(BALANCE_FACTOR); + // Fund each account with at-least his stake but still a sane amount as to not mess up + // the vote calculation. + let amount = default_stake::(MAX_VOTERS) * BalanceOf::::from(BALANCE_FACTOR); let _ = T::Currency::make_free_balance_be(&account, amount); // important to increase the total issuance since T::CurrencyToVote will need it to be sane for // phragmen to work. @@ -56,9 +56,9 @@ fn as_lookup(account: T::AccountId) -> Lookup { } /// Get a reasonable amount of stake based on the execution trait's configuration -fn default_stake(factor: u32) -> BalanceOf { - let factor = BalanceOf::::from(factor); - T::Currency::minimum_balance() * factor +fn default_stake(num_votes: u32) -> BalanceOf { + let min = T::Currency::minimum_balance(); + Elections::::deposit_of(num_votes as usize).max(min) } /// Get the current number of candidates. @@ -90,7 +90,7 @@ fn submit_candidates_with_self_vote( prefix: &'static str, ) -> Result, &'static str> { let candidates = submit_candidates::(c, prefix)?; - let stake = default_stake::(BALANCE_FACTOR); + let stake = default_stake::(c); let _ = candidates .iter() .map(|c| submit_voter::(c.clone(), vec![c.clone()], stake).map(|_| ())) @@ -114,7 +114,7 @@ fn distribute_voters( num_voters: u32, votes: usize, ) -> Result<(), &'static str> { - let stake = default_stake::(BALANCE_FACTOR); + let stake = default_stake::(num_voters); for i in 0..num_voters { // to ensure that votes are different all_candidates.rotate_left(1); @@ -162,7 +162,7 @@ benchmarks! { let all_candidates = submit_candidates::(v, "candidates")?; let caller = endowed_account::("caller", 0); - let stake = default_stake::(BALANCE_FACTOR); + let stake = default_stake::(v); // original votes. let mut votes = all_candidates; @@ -175,14 +175,15 @@ benchmarks! { }: vote(RawOrigin::Signed(caller), votes, stake) vote_more { - let v in 2 .. (MAXIMUM_VOTE as u32); + let v in 2 .. (MAXIMUM_VOTE as u32); clean::(); // create a bunch of candidates. let all_candidates = submit_candidates::(v, "candidates")?; let caller = endowed_account::("caller", 0); - let stake = default_stake::(BALANCE_FACTOR); + // Multiply the stake with 10 since we want to be able to divide it by 10 again. + let stake = default_stake::(v) * BalanceOf::::from(10u32); // original votes. let mut votes = all_candidates.iter().skip(1).cloned().collect::>(); @@ -196,14 +197,14 @@ benchmarks! { }: vote(RawOrigin::Signed(caller), votes, stake / >::from(10u32)) vote_less { - let v in 2 .. (MAXIMUM_VOTE as u32); + let v in 2 .. (MAXIMUM_VOTE as u32); clean::(); // create a bunch of candidates. let all_candidates = submit_candidates::(v, "candidates")?; let caller = endowed_account::("caller", 0); - let stake = default_stake::(BALANCE_FACTOR); + let stake = default_stake::(v); // original votes. let mut votes = all_candidates; @@ -226,7 +227,7 @@ benchmarks! { let caller = endowed_account::("caller", 0); - let stake = default_stake::(BALANCE_FACTOR); + let stake = default_stake::(v); submit_voter::(caller.clone(), all_candidates, stake)?; whitelist!(caller); @@ -240,7 +241,7 @@ benchmarks! { let m = T::DesiredMembers::get() + T::DesiredRunnersUp::get(); clean::(); - let stake = default_stake::(BALANCE_FACTOR); + let stake = default_stake::(c); // create m members and runners combined. let _ = fill_seats_up_to::(m)?; @@ -549,11 +550,11 @@ benchmarks! { MEMBERS.with(|m| *m.borrow_mut() = vec![]); } } -} -impl_benchmark_test_suite!( - Elections, - crate::tests::ExtBuilder::default().desired_members(13).desired_runners_up(7), - crate::tests::Test, - exec_name = build_and_execute, -); + impl_benchmark_test_suite!( + Elections, + crate::tests::ExtBuilder::default().desired_members(13).desired_runners_up(7), + crate::tests::Test, + exec_name = build_and_execute, + ); +} diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs index d7b42383da75..4758c793cfef 100644 --- a/frame/elections-phragmen/src/lib.rs +++ b/frame/elections-phragmen/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -102,9 +102,9 @@ use codec::{Decode, Encode}; use frame_support::{ dispatch::WithPostDispatchInfo, traits::{ - ChangeMembers, Contains, ContainsLengthBound, Currency, CurrencyToVote, Get, - InitializeMembers, LockIdentifier, LockableCurrency, OnUnbalanced, ReservableCurrency, - SortedMembers, StorageVersion, WithdrawReasons, + defensive_prelude::*, ChangeMembers, Contains, ContainsLengthBound, Currency, + CurrencyToVote, Get, InitializeMembers, LockIdentifier, LockableCurrency, OnUnbalanced, + ReservableCurrency, SortedMembers, StorageVersion, WithdrawReasons, }, weights::Weight, }; @@ -147,7 +147,7 @@ pub enum Renouncing { } /// An active voter. -#[derive(Encode, Decode, Clone, Default, RuntimeDebug, PartialEq, TypeInfo)] +#[derive(Encode, Decode, Clone, RuntimeDebug, PartialEq, TypeInfo)] pub struct Voter { /// The members being backed. pub votes: Vec, @@ -159,6 +159,12 @@ pub struct Voter { pub deposit: Balance, } +impl Default for Voter { + fn default() -> Self { + Self { votes: vec![], stake: Default::default(), deposit: Default::default() } + } +} + /// A holder of a seat as either a member or a runner-up. #[derive(Encode, Decode, Clone, Default, RuntimeDebug, PartialEq, TypeInfo)] pub struct SeatHolder { @@ -244,6 +250,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] #[pallet::storage_version(STORAGE_VERSION)] + #[pallet::without_storage_info] pub struct Pallet(PhantomData); #[pallet::hooks] @@ -274,7 +281,7 @@ pub mod pallet { /// - be less than the number of possible candidates. Note that all current members and /// runners-up are also automatically candidates for the next round. /// - /// If `value` is more than `who`'s total balance, then the maximum of the two is used. + /// If `value` is more than `who`'s free balance, then the maximum of the two is used. /// /// The dispatch origin of this call must be signed. /// @@ -336,7 +343,7 @@ pub mod pallet { }; // Amount to be locked up. - let locked_stake = value.min(T::Currency::total_balance(&who)); + let locked_stake = value.min(T::Currency::free_balance(&who)); T::Currency::set_lock(T::PalletId::get(), &who, locked_stake, WithdrawReasons::all()); Voting::::insert(&who, Voter { votes, deposit: new_deposit, stake: locked_stake }); @@ -425,7 +432,7 @@ pub mod pallet { Renouncing::Member => { let _ = Self::remove_and_replace_member(&who, false) .map_err(|_| Error::::InvalidRenouncing)?; - Self::deposit_event(Event::Renounced(who)); + Self::deposit_event(Event::Renounced { candidate: who }); }, Renouncing::RunnerUp => { >::try_mutate::<_, Error, _>(|runners_up| { @@ -437,7 +444,7 @@ pub mod pallet { let SeatHolder { deposit, .. } = runners_up.remove(index); let _remainder = T::Currency::unreserve(&who, deposit); debug_assert!(_remainder.is_zero()); - Self::deposit_event(Event::Renounced(who)); + Self::deposit_event(Event::Renounced { candidate: who }); Ok(()) })?; }, @@ -450,7 +457,7 @@ pub mod pallet { let (_removed, deposit) = candidates.remove(index); let _remainder = T::Currency::unreserve(&who, deposit); debug_assert!(_remainder.is_zero()); - Self::deposit_event(Event::Renounced(who)); + Self::deposit_event(Event::Renounced { candidate: who }); Ok(()) })?; }, @@ -496,7 +503,7 @@ pub mod pallet { let had_replacement = Self::remove_and_replace_member(&who, true)?; debug_assert_eq!(has_replacement, had_replacement); - Self::deposit_event(Event::MemberKicked(who.clone())); + Self::deposit_event(Event::MemberKicked { member: who.clone() }); if !had_replacement { Self::do_phragmen(); @@ -534,29 +541,32 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// A new term with \[new_members\]. This indicates that enough candidates existed to run + /// A new term with new_members. This indicates that enough candidates existed to run /// the election, not that enough have has been elected. The inner value must be examined /// for this purpose. A `NewTerm(\[\])` indicates that some candidates got their bond /// slashed and none were elected, whilst `EmptyTerm` means that no candidates existed to /// begin with. - NewTerm(Vec<(::AccountId, BalanceOf)>), + NewTerm { new_members: Vec<(::AccountId, BalanceOf)> }, /// No (or not enough) candidates existed for this round. This is different from /// `NewTerm(\[\])`. See the description of `NewTerm`. EmptyTerm, /// Internal error happened while trying to perform election. ElectionError, - /// A \[member\] has been removed. This should always be followed by either `NewTerm` or + /// A member has been removed. This should always be followed by either `NewTerm` or /// `EmptyTerm`. - MemberKicked(::AccountId), + MemberKicked { member: ::AccountId }, /// Someone has renounced their candidacy. - Renounced(::AccountId), - /// A \[candidate\] was slashed by \[amount\] due to failing to obtain a seat as member or + Renounced { candidate: ::AccountId }, + /// A candidate was slashed by amount due to failing to obtain a seat as member or /// runner-up. /// /// Note that old members and runners-up are also candidates. - CandidateSlashed(::AccountId, BalanceOf), - /// A \[seat holder\] was slashed by \[amount\] by being forcefully removed from the set. - SeatHolderSlashed(::AccountId, BalanceOf), + CandidateSlashed { candidate: ::AccountId, amount: BalanceOf }, + /// A seat holder was slashed by amount by being forcefully removed from the set. + SeatHolderSlashed { + seat_holder: ::AccountId, + amount: BalanceOf, + }, } #[deprecated(note = "use `Event` instead")] @@ -748,7 +758,10 @@ impl Pallet { let (imbalance, _remainder) = T::Currency::slash_reserved(who, removed.deposit); debug_assert!(_remainder.is_zero()); T::LoserCandidate::on_unbalanced(imbalance); - Self::deposit_event(Event::SeatHolderSlashed(who.clone(), removed.deposit)); + Self::deposit_event(Event::SeatHolderSlashed { + seat_holder: who.clone(), + amount: removed.deposit, + }); } else { T::Currency::unreserve(who, removed.deposit); } @@ -824,7 +837,7 @@ impl Pallet { /// Check if `who` is currently an active runner-up. fn is_runner_up(who: &T::AccountId) -> bool { - Self::runners_up().iter().position(|r| &r.who == who).is_some() + Self::runners_up().iter().any(|r| &r.who == who) } /// Get the members' account ids. @@ -913,13 +926,13 @@ impl Pallet { let weight_candidates = candidates_and_deposit.len() as u32; let weight_voters = voters_and_votes.len() as u32; let weight_edges = num_edges; - let _ = sp_npos_elections::seq_phragmen::( + let _ = sp_npos_elections::seq_phragmen( num_to_elect, candidate_ids, voters_and_votes.clone(), None, ) - .map(|ElectionResult { winners, assignments: _ }| { + .map(|ElectionResult:: { winners, assignments: _ }| { // this is already sorted by id. let old_members_ids_sorted = >::take().into_iter().map(|m| m.who).collect::>(); @@ -1001,7 +1014,10 @@ impl Pallet { { let (imbalance, _) = T::Currency::slash_reserved(c, *d); T::LoserCandidate::on_unbalanced(imbalance); - Self::deposit_event(Event::CandidateSlashed(c.clone(), *d)); + Self::deposit_event(Event::CandidateSlashed { + candidate: c.clone(), + amount: *d, + }); } }); @@ -1012,7 +1028,7 @@ impl Pallet { candidates_and_deposit .iter() .find_map(|(c, d)| if c == x { Some(*d) } else { None }) - .unwrap_or_default() + .defensive_unwrap_or_default() }; // fetch deposits from the one recorded one. This will make sure that a candidate who // submitted candidacy before a change to candidacy deposit will have the correct amount @@ -1041,7 +1057,7 @@ impl Pallet { // clean candidates. >::kill(); - Self::deposit_event(Event::NewTerm(new_members_sorted_by_id)); + Self::deposit_event(Event::NewTerm { new_members: new_members_sorted_by_id }); >::mutate(|v| *v += 1); }) .map_err(|e| { @@ -1078,7 +1094,14 @@ impl SortedMembers for Pallet { fn add(who: &T::AccountId) { Members::::mutate(|members| match members.binary_search_by(|m| m.who.cmp(who)) { Ok(_) => (), - Err(pos) => members.insert(pos, SeatHolder { who: who.clone(), ..Default::default() }), + Err(pos) => { + let s = SeatHolder { + who: who.clone(), + stake: Default::default(), + deposit: Default::default(), + }; + members.insert(pos, s) + }, }) } } @@ -1099,20 +1122,21 @@ mod tests { use super::*; use crate as elections_phragmen; use frame_support::{ - assert_noop, assert_ok, dispatch::DispatchResultWithPostInfo, parameter_types, - traits::OnInitialize, + assert_noop, assert_ok, + dispatch::DispatchResultWithPostInfo, + parameter_types, + traits::{ConstU32, ConstU64, OnInitialize}, }; use frame_system::ensure_signed; use sp_core::H256; use sp_runtime::{ testing::Header, traits::{BlakeTwo256, IdentityLookup}, - BuildStorage, + BuildStorage, ModuleError, }; use substrate_test_utils::assert_eq_uvec; parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -1132,7 +1156,7 @@ mod tests { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -1141,17 +1165,14 @@ mod tests { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); - } - - parameter_types! { - pub const ExistentialDeposit: u64 = 1; + type MaxConsumers = ConstU32<16>; } impl pallet_balances::Config for Test { type Balance = u64; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = frame_system::Pallet; type MaxLocks = (); type MaxReserves = (); @@ -1647,11 +1668,13 @@ mod tests { System::set_block_number(5); Elections::on_initialize(System::block_number()); + assert_eq!(balances(&4), (34, 6)); + assert_eq!(balances(&5), (45, 5)); assert_eq!( Elections::members(), vec![ - SeatHolder { who: 4, stake: 40, deposit: 4 }, - SeatHolder { who: 5, stake: 50, deposit: 3 }, + SeatHolder { who: 4, stake: 34, deposit: 4 }, + SeatHolder { who: 5, stake: 45, deposit: 3 }, ] ); }) @@ -1742,7 +1765,7 @@ mod tests { assert_ok!(vote(Origin::signed(2), vec![5], 20)); assert_eq!(balances(&2), (18, 2)); - assert_eq!(has_lock(&2), 20); + assert_eq!(has_lock(&2), 18); }); } @@ -1769,9 +1792,10 @@ mod tests { assert_ok!(submit_candidacy(Origin::signed(4))); assert_ok!(vote(Origin::signed(2), vec![5], 20)); + // User only locks up to their free balance. assert_eq!(balances(&2), (18, 2)); - assert_eq!(has_lock(&2), 20); - assert_eq!(locked_stake_of(&2), 20); + assert_eq!(has_lock(&2), 18); + assert_eq!(locked_stake_of(&2), 18); // can update; different stake; different lock and reserve. assert_ok!(vote(Origin::signed(2), vec![5, 4], 15)); @@ -1836,8 +1860,8 @@ mod tests { // 2 + 2 assert_eq!(balances(&2), (16, 4)); assert_eq!(Elections::voting(&2).deposit, 4); - assert_eq!(has_lock(&2), 18); - assert_eq!(locked_stake_of(&2), 18); + assert_eq!(has_lock(&2), 16); + assert_eq!(locked_stake_of(&2), 16); // back to 1 vote. assert_ok!(vote(Origin::signed(2), vec![4], 12)); @@ -2011,15 +2035,18 @@ mod tests { } #[test] - fn can_vote_for_more_than_total_balance_but_moot() { + fn can_vote_for_more_than_free_balance_but_moot() { ExtBuilder::default().build_and_execute(|| { assert_ok!(submit_candidacy(Origin::signed(5))); assert_ok!(submit_candidacy(Origin::signed(4))); - assert_ok!(vote(Origin::signed(2), vec![4, 5], 30)); - // you can lie but won't get away with it. - assert_eq!(locked_stake_of(&2), 20); - assert_eq!(has_lock(&2), 20); + // User has 100 free and 50 reserved. + assert_ok!(Balances::set_balance(Origin::root(), 2, 100, 50)); + // User tries to vote with 150 tokens. + assert_ok!(vote(Origin::signed(2), vec![4, 5], 150)); + // We truncate to only their free balance, after reserving additional for voting. + assert_eq!(locked_stake_of(&2), 98); + assert_eq!(has_lock(&2), 98); }); } @@ -2032,8 +2059,10 @@ mod tests { assert_ok!(vote(Origin::signed(3), vec![5], 30)); assert_eq_uvec!(all_voters(), vec![2, 3]); - assert_eq!(locked_stake_of(&2), 20); - assert_eq!(locked_stake_of(&3), 30); + assert_eq!(balances(&2), (12, 8)); + assert_eq!(locked_stake_of(&2), 12); + assert_eq!(balances(&3), (22, 8)); + assert_eq!(locked_stake_of(&3), 22); assert_eq!(votes_of(&2), vec![5]); assert_eq!(votes_of(&3), vec![5]); @@ -2113,7 +2142,10 @@ mod tests { System::set_block_number(5); Elections::on_initialize(System::block_number()); - assert_eq!(members_and_stake(), vec![(3, 30), (5, 20)]); + assert_eq!(balances(&3), (25, 5)); + // votes for 5 + assert_eq!(balances(&2), (18, 2)); + assert_eq!(members_and_stake(), vec![(3, 25), (5, 18)]); assert!(Elections::runners_up().is_empty()); assert_eq_uvec!(all_voters(), vec![2, 3, 4]); @@ -2147,12 +2179,11 @@ mod tests { System::set_block_number(5); Elections::on_initialize(System::block_number()); - System::assert_last_event(Event::Elections(super::Event::NewTerm(vec![ - (4, 40), - (5, 50), - ]))); + System::assert_last_event(Event::Elections(super::Event::NewTerm { + new_members: vec![(4, 35), (5, 45)], + })); - assert_eq!(members_and_stake(), vec![(4, 40), (5, 50)]); + assert_eq!(members_and_stake(), vec![(4, 35), (5, 45)]); assert_eq!(runners_up_and_stake(), vec![]); assert_ok!(Elections::remove_voter(Origin::signed(5))); @@ -2161,7 +2192,9 @@ mod tests { System::set_block_number(10); Elections::on_initialize(System::block_number()); - System::assert_last_event(Event::Elections(super::Event::NewTerm(vec![]))); + System::assert_last_event(Event::Elections(super::Event::NewTerm { + new_members: vec![], + })); // outgoing have lost their bond. assert_eq!(balances(&4), (37, 0)); @@ -2181,7 +2214,7 @@ mod tests { System::set_block_number(5); Elections::on_initialize(System::block_number()); - assert_eq!(members_and_stake(), vec![(5, 50)]); + assert_eq!(members_and_stake(), vec![(5, 45)]); assert_eq!(Elections::election_rounds(), 1); // but now it has a valid target. @@ -2191,7 +2224,7 @@ mod tests { Elections::on_initialize(System::block_number()); // candidate 4 is affected by an old vote. - assert_eq!(members_and_stake(), vec![(4, 30), (5, 50)]); + assert_eq!(members_and_stake(), vec![(4, 28), (5, 45)]); assert_eq!(Elections::election_rounds(), 2); assert_eq_uvec!(all_voters(), vec![3, 5]); }); @@ -2231,7 +2264,9 @@ mod tests { assert_eq!(Elections::election_rounds(), 1); assert!(members_ids().is_empty()); - System::assert_last_event(Event::Elections(super::Event::NewTerm(vec![]))); + System::assert_last_event(Event::Elections(super::Event::NewTerm { + new_members: vec![], + })); }); } @@ -2277,16 +2312,16 @@ mod tests { System::set_block_number(5); Elections::on_initialize(System::block_number()); - assert_eq!(members_and_stake(), vec![(4, 40), (5, 50)]); - assert_eq!(runners_up_and_stake(), vec![(2, 20), (3, 30)]); + assert_eq!(members_and_stake(), vec![(4, 35), (5, 45)]); + assert_eq!(runners_up_and_stake(), vec![(2, 15), (3, 25)]); - assert_ok!(vote(Origin::signed(5), vec![5], 15)); + assert_ok!(vote(Origin::signed(5), vec![5], 10)); System::set_block_number(10); Elections::on_initialize(System::block_number()); - assert_eq!(members_and_stake(), vec![(3, 30), (4, 40)]); - assert_eq!(runners_up_and_stake(), vec![(5, 15), (2, 20)]); + assert_eq!(members_and_stake(), vec![(3, 25), (4, 35)]); + assert_eq!(runners_up_and_stake(), vec![(5, 10), (2, 15)]); }); } @@ -2420,8 +2455,8 @@ mod tests { System::set_block_number(b.into()); Elections::on_initialize(System::block_number()); // we keep re-electing the same folks. - assert_eq!(members_and_stake(), vec![(4, 40), (5, 50)]); - assert_eq!(runners_up_and_stake(), vec![(2, 20), (3, 30)]); + assert_eq!(members_and_stake(), vec![(4, 35), (5, 45)]); + assert_eq!(runners_up_and_stake(), vec![(2, 15), (3, 25)]); // no new candidates but old members and runners-up are always added. assert!(candidate_ids().is_empty()); assert_eq!(Elections::election_rounds(), b / 5); @@ -2479,7 +2514,7 @@ mod tests { let unwrapped_error = Elections::remove_member(Origin::root(), 4, true).unwrap_err(); assert!(matches!( unwrapped_error.error, - DispatchError::Module { message: Some("InvalidReplacement"), .. } + DispatchError::Module(ModuleError { message: Some("InvalidReplacement"), .. }) )); assert!(unwrapped_error.post_info.actual_weight.is_some()); }); @@ -2502,7 +2537,7 @@ mod tests { let unwrapped_error = Elections::remove_member(Origin::root(), 4, false).unwrap_err(); assert!(matches!( unwrapped_error.error, - DispatchError::Module { message: Some("InvalidReplacement"), .. } + DispatchError::Module(ModuleError { message: Some("InvalidReplacement"), .. }) )); assert!(unwrapped_error.post_info.actual_weight.is_some()); }); @@ -2573,7 +2608,7 @@ mod tests { Elections::on_initialize(System::block_number()); // 3, 4 are new members, must still be bonded, nothing slashed. - assert_eq!(members_and_stake(), vec![(3, 30), (4, 48)]); + assert_eq!(members_and_stake(), vec![(3, 25), (4, 43)]); assert_eq!(balances(&3), (25, 5)); assert_eq!(balances(&4), (35, 5)); @@ -2583,10 +2618,9 @@ mod tests { // 5 is an outgoing loser. will also get slashed. assert_eq!(balances(&5), (45, 2)); - System::assert_has_event(Event::Elections(super::Event::NewTerm(vec![ - (4, 40), - (5, 50), - ]))); + System::assert_has_event(Event::Elections(super::Event::NewTerm { + new_members: vec![(4, 35), (5, 45)], + })); }) } @@ -2624,9 +2658,9 @@ mod tests { System::set_block_number(5); Elections::on_initialize(System::block_number()); // id: low -> high. - assert_eq!(members_and_stake(), vec![(4, 50), (5, 40)]); + assert_eq!(members_and_stake(), vec![(4, 45), (5, 35)]); // merit: low -> high. - assert_eq!(runners_up_and_stake(), vec![(3, 20), (2, 30)]); + assert_eq!(runners_up_and_stake(), vec![(3, 15), (2, 25)]); }); } diff --git a/frame/elections-phragmen/src/migrations/mod.rs b/frame/elections-phragmen/src/migrations/mod.rs index 9a1f86a1ad7c..7c62e8fa9306 100644 --- a/frame/elections-phragmen/src/migrations/mod.rs +++ b/frame/elections-phragmen/src/migrations/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,3 +21,5 @@ pub mod v3; /// Version 4. pub mod v4; +/// Version 5. +pub mod v5; diff --git a/frame/elections-phragmen/src/migrations/v3.rs b/frame/elections-phragmen/src/migrations/v3.rs index 728e0c4b0c91..c6a7ce7e7ca1 100644 --- a/frame/elections-phragmen/src/migrations/v3.rs +++ b/frame/elections-phragmen/src/migrations/v3.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/elections-phragmen/src/migrations/v4.rs b/frame/elections-phragmen/src/migrations/v4.rs index 9acc1297294d..e0fc17ec2a12 100644 --- a/frame/elections-phragmen/src/migrations/v4.rs +++ b/frame/elections-phragmen/src/migrations/v4.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/elections-phragmen/src/migrations/v5.rs b/frame/elections-phragmen/src/migrations/v5.rs new file mode 100644 index 000000000000..1898668cd07b --- /dev/null +++ b/frame/elections-phragmen/src/migrations/v5.rs @@ -0,0 +1,70 @@ +use super::super::*; + +/// Migrate the locks and vote stake on accounts (as specified with param `to_migrate`) that have +/// more than their free balance locked. +/// +/// This migration addresses a bug were a voter could lock up to their reserved balance + free +/// balance. Since locks are only designed to operate on free balance, this put those affected in a +/// situation where they could increase their free balance but still not be able to use their funds +/// because they were less than the lock. +pub fn migrate(to_migrate: Vec) -> Weight { + let mut weight = 0; + + for who in to_migrate.iter() { + if let Ok(mut voter) = Voting::::try_get(who) { + let free_balance = T::Currency::free_balance(&who); + + weight = weight.saturating_add(T::DbWeight::get().reads(2)); + + if voter.stake > free_balance { + voter.stake = free_balance; + Voting::::insert(&who, voter); + + let pallet_id = T::PalletId::get(); + T::Currency::set_lock(pallet_id, &who, free_balance, WithdrawReasons::all()); + + weight = weight.saturating_add(T::DbWeight::get().writes(2)); + } + } + } + + weight +} + +/// Given the list of voters to migrate return a function that does some checks and information +/// prior to migration. This can be linked to [`frame_support::traits::OnRuntimeUpgrade:: +/// pre_upgrade`] for further testing. +pub fn pre_migrate_fn(to_migrate: Vec) -> Box ()> { + Box::new(move || { + for who in to_migrate.iter() { + if let Ok(voter) = Voting::::try_get(who) { + let free_balance = T::Currency::free_balance(&who); + + if voter.stake > free_balance { + // all good + } else { + log::warn!("pre-migrate elections-phragmen: voter={:?} has less stake then free balance", who); + } + } else { + log::warn!("pre-migrate elections-phragmen: cannot find voter={:?}", who); + } + } + log::info!("pre-migrate elections-phragmen complete"); + }) +} + +/// Some checks for after migration. This can be linked to +/// [`frame_support::traits::OnRuntimeUpgrade::post_upgrade`] for further testing. +/// +/// Panics if anything goes wrong. +pub fn post_migrate() { + for (who, voter) in Voting::::iter() { + let free_balance = T::Currency::free_balance(&who); + + assert!(voter.stake <= free_balance, "migration should have made locked <= free_balance"); + // Ideally we would also check that the locks and AccountData.misc_frozen where correctly + // updated, but since both of those are generic we can't do that without further bounding T. + } + + log::info!("post-migrate elections-phragmen complete"); +} diff --git a/frame/elections-phragmen/src/weights.rs b/frame/elections-phragmen/src/weights.rs index b60308c4f0a6..e973334b833c 100644 --- a/frame/elections-phragmen/src/weights.rs +++ b/frame/elections-phragmen/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_elections_phragmen //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-03-29, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -35,7 +35,6 @@ // --output=./frame/elections-phragmen/src/weights.rs // --template=./.maintain/frame-weight-template.hbs - #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] #![allow(unused_imports)] @@ -53,8 +52,8 @@ pub trait WeightInfo { fn renounce_candidacy_candidate(c: u32, ) -> Weight; fn renounce_candidacy_members() -> Weight; fn renounce_candidacy_runners_up() -> Weight; - fn remove_member_with_replacement() -> Weight; fn remove_member_without_replacement() -> Weight; + fn remove_member_with_replacement() -> Weight; fn remove_member_wrong_refund() -> Weight; fn clean_defunct_voters(v: u32, d: u32, ) -> Weight; fn election_phragmen(c: u32, v: u32, e: u32, ) -> Weight; @@ -69,9 +68,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Elections Voting (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vote_equal(v: u32, ) -> Weight { - (42_509_000 as Weight) - // Standard Error: 4_000 - .saturating_add((372_000 as Weight).saturating_mul(v as Weight)) + (22_981_000 as Weight) + // Standard Error: 6_000 + .saturating_add((232_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -81,9 +80,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Elections Voting (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vote_more(v: u32, ) -> Weight { - (65_311_000 as Weight) - // Standard Error: 6_000 - .saturating_add((419_000 as Weight).saturating_mul(v as Weight)) + (36_170_000 as Weight) + // Standard Error: 8_000 + .saturating_add((219_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -93,16 +92,16 @@ impl WeightInfo for SubstrateWeight { // Storage: Elections Voting (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vote_less(v: u32, ) -> Weight { - (65_444_000 as Weight) - // Standard Error: 5_000 - .saturating_add((376_000 as Weight).saturating_mul(v as Weight)) + (35_798_000 as Weight) + // Standard Error: 8_000 + .saturating_add((241_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Elections Voting (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn remove_voter() -> Weight { - (61_585_000 as Weight) + (33_060_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -110,53 +109,54 @@ impl WeightInfo for SubstrateWeight { // Storage: Elections Members (r:1 w:0) // Storage: Elections RunnersUp (r:1 w:0) fn submit_candidacy(c: u32, ) -> Weight { - (53_333_000 as Weight) - // Standard Error: 1_000 - .saturating_add((267_000 as Weight).saturating_mul(c as Weight)) + (35_384_000 as Weight) + // Standard Error: 0 + .saturating_add((124_000 as Weight).saturating_mul(c as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Elections Candidates (r:1 w:1) fn renounce_candidacy_candidate(c: u32, ) -> Weight { - (49_128_000 as Weight) + (31_555_000 as Weight) // Standard Error: 1_000 - .saturating_add((144_000 as Weight).saturating_mul(c as Weight)) + .saturating_add((78_000 as Weight).saturating_mul(c as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Elections Members (r:1 w:1) // Storage: Elections RunnersUp (r:1 w:1) - // Storage: Instance1Collective Prime (r:1 w:1) - // Storage: Instance1Collective Proposals (r:1 w:0) - // Storage: Instance1Collective Members (r:0 w:1) + // Storage: Council Prime (r:1 w:1) + // Storage: Council Proposals (r:1 w:0) + // Storage: Council Members (r:0 w:1) fn renounce_candidacy_members() -> Weight { - (70_685_000 as Weight) + (41_531_000 as Weight) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } // Storage: Elections RunnersUp (r:1 w:1) fn renounce_candidacy_runners_up() -> Weight { - (49_766_000 as Weight) + (30_762_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: Benchmark Override (r:0 w:0) + fn remove_member_without_replacement() -> Weight { + (2_000_000_000_000 as Weight) + } // Storage: Elections RunnersUp (r:1 w:1) // Storage: Elections Members (r:1 w:1) // Storage: System Account (r:1 w:1) - // Storage: Instance1Collective Prime (r:1 w:1) - // Storage: Instance1Collective Proposals (r:1 w:0) - // Storage: Instance1Collective Members (r:0 w:1) + // Storage: Council Prime (r:1 w:1) + // Storage: Council Proposals (r:1 w:0) + // Storage: Council Members (r:0 w:1) fn remove_member_with_replacement() -> Weight { - (76_153_000 as Weight) + (48_287_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } - fn remove_member_without_replacement() -> Weight { - T::BlockWeights::get().max_block - } // Storage: Elections RunnersUp (r:1 w:0) fn remove_member_wrong_refund() -> Weight { - (6_697_000 as Weight) + (4_747_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) } // Storage: Elections Voting (r:251 w:250) @@ -167,8 +167,8 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:250 w:250) fn clean_defunct_voters(v: u32, _d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 60_000 - .saturating_add((107_467_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 37_000 + .saturating_add((49_564_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(v as Weight))) @@ -177,19 +177,19 @@ impl WeightInfo for SubstrateWeight { // Storage: Elections Members (r:1 w:1) // Storage: Elections RunnersUp (r:1 w:1) // Storage: Elections Voting (r:502 w:0) - // Storage: Instance1Collective Proposals (r:1 w:0) + // Storage: Council Proposals (r:1 w:0) // Storage: Elections ElectionRounds (r:1 w:1) - // Storage: Instance1Collective Members (r:0 w:1) - // Storage: Instance1Collective Prime (r:0 w:1) + // Storage: Council Members (r:0 w:1) + // Storage: Council Prime (r:0 w:1) // Storage: System Account (r:2 w:2) fn election_phragmen(c: u32, v: u32, e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 1_846_000 - .saturating_add((39_843_000 as Weight).saturating_mul(c as Weight)) - // Standard Error: 768_000 - .saturating_add((60_623_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 52_000 - .saturating_add((3_884_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 1_656_000 + .saturating_add((29_011_000 as Weight).saturating_mul(c as Weight)) + // Standard Error: 689_000 + .saturating_add((49_204_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 47_000 + .saturating_add((3_352_000 as Weight).saturating_mul(e as Weight)) .saturating_add(T::DbWeight::get().reads((2 as Weight).saturating_mul(c as Weight))) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(c as Weight))) @@ -204,9 +204,9 @@ impl WeightInfo for () { // Storage: Elections Voting (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vote_equal(v: u32, ) -> Weight { - (42_509_000 as Weight) - // Standard Error: 4_000 - .saturating_add((372_000 as Weight).saturating_mul(v as Weight)) + (22_981_000 as Weight) + // Standard Error: 6_000 + .saturating_add((232_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -216,9 +216,9 @@ impl WeightInfo for () { // Storage: Elections Voting (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vote_more(v: u32, ) -> Weight { - (65_311_000 as Weight) - // Standard Error: 6_000 - .saturating_add((419_000 as Weight).saturating_mul(v as Weight)) + (36_170_000 as Weight) + // Standard Error: 8_000 + .saturating_add((219_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -228,16 +228,16 @@ impl WeightInfo for () { // Storage: Elections Voting (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vote_less(v: u32, ) -> Weight { - (65_444_000 as Weight) - // Standard Error: 5_000 - .saturating_add((376_000 as Weight).saturating_mul(v as Weight)) + (35_798_000 as Weight) + // Standard Error: 8_000 + .saturating_add((241_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Elections Voting (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn remove_voter() -> Weight { - (61_585_000 as Weight) + (33_060_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -245,55 +245,54 @@ impl WeightInfo for () { // Storage: Elections Members (r:1 w:0) // Storage: Elections RunnersUp (r:1 w:0) fn submit_candidacy(c: u32, ) -> Weight { - (53_333_000 as Weight) - // Standard Error: 1_000 - .saturating_add((267_000 as Weight).saturating_mul(c as Weight)) + (35_384_000 as Weight) + // Standard Error: 0 + .saturating_add((124_000 as Weight).saturating_mul(c as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Elections Candidates (r:1 w:1) fn renounce_candidacy_candidate(c: u32, ) -> Weight { - (49_128_000 as Weight) + (31_555_000 as Weight) // Standard Error: 1_000 - .saturating_add((144_000 as Weight).saturating_mul(c as Weight)) + .saturating_add((78_000 as Weight).saturating_mul(c as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Elections Members (r:1 w:1) // Storage: Elections RunnersUp (r:1 w:1) - // Storage: Instance1Collective Prime (r:1 w:1) - // Storage: Instance1Collective Proposals (r:1 w:0) - // Storage: Instance1Collective Members (r:0 w:1) + // Storage: Council Prime (r:1 w:1) + // Storage: Council Proposals (r:1 w:0) + // Storage: Council Members (r:0 w:1) fn renounce_candidacy_members() -> Weight { - (70_685_000 as Weight) + (41_531_000 as Weight) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } // Storage: Elections RunnersUp (r:1 w:1) fn renounce_candidacy_runners_up() -> Weight { - (49_766_000 as Weight) + (30_762_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: Benchmark Override (r:0 w:0) + fn remove_member_without_replacement() -> Weight { + (2_000_000_000_000 as Weight) + } // Storage: Elections RunnersUp (r:1 w:1) // Storage: Elections Members (r:1 w:1) // Storage: System Account (r:1 w:1) - // Storage: Instance1Collective Prime (r:1 w:1) - // Storage: Instance1Collective Proposals (r:1 w:0) - // Storage: Instance1Collective Members (r:0 w:1) + // Storage: Council Prime (r:1 w:1) + // Storage: Council Proposals (r:1 w:0) + // Storage: Council Members (r:0 w:1) fn remove_member_with_replacement() -> Weight { - (76_153_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(5 as Weight)) - .saturating_add(RocksDbWeight::get().writes(5 as Weight)) - } - fn remove_member_without_replacement() -> Weight { - (76_153_000 as Weight) + (48_287_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } // Storage: Elections RunnersUp (r:1 w:0) fn remove_member_wrong_refund() -> Weight { - (6_697_000 as Weight) + (4_747_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) } // Storage: Elections Voting (r:251 w:250) @@ -304,8 +303,8 @@ impl WeightInfo for () { // Storage: System Account (r:250 w:250) fn clean_defunct_voters(v: u32, _d: u32, ) -> Weight { (0 as Weight) - // Standard Error: 60_000 - .saturating_add((107_467_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 37_000 + .saturating_add((49_564_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(v as Weight))) @@ -314,19 +313,19 @@ impl WeightInfo for () { // Storage: Elections Members (r:1 w:1) // Storage: Elections RunnersUp (r:1 w:1) // Storage: Elections Voting (r:502 w:0) - // Storage: Instance1Collective Proposals (r:1 w:0) + // Storage: Council Proposals (r:1 w:0) // Storage: Elections ElectionRounds (r:1 w:1) - // Storage: Instance1Collective Members (r:0 w:1) - // Storage: Instance1Collective Prime (r:0 w:1) + // Storage: Council Members (r:0 w:1) + // Storage: Council Prime (r:0 w:1) // Storage: System Account (r:2 w:2) fn election_phragmen(c: u32, v: u32, e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 1_846_000 - .saturating_add((39_843_000 as Weight).saturating_mul(c as Weight)) - // Standard Error: 768_000 - .saturating_add((60_623_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 52_000 - .saturating_add((3_884_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 1_656_000 + .saturating_add((29_011_000 as Weight).saturating_mul(c as Weight)) + // Standard Error: 689_000 + .saturating_add((49_204_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 47_000 + .saturating_add((3_352_000 as Weight).saturating_mul(e as Weight)) .saturating_add(RocksDbWeight::get().reads((2 as Weight).saturating_mul(c as Weight))) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(c as Weight))) diff --git a/frame/elections/Cargo.toml b/frame/elections/Cargo.toml deleted file mode 100644 index 8557cfba6b58..000000000000 --- a/frame/elections/Cargo.toml +++ /dev/null @@ -1,42 +0,0 @@ -[package] -name = "pallet-elections" -version = "4.0.0-dev" -authors = ["Parity Technologies "] -edition = "2018" -license = "Apache-2.0" -homepage = "https://substrate.dev" -repository = "https://github.com/paritytech/substrate/" -description = "FRAME pallet for elections" -readme = "README.md" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ - "derive", -] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } -frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } - -[dev-dependencies] -pallet-balances = { version = "4.0.0-dev", path = "../balances" } - -[features] -default = ["std"] -std = [ - "codec/std", - "scale-info/std", - "sp-core/std", - "sp-std/std", - "sp-io/std", - "frame-support/std", - "sp-runtime/std", - "frame-system/std", -] -try-runtime = ["frame-support/try-runtime"] diff --git a/frame/elections/README.md b/frame/elections/README.md deleted file mode 100644 index 1f6fd42331c1..000000000000 --- a/frame/elections/README.md +++ /dev/null @@ -1,7 +0,0 @@ -Election module for stake-weighted membership selection of a collective. - -The composition of a set of account IDs works according to one or more approval votes -weighted by stake. There is a partial carry-over facility to give greater weight to those -whose voting is serially unsuccessful. - -License: Apache-2.0 \ No newline at end of file diff --git a/frame/elections/src/lib.rs b/frame/elections/src/lib.rs deleted file mode 100644 index ac13bce31b0f..000000000000 --- a/frame/elections/src/lib.rs +++ /dev/null @@ -1,1325 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! # WARNING: NOT ACTIVELY MAINTAINED -//! -//! This pallet is currently not maintained and should not be used in production until further -//! notice. -//! -//! --- -//! -//! Election pallet for stake-weighted membership selection of a collective. -//! -//! The composition of a set of account IDs works according to one or more approval votes -//! weighted by stake. There is a partial carry-over facility to give greater weight to those -//! whose voting is serially unsuccessful. - -#![cfg_attr(not(feature = "std"), no_std)] -#![recursion_limit = "128"] - -use codec::{Decode, Encode}; -use frame_support::{ - ensure, - pallet_prelude::*, - traits::{ - BalanceStatus, ChangeMembers, Currency, ExistenceRequirement, LockIdentifier, - LockableCurrency, OnUnbalanced, ReservableCurrency, WithdrawReasons, - }, - weights::{DispatchClass, Weight}, -}; -use frame_system::pallet_prelude::*; -pub use pallet::*; -use sp_runtime::{ - print, - traits::{One, Saturating, StaticLookup, Zero}, - RuntimeDebug, -}; -use sp_std::prelude::*; - -mod mock; -mod tests; - -// no polynomial attacks: -// -// all unbonded public operations should be constant time. -// all other public operations must be linear time in terms of prior public operations and: -// - those "valid" ones that cost nothing be limited to a constant number per single protected -// operation -// - the rest costing the same order as the computational complexity -// all protected operations must complete in at most O(public operations) -// -// we assume "beneficial" transactions will have the same access as attack transactions. -// -// any storage requirements should be bonded by the same order as the volume. - -// public operations: -// - express approvals (you pay in a "voter" bond the first time you do this; O(1); one extra DB -// entry, one DB change) -// - remove active voter (you get your "voter" bond back; O(1); one fewer DB entry, one DB change) -// - remove inactive voter (either you or the target is removed; if the target, you get their -// "voter" bond back; O(1); one fewer DB entry, one DB change) -// - submit candidacy (you pay a "candidate" bond; O(1); one extra DB entry, two DB changes) -// - present winner/runner-up (you may pay a "presentation" bond of O(voters) if the presentation is -// invalid; O(voters) compute; ) protected operations: -// - remove candidacy (remove all votes for a candidate) (one fewer DB entry, two DB changes) - -// to avoid a potentially problematic case of not-enough approvals prior to voting causing a -// back-to-back votes that have no way of ending, then there's a forced grace period between votes. -// to keep the system as stateless as possible (making it a bit easier to reason about), we just -// restrict when votes can begin to blocks that lie on boundaries (`voting_period`). - -// for an approval vote of C members: - -// top K runners-up are maintained between votes. all others are discarded. -// - candidate removed & bond returned when elected. -// - candidate removed & bond burned when discarded. - -// at the point that the vote ends (), all voters' balances are snapshotted. - -// for B blocks following, there's a counting period whereby each of the candidates that believe -// they fall in the top K+C voted can present themselves. they get the total stake -// recorded (based on the snapshot); an ordered list is maintained (the leaderboard). No one may -// present themselves that, if elected, would result in being included twice in the collective -// (important since existing members will have their approval votes as it may be that they -// don't get removed), nor if existing presenters would mean they're not in the top K+C. - -// following B blocks, the top C candidates are elected and have their bond returned. the top C -// candidates and all other candidates beyond the top C+K are cleared. - -// vote-clearing happens lazily; for an approval to count, the most recent vote at the time of the -// voter's most recent vote must be no later than the most recent vote at the time that the -// candidate in the approval position was registered there. as candidates are removed from the -// register and others join in their place, this prevents an approval meant for an earlier candidate -// being used to elect a new candidate. - -// the candidate list increases as needed, but the contents (though not really the capacity) reduce -// after each vote as all but K entries are cleared. newly registering candidates must use cleared -// entries before they increase the capacity. - -/// The activity status of a voter. -#[derive( - PartialEq, Eq, Copy, Clone, Encode, Decode, Default, RuntimeDebug, scale_info::TypeInfo, -)] -pub struct VoterInfo { - /// Last VoteIndex in which this voter assigned (or initialized) approvals. - last_active: VoteIndex, - /// Last VoteIndex in which one of this voter's approvals won. - /// Note that `last_win = N` indicates a last win at index `N-1`, hence `last_win = 0` means no - /// win ever. - last_win: VoteIndex, - /// The amount of stored weight as a result of not winning but changing approvals. - pot: Balance, - /// Current staked amount. A lock equal to this value always exists. - stake: Balance, -} - -/// Used to demonstrate the status of a particular index in the global voter list. -#[derive(PartialEq, Eq, RuntimeDebug)] -pub enum CellStatus { - /// Any out of bound index. Means a push a must happen to the chunk pointed by - /// `NextVoterSet`. Voting fee is applied in case a new chunk is created. - Head, - /// Already occupied by another voter. Voting fee is applied. - Occupied, - /// Empty hole which should be filled. No fee will be applied. - Hole, -} - -/// Number of voters grouped in one chunk. -pub const VOTER_SET_SIZE: usize = 64; -/// NUmber of approvals grouped in one chunk. -pub const APPROVAL_SET_SIZE: usize = 8; - -type BalanceOf = - <::Currency as Currency<::AccountId>>::Balance; -type NegativeImbalanceOf = <::Currency as Currency< - ::AccountId, ->>::NegativeImbalance; - -/// Index used to access chunks. -type SetIndex = u32; -/// Index used to count voting rounds. -pub type VoteIndex = u32; -/// Underlying data type of the approvals. -type ApprovalFlag = u32; -/// Number of approval flags that can fit into [`ApprovalFlag`] type. -const APPROVAL_FLAG_LEN: usize = 32; - -#[frame_support::pallet] -pub mod pallet { - use super::*; - - #[pallet::pallet] - #[pallet::generate_store(pub(super) trait Store)] - pub struct Pallet(_); - - #[pallet::config] - pub trait Config: frame_system::Config { - type Event: From> + IsType<::Event>; - - /// Identifier for the elections pallet's lock - #[pallet::constant] - type PalletId: Get; - - /// The currency that people are electing with. - type Currency: LockableCurrency - + ReservableCurrency; - - /// Handler for the unbalanced reduction when slashing a validator. - type BadPresentation: OnUnbalanced>; - - /// Handler for the unbalanced reduction when slashing an invalid reaping attempt. - type BadReaper: OnUnbalanced>; - - /// Handler for the unbalanced reduction when submitting a bad `voter_index`. - type BadVoterIndex: OnUnbalanced>; - - /// Handler for the unbalanced reduction when a candidate has lost (and is not a runner up) - type LoserCandidate: OnUnbalanced>; - - /// What to do when the members change. - type ChangeMembers: ChangeMembers; - - /// How much should be locked up in order to submit one's candidacy. A reasonable - /// default value is 9. - #[pallet::constant] - type CandidacyBond: Get>; - - /// How much should be locked up in order to be able to submit votes. - #[pallet::constant] - type VotingBond: Get>; - - /// The amount of fee paid upon each vote submission, unless if they submit a - /// _hole_ index and replace it. - #[pallet::constant] - type VotingFee: Get>; - - /// Minimum about that can be used as the locked value for voting. - #[pallet::constant] - type MinimumVotingLock: Get>; - - /// The punishment, per voter, if you provide an invalid presentation. A - /// reasonable default value is 1. - #[pallet::constant] - type PresentSlashPerVoter: Get>; - - /// How many runners-up should have their approvals persist until the next - /// vote. A reasonable default value is 2. - #[pallet::constant] - type CarryCount: Get; - - /// How many vote indices need to go by after a target voter's last vote before - /// they can be reaped if their approvals are moot. A reasonable default value - /// is 1. - #[pallet::constant] - type InactiveGracePeriod: Get; - - /// How often (in blocks) to check for new votes. A reasonable default value - /// is 1000. - #[pallet::constant] - type VotingPeriod: Get; - - /// Decay factor of weight when being accumulated. It should typically be set to - /// __at least__ `membership_size -1` to keep the collective secure. - /// When set to `N`, it indicates `(1/N)^t` of staked is decayed at weight - /// increment step `t`. 0 will result in no weight being added at all (normal - /// approval voting). A reasonable default value is 24. - #[pallet::constant] - type DecayRatio: Get; - } - - #[pallet::extra_constants] - impl Pallet { - // TODO: rename to snake case after https://github.com/paritytech/substrate/issues/8826 fixed. - /// The chunk size of the voter vector. - #[allow(non_snake_case)] - fn VOTER_SET_SIZE() -> u32 { - VOTER_SET_SIZE as u32 - } - - // TODO: rename to snake case after https://github.com/paritytech/substrate/issues/8826 fixed. - /// The chunk size of the approval vector. - #[allow(non_snake_case)] - fn APPROVAL_SET_SIZE() -> u32 { - APPROVAL_SET_SIZE as u32 - } - } - - // ---- permanent state (always relevant, changes only at the finalization of voting) - - /// How long to give each top candidate to present themselves after the vote ends. - #[pallet::storage] - #[pallet::getter(fn presentation_duration)] - pub type PresentationDuration = StorageValue<_, T::BlockNumber, ValueQuery>; - - /// How long each position is active for. - #[pallet::storage] - #[pallet::getter(fn term_duration)] - pub type TermDuration = StorageValue<_, T::BlockNumber, ValueQuery>; - - /// Number of accounts that should constitute the collective. - #[pallet::storage] - #[pallet::getter(fn desired_seats)] - pub type DesiredSeats = StorageValue<_, u32, ValueQuery>; - - // ---- permanent state (always relevant, changes only at the finalization of voting) - - /// The current membership. When there's a vote going on, this should still be used for - /// executive matters. The block number (second element in the tuple) is the block that - /// their position is active until (calculated by the sum of the block number when the - /// member was elected and their term duration). - #[pallet::storage] - #[pallet::getter(fn members)] - pub type Members = StorageValue<_, Vec<(T::AccountId, T::BlockNumber)>, ValueQuery>; - - /// The total number of vote rounds that have happened or are in progress. - #[pallet::storage] - #[pallet::getter(fn vote_index)] - pub type VoteCount = StorageValue<_, VoteIndex, ValueQuery>; - - // ---- persistent state (always relevant, changes constantly) - - // A list of votes for each voter. The votes are stored as numeric values and parsed in a - // bit-wise manner. In order to get a human-readable representation (`Vec`), use - // [`all_approvals_of`]. Furthermore, each vector of scalars is chunked with the cap of - // `APPROVAL_SET_SIZE`. - /// TWOX-NOTE: SAFE as `AccountId` is a crypto hash and `SetIndex` is not - /// attacker-controlled. - #[pallet::storage] - #[pallet::getter(fn approvals_of)] - pub type ApprovalsOf = - StorageMap<_, Twox64Concat, (T::AccountId, SetIndex), Vec, ValueQuery>; - - /// The vote index and list slot that the candidate `who` was registered or `None` if they - /// are not currently registered. - /// - /// TWOX-NOTE: SAFE as `AccountId` is a crypto hash. - #[pallet::storage] - #[pallet::getter(fn candidate_reg_info)] - pub type RegisterInfoOf = - StorageMap<_, Twox64Concat, T::AccountId, (VoteIndex, u32)>; - - /// Basic information about a voter. - /// - /// TWOX-NOTE: SAFE as `AccountId` is a crypto hash. - #[pallet::storage] - #[pallet::getter(fn voter_info)] - pub type VoterInfoOf = - StorageMap<_, Twox64Concat, T::AccountId, VoterInfo>>; - - /// The present voter list (chunked and capped at [`VOTER_SET_SIZE`]). - /// - /// TWOX-NOTE: OKAY ― `SetIndex` is not user-controlled data. - #[pallet::storage] - #[pallet::getter(fn voters)] - pub type Voters = - StorageMap<_, Twox64Concat, SetIndex, Vec>, ValueQuery>; - - /// the next free set to store a voter in. This will keep growing. - #[pallet::storage] - #[pallet::getter(fn next_nonfull_voter_set)] - pub type NextVoterSet = StorageValue<_, SetIndex, ValueQuery>; - - /// Current number of Voters. - #[pallet::storage] - #[pallet::getter(fn voter_count)] - pub type VoterCount = StorageValue<_, SetIndex, ValueQuery>; - - /// The present candidate list. - #[pallet::storage] - #[pallet::getter(fn candidates)] - pub type Candidates = StorageValue<_, Vec, ValueQuery>; // has holes - - /// Current number of active candidates - #[pallet::storage] - #[pallet::getter(fn candidate_count)] - pub type CandidateCount = StorageValue<_, u32, ValueQuery>; - - // ---- temporary state (only relevant during finalization/presentation) - - /// The accounts holding the seats that will become free on the next tally. - #[pallet::storage] - #[pallet::getter(fn next_finalize)] - pub type NextFinalize = StorageValue<_, (T::BlockNumber, u32, Vec)>; - - /// Get the leaderboard if we're in the presentation phase. The first element is the weight - /// of each entry; It may be the direct summed approval stakes, or a weighted version of it. - /// Sorted from low to high. - #[pallet::storage] - #[pallet::getter(fn leaderboard)] - pub type Leaderboard = StorageValue<_, Vec<(BalanceOf, T::AccountId)>>; - - #[pallet::genesis_config] - pub struct GenesisConfig { - pub presentation_duration: T::BlockNumber, - pub term_duration: T::BlockNumber, - pub desired_seats: u32, - pub members: Vec<(T::AccountId, T::BlockNumber)>, - } - - #[cfg(feature = "std")] - impl Default for GenesisConfig { - fn default() -> Self { - GenesisConfig { - presentation_duration: Default::default(), - term_duration: Default::default(), - desired_seats: Default::default(), - members: Default::default(), - } - } - } - - #[pallet::genesis_build] - impl GenesisBuild for GenesisConfig { - fn build(&self) { - PresentationDuration::::put(self.presentation_duration); - TermDuration::::put(self.term_duration); - DesiredSeats::::put(self.desired_seats); - Members::::put(&self.members); - } - } - - #[pallet::error] - pub enum Error { - /// Reporter must be a voter. - NotVoter, - /// Target for inactivity cleanup must be active. - InactiveTarget, - /// Cannot reap during presentation period. - CannotReapPresenting, - /// Cannot reap during grace period. - ReapGrace, - /// Invalid reporter index. - InvalidReporterIndex, - /// Invalid target index. - InvalidTargetIndex, - /// Invalid vote index. - InvalidVoteIndex, - /// Cannot retract when presenting. - CannotRetractPresenting, - /// Cannot retract non-voter. - RetractNonVoter, - /// Invalid retraction index. - InvalidRetractionIndex, - /// Duplicate candidate submission. - DuplicatedCandidate, - /// Invalid candidate slot. - InvalidCandidateSlot, - /// Candidate has not enough funds. - InsufficientCandidateFunds, - /// Presenter must have sufficient slashable funds. - InsufficientPresenterFunds, - /// Stake deposited to present winner and be added to leaderboard should be non-zero. - ZeroDeposit, - /// Candidate not worthy of leaderboard. - UnworthyCandidate, - /// Leaderboard must exist while present phase active. - LeaderboardMustExist, - /// Cannot present outside of presentation period. - NotPresentationPeriod, - /// Presented candidate must be current. - InvalidCandidate, - /// Duplicated presentation. - DuplicatedPresentation, - /// Incorrect total. - IncorrectTotal, - /// Invalid voter index. - InvalidVoterIndex, - /// New voter must have sufficient funds to pay the bond. - InsufficientVoterFunds, - /// Locked value must be more than limit. - InsufficientLockedValue, - /// Amount of candidate votes cannot exceed amount of candidates. - TooManyVotes, - /// Amount of candidates to receive approval votes should be non-zero. - ZeroCandidates, - /// No approval changes during presentation period. - ApprovalPresentation, - } - - #[pallet::hooks] - impl Hooks> for Pallet { - fn on_initialize(n: T::BlockNumber) -> Weight { - if let Err(e) = Self::end_block(n) { - print("Guru meditation"); - print(e); - } - 0 - } - } - - #[pallet::event] - #[pallet::generate_deposit(pub(super) fn deposit_event)] - pub enum Event { - /// Reaped \[voter, reaper\]. - VoterReaped(T::AccountId, T::AccountId), - /// Slashed \[reaper\]. - BadReaperSlashed(T::AccountId), - /// A tally (for approval votes of \[seats\]) has started. - TallyStarted(u32), - /// A tally (for approval votes of seat(s)) has ended (with one or more new members). - /// \[incoming, outgoing\] - TallyFinalized(Vec, Vec), - } - - #[pallet::call] - impl Pallet { - /// Set candidate approvals. Approval slots stay valid as long as candidates in those slots - /// are registered. - /// - /// Locks `value` from the balance of `origin` indefinitely. Only - /// [`retract_voter`](Self::retract_voter) or - /// [`reap_inactive_voter`](Self::reap_inactive_voter) can unlock the balance. - /// - /// `hint` argument is interpreted differently based on: - /// - if `origin` is setting approvals for the first time: The index will be checked for - /// being a valid _hole_ in the voter list. - /// - if the hint is correctly pointing to a hole, no fee is deducted from `origin`. - /// - Otherwise, the call will succeed but the index is ignored and simply a push to the - /// last chunk with free space happens. If the new push causes a new chunk to be - /// created, a fee indicated by [`Config::VotingFee`] is deducted. - /// - if `origin` is already a voter: the index __must__ be valid and point to the correct - /// position of the `origin` in the current voters list. - /// - /// Note that any trailing `false` votes in `votes` is ignored; In approval voting, not - /// voting for a candidate and voting false, are equal. - /// - /// # - /// - O(1). - /// - Two extra DB entries, one DB change. - /// - Argument `votes` is limited in length to number of candidates. - /// # - #[pallet::weight(2_500_000_000)] - pub fn set_approvals( - origin: OriginFor, - votes: Vec, - #[pallet::compact] index: VoteIndex, - hint: SetIndex, - #[pallet::compact] value: BalanceOf, - ) -> DispatchResult { - let who = ensure_signed(origin)?; - Self::do_set_approvals(who, votes, index, hint, value) - } - - /// Remove a voter. For it not to be a bond-consuming no-op, all approved candidate indices - /// must now be either unregistered or registered to a candidate that registered the slot - /// after the voter gave their last approval set. - /// - /// Both indices must be provided according to the following principle: - /// Voter index does not take holes into account. This means that any account submitting an - /// index at any point in time should submit: - /// `VOTER_SET_SIZE * set_index + local_index`, meaning that you are ignoring all holes in - /// the first `set_index` sets. - /// - /// May be called by anyone. Returns the voter deposit to `signed`. - /// - /// # - /// - O(1). - /// - Two fewer DB entries, one DB change. - /// # - #[pallet::weight(2_500_000_000)] - pub fn reap_inactive_voter( - origin: OriginFor, - #[pallet::compact] reporter_index: u32, - who: ::Source, - #[pallet::compact] who_index: u32, - #[pallet::compact] assumed_vote_index: VoteIndex, - ) -> DispatchResult { - let reporter = ensure_signed(origin)?; - let who = T::Lookup::lookup(who)?; - - ensure!(!Self::presentation_active(), Error::::CannotReapPresenting); - ensure!(Self::voter_info(&reporter).is_some(), Error::::NotVoter); - - let info = Self::voter_info(&who).ok_or(Error::::InactiveTarget)?; - let last_active = info.last_active; - - ensure!(assumed_vote_index == Self::vote_index(), Error::::InvalidVoteIndex); - ensure!( - assumed_vote_index > last_active + T::InactiveGracePeriod::get(), - Error::::ReapGrace, - ); - - let reporter_index = reporter_index as usize; - let who_index = who_index as usize; - let assumed_reporter = - Self::voter_at(reporter_index).ok_or(Error::::InvalidReporterIndex)?; - let assumed_who = Self::voter_at(who_index).ok_or(Error::::InvalidTargetIndex)?; - - ensure!(assumed_reporter == reporter, Error::::InvalidReporterIndex); - ensure!(assumed_who == who, Error::::InvalidTargetIndex); - - // will definitely kill one of reporter or who now. - - let valid = !Self::all_approvals_of(&who).iter().zip(Self::candidates().iter()).any( - |(&appr, addr)| { - appr && - *addr != T::AccountId::default() && - // defensive only: all items in candidates list are registered - Self::candidate_reg_info(addr).map_or(false, |x| x.0 <= last_active) - }, - ); - - Self::remove_voter( - if valid { &who } else { &reporter }, - if valid { who_index } else { reporter_index }, - ); - - T::Currency::remove_lock(T::PalletId::get(), if valid { &who } else { &reporter }); - - if valid { - // This only fails if `reporter` doesn't exist, which it clearly must do since its - // the origin. Still, it's no more harmful to propagate any error at this point. - T::Currency::repatriate_reserved( - &who, - &reporter, - T::VotingBond::get(), - BalanceStatus::Free, - )?; - Self::deposit_event(Event::::VoterReaped(who, reporter)); - } else { - let imbalance = T::Currency::slash_reserved(&reporter, T::VotingBond::get()).0; - T::BadReaper::on_unbalanced(imbalance); - Self::deposit_event(Event::::BadReaperSlashed(reporter)); - } - Ok(()) - } - - /// Remove a voter. All votes are cancelled and the voter deposit is returned. - /// - /// The index must be provided according to the following principle: - /// Voter index does not take holes into account. This means that any account submitting an - /// index at any point in time should submit: - /// `VOTER_SET_SIZE * set_index + local_index`, meaning that you are ignoring all holes in - /// the first `set_index` sets. - /// - /// Also removes the lock on the balance of the voter. - /// - /// # - /// - O(1). - /// - Two fewer DB entries, one DB change. - /// # - #[pallet::weight(1_250_000_000)] - pub fn retract_voter( - origin: OriginFor, - #[pallet::compact] index: u32, - ) -> DispatchResult { - let who = ensure_signed(origin)?; - - ensure!(!Self::presentation_active(), Error::::CannotRetractPresenting); - ensure!(>::contains_key(&who), Error::::RetractNonVoter); - let index = index as usize; - let voter = Self::voter_at(index).ok_or(Error::::InvalidRetractionIndex)?; - ensure!(voter == who, Error::::InvalidRetractionIndex); - - Self::remove_voter(&who, index); - T::Currency::unreserve(&who, T::VotingBond::get()); - T::Currency::remove_lock(T::PalletId::get(), &who); - Ok(()) - } - - /// Submit oneself for candidacy. - /// - /// Account must have enough transferrable funds in it to pay the bond. - /// - /// NOTE: if `origin` has already assigned approvals via - /// [`set_approvals`](Self::set_approvals), it will NOT have any usable funds to pass - /// candidacy bond and must first retract. - /// Note that setting approvals will lock the entire balance of the voter until - /// retraction or being reported. - /// - /// # - /// - Independent of input. - /// - Three DB changes. - /// # - #[pallet::weight(2_500_000_000)] - pub fn submit_candidacy( - origin: OriginFor, - #[pallet::compact] slot: u32, - ) -> DispatchResult { - let who = ensure_signed(origin)?; - - ensure!(!Self::is_a_candidate(&who), Error::::DuplicatedCandidate); - let slot = slot as usize; - let count = Self::candidate_count() as usize; - let candidates = Self::candidates(); - ensure!( - (slot == count && count == candidates.len()) || - (slot < candidates.len() && candidates[slot] == T::AccountId::default()), - Error::::InvalidCandidateSlot, - ); - // NOTE: This must be last as it has side-effects. - T::Currency::reserve(&who, T::CandidacyBond::get()) - .map_err(|_| Error::::InsufficientCandidateFunds)?; - - >::insert(&who, (Self::vote_index(), slot as u32)); - let mut candidates = candidates; - if slot == candidates.len() { - candidates.push(who); - } else { - candidates[slot] = who; - } - >::put(candidates); - CandidateCount::::put(count as u32 + 1); - Ok(()) - } - - /// Claim that `candidate` is one of the top `carry_count + desired_seats` candidates. Only - /// works iff the presentation period is active. `candidate` should have at least collected - /// some non-zero `total` votes and `origin` must have enough funds to pay for a potential - /// slash. - /// - /// # - /// - O(voters) compute. - /// - One DB change. - /// # - #[pallet::weight(10_000_000_000)] - pub fn present_winner( - origin: OriginFor, - candidate: ::Source, - #[pallet::compact] total: BalanceOf, - #[pallet::compact] index: VoteIndex, - ) -> DispatchResult { - let who = ensure_signed(origin)?; - ensure!(!total.is_zero(), Error::::ZeroDeposit); - - let candidate = T::Lookup::lookup(candidate)?; - ensure!(index == Self::vote_index(), Error::::InvalidVoteIndex); - let (_, _, expiring) = - Self::next_finalize().ok_or(Error::::NotPresentationPeriod)?; - let bad_presentation_punishment = - T::PresentSlashPerVoter::get() * BalanceOf::::from(Self::voter_count() as u32); - ensure!( - T::Currency::can_slash(&who, bad_presentation_punishment), - Error::::InsufficientPresenterFunds, - ); - - let mut leaderboard = Self::leaderboard().ok_or(Error::::LeaderboardMustExist)?; - ensure!(total > leaderboard[0].0, Error::::UnworthyCandidate); - - if let Some(p) = Self::members().iter().position(|&(ref c, _)| c == &candidate) { - ensure!(p < expiring.len(), Error::::DuplicatedCandidate); - } - - let voters = Self::all_voters(); - let (registered_since, candidate_index): (VoteIndex, u32) = - Self::candidate_reg_info(&candidate).ok_or(Error::::InvalidCandidate)?; - let actual_total = voters - .iter() - .filter_map(|maybe_voter| maybe_voter.as_ref()) - .filter_map(|voter| match Self::voter_info(voter) { - Some(b) if b.last_active >= registered_since => { - let last_win = b.last_win; - let now = Self::vote_index(); - let stake = b.stake; - let offset = Self::get_offset(stake, now - last_win); - let weight = stake + offset + b.pot; - if Self::approvals_of_at(voter, candidate_index as usize) { - Some(weight) - } else { - None - } - }, - _ => None, - }) - .fold(Zero::zero(), |acc, n| acc + n); - let dupe = leaderboard.iter().find(|&&(_, ref c)| c == &candidate).is_some(); - if total == actual_total && !dupe { - // insert into leaderboard - leaderboard[0] = (total, candidate); - leaderboard.sort_by_key(|&(t, _)| t); - >::put(leaderboard); - Ok(()) - } else { - // we can rest assured it will be Ok since we checked `can_slash` earlier; still - // better safe than sorry. - let imbalance = T::Currency::slash(&who, bad_presentation_punishment).0; - T::BadPresentation::on_unbalanced(imbalance); - Err(if dupe { - Error::::DuplicatedPresentation - } else { - Error::::IncorrectTotal - })? - } - } - - /// Set the desired member count; if lower than the current count, then seats will not be up - /// election when they expire. If more, then a new vote will be started if one is not - /// already in progress. - #[pallet::weight((0, DispatchClass::Operational))] - pub fn set_desired_seats( - origin: OriginFor, - #[pallet::compact] count: u32, - ) -> DispatchResult { - ensure_root(origin)?; - DesiredSeats::::put(count); - Ok(()) - } - - /// Remove a particular member from the set. This is effective immediately. - /// - /// Note: A tally should happen instantly (if not already in a presentation - /// period) to fill the seat if removal means that the desired members are not met. - #[pallet::weight((0, DispatchClass::Operational))] - pub fn remove_member( - origin: OriginFor, - who: ::Source, - ) -> DispatchResult { - ensure_root(origin)?; - let who = T::Lookup::lookup(who)?; - let new_set: Vec<(T::AccountId, T::BlockNumber)> = - Self::members().into_iter().filter(|i| i.0 != who).collect(); - >::put(&new_set); - let new_set = new_set.into_iter().map(|x| x.0).collect::>(); - T::ChangeMembers::change_members(&[], &[who], new_set); - Ok(()) - } - - /// Set the presentation duration. If there is currently a vote being presented for, will - /// invoke `finalize_vote`. - #[pallet::weight((0, DispatchClass::Operational))] - pub fn set_presentation_duration( - origin: OriginFor, - #[pallet::compact] count: T::BlockNumber, - ) -> DispatchResult { - ensure_root(origin)?; - >::put(count); - Ok(()) - } - - /// Set the presentation duration. If there is current a vote being presented for, will - /// invoke `finalize_vote`. - #[pallet::weight((0, DispatchClass::Operational))] - pub fn set_term_duration( - origin: OriginFor, - #[pallet::compact] count: T::BlockNumber, - ) -> DispatchResult { - ensure_root(origin)?; - >::put(count); - Ok(()) - } - } -} - -impl Pallet { - // exposed immutables. - - /// True if we're currently in a presentation period. - pub fn presentation_active() -> bool { - >::exists() - } - - /// If `who` a candidate at the moment? - pub fn is_a_candidate(who: &T::AccountId) -> bool { - >::contains_key(who) - } - - /// Iff the member `who` still has a seat at blocknumber `n` returns `true`. - pub fn will_still_be_member_at(who: &T::AccountId, n: T::BlockNumber) -> bool { - Self::members() - .iter() - .find(|&&(ref a, _)| a == who) - .map(|&(_, expires)| expires > n) - .unwrap_or(false) - } - - /// Determine the block that a vote can happen on which is no less than `n`. - pub fn next_vote_from(n: T::BlockNumber) -> T::BlockNumber { - let voting_period = T::VotingPeriod::get(); - (n + voting_period - One::one()) / voting_period * voting_period - } - - /// The block number on which the tally for the next election will happen. `None` only if the - /// desired seats of the set is zero. - pub fn next_tally() -> Option { - let desired_seats = Self::desired_seats(); - if desired_seats == 0 { - None - } else { - let c = Self::members(); - let (next_possible, count, coming) = if let Some((tally_end, comers, leavers)) = - Self::next_finalize() - { - // if there's a tally in progress, then next tally can begin immediately afterwards - (tally_end, c.len() - leavers.len() + comers as usize, comers) - } else { - (>::block_number(), c.len(), 0) - }; - if count < desired_seats as usize { - Some(next_possible) - } else { - // next tally begins once enough members expire to bring members below desired. - if desired_seats <= coming { - // the entire amount of desired seats is less than those new members - we'll - // have to wait until they expire. - Some(next_possible + Self::term_duration()) - } else { - Some(c[c.len() - (desired_seats - coming) as usize].1) - } - } - .map(Self::next_vote_from) - } - } - - // Private - /// Check there's nothing to do this block - fn end_block(block_number: T::BlockNumber) -> DispatchResult { - if (block_number % T::VotingPeriod::get()).is_zero() { - if let Some(number) = Self::next_tally() { - if block_number == number { - Self::start_tally(); - } - } - } - if let Some((number, _, _)) = Self::next_finalize() { - if block_number == number { - Self::finalize_tally()? - } - } - Ok(()) - } - - /// Remove a voter at a specified index from the system. - fn remove_voter(voter: &T::AccountId, index: usize) { - let (set_index, vec_index) = Self::split_index(index, VOTER_SET_SIZE); - let mut set = Self::voters(set_index); - set[vec_index] = None; - >::insert(set_index, set); - VoterCount::::mutate(|c| *c = *c - 1); - Self::remove_all_approvals_of(voter); - >::remove(voter); - } - - /// Actually do the voting. - /// - /// The voter index must be provided as explained in [`voter_at`] function. - fn do_set_approvals( - who: T::AccountId, - votes: Vec, - index: VoteIndex, - hint: SetIndex, - value: BalanceOf, - ) -> DispatchResult { - let candidates_len = ::Candidates::decode_len().unwrap_or(0_usize); - - ensure!(!Self::presentation_active(), Error::::ApprovalPresentation); - ensure!(index == Self::vote_index(), Error::::InvalidVoteIndex); - ensure!(!candidates_len.is_zero(), Error::::ZeroCandidates); - // Prevent a vote from voters that provide a list of votes that exceeds the candidates - // length since otherwise an attacker may be able to submit a very long list of `votes` that - // far exceeds the amount of candidates and waste more computation than a reasonable voting - // bond would cover. - ensure!(candidates_len >= votes.len(), Error::::TooManyVotes); - ensure!(value >= T::MinimumVotingLock::get(), Error::::InsufficientLockedValue); - - // Amount to be locked up. - let mut locked_balance = value.min(T::Currency::total_balance(&who)); - let mut pot_to_set = Zero::zero(); - let hint = hint as usize; - - if let Some(info) = Self::voter_info(&who) { - // already a voter. Index must be valid. No fee. update pot. O(1) - let voter = Self::voter_at(hint).ok_or(Error::::InvalidVoterIndex)?; - ensure!(voter == who, Error::::InvalidVoterIndex); - - // write new accumulated offset. - let last_win = info.last_win; - let now = index; - let offset = Self::get_offset(info.stake, now - last_win); - pot_to_set = info.pot + offset; - } else { - // not yet a voter. Index _could be valid_. Fee might apply. Bond will be reserved O(1). - ensure!( - T::Currency::free_balance(&who) > T::VotingBond::get(), - Error::::InsufficientVoterFunds, - ); - - let (set_index, vec_index) = Self::split_index(hint, VOTER_SET_SIZE); - match Self::cell_status(set_index, vec_index) { - CellStatus::Hole => { - // requested cell was a valid hole. - >::mutate(set_index, |set| set[vec_index] = Some(who.clone())); - }, - CellStatus::Head | CellStatus::Occupied => { - // Either occupied or out-of-range. - let next = Self::next_nonfull_voter_set(); - let set_len = >::decode_len(next).unwrap_or(0_usize); - // Caused a new set to be created. Pay for it. - // This is the last potential error. Writes will begin afterwards. - if set_len == 0 { - let imbalance = T::Currency::withdraw( - &who, - T::VotingFee::get(), - WithdrawReasons::FEE, - ExistenceRequirement::KeepAlive, - )?; - T::BadVoterIndex::on_unbalanced(imbalance); - // NOTE: this is safe since the `withdraw()` will check this. - locked_balance -= T::VotingFee::get(); - } - if set_len + 1 == VOTER_SET_SIZE { - NextVoterSet::::put(next + 1); - } - >::append(next, Some(who.clone())); - }, - } - - T::Currency::reserve(&who, T::VotingBond::get())?; - VoterCount::::mutate(|c| *c = *c + 1); - } - - T::Currency::set_lock(T::PalletId::get(), &who, locked_balance, WithdrawReasons::all()); - - >::insert( - &who, - VoterInfo::> { - last_active: index, - last_win: index, - stake: locked_balance, - pot: pot_to_set, - }, - ); - Self::set_approvals_chunked(&who, votes); - - Ok(()) - } - - /// Close the voting, record the number of seats that are actually up for grabs. - fn start_tally() { - let members = Self::members(); - let desired_seats = Self::desired_seats() as usize; - let number = >::block_number(); - let expiring = members - .iter() - .take_while(|i| i.1 <= number) - .map(|i| i.0.clone()) - .collect::>(); - let retaining_seats = members.len() - expiring.len(); - if retaining_seats < desired_seats { - let empty_seats = desired_seats - retaining_seats; - >::put(( - number + Self::presentation_duration(), - empty_seats as u32, - expiring, - )); - - // initialize leaderboard. - let leaderboard_size = empty_seats + T::CarryCount::get() as usize; - >::put(vec![ - (BalanceOf::::zero(), T::AccountId::default()); - leaderboard_size - ]); - - Self::deposit_event(Event::::TallyStarted(empty_seats as u32)); - } - } - - /// Finalize the vote, removing each of the `removals` and inserting `seats` of the most - /// approved candidates in their place. If the total number of members is less than the desired - /// membership a new vote is started. Clears all presented candidates, returning the bond of the - /// elected ones. - fn finalize_tally() -> DispatchResult { - let (_, coming, expiring): (T::BlockNumber, u32, Vec) = - >::take() - .ok_or("finalize can only be called after a tally is started.")?; - let leaderboard: Vec<(BalanceOf, T::AccountId)> = - >::take().unwrap_or_default(); - let new_expiry = >::block_number() + Self::term_duration(); - - // return bond to winners. - let candidacy_bond = T::CandidacyBond::get(); - let incoming: Vec<_> = leaderboard - .iter() - .rev() - .take_while(|&&(b, _)| !b.is_zero()) - .take(coming as usize) - .map(|(_, a)| a) - .cloned() - .inspect(|a| { - T::Currency::unreserve(a, candidacy_bond); - }) - .collect(); - - // Update last win index for anyone voted for any of the incomings. - incoming.iter().filter_map(|i| Self::candidate_reg_info(i)).for_each(|r| { - let index = r.1 as usize; - Self::all_voters() - .iter() - .filter_map(|mv| mv.as_ref()) - .filter(|v| Self::approvals_of_at(*v, index)) - .for_each(|v| { - >::mutate(v, |a| { - if let Some(activity) = a { - activity.last_win = Self::vote_index() + 1; - } - }) - }); - }); - let members = Self::members(); - let outgoing: Vec<_> = members.iter().take(expiring.len()).map(|a| a.0.clone()).collect(); - - // set the new membership set. - let mut new_set: Vec<_> = members - .into_iter() - .skip(expiring.len()) - .chain(incoming.iter().cloned().map(|a| (a, new_expiry))) - .collect(); - new_set.sort_by_key(|&(_, expiry)| expiry); - >::put(&new_set); - - let new_set = new_set.into_iter().map(|x| x.0).collect::>(); - T::ChangeMembers::change_members(&incoming, &outgoing, new_set); - - // clear all except runners-up from candidate list. - let candidates = Self::candidates(); - let mut new_candidates = vec![T::AccountId::default(); candidates.len()]; // shrink later. - let runners_up = leaderboard - .into_iter() - .rev() - .take_while(|&(b, _)| !b.is_zero()) - .skip(coming as usize) - .filter_map(|(_, a)| Self::candidate_reg_info(&a).map(|i| (a, i.1))); - let mut count = 0u32; - for (address, slot) in runners_up { - new_candidates[slot as usize] = address; - count += 1; - } - for (old, new) in candidates.iter().zip(new_candidates.iter()) { - // candidate is not a runner up. - if old != new { - // removed - kill it - >::remove(old); - - // and candidate is not a winner. - if incoming.iter().find(|e| *e == old).is_none() { - // slash the bond. - let (imbalance, _) = T::Currency::slash_reserved(&old, T::CandidacyBond::get()); - T::LoserCandidate::on_unbalanced(imbalance); - } - } - } - // discard any superfluous slots. - if let Some(last_index) = new_candidates.iter().rposition(|c| *c != T::AccountId::default()) - { - new_candidates.truncate(last_index + 1); - } - - Self::deposit_event(Event::::TallyFinalized(incoming, outgoing)); - - >::put(new_candidates); - CandidateCount::::put(count); - VoteCount::::put(Self::vote_index() + 1); - Ok(()) - } - - /// Get the set and vector index of a global voter index. - /// - /// Note that this function does not take holes into account. - /// See [`voter_at`]. - fn split_index(index: usize, scale: usize) -> (SetIndex, usize) { - let set_index = (index / scale) as u32; - let vec_index = index % scale; - (set_index, vec_index) - } - - /// Return a concatenated vector over all voter sets. - fn all_voters() -> Vec> { - let mut all = >::get(0); - let mut index = 1; - // NOTE: we could also use `Self::next_nonfull_voter_set()` here but that might change based - // on how we do chunking. This is more generic. - loop { - let next_set = >::get(index); - if next_set.is_empty() { - break - } else { - index += 1; - all.extend(next_set); - } - } - all - } - - /// Shorthand for fetching a voter at a specific (global) index. - /// - /// NOTE: this function is used for checking indices. Yet, it does not take holes into account. - /// This means that any account submitting an index at any point in time should submit: - /// `VOTER_SET_SIZE * set_index + local_index`, meaning that you are ignoring all holes in the - /// first `set_index` sets. - fn voter_at(index: usize) -> Option { - let (set_index, vec_index) = Self::split_index(index, VOTER_SET_SIZE); - let set = Self::voters(set_index); - if vec_index < set.len() { - set[vec_index].clone() - } else { - None - } - } - - /// A more sophisticated version of `voter_at`. Will be kept separate as most often it is an - /// overdue compared to `voter_at`. Only used when setting approvals. - fn cell_status(set_index: SetIndex, vec_index: usize) -> CellStatus { - let set = Self::voters(set_index); - if vec_index < set.len() { - if let Some(_) = set[vec_index] { - CellStatus::Occupied - } else { - CellStatus::Hole - } - } else { - CellStatus::Head - } - } - - /// Sets the approval of a voter in a chunked manner. - fn set_approvals_chunked(who: &T::AccountId, approvals: Vec) { - let approvals_flag_vec = Self::bool_to_flag(approvals); - approvals_flag_vec - .chunks(APPROVAL_SET_SIZE) - .enumerate() - .for_each(|(index, slice)| >::insert((&who, index as SetIndex), slice)); - } - - /// shorthand for fetching a specific approval of a voter at a specific (global) index. - /// - /// Using this function to read a vote is preferred as it reads `APPROVAL_SET_SIZE` items of - /// type `ApprovalFlag` from storage at most; not all of them. - /// - /// Note that false is returned in case of no-vote or an explicit `false`. - fn approvals_of_at(who: &T::AccountId, index: usize) -> bool { - let (flag_index, bit) = Self::split_index(index, APPROVAL_FLAG_LEN); - let (set_index, vec_index) = Self::split_index(flag_index as usize, APPROVAL_SET_SIZE); - let set = Self::approvals_of((who.clone(), set_index)); - if vec_index < set.len() { - // This is because bit_at treats numbers in lsb -> msb order. - let reversed_index = set.len() - 1 - vec_index; - Self::bit_at(set[reversed_index], bit) - } else { - false - } - } - - /// Return true of the bit `n` of scalar `x` is set to `1` and false otherwise. - fn bit_at(x: ApprovalFlag, n: usize) -> bool { - if n < APPROVAL_FLAG_LEN { - x & (1 << n) != 0 - } else { - false - } - } - - /// Convert a vec of boolean approval flags to a vec of integers, as denoted by - /// the type `ApprovalFlag`. see `bool_to_flag_should_work` test for examples. - pub fn bool_to_flag(x: Vec) -> Vec { - let mut result: Vec = Vec::with_capacity(x.len() / APPROVAL_FLAG_LEN); - if x.is_empty() { - return result - } - result.push(0); - let mut index = 0; - let mut counter = 0; - loop { - let shl_index = counter % APPROVAL_FLAG_LEN; - result[index] += (if x[counter] { 1 } else { 0 }) << shl_index; - counter += 1; - if counter > x.len() - 1 { - break - } - if counter % APPROVAL_FLAG_LEN == 0 { - result.push(0); - index += 1; - } - } - result - } - - /// Convert a vec of flags (u32) to boolean. - pub fn flag_to_bool(chunk: Vec) -> Vec { - let mut result = Vec::with_capacity(chunk.len()); - if chunk.is_empty() { - return vec![] - } - chunk - .into_iter() - .map(|num| { - (0..APPROVAL_FLAG_LEN).map(|bit| Self::bit_at(num, bit)).collect::>() - }) - .for_each(|c| { - let last_approve = match c.iter().rposition(|n| *n) { - Some(index) => index + 1, - None => 0, - }; - result.extend(c.into_iter().take(last_approve)); - }); - result - } - - /// Return a concatenated vector over all approvals of a voter as boolean. - /// The trailing zeros are removed. - fn all_approvals_of(who: &T::AccountId) -> Vec { - let mut all: Vec = vec![]; - let mut index = 0_u32; - loop { - let chunk = Self::approvals_of((who.clone(), index)); - if chunk.is_empty() { - break - } - all.extend(Self::flag_to_bool(chunk)); - index += 1; - } - all - } - - /// Remove all approvals associated with one account. - fn remove_all_approvals_of(who: &T::AccountId) { - let mut index = 0; - loop { - let set = Self::approvals_of((who.clone(), index)); - if set.len() > 0 { - >::remove((who.clone(), index)); - index += 1; - } else { - break - } - } - } - - /// Calculates the offset value (stored pot) of a stake, based on the distance - /// to the last win_index, `t`. Regardless of the internal implementation, - /// it should always be used with the following structure: - /// - /// Given Stake of voter `V` being `x` and distance to last_win index `t`, the new weight - /// of `V` is `x + get_offset(x, t)`. - /// - /// In other words, this function returns everything extra that should be added - /// to a voter's stake value to get the correct weight. Indeed, zero is - /// returned if `t` is zero. - fn get_offset(stake: BalanceOf, t: VoteIndex) -> BalanceOf { - let decay_ratio: BalanceOf = T::DecayRatio::get().into(); - if t > 150 { - return stake * decay_ratio - } - let mut offset = stake; - let mut r = Zero::zero(); - let decay = decay_ratio + One::one(); - for _ in 0..t { - offset = offset.saturating_sub(offset / decay); - r += offset - } - r - } -} diff --git a/frame/elections/src/mock.rs b/frame/elections/src/mock.rs deleted file mode 100644 index 91318e1e07bc..000000000000 --- a/frame/elections/src/mock.rs +++ /dev/null @@ -1,273 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Mock file for election module. - -#![cfg(test)] - -use crate as elections; -use frame_support::{ - assert_ok, parameter_types, - traits::{ChangeMembers, Currency, LockIdentifier}, -}; -use sp_core::H256; -use sp_runtime::{ - testing::Header, - traits::{BlakeTwo256, IdentityLookup}, - BuildStorage, -}; - -parameter_types! { - pub const BlockHashCount: u64 = 250; - pub BlockWeights: frame_system::limits::BlockWeights = - frame_system::limits::BlockWeights::simple_max(1024); -} -impl frame_system::Config for Test { - type BaseCallFilter = frame_support::traits::Everything; - type BlockWeights = (); - type BlockLength = (); - type DbWeight = (); - type Origin = Origin; - type Call = Call; - type Index = u64; - type BlockNumber = u64; - type Hash = H256; - type Hashing = BlakeTwo256; - type AccountId = u64; - type Lookup = IdentityLookup; - type Header = Header; - type Event = Event; - type BlockHashCount = BlockHashCount; - type Version = (); - type PalletInfo = PalletInfo; - type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - type OnSetCode = (); -} - -parameter_types! { - pub const ExistentialDeposit: u64 = 1; -} -impl pallet_balances::Config for Test { - type MaxLocks = (); - type MaxReserves = (); - type ReserveIdentifier = [u8; 8]; - type Balance = u64; - type DustRemoval = (); - type Event = Event; - type ExistentialDeposit = ExistentialDeposit; - type AccountStore = System; - type WeightInfo = (); -} - -parameter_types! { - pub const CandidacyBond: u64 = 3; - pub const CarryCount: u32 = 2; - pub const InactiveGracePeriod: u32 = 1; - pub const VotingPeriod: u64 = 4; - pub const MinimumVotingLock: u64 = 5; - pub static VotingBond: u64 = 0; - pub static VotingFee: u64 = 0; - pub static PresentSlashPerVoter: u64 = 0; - pub static DecayRatio: u32 = 0; - pub static Members: Vec = vec![]; -} - -pub struct TestChangeMembers; -impl ChangeMembers for TestChangeMembers { - fn change_members_sorted(incoming: &[u64], outgoing: &[u64], new: &[u64]) { - let mut old_plus_incoming = MEMBERS.with(|m| m.borrow().to_vec()); - old_plus_incoming.extend_from_slice(incoming); - old_plus_incoming.sort(); - let mut new_plus_outgoing = new.to_vec(); - new_plus_outgoing.extend_from_slice(outgoing); - new_plus_outgoing.sort(); - assert_eq!(old_plus_incoming, new_plus_outgoing); - - MEMBERS.with(|m| *m.borrow_mut() = new.to_vec()); - } -} - -parameter_types! { - pub const ElectionPalletId: LockIdentifier = *b"py/elect"; -} - -impl elections::Config for Test { - type Event = Event; - type Currency = Balances; - type BadPresentation = (); - type BadReaper = (); - type BadVoterIndex = (); - type LoserCandidate = (); - type ChangeMembers = TestChangeMembers; - type CandidacyBond = CandidacyBond; - type VotingBond = VotingBond; - type VotingFee = VotingFee; - type MinimumVotingLock = MinimumVotingLock; - type PresentSlashPerVoter = PresentSlashPerVoter; - type CarryCount = CarryCount; - type InactiveGracePeriod = InactiveGracePeriod; - type VotingPeriod = VotingPeriod; - type DecayRatio = DecayRatio; - type PalletId = ElectionPalletId; -} - -pub type Block = sp_runtime::generic::Block; -pub type UncheckedExtrinsic = sp_runtime::generic::UncheckedExtrinsic; - -use frame_system as system; -frame_support::construct_runtime!( - pub enum Test where - Block = Block, - NodeBlock = Block, - UncheckedExtrinsic = UncheckedExtrinsic - { - System: system::{Pallet, Call, Event}, - Balances: pallet_balances::{Pallet, Call, Event, Config}, - Elections: elections::{Pallet, Call, Event, Config}, - } -); - -pub struct ExtBuilder { - balance_factor: u64, - decay_ratio: u32, - desired_seats: u32, - voting_fee: u64, - voting_bond: u64, - bad_presentation_punishment: u64, -} - -impl Default for ExtBuilder { - fn default() -> Self { - Self { - balance_factor: 1, - decay_ratio: 24, - desired_seats: 2, - voting_fee: 0, - voting_bond: 0, - bad_presentation_punishment: 1, - } - } -} - -impl ExtBuilder { - pub fn balance_factor(mut self, factor: u64) -> Self { - self.balance_factor = factor; - self - } - pub fn decay_ratio(mut self, ratio: u32) -> Self { - self.decay_ratio = ratio; - self - } - pub fn voting_fee(mut self, fee: u64) -> Self { - self.voting_fee = fee; - self - } - pub fn bad_presentation_punishment(mut self, fee: u64) -> Self { - self.bad_presentation_punishment = fee; - self - } - pub fn voting_bond(mut self, fee: u64) -> Self { - self.voting_bond = fee; - self - } - pub fn desired_seats(mut self, seats: u32) -> Self { - self.desired_seats = seats; - self - } - pub fn build(self) -> sp_io::TestExternalities { - VOTING_BOND.with(|v| *v.borrow_mut() = self.voting_bond); - VOTING_FEE.with(|v| *v.borrow_mut() = self.voting_fee); - PRESENT_SLASH_PER_VOTER.with(|v| *v.borrow_mut() = self.bad_presentation_punishment); - DECAY_RATIO.with(|v| *v.borrow_mut() = self.decay_ratio); - let mut ext: sp_io::TestExternalities = GenesisConfig { - balances: pallet_balances::GenesisConfig:: { - balances: vec![ - (1, 10 * self.balance_factor), - (2, 20 * self.balance_factor), - (3, 30 * self.balance_factor), - (4, 40 * self.balance_factor), - (5, 50 * self.balance_factor), - (6, 60 * self.balance_factor), - ], - }, - elections: elections::GenesisConfig:: { - members: vec![], - desired_seats: self.desired_seats, - presentation_duration: 2, - term_duration: 5, - }, - } - .build_storage() - .unwrap() - .into(); - ext.execute_with(|| System::set_block_number(1)); - ext - } -} - -pub(crate) fn voter_ids() -> Vec { - Elections::all_voters().iter().map(|v| v.unwrap_or(0)).collect::>() -} - -pub(crate) fn vote(i: u64, l: usize) { - let _ = Balances::make_free_balance_be(&i, 20); - assert_ok!(Elections::set_approvals( - Origin::signed(i), - (0..l).map(|_| true).collect::>(), - 0, - 0, - 20, - )); -} - -pub(crate) fn vote_at(i: u64, l: usize, index: elections::VoteIndex) { - let _ = Balances::make_free_balance_be(&i, 20); - assert_ok!(Elections::set_approvals( - Origin::signed(i), - (0..l).map(|_| true).collect::>(), - 0, - index, - 20, - )); -} - -pub(crate) fn create_candidate(i: u64, index: u32) { - let _ = Balances::make_free_balance_be(&i, 20); - assert_ok!(Elections::submit_candidacy(Origin::signed(i), index)); -} - -pub(crate) fn balances(who: &u64) -> (u64, u64) { - (Balances::free_balance(who), Balances::reserved_balance(who)) -} - -pub(crate) fn locks(who: &u64) -> Vec { - Balances::locks(who).iter().map(|l| l.amount).collect::>() -} - -pub(crate) fn new_test_ext_with_candidate_holes() -> sp_io::TestExternalities { - let mut t = ExtBuilder::default().build(); - t.execute_with(|| { - >::put(vec![0, 0, 1]); - elections::CandidateCount::::put(1); - >::insert(1, (0, 2)); - }); - t -} diff --git a/frame/elections/src/tests.rs b/frame/elections/src/tests.rs deleted file mode 100644 index 0df84c6d79ba..000000000000 --- a/frame/elections/src/tests.rs +++ /dev/null @@ -1,1881 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Tests for election module. - -#![cfg(test)] - -use crate::{mock::*, *}; - -use frame_support::{assert_err, assert_noop, assert_ok}; - -#[test] -fn params_should_work() { - ExtBuilder::default().build().execute_with(|| { - assert_eq!(Elections::next_vote_from(1), 4); - assert_eq!(Elections::next_vote_from(4), 4); - assert_eq!(Elections::next_vote_from(5), 8); - assert_eq!(Elections::vote_index(), 0); - assert_eq!(Elections::presentation_duration(), 2); - assert_eq!(Elections::term_duration(), 5); - assert_eq!(Elections::desired_seats(), 2); - - assert_eq!(Elections::members(), vec![]); - assert_eq!(Elections::next_tally(), Some(4)); - assert_eq!(Elections::presentation_active(), false); - assert_eq!(Elections::next_finalize(), None); - - assert_eq!(Elections::candidates(), Vec::::new()); - assert_eq!(Elections::is_a_candidate(&1), false); - assert_eq!(Elections::candidate_reg_info(1), None); - - assert_eq!(Elections::voters(0), Vec::>::new()); - assert_eq!(Elections::voter_info(1), None); - assert!(Elections::all_approvals_of(&1).is_empty()); - }); -} - -#[test] -fn chunking_bool_to_flag_should_work() { - ExtBuilder::default().build().execute_with(|| { - assert!(Elections::bool_to_flag(vec![]).is_empty()); - assert_eq!(Elections::bool_to_flag(vec![false]), vec![0]); - assert_eq!(Elections::bool_to_flag(vec![true]), vec![1]); - assert_eq!(Elections::bool_to_flag(vec![true, true, true, true]), vec![15]); - assert_eq!(Elections::bool_to_flag(vec![true, true, true, true, true]), vec![15 + 16]); - - let set_1 = vec![ - true, false, false, false, // 0x1 - false, true, true, true, // 0xE - ]; - assert_eq!(Elections::bool_to_flag(set_1.clone()), vec![0x00_00_00_E1_u32]); - assert_eq!(Elections::flag_to_bool(vec![0x00_00_00_E1_u32]), set_1); - - let set_2 = vec![ - false, false, false, true, // 0x8 - false, true, false, true, // 0xA - ]; - assert_eq!(Elections::bool_to_flag(set_2.clone()), vec![0x00_00_00_A8_u32]); - assert_eq!(Elections::flag_to_bool(vec![0x00_00_00_A8_u32]), set_2); - - let mut rhs = (0..100 / APPROVAL_FLAG_LEN).map(|_| 0xFFFFFFFF_u32).collect::>(); - // NOTE: this might be need change based on `APPROVAL_FLAG_LEN`. - rhs.extend(vec![0x00_00_00_0F]); - assert_eq!(Elections::bool_to_flag((0..100).map(|_| true).collect()), rhs) - }) -} - -#[test] -fn chunking_voter_set_growth_should_work() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - - // create 65. 64 (set0) + 1 (set1) - (1..=63).for_each(|i| vote(i, 0)); - assert_eq!(Elections::next_nonfull_voter_set(), 0); - vote(64, 0); - assert_eq!(Elections::next_nonfull_voter_set(), 1); - vote(65, 0); - - let set1 = Elections::voters(0); - let set2 = Elections::voters(1); - - assert_eq!(set1.len(), 64); - assert_eq!(set2.len(), 1); - - assert_eq!(set1[0], Some(1)); - assert_eq!(set1[10], Some(11)); - assert_eq!(set2[0], Some(65)); - }) -} - -#[test] -fn chunking_voter_set_reclaim_should_work() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - - (1..=129).for_each(|i| vote(i, 0)); - assert_eq!(Elections::next_nonfull_voter_set(), 2); - - assert_ok!(Elections::retract_voter(Origin::signed(11), 10)); - - assert_ok!(Elections::retract_voter(Origin::signed(66), 65)); - assert_ok!(Elections::retract_voter(Origin::signed(67), 66)); - - // length does not show it but holes do exist. - assert_eq!(Elections::voters(0).len(), 64); - assert_eq!(Elections::voters(1).len(), 64); - assert_eq!(Elections::voters(2).len(), 1); - - assert_eq!(Elections::voters(0)[10], None); - assert_eq!(Elections::voters(1)[1], None); - assert_eq!(Elections::voters(1)[2], None); - // Next set with capacity is 2. - assert_eq!(Elections::next_nonfull_voter_set(), 2); - - // But we can fill a hole. - vote_at(130, 0, 10); - - // Nothing added to set 2. A hole was filled. - assert_eq!(Elections::voters(0).len(), 64); - assert_eq!(Elections::voters(1).len(), 64); - assert_eq!(Elections::voters(2).len(), 1); - - // and the next two (scheduled) to the second set. - assert_eq!(Elections::next_nonfull_voter_set(), 2); - }) -} - -#[test] -fn chunking_approvals_set_growth_should_work() { - ExtBuilder::default().build().execute_with(|| { - // create candidates and voters. - (1..=250).for_each(|i| create_candidate(i, (i - 1) as u32)); - (1..=250).for_each(|i| vote(i, i as usize)); - - // all approvals of should return the exact expected vector. - assert_eq!( - Elections::all_approvals_of(&180), - (0..180).map(|_| true).collect::>() - ); - assert_eq!(Elections::all_approvals_of(&32), (0..32).map(|_| true).collect::>()); - assert_eq!(Elections::all_approvals_of(&8), (0..8).map(|_| true).collect::>()); - assert_eq!(Elections::all_approvals_of(&64), (0..64).map(|_| true).collect::>()); - assert_eq!(Elections::all_approvals_of(&65), (0..65).map(|_| true).collect::>()); - assert_eq!(Elections::all_approvals_of(&63), (0..63).map(|_| true).collect::>()); - - // NOTE: assuming that APPROVAL_SET_SIZE is more or less small-ish. Might fail otherwise. - let full_sets = (180 / APPROVAL_FLAG_LEN) / APPROVAL_SET_SIZE; - let left_over = (180 / APPROVAL_FLAG_LEN) / APPROVAL_SET_SIZE; - let rem = 180 % APPROVAL_FLAG_LEN; - - // grab and check the last full set, if it exists. - if full_sets > 0 { - assert_eq!( - Elections::approvals_of((180, (full_sets - 1) as SetIndex)), - Elections::bool_to_flag( - (0..APPROVAL_SET_SIZE * APPROVAL_FLAG_LEN).map(|_| true).collect::>() - ) - ); - } - - // grab and check the last, half-empty, set. - if left_over > 0 { - assert_eq!( - Elections::approvals_of((180, full_sets as SetIndex)), - Elections::bool_to_flag( - (0..left_over * APPROVAL_FLAG_LEN + rem).map(|_| true).collect::>() - ) - ); - } - }) -} - -#[test] -fn chunking_cell_status_works() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - - (1..=63).for_each(|i| vote(i, 0)); - - assert_ok!(Elections::retract_voter(Origin::signed(11), 10)); - assert_ok!(Elections::retract_voter(Origin::signed(21), 20)); - - assert_eq!(Elections::cell_status(0, 10), CellStatus::Hole); - assert_eq!(Elections::cell_status(0, 0), CellStatus::Occupied); - assert_eq!(Elections::cell_status(0, 20), CellStatus::Hole); - assert_eq!(Elections::cell_status(0, 63), CellStatus::Head); - assert_eq!(Elections::cell_status(1, 0), CellStatus::Head); - assert_eq!(Elections::cell_status(1, 10), CellStatus::Head); - }) -} - -#[test] -fn chunking_voter_index_does_not_take_holes_into_account() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - - // create 65. 64 (set0) + 1 (set1) - (1..=65).for_each(|i| vote(i, 0)); - - // account 65 has global index 65. - assert_eq!(Elections::voter_at(64).unwrap(), 65); - - assert_ok!(Elections::retract_voter(Origin::signed(1), 0)); - assert_ok!(Elections::retract_voter(Origin::signed(2), 1)); - - // still the same. These holes are in some other set. - assert_eq!(Elections::voter_at(64).unwrap(), 65); - // proof: can submit a new approval with the old index. - assert_noop!( - Elections::set_approvals(Origin::signed(65), vec![], 0, 64 - 2, 10), - Error::::InvalidVoterIndex, - ); - assert_ok!(Elections::set_approvals(Origin::signed(65), vec![], 0, 64, 10)); - }) -} - -#[test] -fn chunking_approval_storage_should_work() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 1)); - - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true, false], 0, 0, 20)); - assert_ok!(Elections::set_approvals(Origin::signed(3), vec![false, false], 0, 0, 30)); - assert_ok!(Elections::set_approvals(Origin::signed(4), vec![], 0, 0, 40)); - - assert_eq!(Elections::all_approvals_of(&2), vec![true]); - // NOTE: these two are stored in mem differently though. - assert!(Elections::all_approvals_of(&3).is_empty()); - assert!(Elections::all_approvals_of(&4).is_empty()); - - assert_eq!(Elections::approvals_of((3, 0)), vec![0]); - assert!(Elections::approvals_of((4, 0)).is_empty()); - }); -} - -#[test] -fn voting_initial_set_approvals_ignores_voter_index() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - - // Last argument is essentially irrelevant. You might get or miss a tip. - assert_ok!(Elections::set_approvals(Origin::signed(3), vec![], 0, 0, 30)); - assert_ok!(Elections::set_approvals(Origin::signed(4), vec![], 0, 5, 40)); - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![], 0, 100, 50)); - - // indices are more or less ignored. all is pushed. - assert_eq!(voter_ids(), vec![3, 4, 5]); - }) -} -#[test] -fn voting_bad_approval_index_slashes_voters_and_bond_reduces_stake() { - ExtBuilder::default().voting_fee(5).voting_bond(2).build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - - (1..=63).for_each(|i| vote(i, 0)); - assert_eq!(balances(&1), (13, 2)); - assert_eq!(balances(&10), (18, 2)); - assert_eq!(balances(&60), (18, 2)); - - // still no fee - vote(64, 0); - assert_eq!(balances(&64), (18, 2)); - assert_eq!( - Elections::voter_info(&64).unwrap(), - VoterInfo { last_win: 0, last_active: 0, stake: 20, pot: 0 } - ); - - assert_eq!(Elections::next_nonfull_voter_set(), 1); - - // now we charge the next voter. - vote(65, 0); - assert_eq!(balances(&65), (13, 2)); - assert_eq!( - Elections::voter_info(&65).unwrap(), - VoterInfo { last_win: 0, last_active: 0, stake: 15, pot: 0 } - ); - }); -} - -#[test] -fn voting_subsequent_set_approvals_checks_voter_index() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - - assert_ok!(Elections::set_approvals(Origin::signed(3), vec![], 0, 0, 30)); - assert_ok!(Elections::set_approvals(Origin::signed(4), vec![], 0, 5, 40)); - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![], 0, 100, 50)); - - // invalid index - assert_noop!( - Elections::set_approvals(Origin::signed(4), vec![true], 0, 5, 40), - Error::::InvalidVoterIndex, - ); - // wrong index - assert_noop!( - Elections::set_approvals(Origin::signed(4), vec![true], 0, 0, 40), - Error::::InvalidVoterIndex, - ); - // correct - assert_ok!(Elections::set_approvals(Origin::signed(4), vec![true], 0, 1, 40)); - }) -} - -#[test] -fn voting_cannot_lock_less_than_limit() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - - assert_noop!( - Elections::set_approvals(Origin::signed(3), vec![], 0, 0, 4), - Error::::InsufficientLockedValue, - ); - assert_ok!(Elections::set_approvals(Origin::signed(3), vec![], 0, 0, 5)); - }); -} - -#[test] -fn voting_locking_more_than_total_balance_is_moot() { - ExtBuilder::default().voting_bond(2).build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - - assert_eq!(balances(&3), (30, 0)); - assert_ok!(Elections::set_approvals(Origin::signed(3), vec![], 0, 0, 35)); - - assert_eq!(balances(&3), (28, 2)); - assert_eq!( - Elections::voter_info(&3).unwrap(), - VoterInfo { last_win: 0, last_active: 0, stake: 30, pot: 0 } - ); - }); -} - -#[test] -fn voting_locking_stake_and_reserving_bond_works() { - ExtBuilder::default().voting_bond(2).build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); - - assert_eq!(balances(&2), (20, 0)); - assert!(locks(&2).is_empty()); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![], 0, 0, 15)); - assert_eq!(balances(&2), (18, 2)); - assert_eq!(locks(&2), vec![15]); - - // deposit a bit more. - let _ = Balances::make_free_balance_be(&2, 100); - - // change vote - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true], 0, 0, 70)); - assert_eq!(balances(&2), (100, 2)); - assert_eq!(locks(&2), vec![70]); - - assert_ok!(Elections::retract_voter(Origin::signed(2), 0)); - - assert_eq!(balances(&2), (102, 0)); - assert!(locks(&2).is_empty()); - }); -} - -#[test] -fn voting_without_any_candidate_count_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - assert_eq!(Elections::candidates().len(), 0); - - assert_noop!( - Elections::set_approvals(Origin::signed(4), vec![], 0, 0, 40), - Error::::ZeroCandidates, - ); - }); -} - -#[test] -fn voting_setting_an_approval_vote_count_more_than_candidate_count_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); - assert_eq!(Elections::candidates().len(), 1); - - assert_noop!( - Elections::set_approvals(Origin::signed(4), vec![true, true], 0, 0, 40), - Error::::TooManyVotes, - ); - }); -} - -#[test] -fn voting_resubmitting_approvals_should_work() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(4), vec![true], 0, 0, 40)); - - assert_eq!(Elections::all_approvals_of(&4), vec![true]); - - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 2)); - assert_eq!(Elections::candidates().len(), 3); - assert_ok!(Elections::set_approvals(Origin::signed(4), vec![true, false, true], 0, 0, 40)); - - assert_eq!(Elections::all_approvals_of(&4), vec![true, false, true]); - }); -} - -#[test] -fn voting_retracting_voter_should_work() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); - assert_eq!(Elections::candidates().len(), 1); - - assert_ok!(Elections::set_approvals(Origin::signed(1), vec![true], 0, 0, 10)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true], 0, 1, 20)); - assert_ok!(Elections::set_approvals(Origin::signed(3), vec![true], 0, 2, 30)); - assert_ok!(Elections::set_approvals(Origin::signed(4), vec![true], 0, 3, 40)); - - assert_eq!(voter_ids(), vec![1, 2, 3, 4]); - assert_eq!(Elections::all_approvals_of(&1), vec![true]); - assert_eq!(Elections::all_approvals_of(&2), vec![true]); - assert_eq!(Elections::all_approvals_of(&3), vec![true]); - assert_eq!(Elections::all_approvals_of(&4), vec![true]); - - assert_ok!(Elections::retract_voter(Origin::signed(1), 0)); - - assert_eq!(voter_ids(), vec![0, 2, 3, 4]); - assert_eq!(Elections::all_approvals_of(&1), Vec::::new()); - assert_eq!(Elections::all_approvals_of(&2), vec![true]); - assert_eq!(Elections::all_approvals_of(&3), vec![true]); - assert_eq!(Elections::all_approvals_of(&4), vec![true]); - - assert_ok!(Elections::retract_voter(Origin::signed(2), 1)); - - assert_eq!(voter_ids(), vec![0, 0, 3, 4]); - assert_eq!(Elections::all_approvals_of(&1), Vec::::new()); - assert_eq!(Elections::all_approvals_of(&2), Vec::::new()); - assert_eq!(Elections::all_approvals_of(&3), vec![true]); - assert_eq!(Elections::all_approvals_of(&4), vec![true]); - - assert_ok!(Elections::retract_voter(Origin::signed(3), 2)); - - assert_eq!(voter_ids(), vec![0, 0, 0, 4]); - assert_eq!(Elections::all_approvals_of(&1), Vec::::new()); - assert_eq!(Elections::all_approvals_of(&2), Vec::::new()); - assert_eq!(Elections::all_approvals_of(&3), Vec::::new()); - assert_eq!(Elections::all_approvals_of(&4), vec![true]); - }); -} - -#[test] -fn voting_invalid_retraction_index_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 0)); - - assert_ok!(Elections::set_approvals(Origin::signed(1), vec![true], 0, 0, 10)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true], 0, 0, 20)); - assert_eq!(voter_ids(), vec![1, 2]); - assert_noop!( - Elections::retract_voter(Origin::signed(1), 1), - Error::::InvalidRetractionIndex - ); - }); -} - -#[test] -fn voting_overflow_retraction_index_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 0)); - - assert_ok!(Elections::set_approvals(Origin::signed(1), vec![true], 0, 0, 10)); - assert_noop!( - Elections::retract_voter(Origin::signed(1), 1), - Error::::InvalidRetractionIndex - ); - }); -} - -#[test] -fn voting_non_voter_retraction_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 0)); - - assert_ok!(Elections::set_approvals(Origin::signed(1), vec![true], 0, 0, 10)); - assert_noop!( - Elections::retract_voter(Origin::signed(2), 0), - Error::::RetractNonVoter - ); - }); -} - -#[test] -fn retracting_inactive_voter_should_work() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true], 0, 0, 20)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_ok!(Elections::present_winner(Origin::signed(4), 2, 20, 0)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(8); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![true], 1, 0, 50)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(10); - assert_ok!(Elections::present_winner(Origin::signed(4), 5, 50, 1)); - assert_ok!(Elections::end_block(System::block_number())); - - assert_ok!(Elections::reap_inactive_voter( - Origin::signed(5), - (voter_ids().iter().position(|&i| i == 5).unwrap() as u32).into(), - 2, - (voter_ids().iter().position(|&i| i == 2).unwrap() as u32).into(), - 2 - )); - - assert_eq!(voter_ids(), vec![0, 5]); - assert_eq!(Elections::all_approvals_of(&2).len(), 0); - assert_eq!(Balances::total_balance(&2), 20); - assert_eq!(Balances::total_balance(&5), 50); - }); -} - -#[test] -fn retracting_inactive_voter_with_other_candidates_in_slots_should_work() { - ExtBuilder::default().voting_bond(2).build().execute_with(|| { - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true], 0, 0, 20)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_ok!(Elections::present_winner(Origin::signed(4), 2, 20, 0)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(8); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![true], 1, 0, 50)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(10); - assert_ok!(Elections::present_winner(Origin::signed(4), 5, 50, 1)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(11); - assert_ok!(Elections::submit_candidacy(Origin::signed(1), 0)); - - assert_ok!(Elections::reap_inactive_voter( - Origin::signed(5), - (voter_ids().iter().position(|&i| i == 5).unwrap() as u32).into(), - 2, - (voter_ids().iter().position(|&i| i == 2).unwrap() as u32).into(), - 2 - )); - - assert_eq!(voter_ids(), vec![0, 5]); - assert_eq!(Elections::all_approvals_of(&2).len(), 0); - }); -} - -#[test] -fn retracting_inactive_voter_with_bad_reporter_index_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true], 0, 0, 20)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_ok!(Elections::present_winner(Origin::signed(4), 2, 20, 0)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(8); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![true], 1, 0, 50)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(10); - assert_ok!(Elections::present_winner(Origin::signed(4), 5, 50, 1)); - assert_ok!(Elections::end_block(System::block_number())); - - assert_noop!( - Elections::reap_inactive_voter( - Origin::signed(2), - 42, - 2, - (voter_ids().iter().position(|&i| i == 2).unwrap() as u32).into(), - 2 - ), - Error::::InvalidReporterIndex - ); - }); -} - -#[test] -fn retracting_inactive_voter_with_bad_target_index_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true], 0, 0, 20)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_ok!(Elections::present_winner(Origin::signed(4), 2, 20, 0)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(8); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![true], 1, 0, 50)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(10); - assert_ok!(Elections::present_winner(Origin::signed(4), 5, 50, 1)); - assert_ok!(Elections::end_block(System::block_number())); - - assert_noop!( - Elections::reap_inactive_voter( - Origin::signed(2), - (voter_ids().iter().position(|&i| i == 2).unwrap() as u32).into(), - 2, - 42, - 2 - ), - Error::::InvalidTargetIndex - ); - }); -} - -#[test] -fn retracting_active_voter_should_slash_reporter() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(4), 2)); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 3)); - assert_ok!(Elections::set_approvals( - Origin::signed(2), - vec![true, false, false, false], - 0, - 0, - 20 - )); - assert_ok!(Elections::set_approvals( - Origin::signed(3), - vec![false, true, false, false], - 0, - 0, - 30 - )); - assert_ok!(Elections::set_approvals( - Origin::signed(4), - vec![false, false, true, false], - 0, - 0, - 40 - )); - assert_ok!(Elections::set_approvals( - Origin::signed(5), - vec![false, false, false, true], - 0, - 0, - 50 - )); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_ok!(Elections::present_winner(Origin::signed(4), 2, 20, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 3, 30, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 4, 40, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 5, 50, 0)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(8); - assert_ok!(Elections::set_desired_seats(Origin::root(), 3)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(10); - assert_ok!(Elections::present_winner( - Origin::signed(4), - 2, - 20 + Elections::get_offset(20, 1), - 1 - )); - assert_ok!(Elections::present_winner( - Origin::signed(4), - 3, - 30 + Elections::get_offset(30, 1), - 1 - )); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::vote_index(), 2); - assert_eq!(::InactiveGracePeriod::get(), 1); - assert_eq!(::VotingPeriod::get(), 4); - assert_eq!( - Elections::voter_info(4), - Some(VoterInfo { last_win: 1, last_active: 0, stake: 40, pot: 0 }) - ); - - assert_ok!(Elections::reap_inactive_voter( - Origin::signed(4), - (voter_ids().iter().position(|&i| i == 4).unwrap() as u32).into(), - 2, - (voter_ids().iter().position(|&i| i == 2).unwrap() as u32).into(), - 2 - )); - - assert_eq!(voter_ids(), vec![2, 3, 0, 5]); - assert_eq!(Elections::all_approvals_of(&4).len(), 0); - assert_eq!(Balances::total_balance(&4), 40); - }); -} - -#[test] -fn retracting_inactive_voter_by_nonvoter_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true], 0, 0, 20)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_ok!(Elections::present_winner(Origin::signed(4), 2, 20, 0)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(8); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![true], 1, 0, 50)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(10); - assert_ok!(Elections::present_winner(Origin::signed(4), 5, 50, 1)); - assert_ok!(Elections::end_block(System::block_number())); - - assert_noop!( - Elections::reap_inactive_voter( - Origin::signed(4), - 0, - 2, - (voter_ids().iter().position(|&i| i == 2).unwrap() as u32).into(), - 2 - ), - Error::::NotVoter - ); - }); -} - -#[test] -fn candidacy_simple_candidate_submission_should_work() { - ExtBuilder::default().build().execute_with(|| { - assert_eq!(Elections::candidates(), Vec::::new()); - assert_eq!(Elections::candidate_reg_info(1), None); - assert_eq!(Elections::candidate_reg_info(2), None); - assert_eq!(Elections::is_a_candidate(&1), false); - assert_eq!(Elections::is_a_candidate(&2), false); - - assert_ok!(Elections::submit_candidacy(Origin::signed(1), 0)); - assert_eq!(Elections::candidates(), vec![1]); - assert_eq!(Elections::candidate_reg_info(1), Some((0, 0))); - assert_eq!(Elections::candidate_reg_info(2), None); - assert_eq!(Elections::is_a_candidate(&1), true); - assert_eq!(Elections::is_a_candidate(&2), false); - - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 1)); - assert_eq!(Elections::candidates(), vec![1, 2]); - assert_eq!(Elections::candidate_reg_info(1), Some((0, 0))); - assert_eq!(Elections::candidate_reg_info(2), Some((0, 1))); - assert_eq!(Elections::is_a_candidate(&1), true); - assert_eq!(Elections::is_a_candidate(&2), true); - }); -} - -#[test] -fn candidacy_submission_using_free_slot_should_work() { - let mut t = new_test_ext_with_candidate_holes(); - - t.execute_with(|| { - assert_eq!(Elections::candidates(), vec![0, 0, 1]); - - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 1)); - assert_eq!(Elections::candidates(), vec![0, 2, 1]); - - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 0)); - assert_eq!(Elections::candidates(), vec![3, 2, 1]); - }); -} - -#[test] -fn candidacy_submission_using_alternative_free_slot_should_work() { - let mut t = new_test_ext_with_candidate_holes(); - - t.execute_with(|| { - assert_eq!(Elections::candidates(), vec![0, 0, 1]); - - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_eq!(Elections::candidates(), vec![2, 0, 1]); - - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 1)); - assert_eq!(Elections::candidates(), vec![2, 3, 1]); - }); -} - -#[test] -fn candidacy_submission_not_using_free_slot_should_not_work() { - let mut t = new_test_ext_with_candidate_holes(); - - t.execute_with(|| { - assert_noop!( - Elections::submit_candidacy(Origin::signed(4), 3), - Error::::InvalidCandidateSlot - ); - }); -} - -#[test] -fn candidacy_bad_candidate_slot_submission_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - assert_eq!(Elections::candidates(), Vec::::new()); - assert_noop!( - Elections::submit_candidacy(Origin::signed(1), 1), - Error::::InvalidCandidateSlot - ); - }); -} - -#[test] -fn candidacy_non_free_candidate_slot_submission_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - assert_eq!(Elections::candidates(), Vec::::new()); - assert_ok!(Elections::submit_candidacy(Origin::signed(1), 0)); - assert_eq!(Elections::candidates(), vec![1]); - assert_noop!( - Elections::submit_candidacy(Origin::signed(2), 0), - Error::::InvalidCandidateSlot - ); - }); -} - -#[test] -fn candidacy_dupe_candidate_submission_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - assert_eq!(Elections::candidates(), Vec::::new()); - assert_ok!(Elections::submit_candidacy(Origin::signed(1), 0)); - assert_eq!(Elections::candidates(), vec![1]); - assert_noop!( - Elections::submit_candidacy(Origin::signed(1), 1), - Error::::DuplicatedCandidate, - ); - }); -} - -#[test] -fn candidacy_poor_candidate_submission_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - assert_eq!(Elections::candidates(), Vec::::new()); - assert_noop!( - Elections::submit_candidacy(Origin::signed(7), 0), - Error::::InsufficientCandidateFunds, - ); - }); -} - -#[test] -fn election_voting_should_work() { - ExtBuilder::default().build().execute_with(|| { - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 0)); - - assert_ok!(Elections::set_approvals(Origin::signed(1), vec![true], 0, 0, 10)); - assert_ok!(Elections::set_approvals(Origin::signed(4), vec![true], 0, 1, 40)); - - assert_eq!(Elections::all_approvals_of(&1), vec![true]); - assert_eq!(Elections::all_approvals_of(&4), vec![true]); - assert_eq!(voter_ids(), vec![1, 4]); - - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 2)); - - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![false, true, true], 0, 2, 20)); - assert_ok!(Elections::set_approvals(Origin::signed(3), vec![false, true, true], 0, 3, 30)); - - assert_eq!(Elections::all_approvals_of(&1), vec![true]); - assert_eq!(Elections::all_approvals_of(&4), vec![true]); - assert_eq!(Elections::all_approvals_of(&2), vec![false, true, true]); - assert_eq!(Elections::all_approvals_of(&3), vec![false, true, true]); - - assert_eq!(voter_ids(), vec![1, 4, 2, 3]); - }); -} - -#[test] -fn election_simple_tally_should_work() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert!(!Elections::presentation_active()); - - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 1)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true], 0, 0, 20)); - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![false, true], 0, 0, 50)); - assert_eq!(voter_ids(), vec![2, 5]); - assert_eq!(Elections::all_approvals_of(&2), vec![true]); - assert_eq!(Elections::all_approvals_of(&5), vec![false, true]); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert!(Elections::presentation_active()); - assert_eq!(Elections::present_winner(Origin::signed(4), 2, 20, 0), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(4), 5, 50, 0), Ok(())); - assert_eq!(Elections::leaderboard(), Some(vec![(0, 0), (0, 0), (20, 2), (50, 5)])); - assert_ok!(Elections::end_block(System::block_number())); - - assert!(!Elections::presentation_active()); - assert_eq!(Elections::members(), vec![(5, 11), (2, 11)]); - - assert!(!Elections::is_a_candidate(&2)); - assert!(!Elections::is_a_candidate(&5)); - assert_eq!(Elections::vote_index(), 1); - assert_eq!( - Elections::voter_info(2), - Some(VoterInfo { last_win: 1, last_active: 0, stake: 20, pot: 0 }) - ); - assert_eq!( - Elections::voter_info(5), - Some(VoterInfo { last_win: 1, last_active: 0, stake: 50, pot: 0 }) - ); - }); -} - -#[test] -fn election_seats_should_be_released() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 1)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true, false], 0, 0, 20)); - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![false, true], 0, 0, 50)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert!(Elections::presentation_active()); - assert_eq!(Elections::present_winner(Origin::signed(4), 2, 20, 0), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(4), 5, 50, 0), Ok(())); - assert_eq!(Elections::leaderboard(), Some(vec![(0, 0), (0, 0), (20, 2), (50, 5)])); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::members(), vec![(5, 11), (2, 11)]); - let mut current = System::block_number(); - let free_block; - loop { - current += 1; - System::set_block_number(current); - assert_ok!(Elections::end_block(System::block_number())); - if Elections::members().len() == 0 { - free_block = current; - break - } - } - // 11 + 2 which is the next voting period. - assert_eq!(free_block, 14); - }); -} - -#[test] -fn election_presentations_with_zero_staked_deposit_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true], 0, 0, 20)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_noop!( - Elections::present_winner(Origin::signed(4), 2, 0, 0), - Error::::ZeroDeposit, - ); - }); -} - -#[test] -fn election_double_presentations_should_be_punished() { - ExtBuilder::default().build().execute_with(|| { - assert!(Balances::can_slash(&4, 10)); - - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 1)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true, false], 0, 0, 20)); - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![false, true], 0, 0, 50)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_ok!(Elections::present_winner(Origin::signed(4), 2, 20, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 5, 50, 0)); - assert_eq!( - Elections::present_winner(Origin::signed(4), 5, 50, 0), - Err(Error::::DuplicatedPresentation.into()), - ); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::members(), vec![(5, 11), (2, 11)]); - assert_eq!(Balances::total_balance(&4), 38); - }); -} - -#[test] -fn election_presenting_for_double_election_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert_eq!(Elections::submit_candidacy(Origin::signed(2), 0), Ok(())); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true], 0, 0, 20)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_ok!(Elections::present_winner(Origin::signed(4), 2, 20, 0)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(8); - // NOTE: This is now mandatory to disable the lock - assert_ok!(Elections::retract_voter(Origin::signed(2), 0)); - assert_eq!(Elections::submit_candidacy(Origin::signed(2), 0), Ok(())); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true], 1, 0, 20)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(10); - assert_noop!( - Elections::present_winner(Origin::signed(4), 2, 20, 1), - Error::::DuplicatedCandidate, - ); - }); -} - -#[test] -fn election_presenting_loser_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(1), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(6), vec![true], 0, 0, 60)); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 1)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![false, true], 0, 0, 20)); - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 2)); - assert_ok!(Elections::set_approvals(Origin::signed(3), vec![false, false, true], 0, 0, 30)); - assert_ok!(Elections::submit_candidacy(Origin::signed(4), 3)); - assert_ok!(Elections::set_approvals( - Origin::signed(4), - vec![false, false, false, true], - 0, - 0, - 40 - )); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 4)); - assert_ok!(Elections::set_approvals( - Origin::signed(5), - vec![false, false, false, false, true], - 0, - 0, - 50 - )); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_ok!(Elections::present_winner(Origin::signed(4), 1, 60, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 3, 30, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 4, 40, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 5, 50, 0)); - - assert_eq!(Elections::leaderboard(), Some(vec![(30, 3), (40, 4), (50, 5), (60, 1)])); - - assert_noop!( - Elections::present_winner(Origin::signed(4), 2, 20, 0), - Error::::UnworthyCandidate - ); - }); -} - -#[test] -fn election_presenting_loser_first_should_not_matter() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(1), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(6), vec![true], 0, 0, 60)); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 1)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![false, true], 0, 0, 20)); - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 2)); - assert_ok!(Elections::set_approvals(Origin::signed(3), vec![false, false, true], 0, 0, 30)); - assert_ok!(Elections::submit_candidacy(Origin::signed(4), 3)); - assert_ok!(Elections::set_approvals( - Origin::signed(4), - vec![false, false, false, true], - 0, - 0, - 40 - )); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 4)); - assert_ok!(Elections::set_approvals( - Origin::signed(5), - vec![false, false, false, false, true], - 0, - 0, - 50 - )); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_ok!(Elections::present_winner(Origin::signed(4), 2, 20, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 1, 60, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 3, 30, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 4, 40, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 5, 50, 0)); - - assert_eq!(Elections::leaderboard(), Some(vec![(30, 3), (40, 4), (50, 5), (60, 1)])); - }); -} - -#[test] -fn election_present_outside_of_presentation_period_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert!(!Elections::presentation_active()); - assert_noop!( - Elections::present_winner(Origin::signed(5), 5, 1, 0), - Error::::NotPresentationPeriod, - ); - }); -} - -#[test] -fn election_present_with_invalid_vote_index_should_not_work() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 1)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true, false], 0, 0, 20)); - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![false, true], 0, 0, 50)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_noop!( - Elections::present_winner(Origin::signed(4), 2, 20, 1), - Error::::InvalidVoteIndex - ); - }); -} - -#[test] -fn election_present_when_presenter_is_poor_should_not_work() { - let test_present = |p| { - ExtBuilder::default() - .voting_fee(5) - .voting_bond(2) - .bad_presentation_punishment(p) - .build() - .execute_with(|| { - System::set_block_number(4); - let _ = Balances::make_free_balance_be(&1, 15); - assert!(!Elections::presentation_active()); - - // -3 - assert_ok!(Elections::submit_candidacy(Origin::signed(1), 0)); - assert_eq!(Balances::free_balance(1), 12); - // -2 -5 - assert_ok!(Elections::set_approvals(Origin::signed(1), vec![true], 0, 0, 15)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_eq!(Balances::free_balance(1), 5); - assert_eq!(Balances::reserved_balance(1), 5); - if p > 5 { - assert_noop!( - Elections::present_winner(Origin::signed(1), 1, 10, 0), - Error::::InsufficientPresenterFunds, - ); - } else { - assert_ok!(Elections::present_winner(Origin::signed(1), 1, 10, 0)); - } - }); - }; - test_present(4); - test_present(6); -} - -#[test] -fn election_invalid_present_tally_should_slash() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert!(!Elections::presentation_active()); - assert_eq!(Balances::total_balance(&4), 40); - - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 1)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![true, false], 0, 0, 20)); - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![false, true], 0, 0, 50)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_err!( - Elections::present_winner(Origin::signed(4), 2, 80, 0), - Error::::IncorrectTotal - ); - - assert_eq!(Balances::total_balance(&4), 38); - }); -} - -#[test] -fn election_runners_up_should_be_kept() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert!(!Elections::presentation_active()); - - assert_ok!(Elections::submit_candidacy(Origin::signed(1), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(6), vec![true], 0, 0, 60)); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 1)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![false, true], 0, 0, 20)); - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 2)); - assert_ok!(Elections::set_approvals(Origin::signed(3), vec![false, false, true], 0, 0, 30)); - assert_ok!(Elections::submit_candidacy(Origin::signed(4), 3)); - assert_ok!(Elections::set_approvals( - Origin::signed(4), - vec![false, false, false, true], - 0, - 0, - 40 - )); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 4)); - assert_ok!(Elections::set_approvals( - Origin::signed(5), - vec![false, false, false, false, true], - 0, - 0, - 50 - )); - - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert!(Elections::presentation_active()); - assert_ok!(Elections::present_winner(Origin::signed(4), 1, 60, 0)); - // leaderboard length is the empty seats plus the carry count (i.e. 5 + 2), where those - // to be carried are the lowest and stored in lowest indices - assert_eq!(Elections::leaderboard(), Some(vec![(0, 0), (0, 0), (0, 0), (60, 1)])); - assert_ok!(Elections::present_winner(Origin::signed(4), 3, 30, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 4, 40, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 5, 50, 0)); - assert_eq!(Elections::leaderboard(), Some(vec![(30, 3), (40, 4), (50, 5), (60, 1)])); - - assert_ok!(Elections::end_block(System::block_number())); - - assert!(!Elections::presentation_active()); - assert_eq!(Elections::members(), vec![(1, 11), (5, 11)]); - - assert!(!Elections::is_a_candidate(&1)); - assert!(!Elections::is_a_candidate(&5)); - assert!(!Elections::is_a_candidate(&2)); - assert!(Elections::is_a_candidate(&3)); - assert!(Elections::is_a_candidate(&4)); - assert_eq!(Elections::vote_index(), 1); - assert_eq!( - Elections::voter_info(2), - Some(VoterInfo { last_win: 0, last_active: 0, stake: 20, pot: 0 }) - ); - assert_eq!( - Elections::voter_info(3), - Some(VoterInfo { last_win: 0, last_active: 0, stake: 30, pot: 0 }) - ); - assert_eq!( - Elections::voter_info(4), - Some(VoterInfo { last_win: 0, last_active: 0, stake: 40, pot: 0 }) - ); - assert_eq!( - Elections::voter_info(5), - Some(VoterInfo { last_win: 1, last_active: 0, stake: 50, pot: 0 }) - ); - assert_eq!( - Elections::voter_info(6), - Some(VoterInfo { last_win: 1, last_active: 0, stake: 60, pot: 0 }) - ); - assert_eq!(Elections::candidate_reg_info(3), Some((0, 2))); - assert_eq!(Elections::candidate_reg_info(4), Some((0, 3))); - }); -} - -#[test] -fn election_second_tally_should_use_runners_up() { - ExtBuilder::default().build().execute_with(|| { - System::set_block_number(4); - assert_ok!(Elections::submit_candidacy(Origin::signed(1), 0)); - assert_ok!(Elections::set_approvals(Origin::signed(6), vec![true], 0, 0, 60)); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 1)); - assert_ok!(Elections::set_approvals(Origin::signed(2), vec![false, true], 0, 0, 20)); - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 2)); - assert_ok!(Elections::set_approvals(Origin::signed(3), vec![false, false, true], 0, 0, 30)); - assert_ok!(Elections::submit_candidacy(Origin::signed(4), 3)); - assert_ok!(Elections::set_approvals( - Origin::signed(4), - vec![false, false, false, true], - 0, - 0, - 40 - )); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 4)); - assert_ok!(Elections::set_approvals( - Origin::signed(5), - vec![false, false, false, false, true], - 0, - 0, - 50 - )); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert_ok!(Elections::present_winner(Origin::signed(4), 1, 60, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 3, 30, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 4, 40, 0)); - assert_ok!(Elections::present_winner(Origin::signed(4), 5, 50, 0)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(8); - assert_ok!(Elections::set_approvals( - Origin::signed(6), - vec![false, false, true, false], - 1, - 0, - 60 - )); - assert_ok!(Elections::set_desired_seats(Origin::root(), 3)); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(10); - assert_ok!(Elections::present_winner( - Origin::signed(4), - 3, - 30 + Elections::get_offset(30, 1) + 60, - 1 - )); - assert_ok!(Elections::present_winner( - Origin::signed(4), - 4, - 40 + Elections::get_offset(40, 1), - 1 - )); - assert_ok!(Elections::end_block(System::block_number())); - - assert!(!Elections::presentation_active()); - assert_eq!(Elections::members(), vec![(1, 11), (5, 11), (3, 15)]); - - assert!(!Elections::is_a_candidate(&1)); - assert!(!Elections::is_a_candidate(&2)); - assert!(!Elections::is_a_candidate(&3)); - assert!(!Elections::is_a_candidate(&5)); - assert!(Elections::is_a_candidate(&4)); - assert_eq!(Elections::vote_index(), 2); - assert_eq!( - Elections::voter_info(2), - Some(VoterInfo { last_win: 0, last_active: 0, stake: 20, pot: 0 }) - ); - assert_eq!( - Elections::voter_info(3), - Some(VoterInfo { last_win: 2, last_active: 0, stake: 30, pot: 0 }) - ); - assert_eq!( - Elections::voter_info(4), - Some(VoterInfo { last_win: 0, last_active: 0, stake: 40, pot: 0 }) - ); - assert_eq!( - Elections::voter_info(5), - Some(VoterInfo { last_win: 1, last_active: 0, stake: 50, pot: 0 }) - ); - assert_eq!( - Elections::voter_info(6), - Some(VoterInfo { last_win: 2, last_active: 1, stake: 60, pot: 0 }) - ); - - assert_eq!(Elections::candidate_reg_info(4), Some((0, 3))); - }); -} - -#[test] -fn election_loser_candidates_bond_gets_slashed() { - ExtBuilder::default().desired_seats(1).build().execute_with(|| { - System::set_block_number(4); - assert!(!Elections::presentation_active()); - - assert_ok!(Elections::submit_candidacy(Origin::signed(1), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 2)); - assert_ok!(Elections::submit_candidacy(Origin::signed(4), 3)); - - assert_eq!(balances(&2), (17, 3)); - - assert_ok!(Elections::set_approvals(Origin::signed(5), vec![true], 0, 0, 50)); - assert_ok!(Elections::set_approvals( - Origin::signed(1), - vec![false, true, true, true], - 0, - 0, - 10 - )); - - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert!(Elections::presentation_active()); - assert_eq!(Elections::present_winner(Origin::signed(4), 4, 10, 0), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(3), 3, 10, 0), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(2), 2, 10, 0), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(1), 1, 50, 0), Ok(())); - - // winner + carry - assert_eq!(Elections::leaderboard(), Some(vec![(10, 3), (10, 4), (50, 1)])); - assert_ok!(Elections::end_block(System::block_number())); - assert!(!Elections::presentation_active()); - assert_eq!(Elections::members(), vec![(1, 11)]); - - // account 2 is not a runner up or in leaderboard. - assert_eq!(balances(&2), (17, 0)); - }); -} - -#[test] -fn pot_accumulating_weight_and_decaying_should_work() { - ExtBuilder::default().balance_factor(10).build().execute_with(|| { - System::set_block_number(4); - assert!(!Elections::presentation_active()); - - assert_ok!(Elections::submit_candidacy(Origin::signed(6), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(1), 2)); - - assert_ok!(Elections::set_approvals( - Origin::signed(6), - vec![true, false, false], - 0, - 0, - 600 - )); - assert_ok!(Elections::set_approvals( - Origin::signed(5), - vec![false, true, false], - 0, - 0, - 500 - )); - assert_ok!(Elections::set_approvals( - Origin::signed(1), - vec![false, false, true], - 0, - 0, - 100 - )); - - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert!(Elections::presentation_active()); - - assert_eq!(Elections::present_winner(Origin::signed(6), 6, 600, 0), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(5), 5, 500, 0), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(1), 1, 100, 0), Ok(())); - assert_eq!(Elections::leaderboard(), Some(vec![(0, 0), (100, 1), (500, 5), (600, 6)])); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::members(), vec![(6, 11), (5, 11)]); - assert_eq!( - Elections::voter_info(6).unwrap(), - VoterInfo { last_win: 1, last_active: 0, stake: 600, pot: 0 }, - ); - assert_eq!( - Elections::voter_info(5).unwrap(), - VoterInfo { last_win: 1, last_active: 0, stake: 500, pot: 0 }, - ); - assert_eq!( - Elections::voter_info(1).unwrap(), - VoterInfo { last_win: 0, last_active: 0, stake: 100, pot: 0 }, - ); - - System::set_block_number(12); - // retract needed to unlock approval funds => submit candidacy again. - assert_ok!(Elections::retract_voter(Origin::signed(6), 0)); - assert_ok!(Elections::retract_voter(Origin::signed(5), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(6), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 1)); - assert_ok!(Elections::set_approvals( - Origin::signed(6), - vec![true, false, false], - 1, - 0, - 600 - )); - assert_ok!(Elections::set_approvals( - Origin::signed(5), - vec![false, true, false], - 1, - 1, - 500 - )); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(14); - assert!(Elections::presentation_active()); - assert_eq!(Elections::present_winner(Origin::signed(6), 6, 600, 1), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(5), 5, 500, 1), Ok(())); - assert_eq!( - Elections::present_winner(Origin::signed(1), 1, 100 + Elections::get_offset(100, 1), 1), - Ok(()) - ); - assert_eq!(Elections::leaderboard(), Some(vec![(0, 0), (100 + 96, 1), (500, 5), (600, 6)])); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::members(), vec![(6, 19), (5, 19)]); - assert_eq!( - Elections::voter_info(6).unwrap(), - VoterInfo { last_win: 2, last_active: 1, stake: 600, pot: 0 } - ); - assert_eq!( - Elections::voter_info(5).unwrap(), - VoterInfo { last_win: 2, last_active: 1, stake: 500, pot: 0 } - ); - assert_eq!( - Elections::voter_info(1).unwrap(), - VoterInfo { last_win: 0, last_active: 0, stake: 100, pot: 0 } - ); - - System::set_block_number(20); - assert_ok!(Elections::retract_voter(Origin::signed(6), 0)); - assert_ok!(Elections::retract_voter(Origin::signed(5), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(6), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 1)); - assert_ok!(Elections::set_approvals( - Origin::signed(6), - vec![true, false, false], - 2, - 0, - 600 - )); - assert_ok!(Elections::set_approvals( - Origin::signed(5), - vec![false, true, false], - 2, - 1, - 500 - )); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(22); - assert!(Elections::presentation_active()); - assert_eq!(Elections::present_winner(Origin::signed(6), 6, 600, 2), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(5), 5, 500, 2), Ok(())); - assert_eq!( - Elections::present_winner(Origin::signed(1), 1, 100 + Elections::get_offset(100, 2), 2), - Ok(()) - ); - assert_eq!( - Elections::leaderboard(), - Some(vec![(0, 0), (100 + 96 + 93, 1), (500, 5), (600, 6)]) - ); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::members(), vec![(6, 27), (5, 27)]); - assert_eq!( - Elections::voter_info(6).unwrap(), - VoterInfo { last_win: 3, last_active: 2, stake: 600, pot: 0 } - ); - assert_eq!( - Elections::voter_info(5).unwrap(), - VoterInfo { last_win: 3, last_active: 2, stake: 500, pot: 0 } - ); - assert_eq!( - Elections::voter_info(1).unwrap(), - VoterInfo { last_win: 0, last_active: 0, stake: 100, pot: 0 } - ); - - System::set_block_number(28); - assert_ok!(Elections::retract_voter(Origin::signed(6), 0)); - assert_ok!(Elections::retract_voter(Origin::signed(5), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(6), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 1)); - assert_ok!(Elections::set_approvals( - Origin::signed(6), - vec![true, false, false], - 3, - 0, - 600 - )); - assert_ok!(Elections::set_approvals( - Origin::signed(5), - vec![false, true, false], - 3, - 1, - 500 - )); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(30); - assert!(Elections::presentation_active()); - assert_eq!(Elections::present_winner(Origin::signed(6), 6, 600, 3), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(5), 5, 500, 3), Ok(())); - assert_eq!( - Elections::present_winner(Origin::signed(1), 1, 100 + Elections::get_offset(100, 3), 3), - Ok(()) - ); - assert_eq!( - Elections::leaderboard(), - Some(vec![(0, 0), (100 + 96 + 93 + 90, 1), (500, 5), (600, 6)]) - ); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::members(), vec![(6, 35), (5, 35)]); - assert_eq!( - Elections::voter_info(6).unwrap(), - VoterInfo { last_win: 4, last_active: 3, stake: 600, pot: 0 } - ); - assert_eq!( - Elections::voter_info(5).unwrap(), - VoterInfo { last_win: 4, last_active: 3, stake: 500, pot: 0 } - ); - assert_eq!( - Elections::voter_info(1).unwrap(), - VoterInfo { last_win: 0, last_active: 0, stake: 100, pot: 0 } - ); - }) -} - -#[test] -fn pot_winning_resets_accumulated_pot() { - ExtBuilder::default().balance_factor(10).build().execute_with(|| { - System::set_block_number(4); - assert!(!Elections::presentation_active()); - - assert_ok!(Elections::submit_candidacy(Origin::signed(6), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(4), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(3), 2)); - assert_ok!(Elections::submit_candidacy(Origin::signed(2), 3)); - - assert_ok!(Elections::set_approvals( - Origin::signed(6), - vec![true, false, false, false], - 0, - 0, - 600 - )); - assert_ok!(Elections::set_approvals( - Origin::signed(4), - vec![false, true, false, false], - 0, - 1, - 400 - )); - assert_ok!(Elections::set_approvals( - Origin::signed(3), - vec![false, false, true, true], - 0, - 2, - 300 - )); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert!(Elections::presentation_active()); - assert_eq!(Elections::present_winner(Origin::signed(6), 6, 600, 0), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(4), 4, 400, 0), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(3), 3, 300, 0), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(2), 2, 300, 0), Ok(())); - assert_eq!(Elections::leaderboard(), Some(vec![(300, 2), (300, 3), (400, 4), (600, 6)])); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::members(), vec![(6, 11), (4, 11)]); - - System::set_block_number(12); - assert_ok!(Elections::retract_voter(Origin::signed(6), 0)); - assert_ok!(Elections::retract_voter(Origin::signed(4), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(6), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(4), 1)); - assert_ok!(Elections::set_approvals( - Origin::signed(6), - vec![true, false, false, false], - 1, - 0, - 600 - )); - assert_ok!(Elections::set_approvals( - Origin::signed(4), - vec![false, true, false, false], - 1, - 1, - 400 - )); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(14); - assert!(Elections::presentation_active()); - assert_eq!(Elections::present_winner(Origin::signed(6), 6, 600, 1), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(4), 4, 400, 1), Ok(())); - assert_eq!( - Elections::present_winner(Origin::signed(3), 3, 300 + Elections::get_offset(300, 1), 1), - Ok(()) - ); - assert_eq!( - Elections::present_winner(Origin::signed(2), 2, 300 + Elections::get_offset(300, 1), 1), - Ok(()) - ); - assert_eq!(Elections::leaderboard(), Some(vec![(400, 4), (588, 2), (588, 3), (600, 6)])); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::members(), vec![(6, 19), (3, 19)]); - - System::set_block_number(20); - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(22); - // 2 will not get re-elected with 300 + 288, instead just 300. - // because one of 3's candidates (3) won in previous round - // 4 on the other hand will get extra weight since it was unlucky. - assert_eq!(Elections::present_winner(Origin::signed(3), 2, 300, 2), Ok(())); - assert_eq!( - Elections::present_winner(Origin::signed(4), 4, 400 + Elections::get_offset(400, 1), 2), - Ok(()) - ); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::members(), vec![(4, 27), (2, 27)]); - }) -} - -#[test] -fn pot_resubmitting_approvals_stores_pot() { - ExtBuilder::default() - .voting_bond(0) - .voting_fee(0) - .balance_factor(10) - .build() - .execute_with(|| { - System::set_block_number(4); - assert!(!Elections::presentation_active()); - - assert_ok!(Elections::submit_candidacy(Origin::signed(6), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(1), 2)); - - assert_ok!(Elections::set_approvals( - Origin::signed(6), - vec![true, false, false], - 0, - 0, - 600 - ),); - assert_ok!(Elections::set_approvals( - Origin::signed(5), - vec![false, true, false], - 0, - 1, - 500 - ),); - assert_ok!(Elections::set_approvals( - Origin::signed(1), - vec![false, false, true], - 0, - 2, - 100 - ),); - - assert_ok!(Elections::end_block(System::block_number())); - - System::set_block_number(6); - assert!(Elections::presentation_active()); - - assert_eq!(Elections::present_winner(Origin::signed(6), 6, 600, 0), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(5), 5, 500, 0), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(1), 1, 100, 0), Ok(())); - assert_eq!(Elections::leaderboard(), Some(vec![(0, 0), (100, 1), (500, 5), (600, 6)])); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::members(), vec![(6, 11), (5, 11)]); - - System::set_block_number(12); - assert_ok!(Elections::retract_voter(Origin::signed(6), 0)); - assert_ok!(Elections::retract_voter(Origin::signed(5), 1)); - assert_ok!(Elections::submit_candidacy(Origin::signed(6), 0)); - assert_ok!(Elections::submit_candidacy(Origin::signed(5), 1)); - assert_ok!(Elections::set_approvals( - Origin::signed(6), - vec![true, false, false], - 1, - 0, - 600 - ),); - assert_ok!(Elections::set_approvals( - Origin::signed(5), - vec![false, true, false], - 1, - 1, - 500 - ),); - // give 1 some new high balance - let _ = Balances::make_free_balance_be(&1, 997); - assert_ok!(Elections::set_approvals( - Origin::signed(1), - vec![false, false, true], - 1, - 2, - 1000 - ),); - assert_eq!( - Elections::voter_info(1).unwrap(), - VoterInfo { - stake: 1000, // 997 + 3 which is candidacy bond. - pot: Elections::get_offset(100, 1), - last_active: 1, - last_win: 1, - } - ); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::members(), vec![(6, 11), (5, 11)]); - - System::set_block_number(14); - assert!(Elections::presentation_active()); - assert_eq!(Elections::present_winner(Origin::signed(6), 6, 600, 1), Ok(())); - assert_eq!(Elections::present_winner(Origin::signed(5), 5, 500, 1), Ok(())); - assert_eq!( - Elections::present_winner(Origin::signed(1), 1, 1000 + 96 /* pot */, 1), - Ok(()), - ); - assert_eq!(Elections::leaderboard(), Some(vec![(0, 0), (500, 5), (600, 6), (1096, 1)])); - assert_ok!(Elections::end_block(System::block_number())); - - assert_eq!(Elections::members(), vec![(1, 19), (6, 19)]); - }) -} - -#[test] -fn pot_get_offset_should_work() { - ExtBuilder::default().build().execute_with(|| { - assert_eq!(Elections::get_offset(100, 0), 0); - assert_eq!(Elections::get_offset(100, 1), 96); - assert_eq!(Elections::get_offset(100, 2), 96 + 93); - assert_eq!(Elections::get_offset(100, 3), 96 + 93 + 90); - assert_eq!(Elections::get_offset(100, 4), 96 + 93 + 90 + 87); - // limit - assert_eq!(Elections::get_offset(100, 1000), 100 * 24); - - assert_eq!(Elections::get_offset(50_000_000_000, 0), 0); - assert_eq!(Elections::get_offset(50_000_000_000, 1), 48_000_000_000); - assert_eq!(Elections::get_offset(50_000_000_000, 2), 48_000_000_000 + 46_080_000_000); - assert_eq!( - Elections::get_offset(50_000_000_000, 3), - 48_000_000_000 + 46_080_000_000 + 44_236_800_000 - ); - assert_eq!( - Elections::get_offset(50_000_000_000, 4), - 48_000_000_000 + 46_080_000_000 + 44_236_800_000 + 42_467_328_000 - ); - // limit - assert_eq!(Elections::get_offset(50_000_000_000, 1000), 50_000_000_000 * 24); - }) -} - -#[test] -fn pot_get_offset_with_zero_decay() { - ExtBuilder::default().decay_ratio(0).build().execute_with(|| { - assert_eq!(Elections::get_offset(100, 0), 0); - assert_eq!(Elections::get_offset(100, 1), 0); - assert_eq!(Elections::get_offset(100, 2), 0); - assert_eq!(Elections::get_offset(100, 3), 0); - // limit - assert_eq!(Elections::get_offset(100, 1000), 0); - }) -} diff --git a/frame/example/Cargo.toml b/frame/examples/basic/Cargo.toml similarity index 53% rename from frame/example/Cargo.toml rename to frame/examples/basic/Cargo.toml index 58daaf1c7555..a54301944a01 100644 --- a/frame/example/Cargo.toml +++ b/frame/examples/basic/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "pallet-example" +name = "pallet-example-basic" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Unlicense" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME example pallet" readme = "README.md" @@ -13,19 +13,20 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } -frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../balances" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } log = { version = "0.4.14", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } + +frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../../benchmarking", optional = true } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" } +pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../balances" } +sp-io = { version = "6.0.0", default-features = false, path = "../../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core", default-features = false } +sp-core = { version = "6.0.0", path = "../../../primitives/core", default-features = false } [features] default = ["std"] @@ -41,5 +42,5 @@ std = [ "sp-runtime/std", "sp-std/std" ] -runtime-benchmarks = ["frame-benchmarking"] +runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/example/README.md b/frame/examples/basic/README.md similarity index 94% rename from frame/example/README.md rename to frame/examples/basic/README.md index e06dee78c3f8..358829192f11 100644 --- a/frame/example/README.md +++ b/frame/examples/basic/README.md @@ -1,11 +1,13 @@ -# Example Pallet +# Basic Example Pallet The Example: A simple example of a FRAME pallet demonstrating concepts, APIs and structures common to most FRAME runtimes. -Run `cargo doc --package pallet-example --open` to view this pallet's documentation. +Run `cargo doc --package pallet-example-basic --open` to view this pallet's documentation. + +**This pallet serves as an example and is not meant to be used in production.** ### Documentation Guidelines: @@ -34,7 +36,7 @@ Run `cargo doc --package pallet-example --open` to view this pallet's documentat ### Documentation Template:
-Copy and paste this template from frame/example/src/lib.rs into file +Copy and paste this template from frame/examples/basic/src/lib.rs into file `frame//src/lib.rs` of your own custom pallet and complete it.

 // Add heading with custom pallet name
@@ -46,9 +48,9 @@ Copy and paste this template from frame/example/src/lib.rs into file
 // Include the following links that shows what trait needs to be implemented to use the pallet
 // and the supported dispatchables that are documented in the Call enum.
 
-- \[`::Config`](https://docs.rs/pallet-example/latest/pallet_example/trait.Config.html)
-- \[`Call`](https://docs.rs/pallet-example/latest/pallet_example/enum.Call.html)
-- \[`Module`](https://docs.rs/pallet-example/latest/pallet_example/struct.Module.html)
+- \[`::Config`](https://docs.rs/pallet-example-basic/latest/pallet_example_basic/trait.Config.html)
+- \[`Call`](https://docs.rs/pallet-example-basic/latest/pallet_example_basic/enum.Call.html)
+- \[`Module`](https://docs.rs/pallet-example-basic/latest/pallet_example_basic/struct.Module.html)
 
 \## Overview
 
diff --git a/frame/example/src/benchmarking.rs b/frame/examples/basic/src/benchmarking.rs
similarity index 69%
rename from frame/example/src/benchmarking.rs
rename to frame/examples/basic/src/benchmarking.rs
index cdf6c152a488..b823ccd07296 100644
--- a/frame/example/src/benchmarking.rs
+++ b/frame/examples/basic/src/benchmarking.rs
@@ -1,6 +1,6 @@
 // This file is part of Substrate.
 
-// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd.
+// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd.
 // SPDX-License-Identifier: Apache-2.0
 
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,24 +15,24 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Benchmarking for pallet-example.
+//! Benchmarking for pallet-example-basic.
 
 #![cfg(feature = "runtime-benchmarks")]
 
 use crate::*;
-use frame_benchmarking::{benchmarks, impl_benchmark_test_suite, whitelisted_caller};
+use frame_benchmarking::{benchmarks, whitelisted_caller};
 use frame_system::RawOrigin;
 
-// To actually run this benchmark on pallet-example, we need to put this pallet into the
+// To actually run this benchmark on pallet-example-basic, we need to put this pallet into the
 //   runtime and compile it with `runtime-benchmarks` feature. The detail procedures are
 //   documented at:
-//   https://substrate.dev/docs/en/knowledgebase/runtime/benchmarking#how-to-benchmark
+//   https://docs.substrate.io/v3/runtime/benchmarking#how-to-benchmark
 //
 // The auto-generated weight estimate of this pallet is copied over to the `weights.rs` file.
 // The exact command of how the estimate generated is printed at the top of the file.
 
 // Details on using the benchmarks macro can be seen at:
-//   https://substrate.dev/rustdocs/v3.0.0/frame_benchmarking/macro.benchmarks.html
+//   https://paritytech.github.io/substrate/master/frame_benchmarking/trait.Benchmarking.html#tymethod.benchmarks
 benchmarks! {
 	// This will measure the execution time of `set_dummy` for b in [1..1000] range.
 	set_dummy_benchmark {
@@ -65,12 +65,14 @@ benchmarks! {
 		// The benchmark execution phase could also be a closure with custom code
 		m.sort();
 	}
-}
 
-// This line generates test cases for benchmarking, and could be run by:
-//   `cargo test -p pallet-example --all-features`, you will see an additional line of:
-//   `test benchmarking::benchmark_tests::test_benchmarks ... ok` in the result.
-//
-// The line generates three steps per benchmark, with repeat=1 and the three steps are
-//   [low, mid, high] of the range.
-impl_benchmark_test_suite!(Pallet, crate::tests::new_test_ext(), crate::tests::Test);
+	// This line generates test cases for benchmarking, and could be run by:
+	//   `cargo test -p pallet-example-basic --all-features`, you will see one line per case:
+	//   `test benchmarking::bench_sort_vector ... ok`
+	//   `test benchmarking::bench_accumulate_dummy ... ok`
+	//   `test benchmarking::bench_set_dummy_benchmark ... ok` in the result.
+	//
+	// The line generates three steps per benchmark, with repeat=1 and the three steps are
+	//   [low, mid, high] of the range.
+	impl_benchmark_test_suite!(Pallet, crate::tests::new_test_ext(), crate::tests::Test)
+}
diff --git a/frame/example/src/lib.rs b/frame/examples/basic/src/lib.rs
similarity index 95%
rename from frame/example/src/lib.rs
rename to frame/examples/basic/src/lib.rs
index 23c4951c1a60..56e8db693624 100644
--- a/frame/example/src/lib.rs
+++ b/frame/examples/basic/src/lib.rs
@@ -1,6 +1,6 @@
 // This file is part of Substrate.
 
-// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
+// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd.
 // SPDX-License-Identifier: Apache-2.0
 
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,13 +16,15 @@
 // limitations under the License.
 
 //! 
-//! # Example Pallet
+//! # Basic Example Pallet
 //!
 //! 
 //! The Example: A simple example of a FRAME pallet demonstrating
 //! concepts, APIs and structures common to most FRAME runtimes.
 //!
-//! Run `cargo doc --package pallet-example --open` to view this pallet's documentation.
+//! Run `cargo doc --package pallet-example-basic --open` to view this pallet's documentation.
+//!
+//! **This pallet serves as an example and is not meant to be used in production.**
 //!
 //! ### Documentation Guidelines:
 //!
@@ -59,7 +61,7 @@
 //!
 //! ### Documentation Template:
//! -//! Copy and paste this template from frame/example/src/lib.rs into file +//! Copy and paste this template from frame/examples/basic/src/lib.rs into file //! `frame//src/lib.rs` of your own custom pallet and complete it. //!

 //! // Add heading with custom pallet name
@@ -316,8 +318,7 @@ const MILLICENTS: u32 = 1_000_000_000;
 // - assigns a dispatch class `operational` if the argument of the call is more than 1000.
 //
 // More information can be read at:
-//   - https://substrate.dev/docs/en/knowledgebase/learn-substrate/weight
-//   - https://substrate.dev/docs/en/knowledgebase/runtime/fees#default-weight-annotations
+//   - https://docs.substrate.io/v3/runtime/weights-and-fees
 //
 // Manually configuring weight is an advanced operation and what you really need may well be
 //   fulfilled by running the benchmarking toolchain. Refer to `benchmarking.rs` file.
@@ -487,11 +488,12 @@ pub mod pallet {
 		// the chain in a moderate rate.
 		//
 		// The parenthesized value of the `#[pallet::weight(..)]` attribute can be any type that
-		// implements a set of traits, namely [`WeighData`] and [`ClassifyDispatch`].
-		// The former conveys the weight (a numeric representation of pure execution time and
-		// difficulty) of the transaction and the latter demonstrates the [`DispatchClass`] of the
-		// call. A higher weight means a larger transaction (less of which can be placed in a
-		// single block).
+		// implements a set of traits, namely [`WeighData`], [`ClassifyDispatch`], and
+		// [`PaysFee`]. The first conveys the weight (a numeric representation of pure
+		// execution time and difficulty) of the transaction and the second demonstrates the
+		// [`DispatchClass`] of the call, the third gives whereas extrinsic must pay fees or not.
+		// A higher weight means a larger transaction (less of which can be placed in a single
+		// block).
 		//
 		// The weight for this extrinsic we rely on the auto-generated `WeightInfo` from the
 		// benchmark toolchain.
@@ -523,7 +525,7 @@ pub mod pallet {
 			});
 
 			// Let's deposit an event to let the outside world know this happened.
-			Self::deposit_event(Event::AccumulateDummy(increase_by));
+			Self::deposit_event(Event::AccumulateDummy { balance: increase_by });
 
 			// All good, no refund.
 			Ok(())
@@ -548,13 +550,14 @@ pub mod pallet {
 
 			// Print out log or debug message in the console via log::{error, warn, info, debug,
 			// trace}, accepting format strings similar to `println!`.
-			// https://substrate.dev/rustdocs/v3.0.0/log/index.html
+			// https://paritytech.github.io/substrate/master/sp_io/logging/fn.log.html
+			// https://paritytech.github.io/substrate/master/frame_support/constant.LOG_TARGET.html
 			info!("New value is now: {:?}", new_value);
 
 			// Put the new value into storage.
 			>::put(new_value);
 
-			Self::deposit_event(Event::SetDummy(new_value));
+			Self::deposit_event(Event::SetDummy { balance: new_value });
 
 			// All good, no refund.
 			Ok(())
@@ -571,9 +574,16 @@ pub mod pallet {
 	pub enum Event {
 		// Just a normal `enum`, here's a dummy event to ensure it compiles.
 		/// Dummy event, just here so there's a generic type that's used.
-		AccumulateDummy(BalanceOf),
-		SetDummy(BalanceOf),
-		SetBar(T::AccountId, BalanceOf),
+		AccumulateDummy {
+			balance: BalanceOf,
+		},
+		SetDummy {
+			balance: BalanceOf,
+		},
+		SetBar {
+			account: T::AccountId,
+			balance: BalanceOf,
+		},
 	}
 
 	// pallet::storage attributes allow for type-safe usage of the Substrate storage database,
@@ -721,6 +731,16 @@ where
 		Ok(())
 	}
 
+	fn pre_dispatch(
+		self,
+		who: &Self::AccountId,
+		call: &Self::Call,
+		info: &DispatchInfoOf,
+		len: usize,
+	) -> Result {
+		self.validate(who, call, info, len).map(|_| ())
+	}
+
 	fn validate(
 		&self,
 		_who: &Self::AccountId,
diff --git a/frame/example/src/tests.rs b/frame/examples/basic/src/tests.rs
similarity index 87%
rename from frame/example/src/tests.rs
rename to frame/examples/basic/src/tests.rs
index 4c2274572db8..0f659e12fb44 100644
--- a/frame/example/src/tests.rs
+++ b/frame/examples/basic/src/tests.rs
@@ -1,6 +1,6 @@
 // This file is part of Substrate.
 
-// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd.
+// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd.
 // SPDX-License-Identifier: Apache-2.0
 
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,12 +15,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Tests for pallet-example.
+//! Tests for pallet-example-basic.
 
 use crate::*;
 use frame_support::{
 	assert_ok, parameter_types,
-	traits::OnInitialize,
+	traits::{ConstU64, OnInitialize},
 	weights::{DispatchInfo, GetDispatchInfo},
 };
 use sp_core::H256;
@@ -32,7 +32,7 @@ use sp_runtime::{
 	BuildStorage,
 };
 // Reexport crate as its pallet name for construct_runtime.
-use crate as pallet_example;
+use crate as pallet_example_basic;
 
 type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic;
 type Block = frame_system::mocking::MockBlock;
@@ -46,12 +46,11 @@ frame_support::construct_runtime!(
 	{
 		System: frame_system::{Pallet, Call, Config, Storage, Event},
 		Balances: pallet_balances::{Pallet, Call, Storage, Config, Event},
-		Example: pallet_example::{Pallet, Call, Storage, Config, Event},
+		Example: pallet_example_basic::{Pallet, Call, Storage, Config, Event},
 	}
 );
 
 parameter_types! {
-	pub const BlockHashCount: u64 = 250;
 	pub BlockWeights: frame_system::limits::BlockWeights =
 		frame_system::limits::BlockWeights::simple_max(1024);
 }
@@ -70,7 +69,7 @@ impl frame_system::Config for Test {
 	type Lookup = IdentityLookup;
 	type Header = Header;
 	type Event = Event;
-	type BlockHashCount = BlockHashCount;
+	type BlockHashCount = ConstU64<250>;
 	type Version = ();
 	type PalletInfo = PalletInfo;
 	type AccountData = pallet_balances::AccountData;
@@ -79,10 +78,9 @@ impl frame_system::Config for Test {
 	type SystemWeightInfo = ();
 	type SS58Prefix = ();
 	type OnSetCode = ();
+	type MaxConsumers = frame_support::traits::ConstU32<16>;
 }
-parameter_types! {
-	pub const ExistentialDeposit: u64 = 1;
-}
+
 impl pallet_balances::Config for Test {
 	type MaxLocks = ();
 	type MaxReserves = ();
@@ -90,16 +88,13 @@ impl pallet_balances::Config for Test {
 	type Balance = u64;
 	type DustRemoval = ();
 	type Event = Event;
-	type ExistentialDeposit = ExistentialDeposit;
+	type ExistentialDeposit = ConstU64<1>;
 	type AccountStore = System;
 	type WeightInfo = ();
 }
 
-parameter_types! {
-	pub const MagicNumber: u64 = 1_000_000_000;
-}
 impl Config for Test {
-	type MagicNumber = MagicNumber;
+	type MagicNumber = ConstU64<1_000_000_000>;
 	type Event = Event;
 	type WeightInfo = ();
 }
@@ -111,7 +106,7 @@ pub fn new_test_ext() -> sp_io::TestExternalities {
 		// We use default for brevity, but you can configure as desired if needed.
 		system: Default::default(),
 		balances: Default::default(),
-		example: pallet_example::GenesisConfig {
+		example: pallet_example_basic::GenesisConfig {
 			dummy: 42,
 			// we configure the map with (key, value) pairs.
 			bar: vec![(1, 2), (2, 3)],
@@ -163,7 +158,7 @@ fn set_dummy_works() {
 #[test]
 fn signed_ext_watch_dummy_works() {
 	new_test_ext().execute_with(|| {
-		let call = pallet_example::Call::set_dummy { new_value: 10 }.into();
+		let call = pallet_example_basic::Call::set_dummy { new_value: 10 }.into();
 		let info = DispatchInfo::default();
 
 		assert_eq!(
@@ -192,14 +187,14 @@ fn counted_map_works() {
 #[test]
 fn weights_work() {
 	// must have a defined weight.
-	let default_call = pallet_example::Call::::accumulate_dummy { increase_by: 10 };
+	let default_call = pallet_example_basic::Call::::accumulate_dummy { increase_by: 10 };
 	let info1 = default_call.get_dispatch_info();
 	// aka. `let info =  as GetDispatchInfo>::get_dispatch_info(&default_call);`
 	assert!(info1.weight > 0);
 
 	// `set_dummy` is simpler than `accumulate_dummy`, and the weight
 	//   should be less.
-	let custom_call = pallet_example::Call::::set_dummy { new_value: 20 };
+	let custom_call = pallet_example_basic::Call::::set_dummy { new_value: 20 };
 	let info2 = custom_call.get_dispatch_info();
 	assert!(info1.weight > info2.weight);
 }
diff --git a/frame/example/src/weights.rs b/frame/examples/basic/src/weights.rs
similarity index 91%
rename from frame/example/src/weights.rs
rename to frame/examples/basic/src/weights.rs
index efcfdc6729b5..5fc6434e396e 100644
--- a/frame/example/src/weights.rs
+++ b/frame/examples/basic/src/weights.rs
@@ -1,6 +1,6 @@
 // This file is part of Substrate.
 
-// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd.
 // SPDX-License-Identifier: Apache-2.0
 
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +15,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Autogenerated weights for pallet_example
+//! Autogenerated weights for pallet_example_basic
 //!
 //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0
 //! DATE: 2021-03-15, STEPS: `[100, ]`, REPEAT: 10, LOW RANGE: `[]`, HIGH RANGE: `[]`
@@ -31,7 +31,7 @@
 // --wasm-execution
 // compiled
 // --pallet
-// pallet_example
+// pallet_example_basic
 // --extrinsic
 // *
 // --steps
@@ -52,14 +52,14 @@
 use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}};
 use sp_std::marker::PhantomData;
 
-/// Weight functions needed for pallet_example.
+/// Weight functions needed for pallet_example_basic.
 pub trait WeightInfo {
 	fn set_dummy_benchmark(b: u32, ) -> Weight;
 	fn accumulate_dummy(b: u32, ) -> Weight;
 	fn sort_vector(x: u32, ) -> Weight;
 }
 
-/// Weights for pallet_example using the Substrate node and recommended hardware.
+/// Weights for pallet_example_basic using the Substrate node and recommended hardware.
 pub struct SubstrateWeight(PhantomData);
 impl WeightInfo for SubstrateWeight {
 	fn set_dummy_benchmark(b: u32, ) -> Weight {
diff --git a/frame/example-offchain-worker/Cargo.toml b/frame/examples/offchain-worker/Cargo.toml
similarity index 55%
rename from frame/example-offchain-worker/Cargo.toml
rename to frame/examples/offchain-worker/Cargo.toml
index 1ccd9f33f031..8daf0686d816 100644
--- a/frame/example-offchain-worker/Cargo.toml
+++ b/frame/examples/offchain-worker/Cargo.toml
@@ -2,9 +2,9 @@
 name = "pallet-example-offchain-worker"
 version = "4.0.0-dev"
 authors = ["Parity Technologies "]
-edition = "2018"
+edition = "2021"
 license = "Unlicense"
-homepage = "https://substrate.dev"
+homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME example pallet for offchain worker"
 readme = "README.md"
@@ -13,17 +13,18 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false }
-scale-info = { version = "1.0", default-features = false, features = ["derive"] }
-frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" }
-frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" }
-sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" }
-sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore", optional = true }
-sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" }
-sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" }
-sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" }
+codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false }
 lite-json = { version = "0.1", default-features = false }
 log = { version = "0.4.14", default-features = false }
+scale-info = { version = "2.0.1", default-features = false, features = ["derive"] }
+
+frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" }
+frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" }
+sp-core = { version = "6.0.0", default-features = false, path = "../../../primitives/core" }
+sp-io = { version = "6.0.0", default-features = false, path = "../../../primitives/io" }
+sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore", optional = true }
+sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" }
+sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" }
 
 [features]
 default = ["std"]
diff --git a/frame/example-offchain-worker/README.md b/frame/examples/offchain-worker/README.md
similarity index 87%
rename from frame/example-offchain-worker/README.md
rename to frame/examples/offchain-worker/README.md
index 5299027f3925..587431c92c0e 100644
--- a/frame/example-offchain-worker/README.md
+++ b/frame/examples/offchain-worker/README.md
@@ -1,5 +1,5 @@
 
-# Offchain Worker Example Module
+# Offchain Worker Example Pallet
 
 The Offchain Worker Example: A simple pallet demonstrating
 concepts, APIs and structures common to most offchain workers.
@@ -11,6 +11,8 @@ documentation.
 - [`Call`](./enum.Call.html)
 - [`Module`](./struct.Module.html)
 
+**This pallet serves as an example showcasing Substrate off-chain worker and is not meant to be
+used in production.**
 
 ## Overview
 
diff --git a/frame/example-offchain-worker/src/lib.rs b/frame/examples/offchain-worker/src/lib.rs
similarity index 96%
rename from frame/example-offchain-worker/src/lib.rs
rename to frame/examples/offchain-worker/src/lib.rs
index 644e1ca299a3..3cf718217b06 100644
--- a/frame/example-offchain-worker/src/lib.rs
+++ b/frame/examples/offchain-worker/src/lib.rs
@@ -1,6 +1,6 @@
 // This file is part of Substrate.
 
-// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd.
+// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd.
 // SPDX-License-Identifier: Apache-2.0
 
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,6 +28,8 @@
 //! - [`Call`]
 //! - [`Pallet`]
 //!
+//! **This pallet serves as an example showcasing Substrate off-chain worker and is not meant to
+//! be used in production.**
 //!
 //! ## Overview
 //!
@@ -40,6 +42,7 @@
 //! Additional logic in OCW is put in place to prevent spamming the network with both signed
 //! and unsigned transactions, and custom `UnsignedValidator` makes sure that there is only
 //! one unsigned transaction floating in the network.
+
 #![cfg_attr(not(feature = "std"), no_std)]
 
 use codec::{Decode, Encode};
@@ -86,10 +89,19 @@ pub mod crypto {
 	use sp_runtime::{
 		app_crypto::{app_crypto, sr25519},
 		traits::Verify,
+		MultiSignature, MultiSigner,
 	};
 	app_crypto!(sr25519, KEY_TYPE);
 
 	pub struct TestAuthId;
+
+	impl frame_system::offchain::AppCrypto for TestAuthId {
+		type RuntimeAppPublic = Public;
+		type GenericSignature = sp_core::sr25519::Signature;
+		type GenericPublic = sp_core::sr25519::Public;
+	}
+
+	// implemented for mock runtime in test
 	impl frame_system::offchain::AppCrypto<::Signer, Sr25519Signature>
 		for TestAuthId
 	{
@@ -142,6 +154,10 @@ pub mod pallet {
 		/// multiple pallets send unsigned transactions.
 		#[pallet::constant]
 		type UnsignedPriority: Get;
+
+		/// Maximum number of prices.
+		#[pallet::constant]
+		type MaxPrices: Get;
 	}
 
 	#[pallet::pallet]
@@ -221,7 +237,7 @@ pub mod pallet {
 			// Retrieve sender of the transaction.
 			let who = ensure_signed(origin)?;
 			// Add the price to the on-chain list.
-			Self::add_price(who, price);
+			Self::add_price(Some(who), price);
 			Ok(().into())
 		}
 
@@ -250,7 +266,7 @@ pub mod pallet {
 			// This ensures that the function can only be called via unsigned transaction.
 			ensure_none(origin)?;
 			// Add the price to the on-chain list, but mark it as coming from an empty address.
-			Self::add_price(Default::default(), price);
+			Self::add_price(None, price);
 			// now increment the block number at which we expect next unsigned transaction.
 			let current_block = >::block_number();
 			>::put(current_block + T::UnsignedInterval::get());
@@ -266,7 +282,7 @@ pub mod pallet {
 			// This ensures that the function can only be called via unsigned transaction.
 			ensure_none(origin)?;
 			// Add the price to the on-chain list, but mark it as coming from an empty address.
-			Self::add_price(Default::default(), price_payload.price);
+			Self::add_price(None, price_payload.price);
 			// now increment the block number at which we expect next unsigned transaction.
 			let current_block = >::block_number();
 			>::put(current_block + T::UnsignedInterval::get());
@@ -279,8 +295,7 @@ pub mod pallet {
 	#[pallet::generate_deposit(pub(super) fn deposit_event)]
 	pub enum Event {
 		/// Event generated when new price is accepted to contribute to the average.
-		/// \[price, who\]
-		NewPrice(u32, T::AccountId),
+		NewPrice { price: u32, maybe_who: Option },
 	}
 
 	#[pallet::validate_unsigned]
@@ -318,7 +333,7 @@ pub mod pallet {
 	/// This is used to calculate average price, should have bounded size.
 	#[pallet::storage]
 	#[pallet::getter(fn prices)]
-	pub(super) type Prices = StorageValue<_, Vec, ValueQuery>;
+	pub(super) type Prices = StorageValue<_, BoundedVec, ValueQuery>;
 
 	/// Defines the block when next unsigned transaction will be accepted.
 	///
@@ -630,15 +645,11 @@ impl Pallet {
 	}
 
 	/// Add new price to the list.
-	fn add_price(who: T::AccountId, price: u32) {
+	fn add_price(maybe_who: Option, price: u32) {
 		log::info!("Adding to the average: {}", price);
 		>::mutate(|prices| {
-			const MAX_LEN: usize = 64;
-
-			if prices.len() < MAX_LEN {
-				prices.push(price);
-			} else {
-				prices[price as usize % MAX_LEN] = price;
+			if prices.try_push(price).is_err() {
+				prices[(price % T::MaxPrices::get()) as usize] = price;
 			}
 		});
 
@@ -646,7 +657,7 @@ impl Pallet {
 			.expect("The average is not empty, because it was just mutated; qed");
 		log::info!("Current average price is: {}", average);
 		// here we are raising the NewPrice event
-		Self::deposit_event(Event::NewPrice(price, who));
+		Self::deposit_event(Event::NewPrice { price, maybe_who });
 	}
 
 	/// Calculate current average price.
diff --git a/frame/example-offchain-worker/src/tests.rs b/frame/examples/offchain-worker/src/tests.rs
similarity index 95%
rename from frame/example-offchain-worker/src/tests.rs
rename to frame/examples/offchain-worker/src/tests.rs
index 1dde8a1df60c..e5bd9fabc629 100644
--- a/frame/example-offchain-worker/src/tests.rs
+++ b/frame/examples/offchain-worker/src/tests.rs
@@ -1,6 +1,6 @@
 // This file is part of Substrate.
 
-// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd.
+// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd.
 // SPDX-License-Identifier: Apache-2.0
 
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,7 +18,10 @@
 use crate as example_offchain_worker;
 use crate::*;
 use codec::Decode;
-use frame_support::{assert_ok, parameter_types};
+use frame_support::{
+	assert_ok, parameter_types,
+	traits::{ConstU32, ConstU64},
+};
 use sp_core::{
 	offchain::{testing, OffchainWorkerExt, TransactionPoolExt},
 	sr25519::Signature,
@@ -49,7 +52,6 @@ frame_support::construct_runtime!(
 );
 
 parameter_types! {
-	pub const BlockHashCount: u64 = 250;
 	pub BlockWeights: frame_system::limits::BlockWeights =
 		frame_system::limits::BlockWeights::simple_max(1024);
 }
@@ -68,7 +70,7 @@ impl frame_system::Config for Test {
 	type Lookup = IdentityLookup;
 	type Header = Header;
 	type Event = Event;
-	type BlockHashCount = BlockHashCount;
+	type BlockHashCount = ConstU64<250>;
 	type Version = ();
 	type PalletInfo = PalletInfo;
 	type AccountData = ();
@@ -77,6 +79,7 @@ impl frame_system::Config for Test {
 	type SystemWeightInfo = ();
 	type SS58Prefix = ();
 	type OnSetCode = ();
+	type MaxConsumers = ConstU32<16>;
 }
 
 type Extrinsic = TestXt;
@@ -110,8 +113,6 @@ where
 }
 
 parameter_types! {
-	pub const GracePeriod: u64 = 5;
-	pub const UnsignedInterval: u64 = 128;
 	pub const UnsignedPriority: u64 = 1 << 20;
 }
 
@@ -119,9 +120,14 @@ impl Config for Test {
 	type Event = Event;
 	type AuthorityId = crypto::TestAuthId;
 	type Call = Call;
-	type GracePeriod = GracePeriod;
-	type UnsignedInterval = UnsignedInterval;
+	type GracePeriod = ConstU64<5>;
+	type UnsignedInterval = ConstU64<128>;
 	type UnsignedPriority = UnsignedPriority;
+	type MaxPrices = ConstU32<64>;
+}
+
+fn test_pub() -> sp_core::sr25519::Public {
+	sp_core::sr25519::Public::from_raw([1u8; 32])
 }
 
 #[test]
@@ -129,10 +135,10 @@ fn it_aggregates_the_price() {
 	sp_io::TestExternalities::default().execute_with(|| {
 		assert_eq!(Example::average_price(), None);
 
-		assert_ok!(Example::submit_price(Origin::signed(Default::default()), 27));
+		assert_ok!(Example::submit_price(Origin::signed(test_pub()), 27));
 		assert_eq!(Example::average_price(), Some(27));
 
-		assert_ok!(Example::submit_price(Origin::signed(Default::default()), 43));
+		assert_ok!(Example::submit_price(Origin::signed(test_pub()), 43));
 		assert_eq!(Example::average_price(), Some(35));
 	});
 }
diff --git a/frame/example-parallel/Cargo.toml b/frame/examples/parallel/Cargo.toml
similarity index 54%
rename from frame/example-parallel/Cargo.toml
rename to frame/examples/parallel/Cargo.toml
index 5e0f6d4bc255..367b2e98aaa6 100644
--- a/frame/example-parallel/Cargo.toml
+++ b/frame/examples/parallel/Cargo.toml
@@ -2,9 +2,9 @@
 name = "pallet-example-parallel"
 version = "3.0.0-dev"
 authors = ["Parity Technologies "]
-edition = "2018"
+edition = "2021"
 license = "Unlicense"
-homepage = "https://substrate.dev"
+homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME example pallet using runtime worker threads"
 
@@ -12,15 +12,16 @@ description = "FRAME example pallet using runtime worker threads"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false }
-scale-info = { version = "1.0", default-features = false, features = ["derive"] }
-frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" }
-frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" }
-sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" }
-sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" }
-sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" }
-sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" }
-sp-tasks = { version = "4.0.0-dev", default-features = false, path = "../../primitives/tasks" }
+codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false }
+scale-info = { version = "2.0.1", default-features = false, features = ["derive"] }
+
+frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" }
+frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" }
+sp-core = { version = "6.0.0", default-features = false, path = "../../../primitives/core" }
+sp-io = { version = "6.0.0", default-features = false, path = "../../../primitives/io" }
+sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" }
+sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" }
+sp-tasks = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/tasks" }
 
 [features]
 default = ["std"]
diff --git a/frame/examples/parallel/README.md b/frame/examples/parallel/README.md
new file mode 100644
index 000000000000..44b39a41507d
--- /dev/null
+++ b/frame/examples/parallel/README.md
@@ -0,0 +1,7 @@
+
+# Parallel Tasks Example Pallet
+
+This example pallet demonstrates parallelizing validation of the enlisted participants (see
+`enlist_participants` dispatch).
+
+**This pallet serves as an example and is not meant to be used in production.**
diff --git a/frame/example-parallel/src/lib.rs b/frame/examples/parallel/src/lib.rs
similarity index 88%
rename from frame/example-parallel/src/lib.rs
rename to frame/examples/parallel/src/lib.rs
index c86cac429568..7b8948c2ebd0 100644
--- a/frame/example-parallel/src/lib.rs
+++ b/frame/examples/parallel/src/lib.rs
@@ -1,6 +1,6 @@
 // This file is part of Substrate.
 
-// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd.
+// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd.
 // SPDX-License-Identifier: Apache-2.0
 
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,10 +15,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Parallel tasks example
+//! # Parallel Tasks Example Pallet
 //!
-//! This example pallet parallelizes validation of the enlisted participants
+//! This example pallet demonstrates parallelizing validation of the enlisted participants
 //! (see `enlist_participants` dispatch).
+//!
+//! **This pallet serves as an example and is not meant to be used in production.**
 
 #![cfg_attr(not(feature = "std"), no_std)]
 
@@ -46,6 +48,7 @@ pub mod pallet {
 
 	#[pallet::pallet]
 	#[pallet::generate_store(pub(super) trait Store)]
+	#[pallet::without_storage_info]
 	pub struct Pallet(_);
 
 	/// A public part of the pallet.
@@ -103,14 +106,13 @@ pub struct EnlistedParticipant {
 
 impl EnlistedParticipant {
 	fn verify(&self, event_id: &[u8]) -> bool {
-		use sp_core::Public;
+		use sp_core::ByteArray;
 		use sp_runtime::traits::Verify;
-		use std::convert::TryFrom;
 
 		match sp_core::sr25519::Signature::try_from(&self.signature[..]) {
-			Ok(signature) => {
-				let public = sp_core::sr25519::Public::from_slice(self.account.as_ref());
-				signature.verify(event_id, &public)
+			Ok(signature) => match sp_core::sr25519::Public::from_slice(self.account.as_ref()) {
+				Err(()) => false,
+				Ok(signer) => signature.verify(event_id, &signer),
 			},
 			_ => false,
 		}
diff --git a/frame/example-parallel/src/tests.rs b/frame/examples/parallel/src/tests.rs
similarity index 83%
rename from frame/example-parallel/src/tests.rs
rename to frame/examples/parallel/src/tests.rs
index 4c36f0d6eb85..67d823d8b204 100644
--- a/frame/example-parallel/src/tests.rs
+++ b/frame/examples/parallel/src/tests.rs
@@ -1,6 +1,6 @@
 // This file is part of Substrate.
 
-// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd.
+// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd.
 // SPDX-License-Identifier: Apache-2.0
 
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -40,7 +40,6 @@ frame_support::construct_runtime!(
 );
 
 parameter_types! {
-	pub const BlockHashCount: u64 = 250;
 	pub const AvailableBlockRatio: Perbill = Perbill::one();
 }
 
@@ -57,7 +56,7 @@ impl frame_system::Config for Test {
 	type Lookup = IdentityLookup;
 	type Header = Header;
 	type Event = Event;
-	type BlockHashCount = BlockHashCount;
+	type BlockHashCount = frame_support::traits::ConstU64<250>;
 	type DbWeight = ();
 	type BlockWeights = ();
 	type BlockLength = ();
@@ -68,18 +67,21 @@ impl frame_system::Config for Test {
 	type SystemWeightInfo = ();
 	type SS58Prefix = ();
 	type OnSetCode = ();
-}
-
-parameter_types! {
-	pub const GracePeriod: u64 = 5;
-	pub const UnsignedInterval: u64 = 128;
-	pub const UnsignedPriority: u64 = 1 << 20;
+	type MaxConsumers = frame_support::traits::ConstU32<16>;
 }
 
 impl Config for Test {
 	type Call = Call;
 }
 
+fn test_pub(n: u8) -> sp_core::sr25519::Public {
+	sp_core::sr25519::Public::from_raw([n; 32])
+}
+
+fn test_origin(n: u8) -> Origin {
+	Origin::signed(test_pub(n))
+}
+
 #[test]
 fn it_can_enlist() {
 	use sp_core::Pair;
@@ -90,8 +92,7 @@ fn it_can_enlist() {
 
 		let event_name = b"test";
 
-		Example::run_event(Origin::signed(Default::default()), event_name.to_vec())
-			.expect("Failed to enlist");
+		Example::run_event(test_origin(1), event_name.to_vec()).expect("Failed to enlist");
 
 		let participants = vec![
 			EnlistedParticipant {
@@ -104,7 +105,7 @@ fn it_can_enlist() {
 			},
 		];
 
-		Example::enlist_participants(Origin::signed(Default::default()), participants)
+		Example::enlist_participants(Origin::signed(test_pub(1)), participants)
 			.expect("Failed to enlist");
 
 		assert_eq!(Example::participants().len(), 2);
@@ -122,8 +123,7 @@ fn one_wrong_will_not_enlist_anyone() {
 
 		let event_name = b"test";
 
-		Example::run_event(Origin::signed(Default::default()), event_name.to_vec())
-			.expect("Failed to enlist");
+		Example::run_event(test_origin(1), event_name.to_vec()).expect("Failed to enlist");
 
 		let participants = vec![
 			EnlistedParticipant {
@@ -141,8 +141,7 @@ fn one_wrong_will_not_enlist_anyone() {
 			},
 		];
 
-		Example::enlist_participants(Origin::signed(Default::default()), participants)
-			.expect("Failed to enlist");
+		Example::enlist_participants(test_origin(1), participants).expect("Failed to enlist");
 
 		assert_eq!(Example::participants().len(), 0);
 	});
diff --git a/frame/executive/Cargo.toml b/frame/executive/Cargo.toml
index 1abbf50e6a4c..f920affaab20 100644
--- a/frame/executive/Cargo.toml
+++ b/frame/executive/Cargo.toml
@@ -2,9 +2,9 @@
 name = "frame-executive"
 version = "4.0.0-dev"
 authors = ["Parity Technologies "]
-edition = "2018"
+edition = "2021"
 license = "Apache-2.0"
-homepage = "https://substrate.dev"
+homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME executives engine"
 readme = "README.md"
@@ -13,25 +13,25 @@ readme = "README.md"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [
+codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [
 	"derive",
 ] }
-scale-info = { version = "1.0", default-features = false, features = ["derive"] }
+scale-info = { version = "2.0.1", default-features = false, features = ["derive"] }
 frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" }
 frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" }
-sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" }
-sp-tracing = { version = "4.0.0-dev", default-features = false, path = "../../primitives/tracing" }
-sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" }
-sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" }
-sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" }
+sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" }
+sp-tracing = { version = "5.0.0", default-features = false, path = "../../primitives/tracing" }
+sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" }
+sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" }
+sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" }
 
 [dev-dependencies]
-hex-literal = "0.3.1"
-sp-core = { version = "4.0.0-dev", path = "../../primitives/core" }
-sp-io = { version = "4.0.0-dev", path = "../../primitives/io" }
+hex-literal = "0.3.4"
+sp-core = { version = "6.0.0", path = "../../primitives/core" }
+sp-io = { version = "6.0.0", path = "../../primitives/io" }
 pallet-balances = { version = "4.0.0-dev", path = "../balances" }
 pallet-transaction-payment = { version = "4.0.0-dev", path = "../transaction-payment" }
-sp-version = { version = "4.0.0-dev", path = "../../primitives/version" }
+sp-version = { version = "5.0.0", path = "../../primitives/version" }
 sp-inherents = { version = "4.0.0-dev", path = "../../primitives/inherents" }
 
 [features]
diff --git a/frame/executive/README.md b/frame/executive/README.md
index ae3bbf1a9d99..e96d07b0843f 100644
--- a/frame/executive/README.md
+++ b/frame/executive/README.md
@@ -35,7 +35,13 @@ The default Substrate node template declares the [`Executive`](https://docs.rs/f
 ```rust
 #
 /// Executive: handles dispatch to the various modules.
-pub type Executive = executive::Executive;
+pub type Executive = executive::Executive<
+    Runtime,
+    Block,
+    Context,
+    Runtime,
+    AllPallets,
+>;
 ```
 
 ### Custom `OnRuntimeUpgrade` logic
@@ -54,7 +60,14 @@ impl frame_support::traits::OnRuntimeUpgrade for CustomOnRuntimeUpgrade {
     }
 }
 
-pub type Executive = executive::Executive;
+pub type Executive = executive::Executive<
+    Runtime,
+    Block,
+    Context,
+    Runtime,
+    AllPallets,
+    CustomOnRuntimeUpgrade,
+>;
 ```
 
 License: Apache-2.0
diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs
index 41f679909e6f..de1cc3b91640 100644
--- a/frame/executive/src/lib.rs
+++ b/frame/executive/src/lib.rs
@@ -1,6 +1,6 @@
 // This file is part of Substrate.
 
-// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
+// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd.
 // SPDX-License-Identifier: Apache-2.0
 
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -59,7 +59,7 @@
 //! # type Context = frame_system::ChainContext;
 //! # pub type Block = generic::Block;
 //! # pub type Balances = u64;
-//! # pub type AllPallets = u64;
+//! # pub type AllPalletsWithSystem = u64;
 //! # pub enum Runtime {};
 //! # use sp_runtime::transaction_validity::{
 //! #    TransactionValidity, UnknownTransaction, TransactionSource,
@@ -73,7 +73,7 @@
 //! #     }
 //! # }
 //! /// Executive: handles dispatch to the various modules.
-//! pub type Executive = executive::Executive;
+//! pub type Executive = executive::Executive;
 //! ```
 //!
 //! ### Custom `OnRuntimeUpgrade` logic
@@ -90,7 +90,7 @@
 //! # type Context = frame_system::ChainContext;
 //! # pub type Block = generic::Block;
 //! # pub type Balances = u64;
-//! # pub type AllPallets = u64;
+//! # pub type AllPalletsWithSystem = u64;
 //! # pub enum Runtime {};
 //! # use sp_runtime::transaction_validity::{
 //! #    TransactionValidity, UnknownTransaction, TransactionSource,
@@ -111,7 +111,7 @@
 //!     }
 //! }
 //!
-//! pub type Executive = executive::Executive;
+//! pub type Executive = executive::Executive;
 //! ```
 
 #![cfg_attr(not(feature = "std"), no_std)]
@@ -125,7 +125,6 @@ use frame_support::{
 	},
 	weights::{DispatchClass, DispatchInfo, GetDispatchInfo},
 };
-use frame_system::DigestOf;
 use sp_runtime::{
 	generic::Digest,
 	traits::{
@@ -148,11 +147,26 @@ pub type OriginOf =  as Dispatchable>::Origin;
 /// - `Block`: The block type of the runtime
 /// - `Context`: The context that is used when checking an extrinsic.
 /// - `UnsignedValidator`: The unsigned transaction validator of the runtime.
-/// - `AllPallets`: Tuple that contains all modules. Will be used to call e.g. `on_initialize`.
+/// - `AllPalletsWithSystem`: Tuple that contains all pallets including frame system pallet. Will be
+///   used to call hooks e.g. `on_initialize`.
 /// - `OnRuntimeUpgrade`: Custom logic that should be called after a runtime upgrade. Modules are
-///   already called by `AllPallets`. It will be called before all modules will be called.
-pub struct Executive(
-	PhantomData<(System, Block, Context, UnsignedValidator, AllPallets, OnRuntimeUpgrade)>,
+///   already called by `AllPalletsWithSystem`. It will be called before all modules will be called.
+pub struct Executive<
+	System,
+	Block,
+	Context,
+	UnsignedValidator,
+	AllPalletsWithSystem,
+	OnRuntimeUpgrade = (),
+>(
+	PhantomData<(
+		System,
+		Block,
+		Context,
+		UnsignedValidator,
+		AllPalletsWithSystem,
+		OnRuntimeUpgrade,
+	)>,
 );
 
 impl<
@@ -160,14 +174,14 @@ impl<
 		Block: traits::Block
, Context: Default, UnsignedValidator, - AllPallets: OnRuntimeUpgrade + AllPalletsWithSystem: OnRuntimeUpgrade + OnInitialize + OnIdle + OnFinalize + OffchainWorker, COnRuntimeUpgrade: OnRuntimeUpgrade, > ExecuteBlock - for Executive + for Executive where Block::Extrinsic: Checkable + Codec, CheckedOf: Applyable + GetDispatchInfo, @@ -182,7 +196,7 @@ where Block, Context, UnsignedValidator, - AllPallets, + AllPalletsWithSystem, COnRuntimeUpgrade, >::execute_block(block); } @@ -193,13 +207,13 @@ impl< Block: traits::Block
, Context: Default, UnsignedValidator, - AllPallets: OnRuntimeUpgrade + AllPalletsWithSystem: OnRuntimeUpgrade + OnInitialize + OnIdle + OnFinalize + OffchainWorker, COnRuntimeUpgrade: OnRuntimeUpgrade, - > Executive + > Executive where Block::Extrinsic: Checkable + Codec, CheckedOf: Applyable + GetDispatchInfo, @@ -210,17 +224,10 @@ where { /// Execute all `OnRuntimeUpgrade` of this runtime, and return the aggregate weight. pub fn execute_on_runtime_upgrade() -> frame_support::weights::Weight { - let mut weight = 0; - weight = weight.saturating_add(COnRuntimeUpgrade::on_runtime_upgrade()); - weight = weight.saturating_add( - as OnRuntimeUpgrade>::on_runtime_upgrade(), - ); - weight = weight.saturating_add(::on_runtime_upgrade()); - - weight + <(COnRuntimeUpgrade, AllPalletsWithSystem) as OnRuntimeUpgrade>::on_runtime_upgrade() } - /// Execute given block, but don't do any of the [`final_checks`]. + /// Execute given block, but don't do any of the `final_checks`. /// /// Should only be used for testing. #[cfg(feature = "try-runtime")] @@ -256,19 +263,10 @@ where /// This should only be used for testing. #[cfg(feature = "try-runtime")] pub fn try_runtime_upgrade() -> Result { - < - (frame_system::Pallet::, COnRuntimeUpgrade, AllPallets) - as - OnRuntimeUpgrade - >::pre_upgrade().unwrap(); - + <(COnRuntimeUpgrade, AllPalletsWithSystem) as OnRuntimeUpgrade>::pre_upgrade().unwrap(); let weight = Self::execute_on_runtime_upgrade(); - < - (frame_system::Pallet::, COnRuntimeUpgrade, AllPallets) - as - OnRuntimeUpgrade - >::post_upgrade().unwrap(); + <(COnRuntimeUpgrade, AllPalletsWithSystem) as OnRuntimeUpgrade>::post_upgrade().unwrap(); Ok(weight) } @@ -281,8 +279,8 @@ where Self::initialize_block_impl(header.number(), header.parent_hash(), &digests); } - fn extract_pre_digest(header: &System::Header) -> DigestOf { - let mut digest = >::default(); + fn extract_pre_digest(header: &System::Header) -> Digest { + let mut digest = ::default(); header.digest().logs().iter().for_each(|d| { if d.as_pre_runtime().is_some() { digest.push(d.clone()) @@ -294,24 +292,21 @@ where fn initialize_block_impl( block_number: &System::BlockNumber, parent_hash: &System::Hash, - digest: &Digest, + digest: &Digest, ) { + // Reset events before apply runtime upgrade hook. + // This is required to preserve events from runtime upgrade hook. + // This means the format of all the event related storages must always be compatible. + >::reset_events(); + let mut weight = 0; if Self::runtime_upgraded() { weight = weight.saturating_add(Self::execute_on_runtime_upgrade()); } - >::initialize( - block_number, - parent_hash, - digest, - frame_system::InitKind::Full, - ); - weight = weight.saturating_add( as OnInitialize< + >::initialize(block_number, parent_hash, digest); + weight = weight.saturating_add(>::on_initialize(*block_number)); - weight = weight.saturating_add( - >::on_initialize(*block_number), - ); weight = weight.saturating_add( >::get().base_block, ); @@ -416,30 +411,20 @@ where fn idle_and_finalize_hook(block_number: NumberFor) { let weight = >::block_weight(); let max_weight = >::get().max_block; - let mut remaining_weight = max_weight.saturating_sub(weight.total()); + let remaining_weight = max_weight.saturating_sub(weight.total()); if remaining_weight > 0 { - let mut used_weight = - as OnIdle>::on_idle( - block_number, - remaining_weight, - ); - remaining_weight = remaining_weight.saturating_sub(used_weight); - used_weight = >::on_idle( + let used_weight = >::on_idle( block_number, remaining_weight, - ) - .saturating_add(used_weight); + ); >::register_extra_weight_unchecked( used_weight, DispatchClass::Mandatory, ); } - as OnFinalize>::on_finalize( - block_number, - ); - >::on_finalize(block_number); + >::on_finalize(block_number); } /// Apply extrinsic outside of the block execution function. @@ -525,7 +510,6 @@ where &(frame_system::Pallet::::block_number() + One::one()), &block_hash, &Default::default(), - frame_system::InitKind::Inspection, ); enter_span! { sp_tracing::Level::TRACE, "validate_transaction" }; @@ -556,19 +540,16 @@ where // OffchainWorker RuntimeApi should skip initialization. let digests = header.digest().clone(); - >::initialize( - header.number(), - header.parent_hash(), - &digests, - frame_system::InitKind::Inspection, - ); + >::initialize(header.number(), header.parent_hash(), &digests); // Frame system only inserts the parent hash into the block hashes as normally we don't know // the hash for the header before. However, here we are aware of the hash and we can add it // as well. frame_system::BlockHash::::insert(header.number(), header.hash()); - >::offchain_worker(*header.number()) + >::offchain_worker( + *header.number(), + ) } } @@ -591,8 +572,13 @@ mod tests { use frame_support::{ assert_err, parameter_types, - traits::{Currency, LockIdentifier, LockableCurrency, WithdrawReasons}, - weights::{IdentityFee, RuntimeDbWeight, Weight, WeightToFeePolynomial}, + traits::{ + ConstU32, ConstU64, ConstU8, Currency, LockIdentifier, LockableCurrency, + WithdrawReasons, + }, + weights::{ + ConstantMultiplier, IdentityFee, RuntimeDbWeight, Weight, WeightToFeePolynomial, + }, }; use frame_system::{Call as SystemCall, ChainContext, LastRuntimeUpgradeInfo}; use pallet_balances::Call as BalancesCall; @@ -681,7 +667,7 @@ mod tests { #[pallet::weight(0)] pub fn calculate_storage_root(_origin: OriginFor) -> DispatchResult { - let root = sp_io::storage::root(); + let root = sp_io::storage::root(sp_runtime::StateVersion::V1); sp_io::storage::set("storage_root".as_bytes(), &root); Ok(()) } @@ -744,7 +730,6 @@ mod tests { ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::builder() .base_block(10) @@ -771,7 +756,7 @@ mod tests { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = RuntimeVersion; type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -780,6 +765,7 @@ mod tests { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } type Balance = u64; @@ -803,8 +789,9 @@ mod tests { } impl pallet_transaction_payment::Config for Runtime { type OnChargeTransaction = CurrencyAdapter; - type TransactionByteFee = TransactionByteFee; + type OperationalFeeMultiplier = ConstU8<5>; type WeightToFee = IdentityFee; + type LengthToFee = ConstantMultiplier; type FeeMultiplierUpdate = (); } impl custom::Config for Runtime {} @@ -839,6 +826,7 @@ mod tests { fn on_runtime_upgrade() -> Weight { sp_io::storage::set(TEST_KEY, "custom_upgrade".as_bytes()); sp_io::storage::set(CUSTOM_ON_RUNTIME_KEY, &true.encode()); + System::deposit_event(frame_system::Event::CodeUpdated); 100 } } @@ -848,7 +836,7 @@ mod tests { Block, ChainContext, Runtime, - AllPallets, + AllPalletsWithSystem, CustomOnRuntimeUpgrade, >; @@ -906,17 +894,32 @@ mod tests { t.into() } + fn new_test_ext_v0(balance_factor: Balance) -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } + .assimilate_storage(&mut t) + .unwrap(); + (t, sp_runtime::StateVersion::V0).into() + } + #[test] fn block_import_works() { - new_test_ext(1).execute_with(|| { + block_import_works_inner( + new_test_ext_v0(1), + hex!("1039e1a4bd0cf5deefe65f313577e70169c41c7773d6acf31ca8d671397559f5").into(), + ); + block_import_works_inner( + new_test_ext(1), + hex!("75e7d8f360d375bbe91bcf8019c01ab6362448b4a89e3b329717eb9d910340e5").into(), + ); + } + fn block_import_works_inner(mut ext: sp_io::TestExternalities, state_root: H256) { + ext.execute_with(|| { Executive::execute_block(Block { header: Header { parent_hash: [69u8; 32].into(), number: 1, - state_root: hex!( - "1039e1a4bd0cf5deefe65f313577e70169c41c7773d6acf31ca8d671397559f5" - ) - .into(), + state_root, extrinsics_root: hex!( "03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314" ) @@ -957,7 +960,7 @@ mod tests { parent_hash: [69u8; 32].into(), number: 1, state_root: hex!( - "49cd58a254ccf6abc4a023d9a22dcfc421e385527a250faec69f8ad0d8ed3e48" + "75e7d8f360d375bbe91bcf8019c01ab6362448b4a89e3b329717eb9d910340e5" ) .into(), extrinsics_root: [0u8; 32].into(), @@ -1110,8 +1113,6 @@ mod tests { let invalid = TestXt::new(Call::Custom(custom::Call::unallowed_unsigned {}), None); let mut t = new_test_ext(1); - let mut default_with_prio_3 = ValidTransaction::default(); - default_with_prio_3.priority = 3; t.execute_with(|| { assert_eq!( Executive::validate_transaction( @@ -1119,7 +1120,7 @@ mod tests { valid.clone(), Default::default(), ), - Ok(default_with_prio_3), + Ok(ValidTransaction::default()), ); assert_eq!( Executive::validate_transaction( @@ -1292,6 +1293,30 @@ mod tests { }); } + #[test] + fn event_from_runtime_upgrade_is_included() { + new_test_ext(1).execute_with(|| { + // Make sure `on_runtime_upgrade` is called. + RUNTIME_VERSION.with(|v| { + *v.borrow_mut() = + sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + + // set block number to non zero so events are not exlcuded + System::set_block_number(1); + + Executive::initialize_block(&Header::new( + 2, + H256::default(), + H256::default(), + [69u8; 32].into(), + Digest::default(), + )); + + System::assert_last_event(frame_system::Event::::CodeUpdated.into()); + }); + } + /// Regression test that ensures that the custom on runtime upgrade is called when executive is /// used through the `ExecuteBlock` trait. #[test] @@ -1361,23 +1386,19 @@ mod tests { )); // All weights that show up in the `initialize_block_impl` - let frame_system_upgrade_weight = frame_system::Pallet::::on_runtime_upgrade(); let custom_runtime_upgrade_weight = CustomOnRuntimeUpgrade::on_runtime_upgrade(); - let runtime_upgrade_weight = ::on_runtime_upgrade(); - let frame_system_on_initialize_weight = - frame_system::Pallet::::on_initialize(block_number); + let runtime_upgrade_weight = + ::on_runtime_upgrade(); let on_initialize_weight = - >::on_initialize(block_number); + >::on_initialize(block_number); let base_block_weight = ::BlockWeights::get().base_block; // Weights are recorded correctly assert_eq!( frame_system::Pallet::::block_weight().total(), - frame_system_upgrade_weight + - custom_runtime_upgrade_weight + + custom_runtime_upgrade_weight + runtime_upgrade_weight + - frame_system_on_initialize_weight + on_initialize_weight + base_block_weight, ); }); diff --git a/frame/gilt/Cargo.toml b/frame/gilt/Cargo.toml index c275b693d8f2..837504e516ce 100644 --- a/frame/gilt/Cargo.toml +++ b/frame/gilt/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-gilt" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for rewarding account freezing." readme = "README.md" @@ -13,18 +13,18 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-arithmetic = { version = "4.0.0-dev", default-features = false, path = "../../primitives/arithmetic" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-arithmetic = { version = "5.0.0", default-features = false, path = "../../primitives/arithmetic" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } [features] @@ -40,7 +40,7 @@ std = [ "frame-system/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", ] diff --git a/frame/gilt/src/benchmarking.rs b/frame/gilt/src/benchmarking.rs index cfc503cf897b..3df08372f499 100644 --- a/frame/gilt/src/benchmarking.rs +++ b/frame/gilt/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ #![cfg(feature = "runtime-benchmarks")] use super::*; -use frame_benchmarking::{benchmarks, impl_benchmark_test_suite, whitelisted_caller}; +use frame_benchmarking::{benchmarks, whitelisted_caller}; use frame_support::{ dispatch::UnfilteredDispatchable, traits::{Currency, EnsureOrigin, Get}, @@ -126,6 +126,6 @@ benchmarks! { .dispatch_bypass_filter(T::AdminOrigin::successful_origin())?; }: { Gilt::::pursue_target(q) } -} -impl_benchmark_test_suite!(Gilt, crate::mock::new_test_ext(), crate::mock::Test); + impl_benchmark_test_suite!(Gilt, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/frame/gilt/src/lib.rs b/frame/gilt/src/lib.rs index de114e4bb87d..8956e04857f2 100644 --- a/frame/gilt/src/lib.rs +++ b/frame/gilt/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -79,10 +79,9 @@ pub mod pallet { pub use crate::weights::WeightInfo; use frame_support::{ pallet_prelude::*, - traits::{Currency, OnUnbalanced, ReservableCurrency}, + traits::{Currency, DefensiveSaturating, OnUnbalanced, ReservableCurrency}, }; use frame_system::pallet_prelude::*; - use scale_info::TypeInfo; use sp_arithmetic::{PerThing, Perquintill}; use sp_runtime::traits::{Saturating, Zero}; use sp_std::prelude::*; @@ -113,7 +112,8 @@ pub mod pallet { + sp_std::fmt::Debug + Default + From - + TypeInfo; + + TypeInfo + + MaxEncodedLen; /// Origin required for setting the target proportion to be under gilt. type AdminOrigin: EnsureOrigin; @@ -128,7 +128,6 @@ pub mod pallet { /// The issuance to ignore. This is subtracted from the `Currency`'s `total_issuance` to get /// the issuance by which we inflate or deflate the gilt. - #[pallet::constant] type IgnoredIssuance: Get>; /// Number of duration queues in total. This sets the maximum duration supported, which is @@ -183,7 +182,9 @@ pub mod pallet { pub struct Pallet(_); /// A single bid on a gilt, an item of a *queue* in `Queues`. - #[derive(Clone, Eq, PartialEq, Default, Encode, Decode, RuntimeDebug, TypeInfo)] + #[derive( + Clone, Eq, PartialEq, Default, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen, + )] pub struct GiltBid { /// The amount bid. pub amount: Balance, @@ -192,7 +193,9 @@ pub mod pallet { } /// Information representing an active gilt. - #[derive(Clone, Eq, PartialEq, Default, Encode, Decode, RuntimeDebug, TypeInfo)] + #[derive( + Clone, Eq, PartialEq, Default, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen, + )] pub struct ActiveGilt { /// The proportion of the effective total issuance (i.e. accounting for any eventual gilt /// expansion or contraction that may eventually be claimed). @@ -216,7 +219,9 @@ pub mod pallet { /// `issuance - frozen + proportion * issuance` /// /// where `issuance = total_issuance - IgnoredIssuance` - #[derive(Clone, Eq, PartialEq, Default, Encode, Decode, RuntimeDebug, TypeInfo)] + #[derive( + Clone, Eq, PartialEq, Default, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen, + )] pub struct ActiveGiltsTotal { /// The total amount of funds held in reserve for all active gilts. pub frozen: Balance, @@ -234,12 +239,18 @@ pub mod pallet { /// The vector is indexed by duration in `Period`s, offset by one, so information on the queue /// whose duration is one `Period` would be storage `0`. #[pallet::storage] - pub type QueueTotals = StorageValue<_, Vec<(u32, BalanceOf)>, ValueQuery>; + pub type QueueTotals = + StorageValue<_, BoundedVec<(u32, BalanceOf), T::QueueCount>, ValueQuery>; /// The queues of bids ready to become gilts. Indexed by duration (in `Period`s). #[pallet::storage] - pub type Queues = - StorageMap<_, Blake2_128Concat, u32, Vec, T::AccountId>>, ValueQuery>; + pub type Queues = StorageMap< + _, + Blake2_128Concat, + u32, + BoundedVec, T::AccountId>, T::MaxQueueLen>, + ValueQuery, + >; /// Information relating to the gilts currently active. #[pallet::storage] @@ -266,7 +277,11 @@ pub mod pallet { #[pallet::genesis_build] impl GenesisBuild for GenesisConfig { fn build(&self) { - QueueTotals::::put(vec![(0, BalanceOf::::zero()); T::QueueCount::get() as usize]); + let unbounded = vec![(0, BalanceOf::::zero()); T::QueueCount::get() as usize]; + let bounded: BoundedVec<_, _> = unbounded + .try_into() + .expect("QueueTotals should support up to QueueCount items. qed"); + QueueTotals::::put(bounded); } } @@ -274,17 +289,23 @@ pub mod pallet { #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { /// A bid was successfully placed. - /// \[ who, amount, duration \] - BidPlaced(T::AccountId, BalanceOf, u32), + BidPlaced { who: T::AccountId, amount: BalanceOf, duration: u32 }, /// A bid was successfully removed (before being accepted as a gilt). - /// \[ who, amount, duration \] - BidRetracted(T::AccountId, BalanceOf, u32), + BidRetracted { who: T::AccountId, amount: BalanceOf, duration: u32 }, /// A bid was accepted as a gilt. The balance may not be released until expiry. - /// \[ index, expiry, who, amount \] - GiltIssued(ActiveIndex, T::BlockNumber, T::AccountId, BalanceOf), + GiltIssued { + index: ActiveIndex, + expiry: T::BlockNumber, + who: T::AccountId, + amount: BalanceOf, + }, /// An expired gilt has been thawed. - /// \[ index, who, original_amount, additional_amount \] - GiltThawed(ActiveIndex, T::AccountId, BalanceOf, BalanceOf), + GiltThawed { + index: ActiveIndex, + who: T::AccountId, + original_amount: BalanceOf, + additional_amount: BalanceOf, + }, } #[pallet::error] @@ -361,7 +382,7 @@ pub mod pallet { T::Currency::unreserve(&bid.who, bid.amount); (0, amount - bid.amount) } else { - q.insert(0, bid); + q.try_insert(0, bid).expect("verified queue was not full above. qed."); (1, amount) }; @@ -374,11 +395,11 @@ pub mod pallet { }, )?; QueueTotals::::mutate(|qs| { - qs.resize(queue_count, (0, Zero::zero())); + qs.bounded_resize(queue_count, (0, Zero::zero())); qs[queue_index].0 += net.0; qs[queue_index].1 = qs[queue_index].1.saturating_add(net.1); }); - Self::deposit_event(Event::BidPlaced(who.clone(), amount, duration)); + Self::deposit_event(Event::BidPlaced { who: who.clone(), amount, duration }); Ok(().into()) } @@ -410,13 +431,13 @@ pub mod pallet { })?; QueueTotals::::mutate(|qs| { - qs.resize(queue_count, (0, Zero::zero())); + qs.bounded_resize(queue_count, (0, Zero::zero())); qs[queue_index].0 = new_len; qs[queue_index].1 = qs[queue_index].1.saturating_sub(bid.amount); }); T::Currency::unreserve(&bid.who, bid.amount); - Self::deposit_event(Event::BidRetracted(bid.who, bid.amount, duration)); + Self::deposit_event(Event::BidRetracted { who: bid.who, amount: bid.amount, duration }); Ok(().into()) } @@ -495,7 +516,12 @@ pub mod pallet { debug_assert!(err_amt.is_zero()); } - let e = Event::GiltThawed(index, gilt.who, gilt.amount, gilt_value); + let e = Event::GiltThawed { + index, + who: gilt.who, + original_amount: gilt.amount, + additional_amount: gilt_value, + }; Self::deposit_event(e); }); @@ -582,17 +608,20 @@ pub mod pallet { if remaining < bid.amount { let overflow = bid.amount - remaining; bid.amount = remaining; - q.push(GiltBid { amount: overflow, who: bid.who.clone() }); + q.try_push(GiltBid { amount: overflow, who: bid.who.clone() }) + .expect("just popped, so there must be space. qed"); } let amount = bid.amount; // Can never overflow due to block above. remaining -= amount; // Should never underflow since it should track the total of the // bids exactly, but we'll be defensive. - qs[queue_index].1 = qs[queue_index].1.saturating_sub(bid.amount); + qs[queue_index].1 = + qs[queue_index].1.defensive_saturating_sub(bid.amount); // Now to activate the bid... - let nongilt_issuance = total_issuance.saturating_sub(totals.frozen); + let nongilt_issuance = + total_issuance.defensive_saturating_sub(totals.frozen); let effective_issuance = totals .proportion .left_from_one() @@ -603,9 +632,11 @@ pub mod pallet { let who = bid.who; let index = totals.index; totals.frozen += bid.amount; - totals.proportion = totals.proportion.saturating_add(proportion); + totals.proportion = + totals.proportion.defensive_saturating_add(proportion); totals.index += 1; - let e = Event::GiltIssued(index, expiry, who.clone(), amount); + let e = + Event::GiltIssued { index, expiry, who: who.clone(), amount }; Self::deposit_event(e); let gilt = ActiveGilt { amount, proportion, who, expiry }; Active::::insert(index, gilt); diff --git a/frame/gilt/src/mock.rs b/frame/gilt/src/mock.rs index ac3f4df1b71d..369b34ba77f4 100644 --- a/frame/gilt/src/mock.rs +++ b/frame/gilt/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use crate as pallet_gilt; use frame_support::{ ord_parameter_types, parameter_types, - traits::{Currency, GenesisBuild, OnFinalize, OnInitialize}, + traits::{ConstU16, ConstU32, ConstU64, Currency, GenesisBuild, OnFinalize, OnInitialize}, }; use sp_core::H256; use sp_runtime::{ @@ -45,11 +45,6 @@ frame_support::construct_runtime!( } ); -parameter_types! { - pub const BlockHashCount: u64 = 250; - pub const SS58Prefix: u8 = 42; -} - impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; type BlockWeights = (); @@ -64,7 +59,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type DbWeight = (); type Version = (); type PalletInfo = PalletInfo; @@ -72,19 +67,16 @@ impl frame_system::Config for Test { type OnNewAccount = (); type OnKilledAccount = (); type SystemWeightInfo = (); - type SS58Prefix = SS58Prefix; + type SS58Prefix = ConstU16<42>; type OnSetCode = (); -} - -parameter_types! { - pub const ExistentialDeposit: u64 = 1; + type MaxConsumers = frame_support::traits::ConstU32<16>; } impl pallet_balances::Config for Test { type Balance = u64; type DustRemoval = (); type Event = Event; - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = frame_support::traits::ConstU64<1>; type AccountStore = System; type WeightInfo = (); type MaxLocks = (); @@ -94,13 +86,6 @@ impl pallet_balances::Config for Test { parameter_types! { pub IgnoredIssuance: u64 = Balances::total_balance(&0); // Account zero is ignored. - pub const QueueCount: u32 = 3; - pub const MaxQueueLen: u32 = 3; - pub const FifoQueueLen: u32 = 1; - pub const Period: u64 = 3; - pub const MinFreeze: u64 = 2; - pub const IntakePeriod: u64 = 2; - pub const MaxIntakeBids: u32 = 2; } ord_parameter_types! { pub const One: u64 = 1; @@ -114,13 +99,13 @@ impl pallet_gilt::Config for Test { type Deficit = (); type Surplus = (); type IgnoredIssuance = IgnoredIssuance; - type QueueCount = QueueCount; - type MaxQueueLen = MaxQueueLen; - type FifoQueueLen = FifoQueueLen; - type Period = Period; - type MinFreeze = MinFreeze; - type IntakePeriod = IntakePeriod; - type MaxIntakeBids = MaxIntakeBids; + type QueueCount = ConstU32<3>; + type MaxQueueLen = ConstU32<3>; + type FifoQueueLen = ConstU32<1>; + type Period = ConstU64<3>; + type MinFreeze = ConstU64<2>; + type IntakePeriod = ConstU64<2>; + type MaxIntakeBids = ConstU32<2>; type WeightInfo = (); } diff --git a/frame/gilt/src/tests.rs b/frame/gilt/src/tests.rs index 80315141e232..486601b5b2f2 100644 --- a/frame/gilt/src/tests.rs +++ b/frame/gilt/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/gilt/src/weights.rs b/frame/gilt/src/weights.rs index f54d917cc160..6084d494c5c3 100644 --- a/frame/gilt/src/weights.rs +++ b/frame/gilt/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_gilt //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/gilt/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -61,44 +62,44 @@ impl WeightInfo for SubstrateWeight { // Storage: Gilt Queues (r:1 w:1) // Storage: Gilt QueueTotals (r:1 w:1) fn place_bid(l: u32, ) -> Weight { - (59_219_000 as Weight) + (32_753_000 as Weight) // Standard Error: 0 - .saturating_add((156_000 as Weight).saturating_mul(l as Weight)) + .saturating_add((96_000 as Weight).saturating_mul(l as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Gilt Queues (r:1 w:1) // Storage: Gilt QueueTotals (r:1 w:1) fn place_bid_max() -> Weight { - (184_943_000 as Weight) + (124_814_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Gilt Queues (r:1 w:1) // Storage: Gilt QueueTotals (r:1 w:1) fn retract_bid(l: u32, ) -> Weight { - (59_352_000 as Weight) + (33_544_000 as Weight) // Standard Error: 0 - .saturating_add((129_000 as Weight).saturating_mul(l as Weight)) + .saturating_add((86_000 as Weight).saturating_mul(l as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Gilt ActiveTotal (r:1 w:1) fn set_target() -> Weight { - (5_444_000 as Weight) + (2_648_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Gilt Active (r:1 w:1) // Storage: Gilt ActiveTotal (r:1 w:1) fn thaw() -> Weight { - (71_399_000 as Weight) + (41_549_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Gilt ActiveTotal (r:1 w:0) fn pursue_target_noop() -> Weight { - (3_044_000 as Weight) + (1_583_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) } // Storage: Gilt ActiveTotal (r:1 w:1) @@ -106,9 +107,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Gilt Queues (r:1 w:1) // Storage: Gilt Active (r:0 w:1) fn pursue_target_per_item(b: u32, ) -> Weight { - (54_478_000 as Weight) - // Standard Error: 2_000 - .saturating_add((10_150_000 as Weight).saturating_mul(b as Weight)) + (35_836_000 as Weight) + // Standard Error: 1_000 + .saturating_add((4_008_000 as Weight).saturating_mul(b as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(b as Weight))) @@ -118,9 +119,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Gilt Queues (r:1 w:1) // Storage: Gilt Active (r:0 w:1) fn pursue_target_per_queue(q: u32, ) -> Weight { - (20_099_000 as Weight) - // Standard Error: 7_000 - .saturating_add((16_603_000 as Weight).saturating_mul(q as Weight)) + (12_139_000 as Weight) + // Standard Error: 6_000 + .saturating_add((7_819_000 as Weight).saturating_mul(q as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(q as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) @@ -133,44 +134,44 @@ impl WeightInfo for () { // Storage: Gilt Queues (r:1 w:1) // Storage: Gilt QueueTotals (r:1 w:1) fn place_bid(l: u32, ) -> Weight { - (59_219_000 as Weight) + (32_753_000 as Weight) // Standard Error: 0 - .saturating_add((156_000 as Weight).saturating_mul(l as Weight)) + .saturating_add((96_000 as Weight).saturating_mul(l as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Gilt Queues (r:1 w:1) // Storage: Gilt QueueTotals (r:1 w:1) fn place_bid_max() -> Weight { - (184_943_000 as Weight) + (124_814_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Gilt Queues (r:1 w:1) // Storage: Gilt QueueTotals (r:1 w:1) fn retract_bid(l: u32, ) -> Weight { - (59_352_000 as Weight) + (33_544_000 as Weight) // Standard Error: 0 - .saturating_add((129_000 as Weight).saturating_mul(l as Weight)) + .saturating_add((86_000 as Weight).saturating_mul(l as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Gilt ActiveTotal (r:1 w:1) fn set_target() -> Weight { - (5_444_000 as Weight) + (2_648_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Gilt Active (r:1 w:1) // Storage: Gilt ActiveTotal (r:1 w:1) fn thaw() -> Weight { - (71_399_000 as Weight) + (41_549_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Gilt ActiveTotal (r:1 w:0) fn pursue_target_noop() -> Weight { - (3_044_000 as Weight) + (1_583_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) } // Storage: Gilt ActiveTotal (r:1 w:1) @@ -178,9 +179,9 @@ impl WeightInfo for () { // Storage: Gilt Queues (r:1 w:1) // Storage: Gilt Active (r:0 w:1) fn pursue_target_per_item(b: u32, ) -> Weight { - (54_478_000 as Weight) - // Standard Error: 2_000 - .saturating_add((10_150_000 as Weight).saturating_mul(b as Weight)) + (35_836_000 as Weight) + // Standard Error: 1_000 + .saturating_add((4_008_000 as Weight).saturating_mul(b as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(b as Weight))) @@ -190,9 +191,9 @@ impl WeightInfo for () { // Storage: Gilt Queues (r:1 w:1) // Storage: Gilt Active (r:0 w:1) fn pursue_target_per_queue(q: u32, ) -> Weight { - (20_099_000 as Weight) - // Standard Error: 7_000 - .saturating_add((16_603_000 as Weight).saturating_mul(q as Weight)) + (12_139_000 as Weight) + // Standard Error: 6_000 + .saturating_add((7_819_000 as Weight).saturating_mul(q as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(q as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) diff --git a/frame/grandpa/Cargo.toml b/frame/grandpa/Cargo.toml index 53ab443783e5..cad238a4e365 100644 --- a/frame/grandpa/Cargo.toml +++ b/frame/grandpa/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-grandpa" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for GRANDPA finality gadget" readme = "README.md" @@ -13,15 +13,15 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../../primitives/application-crypto" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } sp-finality-grandpa = { version = "4.0.0-dev", default-features = false, path = "../../primitives/finality-grandpa" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } sp-session = { version = "4.0.0-dev", default-features = false, path = "../../primitives/session" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } @@ -32,8 +32,8 @@ log = { version = "0.4.14", default-features = false } [dev-dependencies] frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking" } -grandpa = { package = "finality-grandpa", version = "0.14.1", features = ["derive-codec"] } -sp-keyring = { version = "4.0.0-dev", path = "../../primitives/keyring" } +grandpa = { package = "finality-grandpa", version = "0.15.0", features = ["derive-codec"] } +sp-keyring = { version = "6.0.0", path = "../../primitives/keyring" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-offences = { version = "4.0.0-dev", path = "../offences" } pallet-staking = { version = "4.0.0-dev", path = "../staking" } @@ -60,5 +60,5 @@ std = [ "pallet-session/std", "log/std", ] -runtime-benchmarks = ["frame-benchmarking"] +runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/grandpa/src/benchmarking.rs b/frame/grandpa/src/benchmarking.rs index 815a18d13531..124085914992 100644 --- a/frame/grandpa/src/benchmarking.rs +++ b/frame/grandpa/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -68,6 +68,12 @@ benchmarks! { verify { assert!(Grandpa::::stalled().is_some()); } + + impl_benchmark_test_suite!( + Pallet, + crate::mock::new_test_ext(vec![(1, 1), (2, 1), (3, 1)]), + crate::mock::Test, + ); } #[cfg(test)] @@ -75,12 +81,6 @@ mod tests { use super::*; use crate::mock::*; - frame_benchmarking::impl_benchmark_test_suite!( - Pallet, - crate::mock::new_test_ext(vec![(1, 1), (2, 1), (3, 1)]), - crate::mock::Test, - ); - #[test] fn test_generate_equivocation_report_blob() { let authorities = crate::tests::test_authorities(); diff --git a/frame/grandpa/src/default_weights.rs b/frame/grandpa/src/default_weights.rs index edc18a7ff8c9..330e9bb25517 100644 --- a/frame/grandpa/src/default_weights.rs +++ b/frame/grandpa/src/default_weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/grandpa/src/equivocation.rs b/frame/grandpa/src/equivocation.rs index 8a23ce6e1ef1..804272c20480 100644 --- a/frame/grandpa/src/equivocation.rs +++ b/frame/grandpa/src/equivocation.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -185,7 +185,7 @@ where } fn block_author() -> Option { - Some(>::author()) + >::author() } } diff --git a/frame/grandpa/src/lib.rs b/frame/grandpa/src/lib.rs index 687207151f4f..e30d65acbc6a 100644 --- a/frame/grandpa/src/lib.rs +++ b/frame/grandpa/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -83,7 +83,6 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] #[pallet::storage_version(STORAGE_VERSION)] - #[pallet::generate_storage_info] pub struct Pallet(_); #[pallet::config] @@ -154,9 +153,9 @@ pub mod pallet { // enact the change if we've reached the enacting block if block_number == pending_change.scheduled_at + pending_change.delay { Self::set_grandpa_authorities(&pending_change.next_authorities); - Self::deposit_event(Event::NewAuthorities( - pending_change.next_authorities.to_vec(), - )); + Self::deposit_event(Event::NewAuthorities { + authority_set: pending_change.next_authorities.to_vec(), + }); >::kill(); } } @@ -255,8 +254,8 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(fn deposit_event)] pub enum Event { - /// New authority set has been applied. \[authority_set\] - NewAuthorities(AuthorityList), + /// New authority set has been applied. + NewAuthorities { authority_set: AuthorityList }, /// Current authority set has been paused. Paused, /// Current authority set has been resumed. @@ -327,18 +326,12 @@ pub mod pallet { #[pallet::getter(fn session_for_set)] pub(super) type SetIdSession = StorageMap<_, Twox64Concat, SetId, SessionIndex>; + #[cfg_attr(feature = "std", derive(Default))] #[pallet::genesis_config] pub struct GenesisConfig { pub authorities: AuthorityList, } - #[cfg(feature = "std")] - impl Default for GenesisConfig { - fn default() -> Self { - Self { authorities: Default::default() } - } - } - #[pallet::genesis_build] impl GenesisBuild for GenesisConfig { fn build(&self) { @@ -371,13 +364,9 @@ pub type BoundedAuthorityList = WeakBoundedVec<(AuthorityId, AuthorityWei /// A stored pending change. /// `Limit` is the bound for `next_authorities` #[derive(Encode, Decode, TypeInfo, MaxEncodedLen)] -#[codec(mel_bound(Limit: Get))] +#[codec(mel_bound(N: MaxEncodedLen, Limit: Get))] #[scale_info(skip_type_params(Limit))] -pub struct StoredPendingChange -where - Limit: Get, - N: MaxEncodedLen, -{ +pub struct StoredPendingChange { /// The block number this was scheduled at. pub scheduled_at: N, /// The delay in blocks until it will be applied. @@ -508,8 +497,8 @@ impl Pallet { /// Deposit one of this module's logs. fn deposit_log(log: ConsensusLog) { - let log: DigestItem = DigestItem::Consensus(GRANDPA_ENGINE_ID, log.encode()); - >::deposit_log(log.into()); + let log = DigestItem::Consensus(GRANDPA_ENGINE_ID, log.encode()); + >::deposit_log(log); } // Perform module initialization, abstracted so that it can be called either through genesis @@ -674,7 +663,7 @@ where SetIdSession::::insert(current_set_id, &session_index); } - fn on_disabled(i: usize) { + fn on_disabled(i: u32) { Self::deposit_log(ConsensusLog::OnDisabled(i as u64)) } } diff --git a/frame/grandpa/src/migrations.rs b/frame/grandpa/src/migrations.rs index 05c24e11b393..7795afcd8034 100644 --- a/frame/grandpa/src/migrations.rs +++ b/frame/grandpa/src/migrations.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/grandpa/src/migrations/v4.rs b/frame/grandpa/src/migrations/v4.rs index 094f276efef3..ab43f7baef4e 100644 --- a/frame/grandpa/src/migrations/v4.rs +++ b/frame/grandpa/src/migrations/v4.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index 4e5e44ce36e7..0296cd2e28d8 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,13 +22,14 @@ use crate::{self as pallet_grandpa, AuthorityId, AuthorityList, Config, ConsensusLog}; use ::grandpa as finality_grandpa; use codec::Encode; -use frame_election_provider_support::onchain; +use frame_election_provider_support::{onchain, SequentialPhragmen}; use frame_support::{ parameter_types, - traits::{GenesisBuild, KeyOwnerProofSystem, OnFinalize, OnInitialize}, + traits::{ + ConstU128, ConstU32, ConstU64, GenesisBuild, KeyOwnerProofSystem, OnFinalize, OnInitialize, + }, }; use pallet_session::historical as pallet_session_historical; -use pallet_staking::EraIndex; use sp_core::{crypto::KeyTypeId, H256}; use sp_finality_grandpa::{RoundNumber, SetId, GRANDPA_ENGINE_ID}; use sp_keyring::Ed25519Keyring; @@ -39,7 +40,7 @@ use sp_runtime::{ traits::{IdentityLookup, OpaqueKeys}, DigestItem, Perbill, }; -use sp_staking::SessionIndex; +use sp_staking::{EraIndex, SessionIndex}; type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; type Block = frame_system::mocking::MockBlock; @@ -69,7 +70,6 @@ impl_opaque_keys! { } parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -89,7 +89,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -98,6 +98,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } impl frame_system::offchain::SendTransactionTypes for Test @@ -111,7 +112,6 @@ where parameter_types! { pub const Period: u64 = 1; pub const Offset: u64 = 0; - pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(17); } /// Custom `SessionHandler` since we use `TestSessionKeys` as `Keys`. @@ -119,12 +119,11 @@ impl pallet_session::Config for Test { type Event = Event; type ValidatorId = u64; type ValidatorIdOf = pallet_staking::StashOf; - type ShouldEndSession = pallet_session::PeriodicSessions; - type NextSessionRotation = pallet_session::PeriodicSessions; + type ShouldEndSession = pallet_session::PeriodicSessions, ConstU64<0>>; + type NextSessionRotation = pallet_session::PeriodicSessions, ConstU64<0>>; type SessionManager = pallet_session::historical::NoteHistoricalRoot; type SessionHandler = ::KeyTypeIdProviders; type Keys = TestSessionKeys; - type DisabledValidatorsThreshold = DisabledValidatorsThreshold; type WeightInfo = (); } @@ -133,21 +132,13 @@ impl pallet_session::historical::Config for Test { type FullIdentificationOf = pallet_staking::ExposureOf; } -parameter_types! { - pub const UncleGenerations: u64 = 0; -} - impl pallet_authorship::Config for Test { type FindAuthor = (); - type UncleGenerations = UncleGenerations; + type UncleGenerations = ConstU64<0>; type FilterUncle = (); type EventHandler = (); } -parameter_types! { - pub const ExistentialDeposit: u128 = 1; -} - impl pallet_balances::Config for Test { type MaxLocks = (); type MaxReserves = (); @@ -155,19 +146,15 @@ impl pallet_balances::Config for Test { type Balance = u128; type DustRemoval = (); type Event = Event; - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU128<1>; type AccountStore = System; type WeightInfo = (); } -parameter_types! { - pub const MinimumPeriod: u64 = 3; -} - impl pallet_timestamp::Config for Test { type Moment = u64; type OnTimestampSet = (); - type MinimumPeriod = MinimumPeriod; + type MinimumPeriod = ConstU64<3>; type WeightInfo = (); } @@ -188,18 +175,20 @@ parameter_types! { pub const SlashDeferDuration: EraIndex = 0; pub const AttestationPeriod: u64 = 100; pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; - pub const MaxNominatorRewardedPerValidator: u32 = 64; pub const ElectionLookahead: u64 = 0; pub const StakingUnsignedPriority: u64 = u64::MAX / 2; + pub const OffendingValidatorsThreshold: Perbill = Perbill::from_percent(17); } -impl onchain::Config for Test { - type Accuracy = Perbill; +pub struct OnChainSeqPhragmen; +impl onchain::ExecutionConfig for OnChainSeqPhragmen { + type System = Test; + type Solver = SequentialPhragmen<::AccountId, Perbill>; type DataProvider = Staking; } impl pallet_staking::Config for Test { - const MAX_NOMINATIONS: u32 = 16; + type MaxNominations = ConstU32<16>; type RewardRemainder = (); type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; type Event = Event; @@ -213,11 +202,14 @@ impl pallet_staking::Config for Test { type SessionInterface = Self; type UnixTime = pallet_timestamp::Pallet; type EraPayout = pallet_staking::ConvertCurve; - type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type MaxNominatorRewardedPerValidator = ConstU32<64>; + type OffendingValidatorsThreshold = OffendingValidatorsThreshold; type NextNewSession = Session; - type ElectionProvider = onchain::OnChainSequentialPhragmen; + type ElectionProvider = onchain::UnboundedExecution; type GenesisElectionProvider = Self::ElectionProvider; - type SortedListProvider = pallet_staking::UseNominatorsMap; + type VoterList = pallet_staking::UseNominatorsAndValidatorsMap; + type MaxUnlockingChunks = ConstU32<32>; + type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig; type WeightInfo = (); } @@ -230,7 +222,6 @@ impl pallet_offences::Config for Test { parameter_types! { pub const ReportLongevity: u64 = BondingDuration::get() as u64 * SessionsPerEra::get() as u64 * Period::get(); - pub const MaxAuthorities: u32 = 100; } impl Config for Test { @@ -251,10 +242,10 @@ impl Config for Test { super::EquivocationHandler; type WeightInfo = (); - type MaxAuthorities = MaxAuthorities; + type MaxAuthorities = ConstU32<100>; } -pub fn grandpa_log(log: ConsensusLog) -> DigestItem { +pub fn grandpa_log(log: ConsensusLog) -> DigestItem { DigestItem::Consensus(GRANDPA_ENGINE_ID, log.encode()) } @@ -337,7 +328,8 @@ pub fn start_session(session_index: SessionIndex) { System::parent_hash() }; - System::initialize(&(i as u64 + 1), &parent_hash, &Default::default(), Default::default()); + System::reset_events(); + System::initialize(&(i as u64 + 1), &parent_hash, &Default::default()); System::set_block_number((i + 1).into()); Timestamp::set_timestamp(System::block_number() * 6000); @@ -356,7 +348,8 @@ pub fn start_era(era_index: EraIndex) { } pub fn initialize_block(number: u64, parent_hash: H256) { - System::initialize(&number, &parent_hash, &Default::default(), Default::default()); + System::reset_events(); + System::initialize(&number, &parent_hash, &Default::default()); } pub fn generate_equivocation_proof( diff --git a/frame/grandpa/src/tests.rs b/frame/grandpa/src/tests.rs index 98f54f966fad..ab0a9c677b00 100644 --- a/frame/grandpa/src/tests.rs +++ b/frame/grandpa/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -57,7 +57,10 @@ fn authorities_change_logged() { System::events(), vec![EventRecord { phase: Phase::Finalization, - event: Event::NewAuthorities(to_authorities(vec![(4, 1), (5, 1), (6, 1)])).into(), + event: Event::NewAuthorities { + authority_set: to_authorities(vec![(4, 1), (5, 1), (6, 1)]) + } + .into(), topics: vec![], },] ); @@ -93,7 +96,10 @@ fn authorities_change_logged_after_delay() { System::events(), vec![EventRecord { phase: Phase::Finalization, - event: Event::NewAuthorities(to_authorities(vec![(4, 1), (5, 1), (6, 1)])).into(), + event: Event::NewAuthorities { + authority_set: to_authorities(vec![(4, 1), (5, 1), (6, 1)]) + } + .into(), topics: vec![], },] ); diff --git a/frame/identity/Cargo.toml b/frame/identity/Cargo.toml index 598be25c5ef3..5dff7acc73e2 100644 --- a/frame/identity/Cargo.toml +++ b/frame/identity/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-identity" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME identity management pallet" readme = "README.md" @@ -13,18 +13,18 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.2.0", default-features = false, features = ["derive", "max-encoded-len"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -enumflags2 = { version = "0.6.2" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +enumflags2 = { version = "0.7.4" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } [features] @@ -40,7 +40,7 @@ std = [ "frame-system/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", ] diff --git a/frame/identity/src/benchmarking.rs b/frame/identity/src/benchmarking.rs index 8bda24ddc73e..2145779ecf54 100644 --- a/frame/identity/src/benchmarking.rs +++ b/frame/identity/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,7 @@ use super::*; use crate::Pallet as Identity; -use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite, whitelisted_caller}; +use frame_benchmarking::{account, benchmarks, whitelisted_caller}; use frame_support::{ensure, traits::Get}; use frame_system::RawOrigin; use sp_runtime::traits::Bounded; @@ -75,7 +75,7 @@ fn create_sub_accounts( } // Set identity so `set_subs` does not fail. - let _ = T::Currency::make_free_balance_be(&who, BalanceOf::::max_value()); + let _ = T::Currency::make_free_balance_be(&who, BalanceOf::::max_value() / 2u32.into()); let info = create_identity_info::(1); Identity::::set_identity(who_origin.clone().into(), Box::new(info))?; @@ -153,7 +153,7 @@ benchmarks! { }; }: _(RawOrigin::Signed(caller.clone()), Box::new(create_identity_info::(x))) verify { - assert_last_event::(Event::::IdentitySet(caller).into()); + assert_last_event::(Event::::IdentitySet { who: caller }.into()); } // We need to split `set_subs` into two benchmarks to accurately isolate the potential @@ -237,7 +237,7 @@ benchmarks! { }; }: _(RawOrigin::Signed(caller.clone()), r - 1, 10u32.into()) verify { - assert_last_event::(Event::::JudgementRequested(caller, r-1).into()); + assert_last_event::(Event::::JudgementRequested { who: caller, registrar_index: r-1 }.into()); } cancel_request { @@ -257,7 +257,7 @@ benchmarks! { Identity::::request_judgement(caller_origin, r - 1, 10u32.into())?; }: _(RawOrigin::Signed(caller.clone()), r - 1) verify { - assert_last_event::(Event::::JudgementUnrequested(caller, r-1).into()); + assert_last_event::(Event::::JudgementUnrequested { who: caller, registrar_index: r-1 }.into()); } set_fee { @@ -328,7 +328,7 @@ benchmarks! { Identity::::request_judgement(user_origin.clone(), r, 10u32.into())?; }: _(RawOrigin::Signed(caller), r, user_lookup, Judgement::Reasonable) verify { - assert_last_event::(Event::::JudgementGiven(user, r).into()) + assert_last_event::(Event::::JudgementGiven { target: user, registrar_index: r }.into()) } kill_identity { @@ -411,6 +411,5 @@ benchmarks! { ensure!(!SuperOf::::contains_key(&caller), "Sub not removed"); } + impl_benchmark_test_suite!(Identity, crate::tests::new_test_ext(), crate::tests::Test); } - -impl_benchmark_test_suite!(Identity, crate::tests::new_test_ext(), crate::tests::Test); diff --git a/frame/identity/src/lib.rs b/frame/identity/src/lib.rs index a91381f1edd8..51e63541a89b 100644 --- a/frame/identity/src/lib.rs +++ b/frame/identity/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -80,7 +80,7 @@ pub mod weights; use frame_support::traits::{BalanceStatus, Currency, OnUnbalanced, ReservableCurrency}; use sp_runtime::traits::{AppendZerosInput, Saturating, StaticLookup, Zero}; -use sp_std::{convert::TryInto, prelude::*}; +use sp_std::prelude::*; pub use weights::WeightInfo; pub use pallet::*; @@ -152,7 +152,6 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] - #[pallet::generate_storage_info] pub struct Pallet(_); /// Information that is pertinent to identify the entity behind an account. @@ -241,28 +240,27 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// A name was set or reset (which will remove all judgements). \[who\] - IdentitySet(T::AccountId), - /// A name was cleared, and the given balance returned. \[who, deposit\] - IdentityCleared(T::AccountId, BalanceOf), - /// A name was removed and the given balance slashed. \[who, deposit\] - IdentityKilled(T::AccountId, BalanceOf), - /// A judgement was asked from a registrar. \[who, registrar_index\] - JudgementRequested(T::AccountId, RegistrarIndex), - /// A judgement request was retracted. \[who, registrar_index\] - JudgementUnrequested(T::AccountId, RegistrarIndex), - /// A judgement was given by a registrar. \[target, registrar_index\] - JudgementGiven(T::AccountId, RegistrarIndex), - /// A registrar was added. \[registrar_index\] - RegistrarAdded(RegistrarIndex), - /// A sub-identity was added to an identity and the deposit paid. \[sub, main, deposit\] - SubIdentityAdded(T::AccountId, T::AccountId, BalanceOf), + /// A name was set or reset (which will remove all judgements). + IdentitySet { who: T::AccountId }, + /// A name was cleared, and the given balance returned. + IdentityCleared { who: T::AccountId, deposit: BalanceOf }, + /// A name was removed and the given balance slashed. + IdentityKilled { who: T::AccountId, deposit: BalanceOf }, + /// A judgement was asked from a registrar. + JudgementRequested { who: T::AccountId, registrar_index: RegistrarIndex }, + /// A judgement request was retracted. + JudgementUnrequested { who: T::AccountId, registrar_index: RegistrarIndex }, + /// A judgement was given by a registrar. + JudgementGiven { target: T::AccountId, registrar_index: RegistrarIndex }, + /// A registrar was added. + RegistrarAdded { registrar_index: RegistrarIndex }, + /// A sub-identity was added to an identity and the deposit paid. + SubIdentityAdded { sub: T::AccountId, main: T::AccountId, deposit: BalanceOf }, /// A sub-identity was removed from an identity and the deposit freed. - /// \[sub, main, deposit\] - SubIdentityRemoved(T::AccountId, T::AccountId, BalanceOf), + SubIdentityRemoved { sub: T::AccountId, main: T::AccountId, deposit: BalanceOf }, /// A sub-identity was cleared, and the given deposit repatriated from the - /// main identity account to the sub-identity account. \[sub, main, deposit\] - SubIdentityRevoked(T::AccountId, T::AccountId, BalanceOf), + /// main identity account to the sub-identity account. + SubIdentityRevoked { sub: T::AccountId, main: T::AccountId, deposit: BalanceOf }, } #[pallet::call] @@ -301,7 +299,7 @@ pub mod pallet { }, )?; - Self::deposit_event(Event::RegistrarAdded(i)); + Self::deposit_event(Event::RegistrarAdded { registrar_index: i }); Ok(Some(T::WeightInfo::add_registrar(registrar_count as u32)).into()) } @@ -364,7 +362,7 @@ pub mod pallet { let judgements = id.judgements.len(); >::insert(&sender, id); - Self::deposit_event(Event::IdentitySet(sender)); + Self::deposit_event(Event::IdentitySet { who: sender }); Ok(Some(T::WeightInfo::set_identity( judgements as u32, // R @@ -489,7 +487,7 @@ pub mod pallet { let err_amount = T::Currency::unreserve(&sender, deposit.clone()); debug_assert!(err_amount.is_zero()); - Self::deposit_event(Event::IdentityCleared(sender, deposit)); + Self::deposit_event(Event::IdentityCleared { who: sender, deposit }); Ok(Some(T::WeightInfo::clear_identity( id.judgements.len() as u32, // R @@ -558,7 +556,10 @@ pub mod pallet { let extra_fields = id.info.additional.len(); >::insert(&sender, id); - Self::deposit_event(Event::JudgementRequested(sender, reg_index)); + Self::deposit_event(Event::JudgementRequested { + who: sender, + registrar_index: reg_index, + }); Ok(Some(T::WeightInfo::request_judgement(judgements as u32, extra_fields as u32)) .into()) @@ -608,7 +609,10 @@ pub mod pallet { let extra_fields = id.info.additional.len(); >::insert(&sender, id); - Self::deposit_event(Event::JudgementUnrequested(sender, reg_index)); + Self::deposit_event(Event::JudgementUnrequested { + who: sender, + registrar_index: reg_index, + }); Ok(Some(T::WeightInfo::cancel_request(judgements as u32, extra_fields as u32)).into()) } @@ -791,7 +795,7 @@ pub mod pallet { let judgements = id.judgements.len(); let extra_fields = id.info.additional.len(); >::insert(&target, id); - Self::deposit_event(Event::JudgementGiven(target, reg_index)); + Self::deposit_event(Event::JudgementGiven { target, registrar_index: reg_index }); Ok(Some(T::WeightInfo::provide_judgement(judgements as u32, extra_fields as u32)) .into()) @@ -839,7 +843,7 @@ pub mod pallet { // Slash their deposit from them. T::Slashed::on_unbalanced(T::Currency::slash_reserved(&target, deposit).0); - Self::deposit_event(Event::IdentityKilled(target, deposit)); + Self::deposit_event(Event::IdentityKilled { who: target, deposit }); Ok(Some(T::WeightInfo::kill_identity( id.judgements.len() as u32, // R @@ -882,7 +886,7 @@ pub mod pallet { sub_ids.try_push(sub.clone()).expect("sub ids length checked above; qed"); *subs_deposit = subs_deposit.saturating_add(deposit); - Self::deposit_event(Event::SubIdentityAdded(sub, sender.clone(), deposit)); + Self::deposit_event(Event::SubIdentityAdded { sub, main: sender.clone(), deposit }); Ok(()) }) } @@ -929,7 +933,7 @@ pub mod pallet { *subs_deposit -= deposit; let err_amount = T::Currency::unreserve(&sender, deposit); debug_assert!(err_amount.is_zero()); - Self::deposit_event(Event::SubIdentityRemoved(sub, sender, deposit)); + Self::deposit_event(Event::SubIdentityRemoved { sub, main: sender, deposit }); }); Ok(()) } @@ -954,7 +958,11 @@ pub mod pallet { *subs_deposit -= deposit; let _ = T::Currency::repatriate_reserved(&sup, &sender, deposit, BalanceStatus::Free); - Self::deposit_event(Event::SubIdentityRevoked(sender, sup.clone(), deposit)); + Self::deposit_event(Event::SubIdentityRevoked { + sub: sender, + main: sup.clone(), + deposit, + }); }); Ok(()) } diff --git a/frame/identity/src/tests.rs b/frame/identity/src/tests.rs index c842b0e2f64b..bf41b451cbaa 100644 --- a/frame/identity/src/tests.rs +++ b/frame/identity/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,8 +21,12 @@ use super::*; use crate as pallet_identity; use codec::{Decode, Encode}; -use frame_support::{assert_noop, assert_ok, ord_parameter_types, parameter_types, BoundedVec}; -use frame_system::{EnsureOneOf, EnsureRoot, EnsureSignedBy}; +use frame_support::{ + assert_noop, assert_ok, ord_parameter_types, parameter_types, + traits::{ConstU32, ConstU64, EnsureOneOf}, + BoundedVec, +}; +use frame_system::{EnsureRoot, EnsureSignedBy}; use sp_core::H256; use sp_runtime::{ testing::Header, @@ -45,7 +49,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -63,7 +66,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type DbWeight = (); type Version = (); type PalletInfo = PalletInfo; @@ -73,43 +76,40 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } -parameter_types! { - pub const ExistentialDeposit: u64 = 1; -} + impl pallet_balances::Config for Test { type Balance = u64; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type MaxLocks = (); type MaxReserves = (); type ReserveIdentifier = [u8; 8]; type WeightInfo = (); } + parameter_types! { - pub const BasicDeposit: u64 = 10; - pub const FieldDeposit: u64 = 10; - pub const SubAccountDeposit: u64 = 10; - pub const MaxSubAccounts: u32 = 2; pub const MaxAdditionalFields: u32 = 2; pub const MaxRegistrars: u32 = 20; } + ord_parameter_types! { pub const One: u64 = 1; pub const Two: u64 = 2; } -type EnsureOneOrRoot = EnsureOneOf, EnsureSignedBy>; -type EnsureTwoOrRoot = EnsureOneOf, EnsureSignedBy>; +type EnsureOneOrRoot = EnsureOneOf, EnsureSignedBy>; +type EnsureTwoOrRoot = EnsureOneOf, EnsureSignedBy>; impl pallet_identity::Config for Test { type Event = Event; type Currency = Balances; type Slashed = (); - type BasicDeposit = BasicDeposit; - type FieldDeposit = FieldDeposit; - type SubAccountDeposit = SubAccountDeposit; - type MaxSubAccounts = MaxSubAccounts; + type BasicDeposit = ConstU64<10>; + type FieldDeposit = ConstU64<10>; + type SubAccountDeposit = ConstU64<10>; + type MaxSubAccounts = ConstU32<2>; type MaxAdditionalFields = MaxAdditionalFields; type MaxRegistrars = MaxRegistrars; type RegistrarOrigin = EnsureOneOrRoot; diff --git a/frame/identity/src/types.rs b/frame/identity/src/types.rs index ed6aeb18e96a..18fc54c941cb 100644 --- a/frame/identity/src/types.rs +++ b/frame/identity/src/types.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +17,7 @@ use super::*; use codec::{Decode, Encode, MaxEncodedLen}; -use enumflags2::BitFlags; +use enumflags2::{bitflags, BitFlags}; use frame_support::{ traits::{ConstU32, Get}, BoundedVec, CloneNoBound, PartialEqNoBound, RuntimeDebugNoBound, @@ -230,8 +230,9 @@ impl` that implements `Codec`. #[derive(Clone, Copy, PartialEq, Default, RuntimeDebug)] -pub struct IdentityFields(pub(crate) BitFlags); +pub struct IdentityFields(pub BitFlags); impl MaxEncodedLen for IdentityFields { fn max_encoded_len() -> usize { @@ -283,7 +284,7 @@ impl TypeInfo for IdentityFields { #[derive( CloneNoBound, Encode, Decode, Eq, MaxEncodedLen, PartialEqNoBound, RuntimeDebugNoBound, TypeInfo, )] -#[codec(mel_bound(FieldLimit: Get))] +#[codec(mel_bound())] #[cfg_attr(test, derive(frame_support::DefaultNoBound))] #[scale_info(skip_type_params(FieldLimit))] pub struct IdentityInfo> { @@ -339,11 +340,7 @@ pub struct IdentityInfo> { #[derive( CloneNoBound, Encode, Eq, MaxEncodedLen, PartialEqNoBound, RuntimeDebugNoBound, TypeInfo, )] -#[codec(mel_bound( - Balance: Encode + Decode + MaxEncodedLen + Copy + Clone + Debug + Eq + PartialEq + Zero + Add, - MaxJudgements: Get, - MaxAdditionalFields: Get, -))] +#[codec(mel_bound())] #[scale_info(skip_type_params(MaxJudgements, MaxAdditionalFields))] pub struct Registration< Balance: Encode + Decode + MaxEncodedLen + Copy + Clone + Debug + Eq + PartialEq, diff --git a/frame/identity/src/weights.rs b/frame/identity/src/weights.rs index 611909f326ea..cf164d3cbd73 100644 --- a/frame/identity/src/weights.rs +++ b/frame/identity/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_identity //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/identity/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -68,19 +69,19 @@ pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { // Storage: Identity Registrars (r:1 w:1) fn add_registrar(r: u32, ) -> Weight { - (22_152_000 as Weight) - // Standard Error: 6_000 - .saturating_add((339_000 as Weight).saturating_mul(r as Weight)) + (13_150_000 as Weight) + // Standard Error: 3_000 + .saturating_add((253_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Identity IdentityOf (r:1 w:1) fn set_identity(r: u32, x: u32, ) -> Weight { - (53_017_000 as Weight) - // Standard Error: 14_000 - .saturating_add((279_000 as Weight).saturating_mul(r as Weight)) + (27_745_000 as Weight) + // Standard Error: 7_000 + .saturating_add((210_000 as Weight).saturating_mul(r as Weight)) // Standard Error: 1_000 - .saturating_add((1_081_000 as Weight).saturating_mul(x as Weight)) + .saturating_add((289_000 as Weight).saturating_mul(x as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -88,9 +89,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Identity SubsOf (r:1 w:1) // Storage: Identity SuperOf (r:1 w:1) fn set_subs_new(s: u32, ) -> Weight { - (44_693_000 as Weight) + (25_184_000 as Weight) // Standard Error: 1_000 - .saturating_add((6_631_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_722_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) .saturating_add(T::DbWeight::get().writes(1 as Weight)) @@ -100,9 +101,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Identity SubsOf (r:1 w:1) // Storage: Identity SuperOf (r:0 w:1) fn set_subs_old(p: u32, ) -> Weight { - (42_017_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_193_000 as Weight).saturating_mul(p as Weight)) + (24_144_000 as Weight) + // Standard Error: 0 + .saturating_add((890_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(p as Weight))) @@ -111,13 +112,13 @@ impl WeightInfo for SubstrateWeight { // Storage: Identity IdentityOf (r:1 w:1) // Storage: Identity SuperOf (r:0 w:100) fn clear_identity(r: u32, s: u32, x: u32, ) -> Weight { - (50_989_000 as Weight) - // Standard Error: 11_000 - .saturating_add((258_000 as Weight).saturating_mul(r as Weight)) - // Standard Error: 1_000 - .saturating_add((2_184_000 as Weight).saturating_mul(s as Weight)) - // Standard Error: 1_000 - .saturating_add((579_000 as Weight).saturating_mul(x as Weight)) + (32_167_000 as Weight) + // Standard Error: 8_000 + .saturating_add((46_000 as Weight).saturating_mul(r as Weight)) + // Standard Error: 0 + .saturating_add((875_000 as Weight).saturating_mul(s as Weight)) + // Standard Error: 0 + .saturating_add((184_000 as Weight).saturating_mul(x as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) @@ -125,56 +126,56 @@ impl WeightInfo for SubstrateWeight { // Storage: Identity Registrars (r:1 w:0) // Storage: Identity IdentityOf (r:1 w:1) fn request_judgement(r: u32, x: u32, ) -> Weight { - (55_562_000 as Weight) - // Standard Error: 5_000 - .saturating_add((317_000 as Weight).saturating_mul(r as Weight)) + (30_009_000 as Weight) + // Standard Error: 3_000 + .saturating_add((287_000 as Weight).saturating_mul(r as Weight)) // Standard Error: 0 - .saturating_add((1_137_000 as Weight).saturating_mul(x as Weight)) + .saturating_add((331_000 as Weight).saturating_mul(x as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Identity IdentityOf (r:1 w:1) fn cancel_request(r: u32, x: u32, ) -> Weight { - (51_744_000 as Weight) - // Standard Error: 6_000 - .saturating_add((192_000 as Weight).saturating_mul(r as Weight)) + (28_195_000 as Weight) + // Standard Error: 3_000 + .saturating_add((137_000 as Weight).saturating_mul(r as Weight)) // Standard Error: 0 - .saturating_add((1_131_000 as Weight).saturating_mul(x as Weight)) + .saturating_add((328_000 as Weight).saturating_mul(x as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Identity Registrars (r:1 w:1) fn set_fee(r: u32, ) -> Weight { - (9_472_000 as Weight) - // Standard Error: 3_000 - .saturating_add((321_000 as Weight).saturating_mul(r as Weight)) + (5_080_000 as Weight) + // Standard Error: 2_000 + .saturating_add((202_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Identity Registrars (r:1 w:1) fn set_account_id(r: u32, ) -> Weight { - (9_705_000 as Weight) - // Standard Error: 3_000 - .saturating_add((312_000 as Weight).saturating_mul(r as Weight)) + (5_298_000 as Weight) + // Standard Error: 2_000 + .saturating_add((195_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Identity Registrars (r:1 w:1) fn set_fields(r: u32, ) -> Weight { - (9_537_000 as Weight) - // Standard Error: 3_000 - .saturating_add((318_000 as Weight).saturating_mul(r as Weight)) + (5_263_000 as Weight) + // Standard Error: 2_000 + .saturating_add((191_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Identity Registrars (r:1 w:0) // Storage: Identity IdentityOf (r:1 w:1) fn provide_judgement(r: u32, x: u32, ) -> Weight { - (36_298_000 as Weight) - // Standard Error: 5_000 - .saturating_add((284_000 as Weight).saturating_mul(r as Weight)) + (21_511_000 as Weight) + // Standard Error: 3_000 + .saturating_add((219_000 as Weight).saturating_mul(r as Weight)) // Standard Error: 0 - .saturating_add((1_141_000 as Weight).saturating_mul(x as Weight)) + .saturating_add((327_000 as Weight).saturating_mul(x as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -183,11 +184,11 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:1 w:1) // Storage: Identity SuperOf (r:0 w:100) fn kill_identity(r: u32, s: u32, _x: u32, ) -> Weight { - (63_238_000 as Weight) - // Standard Error: 10_000 - .saturating_add((246_000 as Weight).saturating_mul(r as Weight)) - // Standard Error: 1_000 - .saturating_add((2_184_000 as Weight).saturating_mul(s as Weight)) + (40_654_000 as Weight) + // Standard Error: 6_000 + .saturating_add((179_000 as Weight).saturating_mul(r as Weight)) + // Standard Error: 0 + .saturating_add((908_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) @@ -196,18 +197,18 @@ impl WeightInfo for SubstrateWeight { // Storage: Identity SuperOf (r:1 w:1) // Storage: Identity SubsOf (r:1 w:1) fn add_sub(s: u32, ) -> Weight { - (57_394_000 as Weight) + (31_573_000 as Weight) // Standard Error: 1_000 - .saturating_add((208_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((151_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Identity IdentityOf (r:1 w:0) // Storage: Identity SuperOf (r:1 w:1) fn rename_sub(s: u32, ) -> Weight { - (18_274_000 as Weight) + (10_452_000 as Weight) // Standard Error: 0 - .saturating_add((52_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((45_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -215,18 +216,18 @@ impl WeightInfo for SubstrateWeight { // Storage: Identity SuperOf (r:1 w:1) // Storage: Identity SubsOf (r:1 w:1) fn remove_sub(s: u32, ) -> Weight { - (58_184_000 as Weight) + (32_711_000 as Weight) // Standard Error: 1_000 - .saturating_add((195_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((150_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Identity SuperOf (r:1 w:1) // Storage: Identity SubsOf (r:1 w:1) fn quit_sub(s: u32, ) -> Weight { - (36_304_000 as Weight) - // Standard Error: 1_000 - .saturating_add((191_000 as Weight).saturating_mul(s as Weight)) + (21_975_000 as Weight) + // Standard Error: 0 + .saturating_add((149_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -236,19 +237,19 @@ impl WeightInfo for SubstrateWeight { impl WeightInfo for () { // Storage: Identity Registrars (r:1 w:1) fn add_registrar(r: u32, ) -> Weight { - (22_152_000 as Weight) - // Standard Error: 6_000 - .saturating_add((339_000 as Weight).saturating_mul(r as Weight)) + (13_150_000 as Weight) + // Standard Error: 3_000 + .saturating_add((253_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Identity IdentityOf (r:1 w:1) fn set_identity(r: u32, x: u32, ) -> Weight { - (53_017_000 as Weight) - // Standard Error: 14_000 - .saturating_add((279_000 as Weight).saturating_mul(r as Weight)) + (27_745_000 as Weight) + // Standard Error: 7_000 + .saturating_add((210_000 as Weight).saturating_mul(r as Weight)) // Standard Error: 1_000 - .saturating_add((1_081_000 as Weight).saturating_mul(x as Weight)) + .saturating_add((289_000 as Weight).saturating_mul(x as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -256,9 +257,9 @@ impl WeightInfo for () { // Storage: Identity SubsOf (r:1 w:1) // Storage: Identity SuperOf (r:1 w:1) fn set_subs_new(s: u32, ) -> Weight { - (44_693_000 as Weight) + (25_184_000 as Weight) // Standard Error: 1_000 - .saturating_add((6_631_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_722_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) @@ -268,9 +269,9 @@ impl WeightInfo for () { // Storage: Identity SubsOf (r:1 w:1) // Storage: Identity SuperOf (r:0 w:1) fn set_subs_old(p: u32, ) -> Weight { - (42_017_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_193_000 as Weight).saturating_mul(p as Weight)) + (24_144_000 as Weight) + // Standard Error: 0 + .saturating_add((890_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(p as Weight))) @@ -279,13 +280,13 @@ impl WeightInfo for () { // Storage: Identity IdentityOf (r:1 w:1) // Storage: Identity SuperOf (r:0 w:100) fn clear_identity(r: u32, s: u32, x: u32, ) -> Weight { - (50_989_000 as Weight) - // Standard Error: 11_000 - .saturating_add((258_000 as Weight).saturating_mul(r as Weight)) - // Standard Error: 1_000 - .saturating_add((2_184_000 as Weight).saturating_mul(s as Weight)) - // Standard Error: 1_000 - .saturating_add((579_000 as Weight).saturating_mul(x as Weight)) + (32_167_000 as Weight) + // Standard Error: 8_000 + .saturating_add((46_000 as Weight).saturating_mul(r as Weight)) + // Standard Error: 0 + .saturating_add((875_000 as Weight).saturating_mul(s as Weight)) + // Standard Error: 0 + .saturating_add((184_000 as Weight).saturating_mul(x as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) @@ -293,56 +294,56 @@ impl WeightInfo for () { // Storage: Identity Registrars (r:1 w:0) // Storage: Identity IdentityOf (r:1 w:1) fn request_judgement(r: u32, x: u32, ) -> Weight { - (55_562_000 as Weight) - // Standard Error: 5_000 - .saturating_add((317_000 as Weight).saturating_mul(r as Weight)) + (30_009_000 as Weight) + // Standard Error: 3_000 + .saturating_add((287_000 as Weight).saturating_mul(r as Weight)) // Standard Error: 0 - .saturating_add((1_137_000 as Weight).saturating_mul(x as Weight)) + .saturating_add((331_000 as Weight).saturating_mul(x as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Identity IdentityOf (r:1 w:1) fn cancel_request(r: u32, x: u32, ) -> Weight { - (51_744_000 as Weight) - // Standard Error: 6_000 - .saturating_add((192_000 as Weight).saturating_mul(r as Weight)) + (28_195_000 as Weight) + // Standard Error: 3_000 + .saturating_add((137_000 as Weight).saturating_mul(r as Weight)) // Standard Error: 0 - .saturating_add((1_131_000 as Weight).saturating_mul(x as Weight)) + .saturating_add((328_000 as Weight).saturating_mul(x as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Identity Registrars (r:1 w:1) fn set_fee(r: u32, ) -> Weight { - (9_472_000 as Weight) - // Standard Error: 3_000 - .saturating_add((321_000 as Weight).saturating_mul(r as Weight)) + (5_080_000 as Weight) + // Standard Error: 2_000 + .saturating_add((202_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Identity Registrars (r:1 w:1) fn set_account_id(r: u32, ) -> Weight { - (9_705_000 as Weight) - // Standard Error: 3_000 - .saturating_add((312_000 as Weight).saturating_mul(r as Weight)) + (5_298_000 as Weight) + // Standard Error: 2_000 + .saturating_add((195_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Identity Registrars (r:1 w:1) fn set_fields(r: u32, ) -> Weight { - (9_537_000 as Weight) - // Standard Error: 3_000 - .saturating_add((318_000 as Weight).saturating_mul(r as Weight)) + (5_263_000 as Weight) + // Standard Error: 2_000 + .saturating_add((191_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Identity Registrars (r:1 w:0) // Storage: Identity IdentityOf (r:1 w:1) fn provide_judgement(r: u32, x: u32, ) -> Weight { - (36_298_000 as Weight) - // Standard Error: 5_000 - .saturating_add((284_000 as Weight).saturating_mul(r as Weight)) + (21_511_000 as Weight) + // Standard Error: 3_000 + .saturating_add((219_000 as Weight).saturating_mul(r as Weight)) // Standard Error: 0 - .saturating_add((1_141_000 as Weight).saturating_mul(x as Weight)) + .saturating_add((327_000 as Weight).saturating_mul(x as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -351,11 +352,11 @@ impl WeightInfo for () { // Storage: System Account (r:1 w:1) // Storage: Identity SuperOf (r:0 w:100) fn kill_identity(r: u32, s: u32, _x: u32, ) -> Weight { - (63_238_000 as Weight) - // Standard Error: 10_000 - .saturating_add((246_000 as Weight).saturating_mul(r as Weight)) - // Standard Error: 1_000 - .saturating_add((2_184_000 as Weight).saturating_mul(s as Weight)) + (40_654_000 as Weight) + // Standard Error: 6_000 + .saturating_add((179_000 as Weight).saturating_mul(r as Weight)) + // Standard Error: 0 + .saturating_add((908_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) @@ -364,18 +365,18 @@ impl WeightInfo for () { // Storage: Identity SuperOf (r:1 w:1) // Storage: Identity SubsOf (r:1 w:1) fn add_sub(s: u32, ) -> Weight { - (57_394_000 as Weight) + (31_573_000 as Weight) // Standard Error: 1_000 - .saturating_add((208_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((151_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Identity IdentityOf (r:1 w:0) // Storage: Identity SuperOf (r:1 w:1) fn rename_sub(s: u32, ) -> Weight { - (18_274_000 as Weight) + (10_452_000 as Weight) // Standard Error: 0 - .saturating_add((52_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((45_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -383,18 +384,18 @@ impl WeightInfo for () { // Storage: Identity SuperOf (r:1 w:1) // Storage: Identity SubsOf (r:1 w:1) fn remove_sub(s: u32, ) -> Weight { - (58_184_000 as Weight) + (32_711_000 as Weight) // Standard Error: 1_000 - .saturating_add((195_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((150_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Identity SuperOf (r:1 w:1) // Storage: Identity SubsOf (r:1 w:1) fn quit_sub(s: u32, ) -> Weight { - (36_304_000 as Weight) - // Standard Error: 1_000 - .saturating_add((191_000 as Weight).saturating_mul(s as Weight)) + (21_975_000 as Weight) + // Standard Error: 0 + .saturating_add((149_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } diff --git a/frame/im-online/Cargo.toml b/frame/im-online/Cargo.toml index a1efd626c069..277d4d2fb2de 100644 --- a/frame/im-online/Cargo.toml +++ b/frame/im-online/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-im-online" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME's I'm online pallet" readme = "README.md" @@ -13,14 +13,14 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../../primitives/application-crypto" } pallet-authorship = { version = "4.0.0-dev", default-features = false, path = "../authorship" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } @@ -47,5 +47,5 @@ std = [ "frame-system/std", "log/std", ] -runtime-benchmarks = ["frame-benchmarking"] +runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/im-online/README.md b/frame/im-online/README.md index 46b2268f18b1..be11e0c49dff 100644 --- a/frame/im-online/README.md +++ b/frame/im-online/README.md @@ -26,21 +26,29 @@ It is submitted as an Unsigned Transaction via off-chain workers. ## Usage ```rust -use frame_support::{decl_module, dispatch}; -use frame_system::ensure_signed; use pallet_im_online::{self as im_online}; -pub trait Config: im_online::Config {} - -decl_module! { - pub struct Module for enum Call where origin: T::Origin { - #[weight = 0] - pub fn is_online(origin, authority_index: u32) -> dispatch::DispatchResult { - let _sender = ensure_signed(origin)?; - let _is_online = >::is_online(authority_index); - Ok(()) - } - } +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config + im_online::Config {} + + #[pallet::call] + impl Pallet { + #[pallet::weight(0)] + pub fn is_online(origin: OriginFor, authority_index: u32) -> DispatchResult { + let _sender = ensure_signed(origin)?; + let _is_online = >::is_online(authority_index); + Ok(()) + } + } } ``` diff --git a/frame/im-online/src/benchmarking.rs b/frame/im-online/src/benchmarking.rs index b39b0057c48e..edc21043c34d 100644 --- a/frame/im-online/src/benchmarking.rs +++ b/frame/im-online/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use super::*; -use frame_benchmarking::{benchmarks, impl_benchmark_test_suite}; +use frame_benchmarking::benchmarks; use frame_support::{traits::UnfilteredDispatchable, WeakBoundedVec}; use frame_system::RawOrigin; use sp_core::{offchain::OpaqueMultiaddr, OpaquePeerId}; @@ -100,6 +100,6 @@ benchmarks! { .expect("call is encoded above, encoding must be correct") .dispatch_bypass_filter(RawOrigin::None.into())?; } -} -impl_benchmark_test_suite!(ImOnline, crate::mock::new_test_ext(), crate::mock::Runtime); + impl_benchmark_test_suite!(ImOnline, crate::mock::new_test_ext(), crate::mock::Runtime); +} diff --git a/frame/im-online/src/lib.rs b/frame/im-online/src/lib.rs index 2fcaed1820ff..5ef515db0b23 100644 --- a/frame/im-online/src/lib.rs +++ b/frame/im-online/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,12 +17,12 @@ //! # I'm online Pallet //! -//! If the local node is a validator (i.e. contains an authority key), this module +//! If the local node is a validator (i.e. contains an authority key), this pallet //! gossips a heartbeat transaction with each new session. The heartbeat functions //! as a simple mechanism to signal that the node is online in the current era. //! //! Received heartbeats are tracked for one era and reset with each new era. The -//! module exposes two public functions to query if a heartbeat has been received +//! pallet exposes two public functions to query if a heartbeat has been received //! in the current era or session. //! //! The heartbeat is a signed transaction, which was signed using the session key @@ -43,16 +43,24 @@ //! ## Usage //! //! ``` -//! use frame_support::{decl_module, dispatch}; -//! use frame_system::ensure_signed; //! use pallet_im_online::{self as im_online}; //! -//! pub trait Config: im_online::Config {} +//! #[frame_support::pallet] +//! pub mod pallet { +//! use super::*; +//! use frame_support::pallet_prelude::*; +//! use frame_system::pallet_prelude::*; //! -//! decl_module! { -//! pub struct Module for enum Call where origin: T::Origin { -//! #[weight = 0] -//! pub fn is_online(origin, authority_index: u32) -> dispatch::DispatchResult { +//! #[pallet::pallet] +//! pub struct Pallet(_); +//! +//! #[pallet::config] +//! pub trait Config: frame_system::Config + im_online::Config {} +//! +//! #[pallet::call] +//! impl Pallet { +//! #[pallet::weight(0)] +//! pub fn is_online(origin: OriginFor, authority_index: u32) -> DispatchResult { //! let _sender = ensure_signed(origin)?; //! let _is_online = >::is_online(authority_index); //! Ok(()) @@ -64,7 +72,7 @@ //! //! ## Dependencies //! -//! This module depends on the [Session module](../pallet_session/index.html). +//! This pallet depends on the [Session pallet](../pallet_session/index.html). // Ensure we're `no_std` when compiling for Wasm. #![cfg_attr(not(feature = "std"), no_std)] @@ -75,7 +83,6 @@ mod tests; pub mod weights; use codec::{Decode, Encode, MaxEncodedLen}; -use core::convert::TryFrom; use frame_support::{ traits::{ EstimateNextSessionRotation, Get, OneSessionHandler, ValidatorSet, @@ -97,7 +104,7 @@ use sp_staking::{ offence::{Kind, Offence, ReportOffence}, SessionIndex, }; -use sp_std::{convert::TryInto, prelude::*}; +use sp_std::prelude::*; pub use weights::WeightInfo; pub mod sr25519 { @@ -231,8 +238,7 @@ where /// `MultiAddrEncodingLimit` represents the size limit of the encoding of `MultiAddr` /// `AddressesLimit` represents the size limit of the vector of peers connected #[derive(Clone, Eq, PartialEq, Encode, Decode, MaxEncodedLen, TypeInfo)] -#[codec(mel_bound(PeerIdEncodingLimit: Get, - MultiAddrEncodingLimit: Get, AddressesLimit: Get))] +#[codec(mel_bound())] #[scale_info(skip_type_params(PeerIdEncodingLimit, MultiAddrEncodingLimit, AddressesLimit))] pub struct BoundedOpaqueNetworkState where @@ -303,19 +309,11 @@ type OffchainResult = Result::B #[frame_support::pallet] pub mod pallet { use super::*; - use frame_support::{pallet_prelude::*, traits::Get, Parameter}; - use frame_system::{ensure_none, pallet_prelude::*}; - use sp_runtime::{ - traits::{MaybeSerializeDeserialize, Member}, - transaction_validity::{ - InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity, - ValidTransaction, - }, - }; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] - #[pallet::generate_storage_info] pub struct Pallet(_); #[pallet::config] @@ -324,7 +322,6 @@ pub mod pallet { type AuthorityId: Member + Parameter + RuntimeAppPublic - + Default + Ord + MaybeSerializeDeserialize + MaxEncodedLen; @@ -375,12 +372,12 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// A new heartbeat was received from `AuthorityId` \[authority_id\] - HeartbeatReceived(T::AuthorityId), + /// A new heartbeat was received from `AuthorityId`. + HeartbeatReceived { authority_id: T::AuthorityId }, /// At the end of the session, no offence was committed. AllGood, - /// At the end of the session, at least one validator was found to be \[offline\]. - SomeOffline(Vec>), + /// At the end of the session, at least one validator was found to be offline. + SomeOffline { offline: Vec> }, } #[pallet::error] @@ -412,7 +409,7 @@ pub mod pallet { pub(crate) type Keys = StorageValue<_, WeakBoundedVec, ValueQuery>; - /// For each session index, we keep a mapping of 'SessionIndex` and `AuthIndex` to + /// For each session index, we keep a mapping of `SessionIndex` and `AuthIndex` to /// `WrapperOpaque`. #[pallet::storage] #[pallet::getter(fn received_heartbeats)] @@ -496,7 +493,7 @@ pub mod pallet { let keys = Keys::::get(); let public = keys.get(heartbeat.authority_index as usize); if let (false, Some(public)) = (exists, public) { - Self::deposit_event(Event::::HeartbeatReceived(public.clone())); + Self::deposit_event(Event::::HeartbeatReceived { authority_id: public.clone() }); let network_state_bounded = BoundedOpaqueNetworkState::< T::MaxPeerDataEncodingSize, @@ -909,7 +906,7 @@ impl OneSessionHandler for Pallet { if offenders.is_empty() { Self::deposit_event(Event::::AllGood); } else { - Self::deposit_event(Event::::SomeOffline(offenders.clone())); + Self::deposit_event(Event::::SomeOffline { offline: offenders.clone() }); let validator_set_count = keys.len() as u32; let offence = UnresponsivenessOffence { session_index, validator_set_count, offenders }; @@ -919,7 +916,7 @@ impl OneSessionHandler for Pallet { } } - fn on_disabled(_i: usize) { + fn on_disabled(_i: u32) { // ignore } } diff --git a/frame/im-online/src/mock.rs b/frame/im-online/src/mock.rs index 92d1fe8e3f8b..86904b38d834 100644 --- a/frame/im-online/src/mock.rs +++ b/frame/im-online/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,13 +21,17 @@ use std::cell::RefCell; -use frame_support::{parameter_types, weights::Weight}; +use frame_support::{ + parameter_types, + traits::{ConstU32, ConstU64}, + weights::Weight, +}; use pallet_session::historical as pallet_session_historical; use sp_core::H256; use sp_runtime::{ testing::{Header, TestXt, UintAuthorityId}, traits::{BlakeTwo256, ConvertInto, IdentityLookup}, - Perbill, Permill, + Permill, }; use sp_staking::{ offence::{OffenceError, ReportOffence}, @@ -106,11 +110,18 @@ impl ReportOffence for OffenceHandler { pub fn new_test_ext() -> sp_io::TestExternalities { let t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - t.into() + let mut result: sp_io::TestExternalities = t.into(); + // Set the default keys, otherwise session will discard the validator. + result.execute_with(|| { + for i in 1..=6 { + System::inc_providers(&i); + assert_eq!(Session::set_keys(Origin::signed(i), (i - 1).into(), vec![]), Ok(())); + } + }); + result } parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -130,7 +141,7 @@ impl frame_system::Config for Runtime { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -139,6 +150,7 @@ impl frame_system::Config for Runtime { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } parameter_types! { @@ -146,10 +158,6 @@ parameter_types! { pub const Offset: u64 = 0; } -parameter_types! { - pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(33); -} - impl pallet_session::Config for Runtime { type ShouldEndSession = pallet_session::PeriodicSessions; type SessionManager = @@ -159,7 +167,6 @@ impl pallet_session::Config for Runtime { type ValidatorIdOf = ConvertInto; type Keys = UintAuthorityId; type Event = Event; - type DisabledValidatorsThreshold = DisabledValidatorsThreshold; type NextSessionRotation = pallet_session::PeriodicSessions; type WeightInfo = (); } @@ -169,13 +176,9 @@ impl pallet_session::historical::Config for Runtime { type FullIdentificationOf = ConvertInto; } -parameter_types! { - pub const UncleGenerations: u32 = 5; -} - impl pallet_authorship::Config for Runtime { type FindAuthor = (); - type UncleGenerations = UncleGenerations; + type UncleGenerations = ConstU64<5>; type FilterUncle = (); type EventHandler = ImOnline; } @@ -217,9 +220,6 @@ impl frame_support::traits::EstimateNextSessionRotation for TestNextSession parameter_types! { pub const UnsignedPriority: u64 = 1 << 20; - pub const MaxKeys: u32 = 10_000; - pub const MaxPeerInHeartbeats: u32 = 10_000; - pub const MaxPeerDataEncodingSize: u32 = 1_000; } impl Config for Runtime { @@ -230,9 +230,9 @@ impl Config for Runtime { type ReportUnresponsiveness = OffenceHandler; type UnsignedPriority = UnsignedPriority; type WeightInfo = (); - type MaxKeys = MaxKeys; - type MaxPeerInHeartbeats = MaxPeerInHeartbeats; - type MaxPeerDataEncodingSize = MaxPeerDataEncodingSize; + type MaxKeys = ConstU32<10_000>; + type MaxPeerInHeartbeats = ConstU32<10_000>; + type MaxPeerDataEncodingSize = ConstU32<1_000>; } impl frame_system::offchain::SendTransactionTypes for Runtime diff --git a/frame/im-online/src/tests.rs b/frame/im-online/src/tests.rs index bb2c4c7cae54..288081556a08 100644 --- a/frame/im-online/src/tests.rs +++ b/frame/im-online/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -341,7 +341,7 @@ fn should_not_send_a_report_if_already_online() { UintAuthorityId::set_all_keys(vec![1, 2, 3]); // we expect error, since the authority is already online. let mut res = ImOnline::send_heartbeats(4).unwrap(); - assert_eq!(res.next().unwrap().unwrap(), ()); + res.next().unwrap().unwrap(); assert_eq!(res.next().unwrap().unwrap_err(), OffchainErr::AlreadyOnline(1)); assert_eq!(res.next().unwrap().unwrap_err(), OffchainErr::AlreadyOnline(2)); assert_eq!(res.next(), None); diff --git a/frame/im-online/src/weights.rs b/frame/im-online/src/weights.rs index 1eadd63cc9d6..f7d4d1441da2 100644 --- a/frame/im-online/src/weights.rs +++ b/frame/im-online/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_im_online //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/im-online/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -57,11 +58,11 @@ impl WeightInfo for SubstrateWeight { // Storage: ImOnline AuthoredBlocks (r:1 w:0) // Storage: ImOnline Keys (r:1 w:0) fn validate_unsigned_and_then_heartbeat(k: u32, e: u32, ) -> Weight { - (93_400_000 as Weight) + (74_137_000 as Weight) // Standard Error: 0 - .saturating_add((144_000 as Weight).saturating_mul(k as Weight)) + .saturating_add((125_000 as Weight).saturating_mul(k as Weight)) // Standard Error: 0 - .saturating_add((335_000 as Weight).saturating_mul(e as Weight)) + .saturating_add((291_000 as Weight).saturating_mul(e as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -75,11 +76,11 @@ impl WeightInfo for () { // Storage: ImOnline AuthoredBlocks (r:1 w:0) // Storage: ImOnline Keys (r:1 w:0) fn validate_unsigned_and_then_heartbeat(k: u32, e: u32, ) -> Weight { - (93_400_000 as Weight) + (74_137_000 as Weight) // Standard Error: 0 - .saturating_add((144_000 as Weight).saturating_mul(k as Weight)) + .saturating_add((125_000 as Weight).saturating_mul(k as Weight)) // Standard Error: 0 - .saturating_add((335_000 as Weight).saturating_mul(e as Weight)) + .saturating_add((291_000 as Weight).saturating_mul(e as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } diff --git a/frame/indices/Cargo.toml b/frame/indices/Cargo.toml index 17d04c43fa5d..690fc9db14f9 100644 --- a/frame/indices/Cargo.toml +++ b/frame/indices/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-indices" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME indices management pallet" readme = "README.md" @@ -13,13 +13,13 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-keyring = { version = "4.0.0-dev", optional = true, path = "../../primitives/keyring" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-keyring = { version = "6.0.0", optional = true, path = "../../primitives/keyring" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } @@ -42,7 +42,7 @@ std = [ "frame-system/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", ] diff --git a/frame/indices/src/benchmarking.rs b/frame/indices/src/benchmarking.rs index ba0152008c41..cb06cd809f54 100644 --- a/frame/indices/src/benchmarking.rs +++ b/frame/indices/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ #![cfg(feature = "runtime-benchmarks")] use super::*; -use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite, whitelisted_caller}; +use frame_benchmarking::{account, benchmarks, whitelisted_caller}; use frame_system::RawOrigin; use sp_runtime::traits::Bounded; @@ -91,6 +91,6 @@ benchmarks! { } // TODO in another PR: lookup and unlookup trait weights (not critical) -} -impl_benchmark_test_suite!(Indices, crate::mock::new_test_ext(), crate::mock::Test); + impl_benchmark_test_suite!(Indices, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/frame/indices/src/lib.rs b/frame/indices/src/lib.rs index 0901a89d41ad..9c9e3580f2c0 100644 --- a/frame/indices/src/lib.rs +++ b/frame/indices/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -56,7 +56,8 @@ pub mod pallet { + Codec + Default + AtLeast32Bit - + Copy; + + Copy + + MaxEncodedLen; /// The currency trait. type Currency: ReservableCurrency; @@ -105,7 +106,7 @@ pub mod pallet { *maybe_value = Some((who.clone(), T::Deposit::get(), false)); T::Currency::reserve(&who, T::Deposit::get()) })?; - Self::deposit_event(Event::IndexAssigned(who, index)); + Self::deposit_event(Event::IndexAssigned { who, index }); Ok(()) } @@ -146,7 +147,7 @@ pub mod pallet { *maybe_value = Some((new.clone(), amount.saturating_sub(lost), false)); Ok(()) })?; - Self::deposit_event(Event::IndexAssigned(new, index)); + Self::deposit_event(Event::IndexAssigned { who: new, index }); Ok(()) } @@ -179,7 +180,7 @@ pub mod pallet { T::Currency::unreserve(&who, amount); Ok(()) })?; - Self::deposit_event(Event::IndexFreed(index)); + Self::deposit_event(Event::IndexFreed { index }); Ok(()) } @@ -219,7 +220,7 @@ pub mod pallet { } *maybe_value = Some((new.clone(), Zero::zero(), freeze)); }); - Self::deposit_event(Event::IndexAssigned(new, index)); + Self::deposit_event(Event::IndexAssigned { who: new, index }); Ok(()) } @@ -253,7 +254,7 @@ pub mod pallet { *maybe_value = Some((account, Zero::zero(), true)); Ok(()) })?; - Self::deposit_event(Event::IndexFrozen(index, who)); + Self::deposit_event(Event::IndexFrozen { index, who }); Ok(()) } } @@ -261,12 +262,12 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// A account index was assigned. \[index, who\] - IndexAssigned(T::AccountId, T::AccountIndex), - /// A account index has been freed up (unassigned). \[index\] - IndexFreed(T::AccountIndex), - /// A account index has been frozen to its current account ID. \[index, who\] - IndexFrozen(T::AccountIndex, T::AccountId), + /// A account index was assigned. + IndexAssigned { who: T::AccountId, index: T::AccountIndex }, + /// A account index has been freed up (unassigned). + IndexFreed { index: T::AccountIndex }, + /// A account index has been frozen to its current account ID. + IndexFrozen { index: T::AccountIndex, who: T::AccountId }, } /// Old name generated by `decl_event`. diff --git a/frame/indices/src/mock.rs b/frame/indices/src/mock.rs index f4c87016141b..6bd79708c3dd 100644 --- a/frame/indices/src/mock.rs +++ b/frame/indices/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,10 @@ #![cfg(test)] use crate::{self as pallet_indices, Config}; -use frame_support::parameter_types; +use frame_support::{ + parameter_types, + traits::{ConstU32, ConstU64}, +}; use sp_core::H256; use sp_runtime::testing::Header; @@ -40,7 +43,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -60,7 +62,7 @@ impl frame_system::Config for Test { type Lookup = Indices; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -69,10 +71,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); -} - -parameter_types! { - pub const ExistentialDeposit: u64 = 1; + type MaxConsumers = ConstU32<16>; } impl pallet_balances::Config for Test { @@ -82,19 +81,15 @@ impl pallet_balances::Config for Test { type Balance = u64; type DustRemoval = (); type Event = Event; - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } -parameter_types! { - pub const Deposit: u64 = 1; -} - impl Config for Test { type AccountIndex = u64; type Currency = Balances; - type Deposit = Deposit; + type Deposit = ConstU64<1>; type Event = Event; type WeightInfo = (); } diff --git a/frame/indices/src/tests.rs b/frame/indices/src/tests.rs index 37df20e9b928..73d591c38bb2 100644 --- a/frame/indices/src/tests.rs +++ b/frame/indices/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/indices/src/weights.rs b/frame/indices/src/weights.rs index 97db58973953..1687237394fb 100644 --- a/frame/indices/src/weights.rs +++ b/frame/indices/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_indices //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/indices/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -57,33 +58,33 @@ pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { // Storage: Indices Accounts (r:1 w:1) fn claim() -> Weight { - (38_814_000 as Weight) + (21_121_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Indices Accounts (r:1 w:1) // Storage: System Account (r:1 w:1) fn transfer() -> Weight { - (47_274_000 as Weight) + (26_843_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Indices Accounts (r:1 w:1) fn free() -> Weight { - (39_692_000 as Weight) + (22_174_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Indices Accounts (r:1 w:1) // Storage: System Account (r:1 w:1) fn force_transfer() -> Weight { - (40_250_000 as Weight) + (22_191_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Indices Accounts (r:1 w:1) fn freeze() -> Weight { - (37_358_000 as Weight) + (24_813_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -93,33 +94,33 @@ impl WeightInfo for SubstrateWeight { impl WeightInfo for () { // Storage: Indices Accounts (r:1 w:1) fn claim() -> Weight { - (38_814_000 as Weight) + (21_121_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Indices Accounts (r:1 w:1) // Storage: System Account (r:1 w:1) fn transfer() -> Weight { - (47_274_000 as Weight) + (26_843_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Indices Accounts (r:1 w:1) fn free() -> Weight { - (39_692_000 as Weight) + (22_174_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Indices Accounts (r:1 w:1) // Storage: System Account (r:1 w:1) fn force_transfer() -> Weight { - (40_250_000 as Weight) + (22_191_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Indices Accounts (r:1 w:1) fn freeze() -> Weight { - (37_358_000 as Weight) + (24_813_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } diff --git a/frame/lottery/Cargo.toml b/frame/lottery/Cargo.toml index f14d65310cc7..7932abdc21a0 100644 --- a/frame/lottery/Cargo.toml +++ b/frame/lottery/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-lottery" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME Participation Lottery Pallet" readme = "README.md" @@ -13,12 +13,12 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } @@ -27,8 +27,8 @@ frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = " [dev-dependencies] frame-support-test = { version = "3.0.0", path = "../support/test" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } [features] default = ["std"] @@ -41,7 +41,7 @@ std = [ "frame-system/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-system/runtime-benchmarks", "frame-support/runtime-benchmarks", ] diff --git a/frame/lottery/src/benchmarking.rs b/frame/lottery/src/benchmarking.rs index 7af20bbb0e11..1c850e66f9c6 100644 --- a/frame/lottery/src/benchmarking.rs +++ b/frame/lottery/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,8 +21,11 @@ use super::*; -use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite, whitelisted_caller}; -use frame_support::traits::{EnsureOrigin, OnInitialize}; +use frame_benchmarking::{account, benchmarks, whitelisted_caller}; +use frame_support::{ + storage::bounded_vec::BoundedVec, + traits::{EnsureOrigin, OnInitialize}, +}; use frame_system::RawOrigin; use sp_runtime::traits::{Bounded, Zero}; @@ -55,12 +58,12 @@ benchmarks! { let set_code_index: CallIndex = Lottery::::call_to_index( &frame_system::Call::::set_code{ code: vec![] }.into() )?; - let already_called: (u32, Vec) = ( + let already_called: (u32, BoundedVec) = ( LotteryIndex::::get(), - vec![ + BoundedVec::::try_from(vec![ set_code_index; T::MaxCalls::get().saturating_sub(1) as usize - ], + ]).unwrap(), ); Participants::::insert(&caller, already_called); @@ -163,6 +166,6 @@ benchmarks! { assert_eq!(Lottery::::pot().1, 0u32.into()); assert!(!T::Currency::free_balance(&winner).is_zero()) } -} -impl_benchmark_test_suite!(Lottery, crate::mock::new_test_ext(), crate::mock::Test); + impl_benchmark_test_suite!(Lottery, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/frame/lottery/src/lib.rs b/frame/lottery/src/lib.rs index 260b4c2d76ae..c9a508372ca8 100644 --- a/frame/lottery/src/lib.rs +++ b/frame/lottery/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,7 +28,7 @@ //! users to make those calls on your network. An example of how this could be //! used is to set validator nominations as a valid lottery call. If the lottery //! is set to repeat every month, then users would be encouraged to re-nominate -//! validators every month. A user can ony purchase one ticket per valid call +//! validators every month. A user can only purchase one ticket per valid call //! per lottery. //! //! This pallet can be configured to use dynamically set calls or statically set @@ -58,6 +58,8 @@ use codec::{Decode, Encode}; use frame_support::{ dispatch::{DispatchResult, Dispatchable, GetDispatchInfo}, ensure, + pallet_prelude::MaxEncodedLen, + storage::bounded_vec::BoundedVec, traits::{Currency, ExistenceRequirement::KeepAlive, Get, Randomness, ReservableCurrency}, PalletId, RuntimeDebug, }; @@ -76,7 +78,9 @@ type BalanceOf = // We use this to uniquely match someone's incoming call with the calls configured for the lottery. type CallIndex = (u8, u8); -#[derive(Encode, Decode, Default, Eq, PartialEq, RuntimeDebug, scale_info::TypeInfo)] +#[derive( + Encode, Decode, Default, Eq, PartialEq, RuntimeDebug, scale_info::TypeInfo, MaxEncodedLen, +)] pub struct LotteryConfig { /// Price per entry. price: Balance, @@ -115,8 +119,8 @@ impl ValidateCall for Pallet { #[frame_support::pallet] pub mod pallet { use super::*; - use frame_support::{pallet_prelude::*, traits::EnsureOrigin, weights::Weight, Parameter}; - use frame_system::{ensure_signed, pallet_prelude::*}; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] @@ -176,9 +180,9 @@ pub mod pallet { /// A new set of calls have been set! CallsUpdated, /// A winner has been chosen! - Winner(T::AccountId, BalanceOf), + Winner { winner: T::AccountId, lottery_balance: BalanceOf }, /// A ticket has been bought! - TicketBought(T::AccountId, CallIndex), + TicketBought { who: T::AccountId, call_index: CallIndex }, } #[pallet::error] @@ -209,8 +213,13 @@ pub mod pallet { /// Users who have purchased a ticket. (Lottery Index, Tickets Purchased) #[pallet::storage] - pub(crate) type Participants = - StorageMap<_, Twox64Concat, T::AccountId, (u32, Vec), ValueQuery>; + pub(crate) type Participants = StorageMap< + _, + Twox64Concat, + T::AccountId, + (u32, BoundedVec), + ValueQuery, + >; /// Total number of tickets sold. #[pallet::storage] @@ -226,7 +235,8 @@ pub mod pallet { /// The calls stored in this pallet to be used in an active lottery if configured /// by `Config::ValidateCall`. #[pallet::storage] - pub(crate) type CallIndices = StorageValue<_, Vec, ValueQuery>; + pub(crate) type CallIndices = + StorageValue<_, BoundedVec, ValueQuery>; #[pallet::hooks] impl Hooks> for Pallet { @@ -237,10 +247,8 @@ pub mod pallet { config.start.saturating_add(config.length).saturating_add(config.delay); if payout_block <= n { let (lottery_account, lottery_balance) = Self::pot(); - let ticket_count = TicketsCount::::get(); - let winning_number = Self::choose_winner(ticket_count); - let winner = Tickets::::get(winning_number).unwrap_or(lottery_account); + let winner = Self::choose_account().unwrap_or(lottery_account); // Not much we can do if this fails... let res = T::Currency::transfer( &Self::account_id(), @@ -250,7 +258,7 @@ pub mod pallet { ); debug_assert!(res.is_ok()); - Self::deposit_event(Event::::Winner(winner, lottery_balance)); + Self::deposit_event(Event::::Winner { winner, lottery_balance }); TicketsCount::::kill(); @@ -385,7 +393,7 @@ impl Pallet { } /// Return the pot account and amount of money in the pot. - // The existential deposit is not part of the pot so lottery account never gets deleted. + /// The existential deposit is not part of the pot so lottery account never gets deleted. fn pot() -> (T::AccountId, BalanceOf) { let account_id = Self::account_id(); let balance = @@ -394,17 +402,19 @@ impl Pallet { (account_id, balance) } - // Converts a vector of calls into a vector of call indices. - fn calls_to_indices(calls: &[::Call]) -> Result, DispatchError> { - let mut indices = Vec::with_capacity(calls.len()); + /// Converts a vector of calls into a vector of call indices. + fn calls_to_indices( + calls: &[::Call], + ) -> Result, DispatchError> { + let mut indices = BoundedVec::::with_bounded_capacity(calls.len()); for c in calls.iter() { let index = Self::call_to_index(c)?; - indices.push(index) + indices.try_push(index).map_err(|_| Error::::TooManyCalls)?; } Ok(indices) } - // Convert a call to it's call index by encoding the call and taking the first two bytes. + /// Convert a call to it's call index by encoding the call and taking the first two bytes. fn call_to_index(call: &::Call) -> Result { let encoded_call = call.encode(); if encoded_call.len() < 2 { @@ -413,7 +423,7 @@ impl Pallet { return Ok((encoded_call[0], encoded_call[1])) } - // Logic for buying a ticket. + /// Logic for buying a ticket. fn do_buy_ticket(caller: &T::AccountId, call: &::Call) -> DispatchResult { // Check the call is valid lottery let config = Lottery::::get().ok_or(Error::::NotConfigured)?; @@ -433,7 +443,7 @@ impl Pallet { let index = LotteryIndex::::get(); // If lottery index doesn't match, then reset participating calls and index. if *lottery_index != index { - *participating_calls = Vec::new(); + *participating_calls = Default::default(); *lottery_index = index; } else { // Check that user is not already participating under this call. @@ -442,23 +452,37 @@ impl Pallet { Error::::AlreadyParticipating ); } + participating_calls.try_push(call_index).map_err(|_| Error::::TooManyCalls)?; // Check user has enough funds and send it to the Lottery account. T::Currency::transfer(caller, &Self::account_id(), config.price, KeepAlive)?; // Create a new ticket. TicketsCount::::put(new_ticket_count); Tickets::::insert(ticket_count, caller.clone()); - participating_calls.push(call_index); Ok(()) }, )?; - Self::deposit_event(Event::::TicketBought(caller.clone(), call_index)); + Self::deposit_event(Event::::TicketBought { who: caller.clone(), call_index }); Ok(()) } - // Randomly choose a winner from among the total number of participants. - fn choose_winner(total: u32) -> u32 { + /// Randomly choose a winning ticket and return the account that purchased it. + /// The more tickets an account bought, the higher are its chances of winning. + /// Returns `None` if there is no winner. + fn choose_account() -> Option { + match Self::choose_ticket(TicketsCount::::get()) { + None => None, + Some(ticket) => Tickets::::get(ticket), + } + } + + /// Randomly choose a winning ticket from among the total number of tickets. + /// Returns `None` if there are no tickets. + fn choose_ticket(total: u32) -> Option { + if total == 0 { + return None + } let mut random_number = Self::generate_random_number(0); // Best effort attempt to remove bias from modulus operator. @@ -470,15 +494,15 @@ impl Pallet { random_number = Self::generate_random_number(i); } - random_number % total + Some(random_number % total) } - // Generate a random number from a given seed. - // Note that there is potential bias introduced by using modulus operator. - // You should call this function with different seed values until the random - // number lies within `u32::MAX - u32::MAX % n`. - // TODO: deal with randomness freshness - // https://github.com/paritytech/substrate/issues/8311 + /// Generate a random number from a given seed. + /// Note that there is potential bias introduced by using modulus operator. + /// You should call this function with different seed values until the random + /// number lies within `u32::MAX - u32::MAX % n`. + /// TODO: deal with randomness freshness + /// https://github.com/paritytech/substrate/issues/8311 fn generate_random_number(seed: u32) -> u32 { let (random_seed, _) = T::Randomness::random(&(T::PalletId::get(), seed).encode()); let random_number = ::decode(&mut random_seed.as_ref()) diff --git a/frame/lottery/src/mock.rs b/frame/lottery/src/mock.rs index d1f090aa26dc..592551fb6b93 100644 --- a/frame/lottery/src/mock.rs +++ b/frame/lottery/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,7 @@ use crate as pallet_lottery; use frame_support::{ parameter_types, - traits::{OnFinalize, OnInitialize}, + traits::{ConstU32, ConstU64, OnFinalize, OnInitialize}, }; use frame_support_test::TestRandomness; use frame_system::EnsureRoot; @@ -49,9 +49,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; - pub const MaximumBlockWeight: u32 = 1024; - pub const MaximumBlockLength: u32 = 2 * 1024; pub const AvailableBlockRatio: Perbill = Perbill::one(); } @@ -70,7 +67,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -79,10 +76,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); -} - -parameter_types! { - pub const ExistentialDeposit: u64 = 1; + type MaxConsumers = ConstU32<16>; } impl pallet_balances::Config for Test { @@ -92,15 +86,13 @@ impl pallet_balances::Config for Test { type Balance = u64; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } parameter_types! { pub const LotteryPalletId: PalletId = PalletId(*b"py/lotto"); - pub const MaxCalls: u32 = 2; - pub const MaxGenerateRandom: u32 = 10; } impl Config for Test { @@ -110,9 +102,9 @@ impl Config for Test { type Randomness = TestRandomness; type Event = Event; type ManagerOrigin = EnsureRoot; - type MaxCalls = MaxCalls; + type MaxCalls = ConstU32<2>; type ValidateCall = Lottery; - type MaxGenerateRandom = MaxGenerateRandom; + type MaxGenerateRandom = ConstU32<10>; type WeightInfo = (); } diff --git a/frame/lottery/src/tests.rs b/frame/lottery/src/tests.rs index 623beea4a6b5..d8dd6e4b7fe6 100644 --- a/frame/lottery/src/tests.rs +++ b/frame/lottery/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,7 @@ //! Tests for the module. use super::*; -use frame_support::{assert_noop, assert_ok}; +use frame_support::{assert_noop, assert_ok, assert_storage_noop}; use mock::{ new_test_ext, run_to_block, Balances, BalancesCall, Call, Lottery, Origin, SystemCall, Test, }; @@ -30,7 +30,7 @@ fn initial_state() { new_test_ext().execute_with(|| { assert_eq!(Balances::free_balance(Lottery::account_id()), 0); assert!(crate::Lottery::::get().is_none()); - assert_eq!(Participants::::get(&1), (0, vec![])); + assert_eq!(Participants::::get(&1), (0, Default::default())); assert_eq!(TicketsCount::::get(), 0); assert!(Tickets::::get(0).is_none()); }); @@ -90,6 +90,37 @@ fn basic_end_to_end_works() { }); } +/// Only the manager can stop the Lottery from repeating via `stop_repeat`. +#[test] +fn stop_repeat_works() { + new_test_ext().execute_with(|| { + let price = 10; + let length = 20; + let delay = 5; + + // Set no calls for the lottery. + assert_ok!(Lottery::set_calls(Origin::root(), vec![])); + // Start lottery, it repeats. + assert_ok!(Lottery::start_lottery(Origin::root(), price, length, delay, true)); + + // Non-manager fails to `stop_repeat`. + assert_noop!(Lottery::stop_repeat(Origin::signed(1)), DispatchError::BadOrigin); + // Manager can `stop_repeat`, even twice. + assert_ok!(Lottery::stop_repeat(Origin::root())); + assert_ok!(Lottery::stop_repeat(Origin::root())); + + // Lottery still exists. + assert!(crate::Lottery::::get().is_some()); + // End and pick a winner. + run_to_block(length + delay); + + // Lottery stays dead and does not repeat. + assert!(crate::Lottery::::get().is_none()); + run_to_block(length + delay + 1); + assert!(crate::Lottery::::get().is_none()); + }); +} + #[test] fn set_calls_works() { new_test_ext().execute_with(|| { @@ -120,6 +151,27 @@ fn set_calls_works() { }); } +#[test] +fn call_to_indices_works() { + new_test_ext().execute_with(|| { + let calls = vec![ + Call::Balances(BalancesCall::force_transfer { source: 0, dest: 0, value: 0 }), + Call::Balances(BalancesCall::transfer { dest: 0, value: 0 }), + ]; + let indices = Lottery::calls_to_indices(&calls).unwrap().into_inner(); + // Only comparing the length since it is otherwise dependant on the API + // of `BalancesCall`. + assert_eq!(indices.len(), calls.len()); + + let too_many_calls = vec![ + Call::Balances(BalancesCall::force_transfer { source: 0, dest: 0, value: 0 }), + Call::Balances(BalancesCall::transfer { dest: 0, value: 0 }), + Call::System(SystemCall::remark { remark: vec![] }), + ]; + assert_noop!(Lottery::calls_to_indices(&too_many_calls), Error::::TooManyCalls); + }); +} + #[test] fn start_lottery_works() { new_test_ext().execute_with(|| { @@ -239,6 +291,106 @@ fn buy_ticket_works() { }); } +/// Test that `do_buy_ticket` returns an `AlreadyParticipating` error. +/// Errors of `do_buy_ticket` are ignored by `buy_ticket`, therefore this white-box test. +#[test] +fn do_buy_ticket_already_participating() { + new_test_ext().execute_with(|| { + let calls = vec![Call::Balances(BalancesCall::transfer { dest: 0, value: 0 })]; + assert_ok!(Lottery::set_calls(Origin::root(), calls.clone())); + assert_ok!(Lottery::start_lottery(Origin::root(), 1, 10, 10, false)); + + // Buying once works. + assert_ok!(Lottery::do_buy_ticket(&1, &calls[0])); + // Buying the same ticket again fails. + assert_noop!(Lottery::do_buy_ticket(&1, &calls[0]), Error::::AlreadyParticipating); + }); +} + +/// `buy_ticket` is a storage noop when called with the same ticket again. +#[test] +fn buy_ticket_already_participating() { + new_test_ext().execute_with(|| { + let calls = vec![Call::Balances(BalancesCall::transfer { dest: 0, value: 0 })]; + assert_ok!(Lottery::set_calls(Origin::root(), calls.clone())); + assert_ok!(Lottery::start_lottery(Origin::root(), 1, 10, 10, false)); + + // Buying once works. + let call = Box::new(calls[0].clone()); + assert_ok!(Lottery::buy_ticket(Origin::signed(1), call.clone())); + + // Buying the same ticket again returns Ok, but changes nothing. + assert_storage_noop!(Lottery::buy_ticket(Origin::signed(1), call).unwrap()); + + // Exactly one ticket exists. + assert_eq!(TicketsCount::::get(), 1); + }); +} + +/// `buy_ticket` is a storage noop when called with insufficient balance. +#[test] +fn buy_ticket_insufficient_balance() { + new_test_ext().execute_with(|| { + let calls = vec![Call::Balances(BalancesCall::transfer { dest: 0, value: 0 })]; + assert_ok!(Lottery::set_calls(Origin::root(), calls.clone())); + // Price set to 100. + assert_ok!(Lottery::start_lottery(Origin::root(), 100, 10, 10, false)); + let call = Box::new(calls[0].clone()); + + // Buying a ticket returns Ok, but changes nothing. + assert_storage_noop!(Lottery::buy_ticket(Origin::signed(1), call).unwrap()); + assert!(TicketsCount::::get().is_zero()); + }); +} + +#[test] +fn do_buy_ticket_insufficient_balance() { + new_test_ext().execute_with(|| { + let calls = vec![Call::Balances(BalancesCall::transfer { dest: 0, value: 0 })]; + assert_ok!(Lottery::set_calls(Origin::root(), calls.clone())); + // Price set to 101. + assert_ok!(Lottery::start_lottery(Origin::root(), 101, 10, 10, false)); + + // Buying fails with InsufficientBalance. + assert_noop!( + Lottery::do_buy_ticket(&1, &calls[0]), + BalancesError::::InsufficientBalance + ); + assert!(TicketsCount::::get().is_zero()); + }); +} + +#[test] +fn do_buy_ticket_keep_alive() { + new_test_ext().execute_with(|| { + let calls = vec![Call::Balances(BalancesCall::transfer { dest: 0, value: 0 })]; + assert_ok!(Lottery::set_calls(Origin::root(), calls.clone())); + // Price set to 100. + assert_ok!(Lottery::start_lottery(Origin::root(), 100, 10, 10, false)); + + // Buying fails with KeepAlive. + assert_noop!(Lottery::do_buy_ticket(&1, &calls[0]), BalancesError::::KeepAlive); + assert!(TicketsCount::::get().is_zero()); + }); +} + +/// The lottery handles the case that no one participated. +#[test] +fn no_participants_works() { + new_test_ext().execute_with(|| { + let length = 20; + let delay = 5; + + // Set no calls for the lottery. + assert_ok!(Lottery::set_calls(Origin::root(), vec![])); + // Start lottery. + assert_ok!(Lottery::start_lottery(Origin::root(), 10, length, delay, false)); + + // End the lottery, no one wins. + run_to_block(length + delay); + }); +} + #[test] fn start_lottery_will_create_account() { new_test_ext().execute_with(|| { @@ -251,3 +403,26 @@ fn start_lottery_will_create_account() { assert_eq!(Balances::total_balance(&Lottery::account_id()), 1); }); } + +#[test] +fn choose_ticket_trivial_cases() { + new_test_ext().execute_with(|| { + assert!(Lottery::choose_ticket(0).is_none()); + assert_eq!(Lottery::choose_ticket(1).unwrap(), 0); + }); +} + +#[test] +fn choose_account_one_participant() { + new_test_ext().execute_with(|| { + let calls = vec![Call::Balances(BalancesCall::transfer { dest: 0, value: 0 })]; + assert_ok!(Lottery::set_calls(Origin::root(), calls.clone())); + assert_ok!(Lottery::start_lottery(Origin::root(), 10, 10, 10, false)); + let call = Box::new(calls[0].clone()); + + // Buy one ticket with account 1. + assert_ok!(Lottery::buy_ticket(Origin::signed(1), call)); + // Account 1 is always the winner. + assert_eq!(Lottery::choose_account().unwrap(), 1); + }); +} diff --git a/frame/lottery/src/weights.rs b/frame/lottery/src/weights.rs index 5fbc61a32e57..3b6114a73cfa 100644 --- a/frame/lottery/src/weights.rs +++ b/frame/lottery/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_lottery //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/lottery/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -64,28 +65,28 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:1 w:1) // Storage: Lottery Tickets (r:0 w:1) fn buy_ticket() -> Weight { - (70_034_000 as Weight) + (37_257_000 as Weight) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } // Storage: Lottery CallIndices (r:0 w:1) fn set_calls(n: u32, ) -> Weight { - (15_243_000 as Weight) - // Standard Error: 8_000 - .saturating_add((312_000 as Weight).saturating_mul(n as Weight)) + (8_966_000 as Weight) + // Standard Error: 5_000 + .saturating_add((302_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Lottery Lottery (r:1 w:1) // Storage: Lottery LotteryIndex (r:1 w:1) // Storage: System Account (r:1 w:1) fn start_lottery() -> Weight { - (57_312_000 as Weight) + (32_282_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Lottery Lottery (r:1 w:1) fn stop_repeat() -> Weight { - (6_964_000 as Weight) + (3_903_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -95,7 +96,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Lottery TicketsCount (r:1 w:1) // Storage: Lottery Tickets (r:1 w:0) fn on_initialize_end() -> Weight { - (110_470_000 as Weight) + (51_489_000 as Weight) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } @@ -106,7 +107,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Lottery Tickets (r:1 w:0) // Storage: Lottery LotteryIndex (r:1 w:1) fn on_initialize_repeat() -> Weight { - (114_794_000 as Weight) + (53_046_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } @@ -122,28 +123,28 @@ impl WeightInfo for () { // Storage: System Account (r:1 w:1) // Storage: Lottery Tickets (r:0 w:1) fn buy_ticket() -> Weight { - (70_034_000 as Weight) + (37_257_000 as Weight) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } // Storage: Lottery CallIndices (r:0 w:1) fn set_calls(n: u32, ) -> Weight { - (15_243_000 as Weight) - // Standard Error: 8_000 - .saturating_add((312_000 as Weight).saturating_mul(n as Weight)) + (8_966_000 as Weight) + // Standard Error: 5_000 + .saturating_add((302_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Lottery Lottery (r:1 w:1) // Storage: Lottery LotteryIndex (r:1 w:1) // Storage: System Account (r:1 w:1) fn start_lottery() -> Weight { - (57_312_000 as Weight) + (32_282_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Lottery Lottery (r:1 w:1) fn stop_repeat() -> Weight { - (6_964_000 as Weight) + (3_903_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -153,7 +154,7 @@ impl WeightInfo for () { // Storage: Lottery TicketsCount (r:1 w:1) // Storage: Lottery Tickets (r:1 w:0) fn on_initialize_end() -> Weight { - (110_470_000 as Weight) + (51_489_000 as Weight) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } @@ -164,7 +165,7 @@ impl WeightInfo for () { // Storage: Lottery Tickets (r:1 w:0) // Storage: Lottery LotteryIndex (r:1 w:1) fn on_initialize_repeat() -> Weight { - (114_794_000 as Weight) + (53_046_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } diff --git a/frame/membership/Cargo.toml b/frame/membership/Cargo.toml index acc82f7678de..40ad619e904c 100644 --- a/frame/membership/Cargo.toml +++ b/frame/membership/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-membership" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME membership management pallet" readme = "README.md" @@ -13,14 +13,14 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } log = { version = "0.4.0", default-features = false } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } @@ -41,7 +41,7 @@ std = [ "frame-benchmarking/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", diff --git a/frame/membership/src/lib.rs b/frame/membership/src/lib.rs index 57a12c7c8a45..e8256fab83af 100644 --- a/frame/membership/src/lib.rs +++ b/frame/membership/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,6 +46,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] #[pallet::storage_version(STORAGE_VERSION)] + #[pallet::without_storage_info] pub struct Pallet(PhantomData<(T, I)>); #[pallet::config] @@ -143,7 +144,7 @@ pub mod pallet { /// One of the members' keys changed. KeyChanged, /// Phantom member, never used. - Dummy(PhantomData<(T::AccountId, >::Event)>), + Dummy { _phantom_data: PhantomData<(T::AccountId, >::Event)> }, } /// Old name generated by `decl_event`. @@ -357,9 +358,7 @@ impl, I: 'static> SortedMembers for Pallet { #[cfg(feature = "runtime-benchmarks")] mod benchmark { use super::{Pallet as Membership, *}; - use frame_benchmarking::{ - account, benchmarks_instance_pallet, impl_benchmark_test_suite, whitelist, - }; + use frame_benchmarking::{account, benchmarks_instance_pallet, whitelist}; use frame_support::{assert_ok, traits::EnsureOrigin}; use frame_system::RawOrigin; @@ -494,9 +493,9 @@ mod benchmark { assert!(::get_prime().is_none()); #[cfg(test)] crate::tests::clean(); } - } - impl_benchmark_test_suite!(Membership, crate::tests::new_bench_ext(), crate::tests::Test); + impl_benchmark_test_suite!(Membership, crate::tests::new_bench_ext(), crate::tests::Test); + } } #[cfg(test)] @@ -511,7 +510,8 @@ mod tests { }; use frame_support::{ - assert_noop, assert_ok, ord_parameter_types, parameter_types, traits::GenesisBuild, + assert_noop, assert_ok, ord_parameter_types, parameter_types, + traits::{ConstU32, ConstU64, GenesisBuild}, }; use frame_system::EnsureSignedBy; @@ -530,8 +530,6 @@ mod tests { ); parameter_types! { - pub const BlockHashCount: u64 = 250; - pub const MaxMembers: u32 = 10; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); pub static Members: Vec = vec![]; @@ -553,7 +551,7 @@ mod tests { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -562,6 +560,7 @@ mod tests { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } ord_parameter_types! { pub const One: u64 = 1; @@ -608,7 +607,7 @@ mod tests { type PrimeOrigin = EnsureSignedBy; type MembershipInitialized = TestChangeMembers; type MembershipChanged = TestChangeMembers; - type MaxMembers = MaxMembers; + type MaxMembers = ConstU32<10>; type WeightInfo = (); } diff --git a/frame/membership/src/migrations/mod.rs b/frame/membership/src/migrations/mod.rs index 26d07a0cd5ac..235d0f1c7cf1 100644 --- a/frame/membership/src/migrations/mod.rs +++ b/frame/membership/src/migrations/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/membership/src/migrations/v4.rs b/frame/membership/src/migrations/v4.rs index c1c944be1fd4..b3b52751d959 100644 --- a/frame/membership/src/migrations/v4.rs +++ b/frame/membership/src/migrations/v4.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/membership/src/weights.rs b/frame/membership/src/weights.rs index 81a1b073faac..0d4936cfba1f 100644 --- a/frame/membership/src/weights.rs +++ b/frame/membership/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_membership //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/membership/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -57,162 +58,162 @@ pub trait WeightInfo { /// Weights for pallet_membership using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { - // Storage: Instance1Membership Members (r:1 w:1) - // Storage: Instance2Collective Proposals (r:1 w:0) - // Storage: Instance2Collective Members (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Members (r:1 w:1) + // Storage: TechnicalCommittee Proposals (r:1 w:0) + // Storage: TechnicalCommittee Members (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn add_member(m: u32, ) -> Weight { - (23_668_000 as Weight) - // Standard Error: 3_000 - .saturating_add((142_000 as Weight).saturating_mul(m as Weight)) + (14_884_000 as Weight) + // Standard Error: 0 + .saturating_add((95_000 as Weight).saturating_mul(m as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Membership Members (r:1 w:1) - // Storage: Instance2Collective Proposals (r:1 w:0) - // Storage: Instance1Membership Prime (r:1 w:0) - // Storage: Instance2Collective Members (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Members (r:1 w:1) + // Storage: TechnicalCommittee Proposals (r:1 w:0) + // Storage: TechnicalMembership Prime (r:1 w:0) + // Storage: TechnicalCommittee Members (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn remove_member(m: u32, ) -> Weight { - (29_149_000 as Weight) + (17_612_000 as Weight) // Standard Error: 0 - .saturating_add((111_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((86_000 as Weight).saturating_mul(m as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Membership Members (r:1 w:1) - // Storage: Instance2Collective Proposals (r:1 w:0) - // Storage: Instance1Membership Prime (r:1 w:0) - // Storage: Instance2Collective Members (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Members (r:1 w:1) + // Storage: TechnicalCommittee Proposals (r:1 w:0) + // Storage: TechnicalMembership Prime (r:1 w:0) + // Storage: TechnicalCommittee Members (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn swap_member(m: u32, ) -> Weight { - (29_289_000 as Weight) + (17_580_000 as Weight) // Standard Error: 0 - .saturating_add((126_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((98_000 as Weight).saturating_mul(m as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Membership Members (r:1 w:1) - // Storage: Instance2Collective Proposals (r:1 w:0) - // Storage: Instance1Membership Prime (r:1 w:0) - // Storage: Instance2Collective Members (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Members (r:1 w:1) + // Storage: TechnicalCommittee Proposals (r:1 w:0) + // Storage: TechnicalMembership Prime (r:1 w:0) + // Storage: TechnicalCommittee Members (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn reset_member(m: u32, ) -> Weight { - (30_178_000 as Weight) - // Standard Error: 1_000 - .saturating_add((286_000 as Weight).saturating_mul(m as Weight)) + (17_610_000 as Weight) + // Standard Error: 0 + .saturating_add((199_000 as Weight).saturating_mul(m as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Membership Members (r:1 w:1) - // Storage: Instance2Collective Proposals (r:1 w:0) - // Storage: Instance1Membership Prime (r:1 w:1) - // Storage: Instance2Collective Members (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Members (r:1 w:1) + // Storage: TechnicalCommittee Proposals (r:1 w:0) + // Storage: TechnicalMembership Prime (r:1 w:1) + // Storage: TechnicalCommittee Members (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn change_key(m: u32, ) -> Weight { - (31_049_000 as Weight) + (18_208_000 as Weight) // Standard Error: 0 - .saturating_add((121_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((96_000 as Weight).saturating_mul(m as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } - // Storage: Instance1Membership Members (r:1 w:0) - // Storage: Instance1Membership Prime (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Members (r:1 w:0) + // Storage: TechnicalMembership Prime (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn set_prime(m: u32, ) -> Weight { - (8_006_000 as Weight) + (4_661_000 as Weight) // Standard Error: 0 - .saturating_add((89_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((70_000 as Weight).saturating_mul(m as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } - // Storage: Instance1Membership Prime (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Prime (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn clear_prime(m: u32, ) -> Weight { - (3_452_000 as Weight) + (1_574_000 as Weight) // Standard Error: 0 - .saturating_add((2_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((1_000 as Weight).saturating_mul(m as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } } // For backwards compatibility and tests impl WeightInfo for () { - // Storage: Instance1Membership Members (r:1 w:1) - // Storage: Instance2Collective Proposals (r:1 w:0) - // Storage: Instance2Collective Members (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Members (r:1 w:1) + // Storage: TechnicalCommittee Proposals (r:1 w:0) + // Storage: TechnicalCommittee Members (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn add_member(m: u32, ) -> Weight { - (23_668_000 as Weight) - // Standard Error: 3_000 - .saturating_add((142_000 as Weight).saturating_mul(m as Weight)) + (14_884_000 as Weight) + // Standard Error: 0 + .saturating_add((95_000 as Weight).saturating_mul(m as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Membership Members (r:1 w:1) - // Storage: Instance2Collective Proposals (r:1 w:0) - // Storage: Instance1Membership Prime (r:1 w:0) - // Storage: Instance2Collective Members (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Members (r:1 w:1) + // Storage: TechnicalCommittee Proposals (r:1 w:0) + // Storage: TechnicalMembership Prime (r:1 w:0) + // Storage: TechnicalCommittee Members (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn remove_member(m: u32, ) -> Weight { - (29_149_000 as Weight) + (17_612_000 as Weight) // Standard Error: 0 - .saturating_add((111_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((86_000 as Weight).saturating_mul(m as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Membership Members (r:1 w:1) - // Storage: Instance2Collective Proposals (r:1 w:0) - // Storage: Instance1Membership Prime (r:1 w:0) - // Storage: Instance2Collective Members (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Members (r:1 w:1) + // Storage: TechnicalCommittee Proposals (r:1 w:0) + // Storage: TechnicalMembership Prime (r:1 w:0) + // Storage: TechnicalCommittee Members (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn swap_member(m: u32, ) -> Weight { - (29_289_000 as Weight) + (17_580_000 as Weight) // Standard Error: 0 - .saturating_add((126_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((98_000 as Weight).saturating_mul(m as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Membership Members (r:1 w:1) - // Storage: Instance2Collective Proposals (r:1 w:0) - // Storage: Instance1Membership Prime (r:1 w:0) - // Storage: Instance2Collective Members (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Members (r:1 w:1) + // Storage: TechnicalCommittee Proposals (r:1 w:0) + // Storage: TechnicalMembership Prime (r:1 w:0) + // Storage: TechnicalCommittee Members (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn reset_member(m: u32, ) -> Weight { - (30_178_000 as Weight) - // Standard Error: 1_000 - .saturating_add((286_000 as Weight).saturating_mul(m as Weight)) + (17_610_000 as Weight) + // Standard Error: 0 + .saturating_add((199_000 as Weight).saturating_mul(m as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } - // Storage: Instance1Membership Members (r:1 w:1) - // Storage: Instance2Collective Proposals (r:1 w:0) - // Storage: Instance1Membership Prime (r:1 w:1) - // Storage: Instance2Collective Members (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Members (r:1 w:1) + // Storage: TechnicalCommittee Proposals (r:1 w:0) + // Storage: TechnicalMembership Prime (r:1 w:1) + // Storage: TechnicalCommittee Members (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn change_key(m: u32, ) -> Weight { - (31_049_000 as Weight) + (18_208_000 as Weight) // Standard Error: 0 - .saturating_add((121_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((96_000 as Weight).saturating_mul(m as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } - // Storage: Instance1Membership Members (r:1 w:0) - // Storage: Instance1Membership Prime (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Members (r:1 w:0) + // Storage: TechnicalMembership Prime (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn set_prime(m: u32, ) -> Weight { - (8_006_000 as Weight) + (4_661_000 as Weight) // Standard Error: 0 - .saturating_add((89_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((70_000 as Weight).saturating_mul(m as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } - // Storage: Instance1Membership Prime (r:0 w:1) - // Storage: Instance2Collective Prime (r:0 w:1) + // Storage: TechnicalMembership Prime (r:0 w:1) + // Storage: TechnicalCommittee Prime (r:0 w:1) fn clear_prime(m: u32, ) -> Weight { - (3_452_000 as Weight) + (1_574_000 as Weight) // Standard Error: 0 - .saturating_add((2_000 as Weight).saturating_mul(m as Weight)) + .saturating_add((1_000 as Weight).saturating_mul(m as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } } diff --git a/frame/merkle-mountain-range/Cargo.toml b/frame/merkle-mountain-range/Cargo.toml index 02b4be182ef8..796ab98dc2c3 100644 --- a/frame/merkle-mountain-range/Cargo.toml +++ b/frame/merkle-mountain-range/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-mmr" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME Merkle Mountain Range pallet." @@ -12,14 +12,14 @@ description = "FRAME Merkle Mountain Range pallet." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -mmr-lib = { package = "ckb-merkle-mountain-range", default-features = false, version = "0.3.1" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +mmr-lib = { package = "ckb-merkle-mountain-range", default-features = false, version = "0.3.2" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } @@ -46,5 +46,5 @@ std = [ "frame-system/std", "pallet-mmr-primitives/std", ] -runtime-benchmarks = ["frame-benchmarking"] +runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/merkle-mountain-range/primitives/Cargo.toml b/frame/merkle-mountain-range/primitives/Cargo.toml index 07b2f8ae3a3a..3ce2caa7762c 100644 --- a/frame/merkle-mountain-range/primitives/Cargo.toml +++ b/frame/merkle-mountain-range/primitives/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-mmr-primitives" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME Merkle Mountain Range primitives." @@ -12,14 +12,14 @@ description = "FRAME Merkle Mountain Range primitives." targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } log = { version = "0.4.14", default-features = false } -serde = { version = "1.0.126", optional = true, features = ["derive"] } +serde = { version = "1.0.136", optional = true, features = ["derive"] } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/api" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/std" } +sp-core = { version = "6.0.0", default-features = false, path = "../../../primitives/core" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" } diff --git a/frame/merkle-mountain-range/primitives/src/lib.rs b/frame/merkle-mountain-range/primitives/src/lib.rs index dac57bd42cd3..cc78dfefefe6 100644 --- a/frame/merkle-mountain-range/primitives/src/lib.rs +++ b/frame/merkle-mountain-range/primitives/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,6 +26,16 @@ use sp_std::fmt; #[cfg(not(feature = "std"))] use sp_std::prelude::Vec; +/// A type to describe node position in the MMR (node index). +pub type NodeIndex = u64; + +/// A type to describe leaf position in the MMR. +/// +/// Note this is different from [`NodeIndex`], which can be applied to +/// both leafs and inner nodes. Leafs will always have consecutive `LeafIndex`, +/// but might be actually at different positions in the MMR `NodeIndex`. +pub type LeafIndex = u64; + /// A provider of the MMR's leaf data. pub trait LeafDataProvider { /// A type that should end up in the leaf of MMR. @@ -275,9 +285,9 @@ impl_leaf_data_for_tuple!(A:0, B:1, C:2, D:3, E:4); #[derive(codec::Encode, codec::Decode, RuntimeDebug, Clone, PartialEq, Eq)] pub struct Proof { /// The index of the leaf the proof is for. - pub leaf_index: u64, + pub leaf_index: LeafIndex, /// Number of leaves in MMR, when the proof was generated. - pub leaf_count: u64, + pub leaf_count: NodeIndex, /// Proof elements (hashes of siblings of inner nodes on the path to the leaf). pub items: Vec, } @@ -402,7 +412,7 @@ sp_api::decl_runtime_apis! { /// API to interact with MMR pallet. pub trait MmrApi { /// Generate MMR proof for a leaf under given index. - fn generate_proof(leaf_index: u64) -> Result<(EncodableOpaqueLeaf, Proof), Error>; + fn generate_proof(leaf_index: LeafIndex) -> Result<(EncodableOpaqueLeaf, Proof), Error>; /// Verify MMR proof against on-chain MMR. /// diff --git a/frame/merkle-mountain-range/rpc/Cargo.toml b/frame/merkle-mountain-range/rpc/Cargo.toml index 5a0f114e5017..9ac26c2ed54b 100644 --- a/frame/merkle-mountain-range/rpc/Cargo.toml +++ b/frame/merkle-mountain-range/rpc/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-mmr-rpc" version = "3.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Node-specific RPC methods for interaction with Merkle Mountain Range pallet." publish = false @@ -13,18 +13,18 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0" } +codec = { package = "parity-scale-codec", version = "3.0.0" } jsonrpc-core = "18.0.0" jsonrpc-core-client = "18.0.0" jsonrpc-derive = "18.0.0" -serde = { version = "1.0.126", features = ["derive"] } +serde = { version = "1.0.136", features = ["derive"] } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } pallet-mmr-primitives = { version = "4.0.0-dev", path = "../primitives" } [dev-dependencies] -serde_json = "1.0.68" +serde_json = "1.0.79" diff --git a/frame/merkle-mountain-range/rpc/src/lib.rs b/frame/merkle-mountain-range/rpc/src/lib.rs index 4719893778f6..bf3eb3b694e3 100644 --- a/frame/merkle-mountain-range/rpc/src/lib.rs +++ b/frame/merkle-mountain-range/rpc/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -32,7 +32,7 @@ use sp_blockchain::HeaderBackend; use sp_core::Bytes; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; -pub use pallet_mmr_primitives::MmrApi as MmrRuntimeApi; +pub use pallet_mmr_primitives::{LeafIndex, MmrApi as MmrRuntimeApi}; /// Retrieved MMR leaf and its proof. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -71,7 +71,7 @@ pub trait MmrApi { #[rpc(name = "mmr_generateProof")] fn generate_proof( &self, - leaf_index: u64, + leaf_index: LeafIndex, at: Option, ) -> Result>; } @@ -98,7 +98,7 @@ where { fn generate_proof( &self, - leaf_index: u64, + leaf_index: LeafIndex, at: Option<::Hash>, ) -> Result::Hash>> { let api = self.client.runtime_api(); @@ -144,11 +144,11 @@ fn mmr_error_into_rpc_error(err: MmrError) -> Error { } /// Converts a runtime trap into an RPC error. -fn runtime_error_into_rpc_error(err: impl std::fmt::Debug) -> Error { +fn runtime_error_into_rpc_error(err: impl std::fmt::Display) -> Error { Error { code: ErrorCode::ServerError(RUNTIME_ERROR), message: "Runtime trapped".into(), - data: Some(format!("{:?}", err).into()), + data: Some(err.to_string().into()), } } diff --git a/frame/merkle-mountain-range/src/benchmarking.rs b/frame/merkle-mountain-range/src/benchmarking.rs index c269afb75855..b698e432534d 100644 --- a/frame/merkle-mountain-range/src/benchmarking.rs +++ b/frame/merkle-mountain-range/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,14 +18,14 @@ //! Benchmarks for the MMR pallet. use crate::*; -use frame_benchmarking::{benchmarks_instance_pallet, impl_benchmark_test_suite}; +use frame_benchmarking::benchmarks_instance_pallet; use frame_support::traits::OnInitialize; benchmarks_instance_pallet! { on_initialize { let x in 1 .. 1_000; - let leaves = x as u64; + let leaves = x as NodeIndex; }: { for b in 0..leaves { Pallet::::on_initialize((b as u32).into()); @@ -33,6 +33,6 @@ benchmarks_instance_pallet! { } verify { assert_eq!(crate::NumberOfLeaves::::get(), leaves); } -} -impl_benchmark_test_suite!(Pallet, crate::tests::new_test_ext(), crate::mock::Test); + impl_benchmark_test_suite!(Pallet, crate::tests::new_test_ext(), crate::mock::Test); +} diff --git a/frame/merkle-mountain-range/src/default_weights.rs b/frame/merkle-mountain-range/src/default_weights.rs index 6308975ce7d2..73d1963a4296 100644 --- a/frame/merkle-mountain-range/src/default_weights.rs +++ b/frame/merkle-mountain-range/src/default_weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/merkle-mountain-range/src/lib.rs b/frame/merkle-mountain-range/src/lib.rs index 01bf1b2254f0..f904428e0204 100644 --- a/frame/merkle-mountain-range/src/lib.rs +++ b/frame/merkle-mountain-range/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -70,10 +70,10 @@ mod mock; mod tests; pub use pallet::*; -pub use pallet_mmr_primitives as primitives; +pub use pallet_mmr_primitives::{self as primitives, NodeIndex}; pub trait WeightInfo { - fn on_initialize(peaks: u64) -> Weight; + fn on_initialize(peaks: NodeIndex) -> Weight; } #[frame_support::pallet] @@ -125,7 +125,8 @@ pub mod pallet { + Default + codec::Codec + codec::EncodeLike - + scale_info::TypeInfo; + + scale_info::TypeInfo + + MaxEncodedLen; /// Data stored in the leaf nodes. /// @@ -160,7 +161,7 @@ pub mod pallet { /// Current size of the MMR (number of leaves). #[pallet::storage] #[pallet::getter(fn mmr_leaves)] - pub type NumberOfLeaves = StorageValue<_, u64, ValueQuery>; + pub type NumberOfLeaves = StorageValue<_, NodeIndex, ValueQuery>; /// Hashes of the nodes in the MMR. /// @@ -169,7 +170,7 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn mmr_peak)] pub type Nodes, I: 'static = ()> = - StorageMap<_, Identity, u64, >::Hash, OptionQuery>; + StorageMap<_, Identity, NodeIndex, >::Hash, OptionQuery>; #[pallet::hooks] impl, I: 'static> Hooks> for Pallet { @@ -228,7 +229,7 @@ where } impl, I: 'static> Pallet { - fn offchain_key(pos: u64) -> sp_std::prelude::Vec { + fn offchain_key(pos: NodeIndex) -> sp_std::prelude::Vec { (T::INDEXING_PREFIX, pos).encode() } @@ -239,7 +240,7 @@ impl, I: 'static> Pallet { /// all the leaves to be present. /// It may return an error or panic if used incorrectly. pub fn generate_proof( - leaf_index: u64, + leaf_index: NodeIndex, ) -> Result<(LeafOf, primitives::Proof<>::Hash>), primitives::Error> { let mmr: ModuleMmr = mmr::Mmr::new(Self::mmr_leaves()); mmr.generate_proof(leaf_index) @@ -263,7 +264,7 @@ impl, I: 'static> Pallet { .log_debug("The proof has incorrect number of leaves or proof items.")) } - let mmr: ModuleMmr = mmr::Mmr::new(proof.leaf_count); + let mmr: ModuleMmr = mmr::Mmr::new(proof.leaf_count); let is_valid = mmr.verify_leaf_proof(leaf, proof)?; if is_valid { Ok(()) diff --git a/frame/merkle-mountain-range/src/mmr/mmr.rs b/frame/merkle-mountain-range/src/mmr/mmr.rs index d5036e58f432..a1516ee8607f 100644 --- a/frame/merkle-mountain-range/src/mmr/mmr.rs +++ b/frame/merkle-mountain-range/src/mmr/mmr.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use crate::{ utils::NodesUtils, Hasher, Node, NodeOf, }, - primitives::{self, Error}, + primitives::{self, Error, NodeIndex}, Config, HashingOf, }; #[cfg(not(feature = "std"))] @@ -60,7 +60,7 @@ where Storage: mmr_lib::MMRStore>, { mmr: mmr_lib::MMR, Hasher, L>, Storage>, - leaves: u64, + leaves: NodeIndex, } impl Mmr @@ -71,7 +71,7 @@ where Storage: mmr_lib::MMRStore>, { /// Create a pointer to an existing MMR with given number of leaves. - pub fn new(leaves: u64) -> Self { + pub fn new(leaves: NodeIndex) -> Self { let size = NodesUtils::new(leaves).size(); Self { mmr: mmr_lib::MMR::new(size, Default::default()), leaves } } @@ -94,7 +94,7 @@ where /// Return the internal size of the MMR (number of nodes). #[cfg(test)] - pub fn size(&self) -> u64 { + pub fn size(&self) -> NodeIndex { self.mmr.mmr_size() } } @@ -109,7 +109,7 @@ where /// Push another item to the MMR. /// /// Returns element position (index) in the MMR. - pub fn push(&mut self, leaf: L) -> Option { + pub fn push(&mut self, leaf: L) -> Option { let position = self.mmr.push(Node::Data(leaf)).map_err(|e| Error::Push.log_error(e)).ok()?; @@ -120,7 +120,7 @@ where /// Commit the changes to underlying storage, return current number of leaves and /// calculate the new MMR's root hash. - pub fn finalize(self) -> Result<(u64, >::Hash), Error> { + pub fn finalize(self) -> Result<(NodeIndex, >::Hash), Error> { let root = self.mmr.get_root().map_err(|e| Error::GetRoot.log_error(e))?; self.mmr.commit().map_err(|e| Error::Commit.log_error(e))?; Ok((self.leaves, root.hash())) @@ -140,7 +140,7 @@ where /// (i.e. you can't run the function in the pruned storage). pub fn generate_proof( &self, - leaf_index: u64, + leaf_index: NodeIndex, ) -> Result<(L, primitives::Proof<>::Hash>), Error> { let position = mmr_lib::leaf_index_to_pos(leaf_index); let store = >::default(); diff --git a/frame/merkle-mountain-range/src/mmr/mod.rs b/frame/merkle-mountain-range/src/mmr/mod.rs index ec2dfe245bd4..1a729b08b966 100644 --- a/frame/merkle-mountain-range/src/mmr/mod.rs +++ b/frame/merkle-mountain-range/src/mmr/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/merkle-mountain-range/src/mmr/storage.rs b/frame/merkle-mountain-range/src/mmr/storage.rs index 09e24017816e..535057ca80da 100644 --- a/frame/merkle-mountain-range/src/mmr/storage.rs +++ b/frame/merkle-mountain-range/src/mmr/storage.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,19 +18,23 @@ //! A MMR storage implementations. use codec::Encode; +use mmr_lib::helper; +use sp_io::offchain_index; +use sp_std::iter::Peekable; #[cfg(not(feature = "std"))] -use sp_std::prelude::Vec; +use sp_std::prelude::*; use crate::{ - mmr::{Node, NodeOf}, - primitives, Config, Nodes, NumberOfLeaves, Pallet, + mmr::{utils::NodesUtils, Node, NodeOf}, + primitives::{self, NodeIndex}, + Config, Nodes, NumberOfLeaves, Pallet, }; /// A marker type for runtime-specific storage implementation. /// /// Allows appending new items to the MMR and proof verification. /// MMR nodes are appended to two different storages: -/// 1. We add nodes (leaves) hashes to the on-chain storge (see [crate::Nodes]). +/// 1. We add nodes (leaves) hashes to the on-chain storage (see [crate::Nodes]). /// 2. We add full leaves (and all inner nodes as well) into the `IndexingAPI` during block /// processing, so the values end up in the Offchain DB if indexing is enabled. pub struct RuntimeStorage; @@ -60,14 +64,14 @@ where I: 'static, L: primitives::FullLeaf + codec::Decode, { - fn get_elem(&self, pos: u64) -> mmr_lib::Result>> { + fn get_elem(&self, pos: NodeIndex) -> mmr_lib::Result>> { let key = Pallet::::offchain_key(pos); // Retrieve the element from Off-chain DB. Ok(sp_io::offchain::local_storage_get(sp_core::offchain::StorageKind::PERSISTENT, &key) .and_then(|v| codec::Decode::decode(&mut &*v).ok())) } - fn append(&mut self, _: u64, _: Vec>) -> mmr_lib::Result<()> { + fn append(&mut self, _: NodeIndex, _: Vec>) -> mmr_lib::Result<()> { panic!("MMR must not be altered in the off-chain context.") } } @@ -78,32 +82,90 @@ where I: 'static, L: primitives::FullLeaf, { - fn get_elem(&self, pos: u64) -> mmr_lib::Result>> { + fn get_elem(&self, pos: NodeIndex) -> mmr_lib::Result>> { Ok(>::get(pos).map(Node::Hash)) } - fn append(&mut self, pos: u64, elems: Vec>) -> mmr_lib::Result<()> { - let mut leaves = crate::NumberOfLeaves::::get(); - let mut size = crate::mmr::utils::NodesUtils::new(leaves).size(); + fn append(&mut self, pos: NodeIndex, elems: Vec>) -> mmr_lib::Result<()> { + if elems.is_empty() { + return Ok(()) + } + + sp_std::if_std! { + frame_support::log::trace!("elems: {:?}", elems.iter().map(|elem| elem.hash()).collect::>()); + } + + let leaves = NumberOfLeaves::::get(); + let size = NodesUtils::new(leaves).size(); + if pos != size { return Err(mmr_lib::Error::InconsistentStore) } + let new_size = size + elems.len() as NodeIndex; + + // A sorted (ascending) iterator over peak indices to prune and persist. + let (peaks_to_prune, mut peaks_to_store) = peaks_to_prune_and_store(size, new_size); + + // Now we are going to iterate over elements to insert + // and keep track of the current `node_index` and `leaf_index`. + let mut leaf_index = leaves; + let mut node_index = size; + for elem in elems { - // on-chain we only store the hash (even if it's a leaf) - >::insert(size, elem.hash()); - // Indexing API is used to store the full leaf content. - let key = Pallet::::offchain_key(size); - elem.using_encoded(|elem| sp_io::offchain_index::set(&key, elem)); - size += 1; + // Indexing API is used to store the full node content (both leaf and inner). + elem.using_encoded(|elem| { + offchain_index::set(&Pallet::::offchain_key(node_index), elem) + }); + + // On-chain we are going to only store new peaks. + if peaks_to_store.next_if_eq(&node_index).is_some() { + >::insert(node_index, elem.hash()); + } + // Increase the indices. if let Node::Data(..) = elem { - leaves += 1; + leaf_index += 1; } + node_index += 1; } - NumberOfLeaves::::put(leaves); + // Update current number of leaves. + NumberOfLeaves::::put(leaf_index); + + // And remove all remaining items from `peaks_before` collection. + for pos in peaks_to_prune { + >::remove(pos); + } Ok(()) } } + +fn peaks_to_prune_and_store( + old_size: NodeIndex, + new_size: NodeIndex, +) -> (impl Iterator, Peekable>) { + // A sorted (ascending) collection of peak indices before and after insertion. + // both collections may share a common prefix. + let peaks_before = if old_size == 0 { vec![] } else { helper::get_peaks(old_size) }; + let peaks_after = helper::get_peaks(new_size); + sp_std::if_std! { + frame_support::log::trace!("peaks_before: {:?}", peaks_before); + frame_support::log::trace!("peaks_after: {:?}", peaks_after); + } + let mut peaks_before = peaks_before.into_iter().peekable(); + let mut peaks_after = peaks_after.into_iter().peekable(); + + // Consume a common prefix between `peaks_before` and `peaks_after`, + // since that's something we will not be touching anyway. + while peaks_before.peek() == peaks_after.peek() { + peaks_before.next(); + peaks_after.next(); + } + + // what's left in both collections is: + // 1. Old peaks to remove from storage + // 2. New peaks to persist in storage + (peaks_before, peaks_after) +} diff --git a/frame/merkle-mountain-range/src/mmr/utils.rs b/frame/merkle-mountain-range/src/mmr/utils.rs index 8fc725f11e72..d9f7e3b671be 100644 --- a/frame/merkle-mountain-range/src/mmr/utils.rs +++ b/frame/merkle-mountain-range/src/mmr/utils.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,29 +17,31 @@ //! Merkle Mountain Range utilities. +use crate::primitives::{LeafIndex, NodeIndex}; + /// MMR nodes & size -related utilities. pub struct NodesUtils { - no_of_leaves: u64, + no_of_leaves: LeafIndex, } impl NodesUtils { /// Create new instance of MMR nodes utilities for given number of leaves. - pub fn new(no_of_leaves: u64) -> Self { + pub fn new(no_of_leaves: LeafIndex) -> Self { Self { no_of_leaves } } /// Calculate number of peaks in the MMR. - pub fn number_of_peaks(&self) -> u64 { - self.number_of_leaves().count_ones() as u64 + pub fn number_of_peaks(&self) -> NodeIndex { + self.number_of_leaves().count_ones() as NodeIndex } /// Return the number of leaves in the MMR. - pub fn number_of_leaves(&self) -> u64 { + pub fn number_of_leaves(&self) -> LeafIndex { self.no_of_leaves } /// Calculate the total size of MMR (number of nodes). - pub fn size(&self) -> u64 { + pub fn size(&self) -> NodeIndex { 2 * self.no_of_leaves - self.number_of_peaks() } diff --git a/frame/merkle-mountain-range/src/mock.rs b/frame/merkle-mountain-range/src/mock.rs index 3616a8d1d524..56d3c9c0d77d 100644 --- a/frame/merkle-mountain-range/src/mock.rs +++ b/frame/merkle-mountain-range/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,7 @@ use crate as pallet_mmr; use crate::*; use codec::{Decode, Encode}; -use frame_support::parameter_types; +use frame_support::traits::{ConstU32, ConstU64}; use pallet_mmr_primitives::{Compact, LeafDataProvider}; use sp_core::H256; use sp_runtime::{ @@ -42,9 +42,6 @@ frame_support::construct_runtime!( } ); -parameter_types! { - pub const BlockHashCount: u64 = 250; -} impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; type Origin = Origin; @@ -57,7 +54,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type DbWeight = (); type BlockWeights = (); type BlockLength = (); @@ -69,6 +66,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl Config for Test { diff --git a/frame/merkle-mountain-range/src/tests.rs b/frame/merkle-mountain-range/src/tests.rs index 50512e928695..576a7ace8f1c 100644 --- a/frame/merkle-mountain-range/src/tests.rs +++ b/frame/merkle-mountain-range/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,9 +15,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{mock::*, *}; +use crate::{mmr::utils, mock::*, *}; use frame_support::traits::OnInitialize; +use mmr_lib::helper; use pallet_mmr_primitives::{Compact, Proof}; use sp_core::{ offchain::{testing::TestOffchainExt, OffchainDbExt, OffchainWorkerExt}, @@ -39,15 +40,17 @@ fn new_block() -> u64 { let hash = H256::repeat_byte(number as u8); LEAF_DATA.with(|r| r.borrow_mut().a = number); - frame_system::Pallet::::initialize( - &number, - &hash, - &Default::default(), - frame_system::InitKind::Full, - ); + frame_system::Pallet::::reset_events(); + frame_system::Pallet::::initialize(&number, &hash, &Default::default()); MMR::on_initialize(number) } +fn peaks_from_leaves_count(leaves_count: NodeIndex) -> Vec { + let size = utils::NodesUtils::new(leaves_count).size(); + + helper::get_peaks(size) +} + pub(crate) fn hex(s: &str) -> H256 { s.parse().unwrap() } @@ -115,10 +118,29 @@ fn should_append_to_mmr_when_on_initialize_is_called() { ext.execute_with(|| { // when new_block(); + + // then + assert_eq!(crate::NumberOfLeaves::::get(), 1); + assert_eq!( + ( + crate::Nodes::::get(0), + crate::Nodes::::get(1), + crate::RootHash::::get(), + ), + ( + Some(hex("4320435e8c3318562dba60116bdbcc0b82ffcecb9bb39aae3300cfda3ad0b8b0")), + None, + hex("0x4320435e8c3318562dba60116bdbcc0b82ffcecb9bb39aae3300cfda3ad0b8b0"), + ) + ); + + // when new_block(); // then assert_eq!(crate::NumberOfLeaves::::get(), 2); + let peaks = peaks_from_leaves_count(2); + assert_eq!(peaks, vec![2]); assert_eq!( ( crate::Nodes::::get(0), @@ -128,8 +150,8 @@ fn should_append_to_mmr_when_on_initialize_is_called() { crate::RootHash::::get(), ), ( - Some(hex("4320435e8c3318562dba60116bdbcc0b82ffcecb9bb39aae3300cfda3ad0b8b0")), - Some(hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705")), + None, + None, Some(hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854")), None, hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854"), @@ -166,14 +188,21 @@ fn should_construct_larger_mmr_correctly() { // then assert_eq!(crate::NumberOfLeaves::::get(), 7); + let peaks = peaks_from_leaves_count(7); + assert_eq!(peaks, vec![6, 9, 10]); + for i in (0..=10).filter(|p| !peaks.contains(p)) { + assert!(crate::Nodes::::get(i).is_none()); + } assert_eq!( ( - crate::Nodes::::get(0), + crate::Nodes::::get(6), + crate::Nodes::::get(9), crate::Nodes::::get(10), crate::RootHash::::get(), ), ( - Some(hex("4320435e8c3318562dba60116bdbcc0b82ffcecb9bb39aae3300cfda3ad0b8b0")), + Some(hex("ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252")), + Some(hex("7e4316ae2ebf7c3b6821cb3a46ca8b7a4f9351a9b40fcf014bb0a4fd8e8f29da")), Some(hex("611c2174c6164952a66d985cfe1ec1a623794393e3acff96b136d198f37a648c")), hex("e45e25259f7930626431347fa4dd9aae7ac83b4966126d425ca70ab343709d2c"), ) @@ -265,11 +294,7 @@ fn should_verify() { crate::Pallet::::generate_proof(5).unwrap() }); - // Now to verify the proof, we really shouldn't require offchain storage or extension. - // Hence we initialize the storage once again, using different externalities and then - // verify. - let mut ext2 = new_test_ext(); - ext2.execute_with(|| { + ext.execute_with(|| { init_chain(7); // then assert_eq!(crate::Pallet::::verify_leaf(leaf, proof5), Ok(())); diff --git a/frame/multisig/Cargo.toml b/frame/multisig/Cargo.toml index 177334d4ccf8..a2188ca18b3b 100644 --- a/frame/multisig/Cargo.toml +++ b/frame/multisig/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-multisig" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME multi-signature dispatch pallet" readme = "README.md" @@ -13,18 +13,18 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } [features] @@ -39,7 +39,7 @@ std = [ "sp-std/std" ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", ] diff --git a/frame/multisig/src/benchmarking.rs b/frame/multisig/src/benchmarking.rs index 2e23dff156e0..43abb349b795 100644 --- a/frame/multisig/src/benchmarking.rs +++ b/frame/multisig/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,8 +20,7 @@ #![cfg(feature = "runtime-benchmarks")] use super::*; -use core::convert::TryInto; -use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite}; +use frame_benchmarking::{account, benchmarks}; use frame_system::RawOrigin; use sp_runtime::traits::Bounded; @@ -29,7 +28,10 @@ use crate::Pallet as Multisig; const SEED: u32 = 0; -fn setup_multi(s: u32, z: u32) -> Result<(Vec, Vec), &'static str> { +fn setup_multi( + s: u32, + z: u32, +) -> Result<(Vec, OpaqueCall), &'static str> { let mut signatories: Vec = Vec::new(); for i in 0..s { let signatory = account("signatory", i, SEED); @@ -42,7 +44,7 @@ fn setup_multi(s: u32, z: u32) -> Result<(Vec, Vec) // Must first convert to outer call type. let call: ::Call = frame_system::Call::::remark { remark: vec![0; z as usize] }.into(); - let call_data = call.encode(); + let call_data = OpaqueCall::::from_encoded(call.encode()); return Ok((signatories, call_data)) } @@ -72,7 +74,7 @@ benchmarks! { // Transaction Length let z in 0 .. 10_000; let (mut signatories, call) = setup_multi::(s, z)?; - let call_hash = blake2_256(&call); + let call_hash = blake2_256(&call.encoded()); let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; // Whitelist caller account from further DB operations. @@ -90,7 +92,7 @@ benchmarks! { // Transaction Length let z in 0 .. 10_000; let (mut signatories, call) = setup_multi::(s, z)?; - let call_hash = blake2_256(&call); + let call_hash = blake2_256(&call.encoded()); let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); @@ -109,7 +111,7 @@ benchmarks! { // Transaction Length let z in 0 .. 10_000; let (mut signatories, call) = setup_multi::(s, z)?; - let call_hash = blake2_256(&call); + let call_hash = blake2_256(&call.encoded()); let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); let mut signatories2 = signatories.clone(); let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; @@ -134,7 +136,7 @@ benchmarks! { // Transaction Length let z in 0 .. 10_000; let (mut signatories, call) = setup_multi::(s, z)?; - let call_hash = blake2_256(&call); + let call_hash = blake2_256(&call.encoded()); let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); let mut signatories2 = signatories.clone(); let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; @@ -160,7 +162,7 @@ benchmarks! { // Transaction Length let z in 0 .. 10_000; let (mut signatories, call) = setup_multi::(s, z)?; - let call_hash = blake2_256(&call); + let call_hash = blake2_256(&call.encoded()); let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); let mut signatories2 = signatories.clone(); let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; @@ -193,7 +195,7 @@ benchmarks! { let (mut signatories, call) = setup_multi::(s, z)?; let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; - let call_hash = blake2_256(&call); + let call_hash = blake2_256(&call.encoded()); // Whitelist caller account from further DB operations. let caller_key = frame_system::Account::::hashed_key_for(&caller); frame_benchmarking::benchmarking::add_to_whitelist(caller_key.into()); @@ -212,7 +214,7 @@ benchmarks! { let mut signatories2 = signatories.clone(); let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; - let call_hash = blake2_256(&call); + let call_hash = blake2_256(&call.encoded()); // before the call, get the timepoint let timepoint = Multisig::::timepoint(); // Create the multi @@ -245,7 +247,7 @@ benchmarks! { let mut signatories2 = signatories.clone(); let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); - let call_hash = blake2_256(&call); + let call_hash = blake2_256(&call.encoded()); // before the call, get the timepoint let timepoint = Multisig::::timepoint(); // Create the multi @@ -282,7 +284,7 @@ benchmarks! { let (mut signatories, call) = setup_multi::(s, z)?; let multi_account_id = Multisig::::multi_account_id(&signatories, s.try_into().unwrap()); let caller = signatories.pop().ok_or("signatories should have len 2 or more")?; - let call_hash = blake2_256(&call); + let call_hash = blake2_256(&call.encoded()); let timepoint = Multisig::::timepoint(); // Create the multi let o = RawOrigin::Signed(caller.clone()).into(); @@ -297,6 +299,6 @@ benchmarks! { assert!(!Multisigs::::contains_key(multi_account_id, call_hash)); assert!(!Calls::::contains_key(call_hash)); } -} -impl_benchmark_test_suite!(Multisig, crate::tests::new_test_ext(), crate::tests::Test); + impl_benchmark_test_suite!(Multisig, crate::tests::new_test_ext(), crate::tests::Test); +} diff --git a/frame/multisig/src/lib.rs b/frame/multisig/src/lib.rs index 43040ada45a9..cd59ea881739 100644 --- a/frame/multisig/src/lib.rs +++ b/frame/multisig/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -56,7 +56,7 @@ use frame_support::{ DispatchErrorWithPostInfo, DispatchResult, DispatchResultWithPostInfo, PostDispatchInfo, }, ensure, - traits::{Currency, Get, ReservableCurrency}, + traits::{Currency, Get, ReservableCurrency, WrapperKeepOpaque}, weights::{GetDispatchInfo, Weight}, RuntimeDebug, }; @@ -64,7 +64,7 @@ use frame_system::{self as system, RawOrigin}; use scale_info::TypeInfo; use sp_io::hashing::blake2_256; use sp_runtime::{ - traits::{Dispatchable, Zero}, + traits::{Dispatchable, TrailingZeroInput, Zero}, DispatchError, }; use sp_std::prelude::*; @@ -74,8 +74,6 @@ pub use pallet::*; type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; -/// Just a bunch of bytes, but they should decode to a valid `Call`. -pub type OpaqueCall = Vec; /// A global extrinsic index, formed as the extrinsic index within a block, together with that /// block's height. This allows a transaction in which a multisig operation of a particular @@ -101,10 +99,12 @@ pub struct Multisig { approvals: Vec, } +type OpaqueCall = WrapperKeepOpaque<::Call>; + type CallHash = [u8; 32]; -enum CallOrHash { - Call(OpaqueCall, bool), +enum CallOrHash { + Call(OpaqueCall, bool), Hash([u8; 32]), } @@ -153,6 +153,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); /// The set of open multisig operations. @@ -168,7 +169,7 @@ pub mod pallet { #[pallet::storage] pub type Calls = - StorageMap<_, Identity, [u8; 32], (OpaqueCall, T::AccountId, BalanceOf)>; + StorageMap<_, Identity, [u8; 32], (OpaqueCall, T::AccountId, BalanceOf)>; #[pallet::error] pub enum Error { @@ -205,21 +206,30 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// A new multisig operation has begun. \[approving, multisig, call_hash\] - NewMultisig(T::AccountId, T::AccountId, CallHash), + /// A new multisig operation has begun. + NewMultisig { approving: T::AccountId, multisig: T::AccountId, call_hash: CallHash }, /// A multisig operation has been approved by someone. - /// \[approving, timepoint, multisig, call_hash\] - MultisigApproval(T::AccountId, Timepoint, T::AccountId, CallHash), - /// A multisig operation has been executed. \[approving, timepoint, multisig, call_hash\] - MultisigExecuted( - T::AccountId, - Timepoint, - T::AccountId, - CallHash, - DispatchResult, - ), - /// A multisig operation has been cancelled. \[cancelling, timepoint, multisig, call_hash\] - MultisigCancelled(T::AccountId, Timepoint, T::AccountId, CallHash), + MultisigApproval { + approving: T::AccountId, + timepoint: Timepoint, + multisig: T::AccountId, + call_hash: CallHash, + }, + /// A multisig operation has been executed. + MultisigExecuted { + approving: T::AccountId, + timepoint: Timepoint, + multisig: T::AccountId, + call_hash: CallHash, + result: DispatchResult, + }, + /// A multisig operation has been cancelled. + MultisigCancelled { + cancelling: T::AccountId, + timepoint: Timepoint, + multisig: T::AccountId, + call_hash: CallHash, + }, } #[pallet::hooks] @@ -339,7 +349,7 @@ pub mod pallet { /// # #[pallet::weight({ let s = other_signatories.len() as u32; - let z = call.len() as u32; + let z = call.encoded_len() as u32; T::WeightInfo::as_multi_create(s, z) .max(T::WeightInfo::as_multi_create_store(s, z)) @@ -352,7 +362,7 @@ pub mod pallet { threshold: u16, other_signatories: Vec, maybe_timepoint: Option>, - call: OpaqueCall, + call: OpaqueCall, store_call: bool, max_weight: Weight, ) -> DispatchResultWithPostInfo { @@ -406,9 +416,9 @@ pub mod pallet { let s = other_signatories.len() as u32; T::WeightInfo::approve_as_multi_create(s) - .max(T::WeightInfo::approve_as_multi_approve(s)) - .max(T::WeightInfo::approve_as_multi_complete(s)) - .saturating_add(*max_weight) + .max(T::WeightInfo::approve_as_multi_approve(s)) + .max(T::WeightInfo::approve_as_multi_complete(s)) + .saturating_add(*max_weight) })] pub fn approve_as_multi( origin: OriginFor, @@ -481,7 +491,12 @@ pub mod pallet { >::remove(&id, &call_hash); Self::clear_call(&call_hash); - Self::deposit_event(Event::MultisigCancelled(who, timepoint, id, call_hash)); + Self::deposit_event(Event::MultisigCancelled { + cancelling: who, + timepoint, + multisig: id, + call_hash, + }); Ok(()) } } @@ -494,7 +509,8 @@ impl Pallet { /// NOTE: `who` must be sorted. If it is not, then you'll get the wrong answer. pub fn multi_account_id(who: &[T::AccountId], threshold: u16) -> T::AccountId { let entropy = (b"modlpy/utilisuba", who, threshold).using_encoded(blake2_256); - T::AccountId::decode(&mut &entropy[..]).unwrap_or_default() + Decode::decode(&mut TrailingZeroInput::new(entropy.as_ref())) + .expect("infinite length input; no invalid inputs for type; qed") } fn operate( @@ -502,7 +518,7 @@ impl Pallet { threshold: u16, other_signatories: Vec, maybe_timepoint: Option>, - call_or_hash: CallOrHash, + call_or_hash: CallOrHash, max_weight: Weight, ) -> DispatchResultWithPostInfo { ensure!(threshold >= 2, Error::::MinimumThreshold); @@ -517,8 +533,8 @@ impl Pallet { // Threshold > 1; this means it's a multi-step operation. We extract the `call_hash`. let (call_hash, call_len, maybe_call, store) = match call_or_hash { CallOrHash::Call(call, should_store) => { - let call_hash = blake2_256(&call); - let call_len = call.len(); + let call_hash = blake2_256(call.encoded()); + let call_len = call.encoded_len(); (call_hash, call_len, Some(call), should_store) }, CallOrHash::Hash(h) => (h, 0, None, false), @@ -541,7 +557,7 @@ impl Pallet { // We only bother fetching/decoding call if we know that we're ready to execute. let maybe_approved_call = if approvals >= threshold { - Self::get_call(&call_hash, maybe_call.as_ref().map(|c| c.as_ref())) + Self::get_call(&call_hash, maybe_call.as_ref()) } else { None }; @@ -557,13 +573,13 @@ impl Pallet { T::Currency::unreserve(&m.depositor, m.deposit); let result = call.dispatch(RawOrigin::Signed(id.clone()).into()); - Self::deposit_event(Event::MultisigExecuted( - who, + Self::deposit_event(Event::MultisigExecuted { + approving: who, timepoint, - id, + multisig: id, call_hash, - result.map(|_| ()).map_err(|e| e.error), - )); + result: result.map(|_| ()).map_err(|e| e.error), + }); Ok(get_result_weight(result) .map(|actual_weight| { T::WeightInfo::as_multi_complete( @@ -594,7 +610,12 @@ impl Pallet { // Record approval. m.approvals.insert(pos, who.clone()); >::insert(&id, call_hash, m); - Self::deposit_event(Event::MultisigApproval(who, timepoint, id, call_hash)); + Self::deposit_event(Event::MultisigApproval { + approving: who, + timepoint, + multisig: id, + call_hash, + }); } else { // If we already approved and didn't store the Call, then this was useless and // we report an error. @@ -638,7 +659,7 @@ impl Pallet { approvals: vec![who.clone()], }, ); - Self::deposit_event(Event::NewMultisig(who, id, call_hash)); + Self::deposit_event(Event::NewMultisig { approving: who, multisig: id, call_hash }); let final_weight = if stored { T::WeightInfo::as_multi_create_store(other_signatories_len as u32, call_len as u32) @@ -658,13 +679,14 @@ impl Pallet { fn store_call_and_reserve( who: T::AccountId, hash: &[u8; 32], - data: OpaqueCall, + data: OpaqueCall, other_deposit: BalanceOf, ) -> DispatchResult { ensure!(!Calls::::contains_key(hash), Error::::AlreadyStored); let deposit = other_deposit + T::DepositBase::get() + - T::DepositFactor::get() * BalanceOf::::from(((data.len() + 31) / 32) as u32); + T::DepositFactor::get() * + BalanceOf::::from(((data.encoded_len() + 31) / 32) as u32); T::Currency::reserve(&who, deposit)?; Calls::::insert(&hash, (data, who, deposit)); Ok(()) @@ -673,15 +695,14 @@ impl Pallet { /// Attempt to decode and return the call, provided by the user or from storage. fn get_call( hash: &[u8; 32], - maybe_known: Option<&[u8]>, + maybe_known: Option<&OpaqueCall>, ) -> Option<(::Call, usize)> { maybe_known.map_or_else( || { - Calls::::get(hash).and_then(|(data, ..)| { - Decode::decode(&mut &data[..]).ok().map(|d| (d, data.len())) - }) + Calls::::get(hash) + .and_then(|(data, ..)| Some((data.try_decode()?, data.encoded_len()))) }, - |data| Decode::decode(&mut &data[..]).ok().map(|d| (d, data.len())), + |data| Some((data.try_decode()?, data.encoded_len())), ) } diff --git a/frame/multisig/src/tests.rs b/frame/multisig/src/tests.rs index 3d311cf5d3dc..d67d06e1bce0 100644 --- a/frame/multisig/src/tests.rs +++ b/frame/multisig/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,10 @@ use super::*; use crate as pallet_multisig; -use frame_support::{assert_noop, assert_ok, parameter_types, traits::Contains}; +use frame_support::{ + assert_noop, assert_ok, parameter_types, + traits::{ConstU16, ConstU32, ConstU64, Contains}, +}; use sp_core::H256; use sp_runtime::{ testing::Header, @@ -31,6 +34,7 @@ use sp_runtime::{ type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; type Block = frame_system::mocking::MockBlock; +type OpaqueCall = super::OpaqueCall; frame_support::construct_runtime!( pub enum Test where @@ -45,7 +49,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -64,7 +67,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -73,10 +76,9 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } -parameter_types! { - pub const ExistentialDeposit: u64 = 1; -} + impl pallet_balances::Config for Test { type MaxLocks = (); type MaxReserves = (); @@ -84,15 +86,11 @@ impl pallet_balances::Config for Test { type Balance = u64; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } -parameter_types! { - pub const DepositBase: u64 = 1; - pub const DepositFactor: u64 = 1; - pub const MaxSignatories: u16 = 3; -} + pub struct TestBaseCallFilter; impl Contains for TestBaseCallFilter { fn contains(c: &Call) -> bool { @@ -108,9 +106,9 @@ impl Config for Test { type Event = Event; type Call = Call; type Currency = Balances; - type DepositBase = DepositBase; - type DepositFactor = DepositFactor; - type MaxSignatories = MaxSignatories; + type DepositBase = ConstU64<1>; + type DepositFactor = ConstU64<1>; + type MaxSignatories = ConstU16<3>; type WeightInfo = (); } @@ -152,7 +150,7 @@ fn multisig_deposit_is_taken_and_returned() { 2, vec![2, 3], None, - data.clone(), + OpaqueCall::from_encoded(data.clone()), false, 0 )); @@ -164,7 +162,7 @@ fn multisig_deposit_is_taken_and_returned() { 2, vec![1, 3], Some(now()), - data, + OpaqueCall::from_encoded(data), false, call_weight )); @@ -185,7 +183,15 @@ fn multisig_deposit_is_taken_and_returned_with_call_storage() { let call_weight = call.get_dispatch_info().weight; let data = call.encode(); let hash = blake2_256(&data); - assert_ok!(Multisig::as_multi(Origin::signed(1), 2, vec![2, 3], None, data, true, 0)); + assert_ok!(Multisig::as_multi( + Origin::signed(1), + 2, + vec![2, 3], + None, + OpaqueCall::from_encoded(data), + true, + 0 + )); assert_eq!(Balances::free_balance(1), 0); assert_eq!(Balances::reserved_balance(1), 5); @@ -231,7 +237,7 @@ fn multisig_deposit_is_taken_and_returned_with_alt_call_storage() { 3, vec![1, 3], Some(now()), - data, + OpaqueCall::from_encoded(data), true, 0 )); @@ -316,7 +322,15 @@ fn timepoint_checking_works() { assert_ok!(Multisig::approve_as_multi(Origin::signed(1), 2, vec![2, 3], None, hash, 0)); assert_noop!( - Multisig::as_multi(Origin::signed(2), 2, vec![1, 3], None, call.clone(), false, 0), + Multisig::as_multi( + Origin::signed(2), + 2, + vec![1, 3], + None, + OpaqueCall::from_encoded(call.clone()), + false, + 0 + ), Error::::NoTimepoint, ); let later = Timepoint { index: 1, ..now() }; @@ -326,7 +340,7 @@ fn timepoint_checking_works() { 2, vec![1, 3], Some(later), - call.clone(), + OpaqueCall::from_encoded(call), false, 0 ), @@ -347,7 +361,15 @@ fn multisig_2_of_3_works_with_call_storing() { let call_weight = call.get_dispatch_info().weight; let data = call.encode(); let hash = blake2_256(&data); - assert_ok!(Multisig::as_multi(Origin::signed(1), 2, vec![2, 3], None, data, true, 0)); + assert_ok!(Multisig::as_multi( + Origin::signed(1), + 2, + vec![2, 3], + None, + OpaqueCall::from_encoded(data), + true, + 0 + )); assert_eq!(Balances::free_balance(6), 0); assert_ok!(Multisig::approve_as_multi( @@ -382,7 +404,7 @@ fn multisig_2_of_3_works() { 2, vec![1, 3], Some(now()), - data, + OpaqueCall::from_encoded(data), false, call_weight )); @@ -425,7 +447,7 @@ fn multisig_3_of_3_works() { 3, vec![1, 2], Some(now()), - data, + OpaqueCall::from_encoded(data), false, call_weight )); @@ -473,7 +495,15 @@ fn cancel_multisig_with_call_storage_works() { new_test_ext().execute_with(|| { let call = call_transfer(6, 15).encode(); let hash = blake2_256(&call); - assert_ok!(Multisig::as_multi(Origin::signed(1), 3, vec![2, 3], None, call, true, 0)); + assert_ok!(Multisig::as_multi( + Origin::signed(1), + 3, + vec![2, 3], + None, + OpaqueCall::from_encoded(call), + true, + 0 + )); assert_eq!(Balances::free_balance(1), 4); assert_ok!(Multisig::approve_as_multi( Origin::signed(2), @@ -517,7 +547,7 @@ fn cancel_multisig_with_alt_call_storage_works() { 3, vec![1, 3], Some(now()), - call, + OpaqueCall::from_encoded(call), true, 0 )); @@ -544,7 +574,7 @@ fn multisig_2_of_3_as_multi_works() { 2, vec![2, 3], None, - data.clone(), + OpaqueCall::from_encoded(data.clone()), false, 0 )); @@ -555,7 +585,7 @@ fn multisig_2_of_3_as_multi_works() { 2, vec![1, 3], Some(now()), - data, + OpaqueCall::from_encoded(data), false, call_weight )); @@ -583,7 +613,7 @@ fn multisig_2_of_3_as_multi_with_many_calls_works() { 2, vec![2, 3], None, - data1.clone(), + OpaqueCall::from_encoded(data1.clone()), false, 0 )); @@ -592,7 +622,7 @@ fn multisig_2_of_3_as_multi_with_many_calls_works() { 2, vec![1, 3], None, - data2.clone(), + OpaqueCall::from_encoded(data2.clone()), false, 0 )); @@ -601,7 +631,7 @@ fn multisig_2_of_3_as_multi_with_many_calls_works() { 2, vec![1, 2], Some(now()), - data1, + OpaqueCall::from_encoded(data1), false, call1_weight )); @@ -610,7 +640,7 @@ fn multisig_2_of_3_as_multi_with_many_calls_works() { 2, vec![1, 2], Some(now()), - data2, + OpaqueCall::from_encoded(data2), false, call2_weight )); @@ -637,7 +667,7 @@ fn multisig_2_of_3_cannot_reissue_same_call() { 2, vec![2, 3], None, - data.clone(), + OpaqueCall::from_encoded(data.clone()), false, 0 )); @@ -646,7 +676,7 @@ fn multisig_2_of_3_cannot_reissue_same_call() { 2, vec![1, 3], Some(now()), - data.clone(), + OpaqueCall::from_encoded(data.clone()), false, call_weight )); @@ -657,7 +687,7 @@ fn multisig_2_of_3_cannot_reissue_same_call() { 2, vec![2, 3], None, - data.clone(), + OpaqueCall::from_encoded(data.clone()), false, 0 )); @@ -666,14 +696,21 @@ fn multisig_2_of_3_cannot_reissue_same_call() { 2, vec![1, 2], Some(now()), - data.clone(), + OpaqueCall::from_encoded(data), false, call_weight )); let err = DispatchError::from(BalancesError::::InsufficientBalance).stripped(); System::assert_last_event( - pallet_multisig::Event::MultisigExecuted(3, now(), multi, hash, Err(err)).into(), + pallet_multisig::Event::MultisigExecuted { + approving: 3, + timepoint: now(), + multisig: multi, + call_hash: hash, + result: Err(err), + } + .into(), ); }); } @@ -683,11 +720,27 @@ fn minimum_threshold_check_works() { new_test_ext().execute_with(|| { let call = call_transfer(6, 15).encode(); assert_noop!( - Multisig::as_multi(Origin::signed(1), 0, vec![2], None, call.clone(), false, 0), + Multisig::as_multi( + Origin::signed(1), + 0, + vec![2], + None, + OpaqueCall::from_encoded(call.clone()), + false, + 0 + ), Error::::MinimumThreshold, ); assert_noop!( - Multisig::as_multi(Origin::signed(1), 1, vec![2], None, call.clone(), false, 0), + Multisig::as_multi( + Origin::signed(1), + 1, + vec![2], + None, + OpaqueCall::from_encoded(call.clone()), + false, + 0 + ), Error::::MinimumThreshold, ); }); @@ -698,7 +751,15 @@ fn too_many_signatories_fails() { new_test_ext().execute_with(|| { let call = call_transfer(6, 15).encode(); assert_noop!( - Multisig::as_multi(Origin::signed(1), 2, vec![2, 3, 4], None, call.clone(), false, 0), + Multisig::as_multi( + Origin::signed(1), + 2, + vec![2, 3, 4], + None, + OpaqueCall::from_encoded(call), + false, + 0 + ), Error::::TooManySignatories, ); }); @@ -765,7 +826,15 @@ fn multisig_1_of_3_works() { Error::::MinimumThreshold, ); assert_noop!( - Multisig::as_multi(Origin::signed(1), 1, vec![2, 3], None, call.clone(), false, 0), + Multisig::as_multi( + Origin::signed(1), + 1, + vec![2, 3], + None, + OpaqueCall::from_encoded(call), + false, + 0 + ), Error::::MinimumThreshold, ); let boxed_call = Box::new(call_transfer(6, 15)); @@ -781,7 +850,7 @@ fn multisig_filters() { let call = Box::new(Call::System(frame_system::Call::set_code { code: vec![] })); assert_noop!( Multisig::as_multi_threshold_1(Origin::signed(1), vec![2], call.clone()), - DispatchError::BadOrigin, + DispatchError::from(frame_system::Error::::CallFiltered), ); }); } @@ -801,14 +870,22 @@ fn weight_check_works() { 2, vec![2, 3], None, - data.clone(), + OpaqueCall::from_encoded(data.clone()), false, 0 )); assert_eq!(Balances::free_balance(6), 0); assert_noop!( - Multisig::as_multi(Origin::signed(2), 2, vec![1, 3], Some(now()), data, false, 0), + Multisig::as_multi( + Origin::signed(2), + 2, + vec![1, 3], + Some(now()), + OpaqueCall::from_encoded(data), + false, + 0 + ), Error::::MaxWeightTooLow, ); }); @@ -860,7 +937,7 @@ fn multisig_handles_no_preimage_after_all_approve() { 3, vec![1, 2], Some(now()), - data, + OpaqueCall::from_encoded(data), false, call_weight )); diff --git a/frame/multisig/src/weights.rs b/frame/multisig/src/weights.rs index 1bc72d251808..501fb9c18419 100644 --- a/frame/multisig/src/weights.rs +++ b/frame/multisig/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_multisig //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/multisig/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -61,18 +62,18 @@ pub trait WeightInfo { pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { fn as_multi_threshold_1(z: u32, ) -> Weight { - (19_405_000 as Weight) + (16_534_000 as Weight) // Standard Error: 0 .saturating_add((1_000 as Weight).saturating_mul(z as Weight)) } // Storage: Multisig Multisigs (r:1 w:1) // Storage: unknown [0x3a65787472696e7369635f696e646578] (r:1 w:0) fn as_multi_create(s: u32, z: u32, ) -> Weight { - (54_364_000 as Weight) - // Standard Error: 0 - .saturating_add((163_000 as Weight).saturating_mul(s as Weight)) + (30_917_000 as Weight) + // Standard Error: 1_000 + .saturating_add((131_000 as Weight).saturating_mul(s as Weight)) // Standard Error: 0 - .saturating_add((2_000 as Weight).saturating_mul(z as Weight)) + .saturating_add((1_000 as Weight).saturating_mul(z as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -80,32 +81,32 @@ impl WeightInfo for SubstrateWeight { // Storage: Multisig Calls (r:1 w:1) // Storage: unknown [0x3a65787472696e7369635f696e646578] (r:1 w:0) fn as_multi_create_store(s: u32, z: u32, ) -> Weight { - (59_545_000 as Weight) - // Standard Error: 0 - .saturating_add((168_000 as Weight).saturating_mul(s as Weight)) + (33_570_000 as Weight) + // Standard Error: 1_000 + .saturating_add((138_000 as Weight).saturating_mul(s as Weight)) // Standard Error: 0 - .saturating_add((3_000 as Weight).saturating_mul(z as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(z as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Multisig Multisigs (r:1 w:1) fn as_multi_approve(s: u32, z: u32, ) -> Weight { - (32_721_000 as Weight) + (21_051_000 as Weight) // Standard Error: 0 - .saturating_add((176_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((126_000 as Weight).saturating_mul(s as Weight)) // Standard Error: 0 - .saturating_add((2_000 as Weight).saturating_mul(z as Weight)) + .saturating_add((1_000 as Weight).saturating_mul(z as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Multisig Multisigs (r:1 w:1) // Storage: Multisig Calls (r:1 w:1) fn as_multi_approve_store(s: u32, z: u32, ) -> Weight { - (56_596_000 as Weight) + (32_581_000 as Weight) // Standard Error: 1_000 - .saturating_add((183_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((135_000 as Weight).saturating_mul(s as Weight)) // Standard Error: 0 - .saturating_add((3_000 as Weight).saturating_mul(z as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(z as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -113,29 +114,29 @@ impl WeightInfo for SubstrateWeight { // Storage: Multisig Calls (r:1 w:1) // Storage: System Account (r:1 w:1) fn as_multi_complete(s: u32, z: u32, ) -> Weight { - (72_391_000 as Weight) + (39_215_000 as Weight) // Standard Error: 1_000 - .saturating_add((268_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((221_000 as Weight).saturating_mul(s as Weight)) // Standard Error: 0 - .saturating_add((4_000 as Weight).saturating_mul(z as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(z as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Multisig Multisigs (r:1 w:1) // Storage: unknown [0x3a65787472696e7369635f696e646578] (r:1 w:0) fn approve_as_multi_create(s: u32, ) -> Weight { - (52_543_000 as Weight) - // Standard Error: 0 - .saturating_add((164_000 as Weight).saturating_mul(s as Weight)) + (28_572_000 as Weight) + // Standard Error: 1_000 + .saturating_add((146_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Multisig Multisigs (r:1 w:1) // Storage: Multisig Calls (r:1 w:0) fn approve_as_multi_approve(s: u32, ) -> Weight { - (30_764_000 as Weight) + (18_032_000 as Weight) // Standard Error: 0 - .saturating_add((180_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((151_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -143,18 +144,18 @@ impl WeightInfo for SubstrateWeight { // Storage: Multisig Calls (r:1 w:1) // Storage: System Account (r:1 w:1) fn approve_as_multi_complete(s: u32, ) -> Weight { - (113_631_000 as Weight) - // Standard Error: 3_000 - .saturating_add((283_000 as Weight).saturating_mul(s as Weight)) + (56_353_000 as Weight) + // Standard Error: 1_000 + .saturating_add((239_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Multisig Multisigs (r:1 w:1) // Storage: Multisig Calls (r:1 w:1) fn cancel_as_multi(s: u32, ) -> Weight { - (86_310_000 as Weight) + (44_544_000 as Weight) // Standard Error: 0 - .saturating_add((166_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((158_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -163,18 +164,18 @@ impl WeightInfo for SubstrateWeight { // For backwards compatibility and tests impl WeightInfo for () { fn as_multi_threshold_1(z: u32, ) -> Weight { - (19_405_000 as Weight) + (16_534_000 as Weight) // Standard Error: 0 .saturating_add((1_000 as Weight).saturating_mul(z as Weight)) } // Storage: Multisig Multisigs (r:1 w:1) // Storage: unknown [0x3a65787472696e7369635f696e646578] (r:1 w:0) fn as_multi_create(s: u32, z: u32, ) -> Weight { - (54_364_000 as Weight) - // Standard Error: 0 - .saturating_add((163_000 as Weight).saturating_mul(s as Weight)) + (30_917_000 as Weight) + // Standard Error: 1_000 + .saturating_add((131_000 as Weight).saturating_mul(s as Weight)) // Standard Error: 0 - .saturating_add((2_000 as Weight).saturating_mul(z as Weight)) + .saturating_add((1_000 as Weight).saturating_mul(z as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -182,32 +183,32 @@ impl WeightInfo for () { // Storage: Multisig Calls (r:1 w:1) // Storage: unknown [0x3a65787472696e7369635f696e646578] (r:1 w:0) fn as_multi_create_store(s: u32, z: u32, ) -> Weight { - (59_545_000 as Weight) - // Standard Error: 0 - .saturating_add((168_000 as Weight).saturating_mul(s as Weight)) + (33_570_000 as Weight) + // Standard Error: 1_000 + .saturating_add((138_000 as Weight).saturating_mul(s as Weight)) // Standard Error: 0 - .saturating_add((3_000 as Weight).saturating_mul(z as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(z as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Multisig Multisigs (r:1 w:1) fn as_multi_approve(s: u32, z: u32, ) -> Weight { - (32_721_000 as Weight) + (21_051_000 as Weight) // Standard Error: 0 - .saturating_add((176_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((126_000 as Weight).saturating_mul(s as Weight)) // Standard Error: 0 - .saturating_add((2_000 as Weight).saturating_mul(z as Weight)) + .saturating_add((1_000 as Weight).saturating_mul(z as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Multisig Multisigs (r:1 w:1) // Storage: Multisig Calls (r:1 w:1) fn as_multi_approve_store(s: u32, z: u32, ) -> Weight { - (56_596_000 as Weight) + (32_581_000 as Weight) // Standard Error: 1_000 - .saturating_add((183_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((135_000 as Weight).saturating_mul(s as Weight)) // Standard Error: 0 - .saturating_add((3_000 as Weight).saturating_mul(z as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(z as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -215,29 +216,29 @@ impl WeightInfo for () { // Storage: Multisig Calls (r:1 w:1) // Storage: System Account (r:1 w:1) fn as_multi_complete(s: u32, z: u32, ) -> Weight { - (72_391_000 as Weight) + (39_215_000 as Weight) // Standard Error: 1_000 - .saturating_add((268_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((221_000 as Weight).saturating_mul(s as Weight)) // Standard Error: 0 - .saturating_add((4_000 as Weight).saturating_mul(z as Weight)) + .saturating_add((2_000 as Weight).saturating_mul(z as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Multisig Multisigs (r:1 w:1) // Storage: unknown [0x3a65787472696e7369635f696e646578] (r:1 w:0) fn approve_as_multi_create(s: u32, ) -> Weight { - (52_543_000 as Weight) - // Standard Error: 0 - .saturating_add((164_000 as Weight).saturating_mul(s as Weight)) + (28_572_000 as Weight) + // Standard Error: 1_000 + .saturating_add((146_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Multisig Multisigs (r:1 w:1) // Storage: Multisig Calls (r:1 w:0) fn approve_as_multi_approve(s: u32, ) -> Weight { - (30_764_000 as Weight) + (18_032_000 as Weight) // Standard Error: 0 - .saturating_add((180_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((151_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -245,18 +246,18 @@ impl WeightInfo for () { // Storage: Multisig Calls (r:1 w:1) // Storage: System Account (r:1 w:1) fn approve_as_multi_complete(s: u32, ) -> Weight { - (113_631_000 as Weight) - // Standard Error: 3_000 - .saturating_add((283_000 as Weight).saturating_mul(s as Weight)) + (56_353_000 as Weight) + // Standard Error: 1_000 + .saturating_add((239_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Multisig Multisigs (r:1 w:1) // Storage: Multisig Calls (r:1 w:1) fn cancel_as_multi(s: u32, ) -> Weight { - (86_310_000 as Weight) + (44_544_000 as Weight) // Standard Error: 0 - .saturating_add((166_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((158_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } diff --git a/frame/nicks/Cargo.toml b/frame/nicks/Cargo.toml index 431ee2c84157..9e98864d24f9 100644 --- a/frame/nicks/Cargo.toml +++ b/frame/nicks/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-nicks" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for nick management" readme = "README.md" @@ -13,16 +13,16 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } [features] diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs index 16c7e2042dda..632ff7b0a808 100644 --- a/frame/nicks/src/lib.rs +++ b/frame/nicks/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -35,9 +35,6 @@ //! taken. //! * `clear_name` - Remove an account's associated name; the deposit is returned. //! * `kill_name` - Forcibly remove the associated name; the deposit is lost. -//! -//! [`Call`]: ./enum.Call.html -//! [`Config`]: ./trait.Config.html #![cfg_attr(not(feature = "std"), no_std)] @@ -46,21 +43,16 @@ pub use pallet::*; use sp_runtime::traits::{StaticLookup, Zero}; use sp_std::prelude::*; -type BalanceOf = - <::Currency as Currency<::AccountId>>::Balance; -type NegativeImbalanceOf = <::Currency as Currency< - ::AccountId, ->>::NegativeImbalance; +type AccountIdOf = ::AccountId; +type BalanceOf = <::Currency as Currency>>::Balance; +type NegativeImbalanceOf = + <::Currency as Currency>>::NegativeImbalance; #[frame_support::pallet] pub mod pallet { use super::*; - use frame_support::{ - ensure, - pallet_prelude::*, - traits::{EnsureOrigin, Get}, - }; - use frame_system::{ensure_signed, pallet_prelude::*}; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; #[pallet::config] pub trait Config: frame_system::Config { @@ -92,16 +84,16 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// A name was set. \[who\] - NameSet(T::AccountId), - /// A name was forcibly set. \[target\] - NameForced(T::AccountId), - /// A name was changed. \[who\] - NameChanged(T::AccountId), - /// A name was cleared, and the given balance returned. \[who, deposit\] - NameCleared(T::AccountId, BalanceOf), - /// A name was removed and the given balance slashed. \[target, deposit\] - NameKilled(T::AccountId, BalanceOf), + /// A name was set. + NameSet { who: T::AccountId }, + /// A name was forcibly set. + NameForced { target: T::AccountId }, + /// A name was changed. + NameChanged { who: T::AccountId }, + /// A name was cleared, and the given balance returned. + NameCleared { who: T::AccountId, deposit: BalanceOf }, + /// A name was removed and the given balance slashed. + NameKilled { target: T::AccountId, deposit: BalanceOf }, } /// Error for the nicks pallet. @@ -118,7 +110,7 @@ pub mod pallet { /// The lookup table for names. #[pallet::storage] pub(super) type NameOf = - StorageMap<_, Twox64Concat, T::AccountId, (Vec, BalanceOf)>; + StorageMap<_, Twox64Concat, T::AccountId, (BoundedVec, BalanceOf)>; #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] @@ -146,20 +138,21 @@ pub mod pallet { pub fn set_name(origin: OriginFor, name: Vec) -> DispatchResult { let sender = ensure_signed(origin)?; - ensure!(name.len() >= T::MinLength::get() as usize, Error::::TooShort); - ensure!(name.len() <= T::MaxLength::get() as usize, Error::::TooLong); + let bounded_name: BoundedVec<_, _> = + name.try_into().map_err(|()| Error::::TooLong)?; + ensure!(bounded_name.len() >= T::MinLength::get() as usize, Error::::TooShort); let deposit = if let Some((_, deposit)) = >::get(&sender) { - Self::deposit_event(Event::::NameChanged(sender.clone())); + Self::deposit_event(Event::::NameChanged { who: sender.clone() }); deposit } else { let deposit = T::ReservationFee::get(); T::Currency::reserve(&sender, deposit.clone())?; - Self::deposit_event(Event::::NameSet(sender.clone())); + Self::deposit_event(Event::::NameSet { who: sender.clone() }); deposit }; - >::insert(&sender, (name, deposit)); + >::insert(&sender, (bounded_name, deposit)); Ok(()) } @@ -182,13 +175,13 @@ pub mod pallet { let err_amount = T::Currency::unreserve(&sender, deposit.clone()); debug_assert!(err_amount.is_zero()); - Self::deposit_event(Event::::NameCleared(sender, deposit)); + Self::deposit_event(Event::::NameCleared { who: sender, deposit }); Ok(()) } /// Remove an account's name and take charge of the deposit. /// - /// Fails if `who` has not been named. The deposit is dealt with through `T::Slashed` + /// Fails if `target` has not been named. The deposit is dealt with through `T::Slashed` /// imbalance handler. /// /// The dispatch origin for this call must match `T::ForceOrigin`. @@ -213,7 +206,7 @@ pub mod pallet { // Slash their deposit from them. T::Slashed::on_unbalanced(T::Currency::slash_reserved(&target, deposit.clone()).0); - Self::deposit_event(Event::::NameKilled(target, deposit)); + Self::deposit_event(Event::::NameKilled { target, deposit }); Ok(()) } @@ -237,11 +230,13 @@ pub mod pallet { ) -> DispatchResult { T::ForceOrigin::ensure_origin(origin)?; + let bounded_name: BoundedVec<_, _> = + name.try_into().map_err(|()| Error::::TooLong)?; let target = T::Lookup::lookup(target)?; let deposit = >::get(&target).map(|x| x.1).unwrap_or_else(Zero::zero); - >::insert(&target, (name, deposit)); + >::insert(&target, (bounded_name, deposit)); - Self::deposit_event(Event::::NameForced(target)); + Self::deposit_event(Event::::NameForced { target }); Ok(()) } } @@ -252,7 +247,10 @@ mod tests { use super::*; use crate as pallet_nicks; - use frame_support::{assert_noop, assert_ok, ord_parameter_types, parameter_types}; + use frame_support::{ + assert_noop, assert_ok, ord_parameter_types, parameter_types, + traits::{ConstU32, ConstU64}, + }; use frame_system::EnsureSignedBy; use sp_core::H256; use sp_runtime::{ @@ -269,14 +267,13 @@ mod tests { NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic, { - System: frame_system::{Pallet, Call, Config, Storage, Event}, - Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, - Nicks: pallet_nicks::{Pallet, Call, Storage, Event}, + System: frame_system, + Balances: pallet_balances, + Nicks: pallet_nicks, } ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -295,7 +292,7 @@ mod tests { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -304,10 +301,9 @@ mod tests { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } - parameter_types! { - pub const ExistentialDeposit: u64 = 1; - } + impl pallet_balances::Config for Test { type MaxLocks = (); type MaxReserves = (); @@ -315,26 +311,22 @@ mod tests { type Balance = u64; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } - parameter_types! { - pub const ReservationFee: u64 = 2; - pub const MinLength: u32 = 3; - pub const MaxLength: u32 = 16; - } + ord_parameter_types! { pub const One: u64 = 1; } impl Config for Test { type Event = Event; type Currency = Balances; - type ReservationFee = ReservationFee; + type ReservationFee = ConstU64<2>; type Slashed = (); type ForceOrigin = EnsureSignedBy; - type MinLength = MinLength; - type MaxLength = MaxLength; + type MinLength = ConstU32<3>; + type MaxLength = ConstU32<16>; } fn new_test_ext() -> sp_io::TestExternalities { @@ -366,9 +358,15 @@ mod tests { assert_ok!(Nicks::set_name(Origin::signed(2), b"Dave".to_vec())); assert_eq!(Balances::reserved_balance(2), 2); - assert_ok!(Nicks::force_name(Origin::signed(1), 2, b"Dr. David Brubeck, III".to_vec())); + assert_noop!( + Nicks::force_name(Origin::signed(1), 2, b"Dr. David Brubeck, III".to_vec()), + Error::::TooLong, + ); + assert_ok!(Nicks::force_name(Origin::signed(1), 2, b"Dr. Brubeck, III".to_vec())); assert_eq!(Balances::reserved_balance(2), 2); - assert_eq!(>::get(2).unwrap(), (b"Dr. David Brubeck, III".to_vec(), 2)); + let (name, amount) = >::get(2).unwrap(); + assert_eq!(name, b"Dr. Brubeck, III".to_vec()); + assert_eq!(amount, 2); }); } diff --git a/frame/node-authorization/Cargo.toml b/frame/node-authorization/Cargo.toml index 635e72e3a8b8..6ae1f21336b6 100644 --- a/frame/node-authorization/Cargo.toml +++ b/frame/node-authorization/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-node-authorization" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for node authorization" @@ -12,14 +12,14 @@ description = "FRAME pallet for node authorization" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } log = { version = "0.4.14", default-features = false } [features] diff --git a/frame/node-authorization/src/lib.rs b/frame/node-authorization/src/lib.rs index 016f12d2eb83..07f2e9de37dd 100644 --- a/frame/node-authorization/src/lib.rs +++ b/frame/node-authorization/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -52,11 +52,12 @@ pub use weights::WeightInfo; #[frame_support::pallet] pub mod pallet { use super::*; - use frame_support::{dispatch::DispatchResult, pallet_prelude::*}; + use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); /// The module configuration trait @@ -128,24 +129,24 @@ pub mod pallet { #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { /// The given well known node was added. - NodeAdded(PeerId, T::AccountId), + NodeAdded { peer_id: PeerId, who: T::AccountId }, /// The given well known node was removed. - NodeRemoved(PeerId), + NodeRemoved { peer_id: PeerId }, /// The given well known node was swapped; first item was removed, /// the latter was added. - NodeSwapped(PeerId, PeerId), + NodeSwapped { removed: PeerId, added: PeerId }, /// The given well known nodes were reset. - NodesReset(Vec<(PeerId, T::AccountId)>), + NodesReset { nodes: Vec<(PeerId, T::AccountId)> }, /// The given node was claimed by a user. - NodeClaimed(PeerId, T::AccountId), + NodeClaimed { peer_id: PeerId, who: T::AccountId }, /// The given claim was removed by its owner. - ClaimRemoved(PeerId, T::AccountId), + ClaimRemoved { peer_id: PeerId, who: T::AccountId }, /// The node was transferred to another account. - NodeTransferred(PeerId, T::AccountId), + NodeTransferred { peer_id: PeerId, target: T::AccountId }, /// The allowed connections were added to a node. - ConnectionsAdded(PeerId, Vec), + ConnectionsAdded { peer_id: PeerId, allowed_connections: Vec }, /// The allowed connections were removed from a node. - ConnectionsRemoved(PeerId, Vec), + ConnectionsRemoved { peer_id: PeerId, allowed_connections: Vec }, } #[pallet::error] @@ -224,7 +225,7 @@ pub mod pallet { WellKnownNodes::::put(&nodes); >::insert(&node, &owner); - Self::deposit_event(Event::NodeAdded(node, owner)); + Self::deposit_event(Event::NodeAdded { peer_id: node, who: owner }); Ok(()) } @@ -248,7 +249,7 @@ pub mod pallet { >::remove(&node); AdditionalConnections::::remove(&node); - Self::deposit_event(Event::NodeRemoved(node)); + Self::deposit_event(Event::NodeRemoved { peer_id: node }); Ok(()) } @@ -284,7 +285,7 @@ pub mod pallet { Owners::::swap(&remove, &add); AdditionalConnections::::swap(&remove, &add); - Self::deposit_event(Event::NodeSwapped(remove, add)); + Self::deposit_event(Event::NodeSwapped { removed: remove, added: add }); Ok(()) } @@ -305,7 +306,7 @@ pub mod pallet { Self::initialize_nodes(&nodes); - Self::deposit_event(Event::NodesReset(nodes)); + Self::deposit_event(Event::NodesReset { nodes }); Ok(()) } @@ -321,7 +322,7 @@ pub mod pallet { ensure!(!Owners::::contains_key(&node), Error::::AlreadyClaimed); Owners::::insert(&node, &sender); - Self::deposit_event(Event::NodeClaimed(node, sender)); + Self::deposit_event(Event::NodeClaimed { peer_id: node, who: sender }); Ok(()) } @@ -342,7 +343,7 @@ pub mod pallet { Owners::::remove(&node); AdditionalConnections::::remove(&node); - Self::deposit_event(Event::ClaimRemoved(node, sender)); + Self::deposit_event(Event::ClaimRemoved { peer_id: node, who: sender }); Ok(()) } @@ -364,7 +365,7 @@ pub mod pallet { Owners::::insert(&node, &owner); - Self::deposit_event(Event::NodeTransferred(node, owner)); + Self::deposit_event(Event::NodeTransferred { peer_id: node, target: owner }); Ok(()) } @@ -395,7 +396,10 @@ pub mod pallet { AdditionalConnections::::insert(&node, nodes); - Self::deposit_event(Event::ConnectionsAdded(node, connections)); + Self::deposit_event(Event::ConnectionsAdded { + peer_id: node, + allowed_connections: connections, + }); Ok(()) } @@ -423,7 +427,10 @@ pub mod pallet { AdditionalConnections::::insert(&node, nodes); - Self::deposit_event(Event::ConnectionsRemoved(node, connections)); + Self::deposit_event(Event::ConnectionsRemoved { + peer_id: node, + allowed_connections: connections, + }); Ok(()) } } diff --git a/frame/node-authorization/src/mock.rs b/frame/node-authorization/src/mock.rs index 6c79f601c197..d959d1b8610f 100644 --- a/frame/node-authorization/src/mock.rs +++ b/frame/node-authorization/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,10 @@ use super::*; use crate as pallet_node_authorization; -use frame_support::{ord_parameter_types, parameter_types, traits::GenesisBuild}; +use frame_support::{ + ord_parameter_types, + traits::{ConstU32, ConstU64, GenesisBuild}, +}; use frame_system::EnsureSignedBy; use sp_core::H256; use sp_runtime::{ @@ -44,9 +47,6 @@ frame_support::construct_runtime!( } ); -parameter_types! { - pub const BlockHashCount: u64 = 250; -} impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; type DbWeight = (); @@ -62,7 +62,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -71,6 +71,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } ord_parameter_types! { @@ -79,14 +80,11 @@ ord_parameter_types! { pub const Three: u64 = 3; pub const Four: u64 = 4; } -parameter_types! { - pub const MaxWellKnownNodes: u32 = 4; - pub const MaxPeerIdLength: u32 = 2; -} + impl Config for Test { type Event = Event; - type MaxWellKnownNodes = MaxWellKnownNodes; - type MaxPeerIdLength = MaxPeerIdLength; + type MaxWellKnownNodes = ConstU32<4>; + type MaxPeerIdLength = ConstU32<2>; type AddOrigin = EnsureSignedBy; type RemoveOrigin = EnsureSignedBy; type SwapOrigin = EnsureSignedBy; diff --git a/frame/node-authorization/src/tests.rs b/frame/node-authorization/src/tests.rs index 530904fa7348..ba78d1491213 100644 --- a/frame/node-authorization/src/tests.rs +++ b/frame/node-authorization/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/node-authorization/src/weights.rs b/frame/node-authorization/src/weights.rs index dbb7956cff96..cf182f94273c 100644 --- a/frame/node-authorization/src/weights.rs +++ b/frame/node-authorization/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/offences/Cargo.toml b/frame/offences/Cargo.toml index 8fdcbf46fa3e..01409d78053c 100644 --- a/frame/offences/Cargo.toml +++ b/frame/offences/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-offences" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME offences pallet" readme = "README.md" @@ -14,19 +14,19 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../balances" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -serde = { version = "1.0.126", optional = true } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +serde = { version = "1.0.136", optional = true } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } log = { version = "0.4.14", default-features = false } [dev-dependencies] -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } [features] default = ["std"] diff --git a/frame/offences/benchmarking/Cargo.toml b/frame/offences/benchmarking/Cargo.toml index b21e6cf9b7e1..605e0c60a03c 100644 --- a/frame/offences/benchmarking/Cargo.toml +++ b/frame/offences/benchmarking/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-offences-benchmarking" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME offences pallet benchmarking" readme = "README.md" @@ -13,8 +13,8 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../../benchmarking" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" } @@ -29,16 +29,16 @@ pallet-session = { version = "4.0.0-dev", default-features = false, path = "../. pallet-staking = { version = "4.0.0-dev", default-features = false, features = [ "runtime-benchmarks", ], path = "../../staking" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/staking" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/std" } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../../election-provider-support" } [dev-dependencies] pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../../staking/reward-curve" } pallet-timestamp = { version = "4.0.0-dev", path = "../../timestamp" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../../primitives/io" } [features] default = ["std"] diff --git a/frame/offences/benchmarking/src/lib.rs b/frame/offences/benchmarking/src/lib.rs index 35e3c1aec940..8688090206c3 100644 --- a/frame/offences/benchmarking/src/lib.rs +++ b/frame/offences/benchmarking/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,8 +23,8 @@ mod mock; use sp_std::{prelude::*, vec}; -use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite}; -use frame_support::traits::{Currency, ValidatorSet, ValidatorSetWithIdentification}; +use frame_benchmarking::{account, benchmarks}; +use frame_support::traits::{Currency, Get, ValidatorSet, ValidatorSetWithIdentification}; use frame_system::{Config as SystemConfig, Pallet as System, RawOrigin}; use sp_runtime::{ @@ -224,27 +224,49 @@ fn check_events::Event>>(expec .map(|frame_system::EventRecord { event, .. }| event) .collect::>(); let expected = expected.collect::>(); - let lengths = (events.len(), expected.len()); - let length_mismatch = if lengths.0 != lengths.1 { - fn pretty(header: &str, ev: &[D]) { - println!("{}", header); - for (idx, ev) in ev.iter().enumerate() { - println!("\t[{:04}] {:?}", idx, ev); - } + + fn pretty(header: &str, ev: &[D], offset: usize) { + println!("{}", header); + for (idx, ev) in ev.iter().enumerate() { + println!("\t[{:04}] {:?}", idx + offset, ev); } - pretty("--Got:", &events); - pretty("--Expected:", &expected); - format!("Mismatching length. Got: {}, expected: {}", lengths.0, lengths.1) - } else { - Default::default() - }; + } + fn print_events(idx: usize, events: &[D], expected: &[D]) { + let window = 10; + let start = idx.saturating_sub(window / 2); + let end_got = (idx + window / 2).min(events.len()); + pretty("Got(window):", &events[start..end_got], start); + let end_expected = (idx + window / 2).min(expected.len()); + pretty("Expected(window):", &expected[start..end_expected], start); + println!("---------------"); + let start_got = events.len().saturating_sub(window); + pretty("Got(end):", &events[start_got..], start_got); + let start_expected = expected.len().saturating_sub(window); + pretty("Expected(end):", &expected[start_expected..], start_expected); + } + let events_copy = events.clone(); + let expected_copy = expected.clone(); for (idx, (a, b)) in events.into_iter().zip(expected).enumerate() { - assert_eq!(a, b, "Mismatch at: {}. {}", idx, length_mismatch); + if a != b { + print_events(idx, &events_copy, &expected_copy); + println!("Mismatch at: {}", idx); + println!(" Got: {:?}", b); + println!("Expected: {:?}", a); + if events_copy.len() != expected_copy.len() { + println!( + "Mismatching lengths. Got: {}, Expected: {}", + events_copy.len(), + expected_copy.len() + ) + } + panic!("Mismatching events."); + } } - if !length_mismatch.is_empty() { - panic!("{}", length_mismatch); + if events_copy.len() != expected_copy.len() { + print_events(0, &events_copy, &expected_copy); + panic!("Mismatching lengths. Got: {}, Expected: {}", events_copy.len(), expected_copy.len()) } } @@ -253,7 +275,7 @@ benchmarks! { let r in 1 .. MAX_REPORTERS; // we skip 1 offender, because in such case there is no slashing let o in 2 .. MAX_OFFENDERS; - let n in 0 .. MAX_NOMINATORS.min(::MAX_NOMINATIONS); + let n in 0 .. MAX_NOMINATORS.min(::MaxNominations::get()); // Make r reporters let mut reporters = vec![]; @@ -288,50 +310,78 @@ benchmarks! { let bond_amount: u32 = UniqueSaturatedInto::::unique_saturated_into(bond_amount::()); let slash_amount = slash_fraction * bond_amount; let reward_amount = slash_amount * (1 + n) / 2; + let reward = reward_amount / r; let slash = |id| core::iter::once( ::Event::from(StakingEvent::::Slashed(id, BalanceOf::::from(slash_amount))) ); + let balance_slash = |id| core::iter::once( + ::Event::from(pallet_balances::Event::::Slashed{who: id, amount: slash_amount.into()}) + ); let chill = |id| core::iter::once( ::Event::from(StakingEvent::::Chilled(id)) ); - let mut slash_events = raw_offenders.into_iter() + let balance_deposit = |id, amount: u32| + ::Event::from(pallet_balances::Event::::Deposit{who: id, amount: amount.into()}); + let mut first = true; + let slash_events = raw_offenders.into_iter() .flat_map(|offender| { - let nom_slashes = offender.nominator_stashes.into_iter().flat_map(|nom| slash(nom)); - chill(offender.stash.clone()) - .chain(slash(offender.stash)) - .chain(nom_slashes) + let nom_slashes = offender.nominator_stashes.into_iter().flat_map(|nom| { + balance_slash(nom.clone()).map(Into::into) + .chain(slash(nom.clone()).map(Into::into)) + }).collect::>(); + + let mut events = chill(offender.stash.clone()).map(Into::into) + .chain(balance_slash(offender.stash.clone()).map(Into::into)) + .chain(slash(offender.stash.clone()).map(Into::into)) + .chain(nom_slashes.into_iter()) + .collect::>(); + + // the first deposit creates endowed events, see `endowed_reward_events` + if first { + first = false; + let mut reward_events = reporters.clone().into_iter() + .flat_map(|reporter| vec![ + balance_deposit(reporter.clone(), reward.into()).into(), + frame_system::Event::::NewAccount { account: reporter.clone() }.into(), + ::Event::from( + pallet_balances::Event::::Endowed{account: reporter.clone(), free_balance: reward.into()} + ).into(), + ]) + .collect::>(); + events.append(&mut reward_events); + events.into_iter() + } else { + let mut reward_events = reporters.clone().into_iter() + .map(|reporter| balance_deposit(reporter, reward.into()).into()) + .collect::>(); + events.append(&mut reward_events); + events.into_iter() + } }) .collect::>(); - let reward_events = reporters.into_iter() - .flat_map(|reporter| vec![ - frame_system::Event::::NewAccount(reporter.clone()).into(), - ::Event::from( - pallet_balances::Event::::Endowed(reporter, (reward_amount / r).into()) - ).into() - ]); - - // Rewards are applied after first offender and it's nominators. - // We split after: offender slash + offender chill + nominator slashes. - let slash_rest = slash_events.split_off(2 + n as usize); - // make sure that all slashes have been applied + + #[cfg(test)] - check_events::( - std::iter::empty() - .chain(slash_events.into_iter().map(Into::into)) - .chain(reward_events) - .chain(slash_rest.into_iter().map(Into::into)) - .chain(std::iter::once(::Event::from( - pallet_offences::Event::Offence( - UnresponsivenessOffence::::ID, - 0_u32.to_le_bytes().to_vec(), - ) - ).into())) - ); + { + // In case of error it's useful to see the inputs + println!("Inputs: r: {}, o: {}, n: {}", r, o, n); + // make sure that all slashes have been applied + check_events::( + std::iter::empty() + .chain(slash_events.into_iter().map(Into::into)) + .chain(std::iter::once(::Event::from( + pallet_offences::Event::Offence{ + kind: UnresponsivenessOffence::::ID, + timeslot: 0_u32.to_le_bytes().to_vec(), + } + ).into())) + ); + } } report_offence_grandpa { - let n in 0 .. MAX_NOMINATORS.min(::MAX_NOMINATIONS); + let n in 0 .. MAX_NOMINATORS.min(::MaxNominations::get()); // for grandpa equivocation reports the number of reporters // and offenders is always 1 @@ -355,18 +405,19 @@ benchmarks! { } verify { // make sure that all slashes have been applied + #[cfg(test)] assert_eq!( System::::event_count(), 0 + 1 // offence - + 2 // reporter (reward + endowment) - + 1 // offenders slashed + + 3 // reporter (reward + endowment) + + 2 // offenders slashed + 1 // offenders chilled - + n // nominators slashed + + 2 * n // nominators slashed ); } report_offence_babe { - let n in 0 .. MAX_NOMINATORS.min(::MAX_NOMINATIONS); + let n in 0 .. MAX_NOMINATORS.min(::MaxNominations::get()); // for babe equivocation reports the number of reporters // and offenders is always 1 @@ -390,15 +441,16 @@ benchmarks! { } verify { // make sure that all slashes have been applied + #[cfg(test)] assert_eq!( System::::event_count(), 0 + 1 // offence - + 2 // reporter (reward + endowment) - + 1 // offenders slashed + + 3 // reporter (reward + endowment) + + 2 // offenders slashed + 1 // offenders chilled - + n // nominators slashed + + 2 * n // nominators slashed ); } -} -impl_benchmark_test_suite!(Pallet, crate::mock::new_test_ext(), crate::mock::Test); + impl_benchmark_test_suite!(Pallet, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs index 6973e25371d4..1a4414de0b0b 100644 --- a/frame/offences/benchmarking/src/mock.rs +++ b/frame/offences/benchmarking/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,8 +20,12 @@ #![cfg(test)] use super::*; -use frame_election_provider_support::onchain; -use frame_support::{parameter_types, weights::constants::WEIGHT_PER_SECOND}; +use frame_election_provider_support::{onchain, SequentialPhragmen}; +use frame_support::{ + parameter_types, + traits::{ConstU32, ConstU64}, + weights::constants::WEIGHT_PER_SECOND, +}; use frame_system as system; use pallet_session::historical as pallet_session_historical; use sp_runtime::{ @@ -63,6 +67,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } parameter_types! { pub const ExistentialDeposit: Balance = 10; @@ -79,13 +84,10 @@ impl pallet_balances::Config for Test { type WeightInfo = (); } -parameter_types! { - pub const MinimumPeriod: u64 = 5; -} impl pallet_timestamp::Config for Test { type Moment = u64; type OnTimestampSet = (); - type MinimumPeriod = MinimumPeriod; + type MinimumPeriod = ConstU64<5>; type WeightInfo = (); } impl pallet_session::historical::Config for Test { @@ -112,7 +114,7 @@ impl pallet_session::SessionHandler for TestSessionHandler { ) { } - fn on_disabled(_: usize) {} + fn on_disabled(_: u32) {} } parameter_types! { @@ -129,7 +131,6 @@ impl pallet_session::Config for Test { type Event = Event; type ValidatorId = AccountId; type ValidatorIdOf = pallet_staking::StashOf; - type DisabledValidatorsThreshold = (); type WeightInfo = (); } @@ -145,21 +146,19 @@ pallet_staking_reward_curve::build! { } parameter_types! { pub const RewardCurve: &'static sp_runtime::curve::PiecewiseLinear<'static> = &I_NPOS; - pub const MaxNominatorRewardedPerValidator: u32 = 64; - pub const MaxKeys: u32 = 10_000; - pub const MaxPeerInHeartbeats: u32 = 10_000; - pub const MaxPeerDataEncodingSize: u32 = 1_000; } pub type Extrinsic = sp_runtime::testing::TestXt; -impl onchain::Config for Test { - type Accuracy = Perbill; +pub struct OnChainSeqPhragmen; +impl onchain::ExecutionConfig for OnChainSeqPhragmen { + type System = Test; + type Solver = SequentialPhragmen; type DataProvider = Staking; } impl pallet_staking::Config for Test { - const MAX_NOMINATIONS: u32 = 16; + type MaxNominations = ConstU32<16>; type Currency = Balances; type UnixTime = pallet_timestamp::Pallet; type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; @@ -174,10 +173,13 @@ impl pallet_staking::Config for Test { type SessionInterface = Self; type EraPayout = pallet_staking::ConvertCurve; type NextNewSession = Session; - type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; - type ElectionProvider = onchain::OnChainSequentialPhragmen; + type MaxNominatorRewardedPerValidator = ConstU32<64>; + type OffendingValidatorsThreshold = (); + type ElectionProvider = onchain::UnboundedExecution; type GenesisElectionProvider = Self::ElectionProvider; - type SortedListProvider = pallet_staking::UseNominatorsMap; + type VoterList = pallet_staking::UseNominatorsAndValidatorsMap; + type MaxUnlockingChunks = ConstU32<32>; + type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig; type WeightInfo = (); } @@ -189,9 +191,9 @@ impl pallet_im_online::Config for Test { type ReportUnresponsiveness = Offences; type UnsignedPriority = (); type WeightInfo = (); - type MaxKeys = MaxKeys; - type MaxPeerInHeartbeats = MaxPeerInHeartbeats; - type MaxPeerDataEncodingSize = MaxPeerDataEncodingSize; + type MaxKeys = ConstU32<10_000>; + type MaxPeerInHeartbeats = ConstU32<10_000>; + type MaxPeerDataEncodingSize = ConstU32<1_000>; } impl pallet_offences::Config for Test { diff --git a/frame/offences/src/lib.rs b/frame/offences/src/lib.rs index 3392cd6e4a88..6119fce7769e 100644 --- a/frame/offences/src/lib.rs +++ b/frame/offences/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,7 @@ // Ensure we're `no_std` when compiling for Wasm. #![cfg_attr(not(feature = "std"), no_std)] -mod migration; +pub mod migration; mod mock; mod tests; @@ -43,36 +43,14 @@ type OpaqueTimeSlot = Vec; /// A type alias for a report identifier. type ReportIdOf = ::Hash; -pub trait WeightInfo { - fn report_offence_im_online(r: u32, o: u32, n: u32) -> Weight; - fn report_offence_grandpa(r: u32, n: u32) -> Weight; - fn report_offence_babe(r: u32, n: u32) -> Weight; - fn on_initialize(d: u32) -> Weight; -} - -impl WeightInfo for () { - fn report_offence_im_online(_r: u32, _o: u32, _n: u32) -> Weight { - 1_000_000_000 - } - fn report_offence_grandpa(_r: u32, _n: u32) -> Weight { - 1_000_000_000 - } - fn report_offence_babe(_r: u32, _n: u32) -> Weight { - 1_000_000_000 - } - fn on_initialize(_d: u32) -> Weight { - 1_000_000_000 - } -} - #[frame_support::pallet] pub mod pallet { use super::*; use frame_support::pallet_prelude::*; - use frame_system::pallet_prelude::*; #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); /// The pallet's config trait. @@ -130,14 +108,7 @@ pub mod pallet { /// There is an offence reported of the given `kind` happened at the `session_index` and /// (kind-specific) time slot. This event is not deposited for duplicate slashes. /// \[kind, timeslot\]. - Offence(Kind, OpaqueTimeSlot), - } - - #[pallet::hooks] - impl Hooks> for Pallet { - fn on_runtime_upgrade() -> Weight { - migration::remove_deferred_storage::() - } + Offence { kind: Kind, timeslot: OpaqueTimeSlot }, } } @@ -172,10 +143,11 @@ where &concurrent_offenders, &slash_perbill, offence.session_index(), + offence.disable_strategy(), ); // Deposit the event. - Self::deposit_event(Event::Offence(O::ID, time_slot.encode())); + Self::deposit_event(Event::Offence { kind: O::ID, timeslot: time_slot.encode() }); Ok(()) } diff --git a/frame/offences/src/migration.rs b/frame/offences/src/migration.rs index b6e32cbe69e2..72c24ef7c3e5 100644 --- a/frame/offences/src/migration.rs +++ b/frame/offences/src/migration.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,7 @@ use super::{Config, OffenceDetails, Perbill, SessionIndex}; use frame_support::{ generate_storage_alias, pallet_prelude::ValueQuery, traits::Get, weights::Weight, }; -use sp_staking::offence::OnOffenceHandler; +use sp_staking::offence::{DisableStrategy, OnOffenceHandler}; use sp_std::vec::Vec; /// Type of data stored as a deferred offence @@ -41,7 +41,12 @@ pub fn remove_deferred_storage() -> Weight { let deferred = >::take(); log::info!(target: "runtime::offences", "have {} deferred offences, applying.", deferred.len()); for (offences, perbill, session) in deferred.iter() { - let consumed = T::OnOffenceHandler::on_offence(&offences, &perbill, *session); + let consumed = T::OnOffenceHandler::on_offence( + &offences, + &perbill, + *session, + DisableStrategy::WhenSlashed, + ); weight = weight.saturating_add(consumed); } @@ -51,8 +56,7 @@ pub fn remove_deferred_storage() -> Weight { #[cfg(test)] mod test { use super::*; - use crate::mock::{new_test_ext, with_on_offence_fractions, Offences, Runtime as T}; - use frame_support::traits::OnRuntimeUpgrade; + use crate::mock::{new_test_ext, with_on_offence_fractions, Runtime as T}; use sp_runtime::Perbill; use sp_staking::offence::OffenceDetails; @@ -82,7 +86,7 @@ mod test { // when assert_eq!( - Offences::on_runtime_upgrade(), + remove_deferred_storage::(), ::DbWeight::get().reads_writes(1, 1), ); diff --git a/frame/offences/src/mock.rs b/frame/offences/src/mock.rs index 5e4c94944b6f..b3dfbdd90b19 100644 --- a/frame/offences/src/mock.rs +++ b/frame/offences/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,6 +24,7 @@ use crate::Config; use codec::Encode; use frame_support::{ parameter_types, + traits::{ConstU32, ConstU64}, weights::{ constants::{RocksDbWeight, WEIGHT_PER_SECOND}, Weight, @@ -36,7 +37,7 @@ use sp_runtime::{ Perbill, }; use sp_staking::{ - offence::{self, Kind, OffenceDetails}, + offence::{self, DisableStrategy, Kind, OffenceDetails}, SessionIndex, }; use std::cell::RefCell; @@ -55,6 +56,7 @@ impl offence::OnOffenceHandler _offenders: &[OffenceDetails], slash_fraction: &[Perbill], _offence_session: SessionIndex, + _disable_strategy: DisableStrategy, ) -> Weight { ON_OFFENCE_PERBILL.with(|f| { *f.borrow_mut() = slash_fraction.to_vec(); @@ -83,7 +85,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(2 * WEIGHT_PER_SECOND); } @@ -102,7 +103,7 @@ impl frame_system::Config for Runtime { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -111,6 +112,7 @@ impl frame_system::Config for Runtime { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl Config for Runtime { diff --git a/frame/offences/src/tests.rs b/frame/offences/src/tests.rs index 18cfa9410a6c..49bd2fb5a692 100644 --- a/frame/offences/src/tests.rs +++ b/frame/offences/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -114,7 +114,10 @@ fn should_deposit_event() { System::events(), vec![EventRecord { phase: Phase::Initialization, - event: Event::Offences(crate::Event::Offence(KIND, time_slot.encode())), + event: Event::Offences(crate::Event::Offence { + kind: KIND, + timeslot: time_slot.encode() + }), topics: vec![], }] ); @@ -145,7 +148,10 @@ fn doesnt_deposit_event_for_dups() { System::events(), vec![EventRecord { phase: Phase::Initialization, - event: Event::Offences(crate::Event::Offence(KIND, time_slot.encode())), + event: Event::Offences(crate::Event::Offence { + kind: KIND, + timeslot: time_slot.encode() + }), topics: vec![], }] ); diff --git a/frame/preimage/Cargo.toml b/frame/preimage/Cargo.toml new file mode 100644 index 000000000000..58809f9e9896 --- /dev/null +++ b/frame/preimage/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "pallet-preimage" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet for storing preimages of hashes" +readme = "README.md" + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } + +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-core = { version = "6.0.0", default-features = false, optional = true, path = "../../primitives/core" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } + +[dev-dependencies] +sp-core = { version = "6.0.0", path = "../../primitives/core", default-features = false } +pallet-balances = { version = "4.0.0-dev", path = "../balances" } + +[features] +default = ["std"] +runtime-benchmarks = [ + "frame-benchmarking", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", +] +std = [ + "codec/std", + "scale-info/std", + "sp-std/std", + "sp-io/std", + "sp-core/std", + "sp-runtime/std", + "frame-system/std", + "frame-support/std", + "frame-benchmarking/std", +] +try-runtime = ["frame-support/try-runtime"] diff --git a/frame/preimage/src/benchmarking.rs b/frame/preimage/src/benchmarking.rs new file mode 100644 index 000000000000..64326108420e --- /dev/null +++ b/frame/preimage/src/benchmarking.rs @@ -0,0 +1,161 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Preimage pallet benchmarking. + +use super::*; +use frame_benchmarking::{account, benchmarks, whitelist_account}; +use frame_support::assert_ok; +use frame_system::RawOrigin; +use sp_runtime::traits::Bounded; +use sp_std::{prelude::*, vec}; + +use crate::Pallet as Preimage; + +const SEED: u32 = 0; + +fn funded_account(name: &'static str, index: u32) -> T::AccountId { + let caller: T::AccountId = account(name, index, SEED); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + caller +} + +fn preimage_and_hash() -> (Vec, T::Hash) { + sized_preimage_and_hash::(T::MaxSize::get()) +} + +fn sized_preimage_and_hash(size: u32) -> (Vec, T::Hash) { + let mut preimage = vec![]; + preimage.resize(size as usize, 0); + let hash = ::Hashing::hash(&preimage[..]); + (preimage, hash) +} + +benchmarks! { + // Expensive note - will reserve. + note_preimage { + let s in 0 .. T::MaxSize::get(); + let caller = funded_account::("caller", 0); + whitelist_account!(caller); + let (preimage, hash) = sized_preimage_and_hash::(s); + }: _(RawOrigin::Signed(caller), preimage) + verify { + assert!(Preimage::::have_preimage(&hash)); + } + // Cheap note - will not reserve since it was requested. + note_requested_preimage { + let s in 0 .. T::MaxSize::get(); + let caller = funded_account::("caller", 0); + whitelist_account!(caller); + let (preimage, hash) = sized_preimage_and_hash::(s); + assert_ok!(Preimage::::request_preimage(T::ManagerOrigin::successful_origin(), hash.clone())); + }: note_preimage(RawOrigin::Signed(caller), preimage) + verify { + assert!(Preimage::::have_preimage(&hash)); + } + // Cheap note - will not reserve since it's the manager. + note_no_deposit_preimage { + let s in 0 .. T::MaxSize::get(); + let (preimage, hash) = sized_preimage_and_hash::(s); + assert_ok!(Preimage::::request_preimage(T::ManagerOrigin::successful_origin(), hash.clone())); + }: note_preimage(T::ManagerOrigin::successful_origin(), preimage) + verify { + assert!(Preimage::::have_preimage(&hash)); + } + + // Expensive unnote - will unreserve. + unnote_preimage { + let caller = funded_account::("caller", 0); + whitelist_account!(caller); + let (preimage, hash) = preimage_and_hash::(); + assert_ok!(Preimage::::note_preimage(RawOrigin::Signed(caller.clone()).into(), preimage)); + }: _(RawOrigin::Signed(caller), hash.clone()) + verify { + assert!(!Preimage::::have_preimage(&hash)); + } + // Cheap unnote - will not unreserve since there's no deposit held. + unnote_no_deposit_preimage { + let (preimage, hash) = preimage_and_hash::(); + assert_ok!(Preimage::::note_preimage(T::ManagerOrigin::successful_origin(), preimage)); + }: unnote_preimage(T::ManagerOrigin::successful_origin(), hash.clone()) + verify { + assert!(!Preimage::::have_preimage(&hash)); + } + + // Expensive request - will unreserve the noter's deposit. + request_preimage { + let (preimage, hash) = preimage_and_hash::(); + let noter = funded_account::("noter", 0); + whitelist_account!(noter); + assert_ok!(Preimage::::note_preimage(RawOrigin::Signed(noter).into(), preimage)); + }: _(T::ManagerOrigin::successful_origin(), hash) + verify { + assert_eq!(StatusFor::::get(&hash), Some(RequestStatus::Requested(1))); + } + // Cheap request - would unreserve the deposit but none was held. + request_no_deposit_preimage { + let (preimage, hash) = preimage_and_hash::(); + assert_ok!(Preimage::::note_preimage(T::ManagerOrigin::successful_origin(), preimage)); + }: request_preimage(T::ManagerOrigin::successful_origin(), hash) + verify { + assert_eq!(StatusFor::::get(&hash), Some(RequestStatus::Requested(1))); + } + // Cheap request - the preimage is not yet noted, so deposit to unreserve. + request_unnoted_preimage { + let (_, hash) = preimage_and_hash::(); + }: request_preimage(T::ManagerOrigin::successful_origin(), hash) + verify { + assert_eq!(StatusFor::::get(&hash), Some(RequestStatus::Requested(1))); + } + // Cheap request - the preimage is already requested, so just a counter bump. + request_requested_preimage { + let (_, hash) = preimage_and_hash::(); + assert_ok!(Preimage::::request_preimage(T::ManagerOrigin::successful_origin(), hash.clone())); + }: request_preimage(T::ManagerOrigin::successful_origin(), hash) + verify { + assert_eq!(StatusFor::::get(&hash), Some(RequestStatus::Requested(2))); + } + + // Expensive unrequest - last reference and it's noted, so will destroy the preimage. + unrequest_preimage { + let (preimage, hash) = preimage_and_hash::(); + assert_ok!(Preimage::::request_preimage(T::ManagerOrigin::successful_origin(), hash.clone())); + assert_ok!(Preimage::::note_preimage(T::ManagerOrigin::successful_origin(), preimage)); + }: _(T::ManagerOrigin::successful_origin(), hash.clone()) + verify { + assert_eq!(StatusFor::::get(&hash), None); + } + // Cheap unrequest - last reference, but it's not noted. + unrequest_unnoted_preimage { + let (_, hash) = preimage_and_hash::(); + assert_ok!(Preimage::::request_preimage(T::ManagerOrigin::successful_origin(), hash.clone())); + }: unrequest_preimage(T::ManagerOrigin::successful_origin(), hash.clone()) + verify { + assert_eq!(StatusFor::::get(&hash), None); + } + // Cheap unrequest - not the last reference. + unrequest_multi_referenced_preimage { + let (_, hash) = preimage_and_hash::(); + assert_ok!(Preimage::::request_preimage(T::ManagerOrigin::successful_origin(), hash.clone())); + assert_ok!(Preimage::::request_preimage(T::ManagerOrigin::successful_origin(), hash.clone())); + }: unrequest_preimage(T::ManagerOrigin::successful_origin(), hash.clone()) + verify { + assert_eq!(StatusFor::::get(&hash), Some(RequestStatus::Requested(1))); + } + + impl_benchmark_test_suite!(Preimage, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/frame/preimage/src/lib.rs b/frame/preimage/src/lib.rs new file mode 100644 index 000000000000..a5d8ee28b595 --- /dev/null +++ b/frame/preimage/src/lib.rs @@ -0,0 +1,346 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Preimage Pallet +//! +//! - [`Config`] +//! - [`Call`] +//! +//! ## Overview +//! +//! The Preimage pallet allows for the users and the runtime to store the preimage +//! of a hash on chain. This can be used by other pallets for storing and managing +//! large byte-blobs. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "runtime-benchmarks")] +mod benchmarking; +#[cfg(test)] +mod mock; +#[cfg(test)] +mod tests; +pub mod weights; + +use sp_runtime::traits::{BadOrigin, Hash, Saturating}; +use sp_std::prelude::*; + +use codec::{Decode, Encode, MaxEncodedLen}; +use frame_support::{ + ensure, + pallet_prelude::Get, + traits::{Currency, PreimageProvider, PreimageRecipient, ReservableCurrency}, + weights::Pays, + BoundedVec, +}; +use scale_info::TypeInfo; +pub use weights::WeightInfo; + +use frame_support::pallet_prelude::*; +use frame_system::pallet_prelude::*; + +pub use pallet::*; + +/// A type to note whether a preimage is owned by a user or the system. +#[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen, RuntimeDebug)] +pub enum RequestStatus { + /// The associated preimage has not yet been requested by the system. The given deposit (if + /// some) is being held until either it becomes requested or the user retracts the primage. + Unrequested(Option<(AccountId, Balance)>), + /// There are a non-zero number of outstanding requests for this hash by this chain. If there + /// is a preimage registered, then it may be removed iff this counter becomes zero. + Requested(u32), +} + +type BalanceOf = + <::Currency as Currency<::AccountId>>::Balance; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + + #[pallet::config] + pub trait Config: frame_system::Config { + /// The overarching event type. + type Event: From> + IsType<::Event>; + + /// The Weight information for this pallet. + type WeightInfo: weights::WeightInfo; + + /// Currency type for this pallet. + type Currency: ReservableCurrency; + + /// An origin that can request a preimage be placed on-chain without a deposit or fee, or + /// manage existing preimages. + type ManagerOrigin: EnsureOrigin; + + /// Max size allowed for a preimage. + type MaxSize: Get; + + /// The base deposit for placing a preimage on chain. + type BaseDeposit: Get>; + + /// The per-byte deposit for placing a preimage on chain. + type ByteDeposit: Get>; + } + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(PhantomData); + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// A preimage has been noted. + Noted { hash: T::Hash }, + /// A preimage has been requested. + Requested { hash: T::Hash }, + /// A preimage has ben cleared. + Cleared { hash: T::Hash }, + } + + #[pallet::error] + pub enum Error { + /// Preimage is too large to store on-chain. + TooLarge, + /// Preimage has already been noted on-chain. + AlreadyNoted, + /// The user is not authorized to perform this action. + NotAuthorized, + /// The preimage cannot be removed since it has not yet been noted. + NotNoted, + /// A preimage may not be removed when there are outstanding requests. + Requested, + /// The preimage request cannot be removed since no outstanding requests exist. + NotRequested, + } + + /// The request status of a given hash. + #[pallet::storage] + pub(super) type StatusFor = + StorageMap<_, Identity, T::Hash, RequestStatus>>; + + /// The preimages stored by this pallet. + #[pallet::storage] + pub(super) type PreimageFor = + StorageMap<_, Identity, T::Hash, BoundedVec>; + + #[pallet::call] + impl Pallet { + /// Register a preimage on-chain. + /// + /// If the preimage was previously requested, no fees or deposits are taken for providing + /// the preimage. Otherwise, a deposit is taken proportional to the size of the preimage. + #[pallet::weight(T::WeightInfo::note_preimage(bytes.len() as u32))] + pub fn note_preimage(origin: OriginFor, bytes: Vec) -> DispatchResultWithPostInfo { + // We accept a signed origin which will pay a deposit, or a root origin where a deposit + // is not taken. + let maybe_sender = Self::ensure_signed_or_manager(origin)?; + let bounded_vec = + BoundedVec::::try_from(bytes).map_err(|()| Error::::TooLarge)?; + let system_requested = Self::note_bytes(bounded_vec, maybe_sender.as_ref())?; + if system_requested || maybe_sender.is_none() { + Ok(Pays::No.into()) + } else { + Ok(().into()) + } + } + + /// Clear an unrequested preimage from the runtime storage. + #[pallet::weight(T::WeightInfo::unnote_preimage())] + pub fn unnote_preimage(origin: OriginFor, hash: T::Hash) -> DispatchResult { + let maybe_sender = Self::ensure_signed_or_manager(origin)?; + Self::do_unnote_preimage(&hash, maybe_sender) + } + + /// Request a preimage be uploaded to the chain without paying any fees or deposits. + /// + /// If the preimage requests has already been provided on-chain, we unreserve any deposit + /// a user may have paid, and take the control of the preimage out of their hands. + #[pallet::weight(T::WeightInfo::request_preimage())] + pub fn request_preimage(origin: OriginFor, hash: T::Hash) -> DispatchResult { + T::ManagerOrigin::ensure_origin(origin)?; + Self::do_request_preimage(&hash); + Ok(()) + } + + /// Clear a previously made request for a preimage. + /// + /// NOTE: THIS MUST NOT BE CALLED ON `hash` MORE TIMES THAN `request_preimage`. + #[pallet::weight(T::WeightInfo::unrequest_preimage())] + pub fn unrequest_preimage(origin: OriginFor, hash: T::Hash) -> DispatchResult { + T::ManagerOrigin::ensure_origin(origin)?; + Self::do_unrequest_preimage(&hash) + } + } +} + +impl Pallet { + /// Ensure that the origin is either the `ManagerOrigin` or a signed origin. + fn ensure_signed_or_manager(origin: T::Origin) -> Result, BadOrigin> { + if T::ManagerOrigin::ensure_origin(origin.clone()).is_ok() { + return Ok(None) + } + let who = ensure_signed(origin)?; + Ok(Some(who)) + } + + /// Store some preimage on chain. + /// + /// We verify that the preimage is within the bounds of what the pallet supports. + /// + /// If the preimage was requested to be uploaded, then the user pays no deposits or tx fees. + fn note_bytes( + preimage: BoundedVec, + maybe_depositor: Option<&T::AccountId>, + ) -> Result { + let hash = T::Hashing::hash(&preimage); + ensure!(!PreimageFor::::contains_key(hash), Error::::AlreadyNoted); + + // We take a deposit only if there is a provided depositor, and the preimage was not + // previously requested. This also allows the tx to pay no fee. + let was_requested = match (StatusFor::::get(hash), maybe_depositor) { + (Some(RequestStatus::Requested(..)), _) => true, + (Some(RequestStatus::Unrequested(..)), _) => Err(Error::::AlreadyNoted)?, + (None, None) => { + StatusFor::::insert(hash, RequestStatus::Unrequested(None)); + false + }, + (None, Some(depositor)) => { + let length = preimage.len() as u32; + let deposit = T::BaseDeposit::get() + .saturating_add(T::ByteDeposit::get().saturating_mul(length.into())); + T::Currency::reserve(depositor, deposit)?; + let status = RequestStatus::Unrequested(Some((depositor.clone(), deposit))); + StatusFor::::insert(hash, status); + false + }, + }; + + PreimageFor::::insert(hash, preimage); + Self::deposit_event(Event::Noted { hash }); + + Ok(was_requested) + } + + // This function will add a hash to the list of requested preimages. + // + // If the preimage already exists before the request is made, the deposit for the preimage is + // returned to the user, and removed from their management. + fn do_request_preimage(hash: &T::Hash) { + let count = StatusFor::::get(hash).map_or(1, |x| match x { + RequestStatus::Requested(mut count) => { + count.saturating_inc(); + count + }, + RequestStatus::Unrequested(None) => 1, + RequestStatus::Unrequested(Some((owner, deposit))) => { + // Return the deposit - the preimage now has outstanding requests. + T::Currency::unreserve(&owner, deposit); + 1 + }, + }); + StatusFor::::insert(hash, RequestStatus::Requested(count)); + if count == 1 { + Self::deposit_event(Event::Requested { hash: hash.clone() }); + } + } + + // Clear a preimage from the storage of the chain, returning any deposit that may be reserved. + // + // If `maybe_owner` is provided, we verify that it is the correct owner before clearing the + // data. + fn do_unnote_preimage( + hash: &T::Hash, + maybe_check_owner: Option, + ) -> DispatchResult { + match StatusFor::::get(hash).ok_or(Error::::NotNoted)? { + RequestStatus::Unrequested(Some((owner, deposit))) => { + ensure!( + maybe_check_owner.map_or(true, |c| &c == &owner), + Error::::NotAuthorized + ); + T::Currency::unreserve(&owner, deposit); + }, + RequestStatus::Unrequested(None) => { + ensure!(maybe_check_owner.is_none(), Error::::NotAuthorized); + }, + RequestStatus::Requested(_) => Err(Error::::Requested)?, + } + StatusFor::::remove(hash); + PreimageFor::::remove(hash); + Self::deposit_event(Event::Cleared { hash: hash.clone() }); + Ok(()) + } + + /// Clear a preimage request. + fn do_unrequest_preimage(hash: &T::Hash) -> DispatchResult { + match StatusFor::::get(hash).ok_or(Error::::NotRequested)? { + RequestStatus::Requested(mut count) if count > 1 => { + count.saturating_dec(); + StatusFor::::insert(hash, RequestStatus::Requested(count)); + }, + RequestStatus::Requested(count) => { + debug_assert!(count == 1, "preimage request counter at zero?"); + PreimageFor::::remove(hash); + StatusFor::::remove(hash); + Self::deposit_event(Event::Cleared { hash: hash.clone() }); + }, + RequestStatus::Unrequested(_) => Err(Error::::NotRequested)?, + } + Ok(()) + } +} + +impl PreimageProvider for Pallet { + fn have_preimage(hash: &T::Hash) -> bool { + PreimageFor::::contains_key(hash) + } + + fn preimage_requested(hash: &T::Hash) -> bool { + matches!(StatusFor::::get(hash), Some(RequestStatus::Requested(..))) + } + + fn get_preimage(hash: &T::Hash) -> Option> { + PreimageFor::::get(hash).map(|preimage| preimage.to_vec()) + } + + fn request_preimage(hash: &T::Hash) { + Self::do_request_preimage(hash) + } + + fn unrequest_preimage(hash: &T::Hash) { + let res = Self::do_unrequest_preimage(hash); + debug_assert!(res.is_ok(), "do_unrequest_preimage failed - counter underflow?"); + } +} + +impl PreimageRecipient for Pallet { + type MaxSize = T::MaxSize; + + fn note_preimage(bytes: BoundedVec) { + // We don't really care if this fails, since that's only the case if someone else has + // already noted it. + let _ = Self::note_bytes(bytes, None); + } + + fn unnote_preimage(hash: &T::Hash) { + // Should never fail if authorization check is skipped. + let res = Self::do_unnote_preimage(hash, None); + debug_assert!(res.is_ok(), "unnote_preimage failed - request outstanding?"); + } +} diff --git a/frame/preimage/src/mock.rs b/frame/preimage/src/mock.rs new file mode 100644 index 000000000000..109806049a0f --- /dev/null +++ b/frame/preimage/src/mock.rs @@ -0,0 +1,124 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Scheduler test environment. + +use super::*; + +use crate as pallet_preimage; +use frame_support::{ + ord_parameter_types, parameter_types, + traits::{ConstU32, ConstU64, Everything}, + weights::constants::RocksDbWeight, +}; +use frame_system::EnsureSignedBy; +use sp_core::H256; +use sp_runtime::{ + testing::Header, + traits::{BlakeTwo256, IdentityLookup}, + Perbill, +}; + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; + +frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system, + Balances: pallet_balances, + Preimage: pallet_preimage, + } +); + +parameter_types! { + pub BlockWeights: frame_system::limits::BlockWeights = + frame_system::limits::BlockWeights::simple_max(2_000_000_000_000); +} +impl frame_system::Config for Test { + type BaseCallFilter = Everything; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = RocksDbWeight; + type Origin = Origin; + type Call = Call; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = ConstU64<250>; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = ConstU32<16>; +} + +impl pallet_balances::Config for Test { + type Balance = u64; + type Event = Event; + type DustRemoval = (); + type ExistentialDeposit = ConstU64<5>; + type AccountStore = System; + type WeightInfo = (); + type MaxLocks = (); + type MaxReserves = ConstU32<50>; + type ReserveIdentifier = [u8; 8]; +} + +parameter_types! { + pub MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * BlockWeights::get().max_block; + pub const MaxScheduledPerBlock: u32 = 10; +} + +ord_parameter_types! { + pub const One: u64 = 1; +} + +impl Config for Test { + type WeightInfo = (); + type Event = Event; + type Currency = Balances; + type ManagerOrigin = EnsureSignedBy; + type MaxSize = ConstU32<1024>; + type BaseDeposit = ConstU64<2>; + type ByteDeposit = ConstU64<1>; +} + +pub fn new_test_ext() -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let balances = pallet_balances::GenesisConfig:: { + balances: vec![(1, 100), (2, 100), (3, 100), (4, 100), (5, 100)], + }; + balances.assimilate_storage(&mut t).unwrap(); + t.into() +} + +pub fn hashed(data: impl AsRef<[u8]>) -> H256 { + BlakeTwo256::hash(data.as_ref()) +} diff --git a/frame/preimage/src/tests.rs b/frame/preimage/src/tests.rs new file mode 100644 index 000000000000..721bb128de23 --- /dev/null +++ b/frame/preimage/src/tests.rs @@ -0,0 +1,233 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Scheduler tests. + +use super::*; +use crate::mock::*; + +use frame_support::{assert_noop, assert_ok}; +use pallet_balances::Error as BalancesError; + +#[test] +fn user_note_preimage_works() { + new_test_ext().execute_with(|| { + assert_ok!(Preimage::note_preimage(Origin::signed(2), vec![1])); + assert_eq!(Balances::reserved_balance(2), 3); + assert_eq!(Balances::free_balance(2), 97); + + let h = hashed([1]); + assert!(Preimage::have_preimage(&h)); + assert_eq!(Preimage::get_preimage(&h), Some(vec![1])); + + assert_noop!( + Preimage::note_preimage(Origin::signed(2), vec![1]), + Error::::AlreadyNoted + ); + assert_noop!( + Preimage::note_preimage(Origin::signed(0), vec![2]), + BalancesError::::InsufficientBalance + ); + }); +} + +#[test] +fn manager_note_preimage_works() { + new_test_ext().execute_with(|| { + assert_ok!(Preimage::note_preimage(Origin::signed(1), vec![1])); + assert_eq!(Balances::reserved_balance(1), 0); + assert_eq!(Balances::free_balance(1), 100); + + let h = hashed([1]); + assert!(Preimage::have_preimage(&h)); + assert_eq!(Preimage::get_preimage(&h), Some(vec![1])); + + assert_noop!( + Preimage::note_preimage(Origin::signed(1), vec![1]), + Error::::AlreadyNoted + ); + }); +} + +#[test] +fn user_unnote_preimage_works() { + new_test_ext().execute_with(|| { + assert_ok!(Preimage::note_preimage(Origin::signed(2), vec![1])); + assert_noop!( + Preimage::unnote_preimage(Origin::signed(3), hashed([1])), + Error::::NotAuthorized + ); + assert_noop!( + Preimage::unnote_preimage(Origin::signed(2), hashed([2])), + Error::::NotNoted + ); + + assert_ok!(Preimage::unnote_preimage(Origin::signed(2), hashed([1]))); + assert_noop!( + Preimage::unnote_preimage(Origin::signed(2), hashed([1])), + Error::::NotNoted + ); + + let h = hashed([1]); + assert!(!Preimage::have_preimage(&h)); + assert_eq!(Preimage::get_preimage(&h), None); + }); +} + +#[test] +fn manager_unnote_preimage_works() { + new_test_ext().execute_with(|| { + assert_ok!(Preimage::note_preimage(Origin::signed(1), vec![1])); + assert_ok!(Preimage::unnote_preimage(Origin::signed(1), hashed([1]))); + assert_noop!( + Preimage::unnote_preimage(Origin::signed(1), hashed([1])), + Error::::NotNoted + ); + + let h = hashed([1]); + assert!(!Preimage::have_preimage(&h)); + assert_eq!(Preimage::get_preimage(&h), None); + }); +} + +#[test] +fn manager_unnote_user_preimage_works() { + new_test_ext().execute_with(|| { + assert_ok!(Preimage::note_preimage(Origin::signed(2), vec![1])); + assert_noop!( + Preimage::unnote_preimage(Origin::signed(3), hashed([1])), + Error::::NotAuthorized + ); + assert_noop!( + Preimage::unnote_preimage(Origin::signed(2), hashed([2])), + Error::::NotNoted + ); + + assert_ok!(Preimage::unnote_preimage(Origin::signed(1), hashed([1]))); + + let h = hashed([1]); + assert!(!Preimage::have_preimage(&h)); + assert_eq!(Preimage::get_preimage(&h), None); + }); +} + +#[test] +fn requested_then_noted_preimage_cannot_be_unnoted() { + new_test_ext().execute_with(|| { + assert_ok!(Preimage::note_preimage(Origin::signed(1), vec![1])); + assert_ok!(Preimage::request_preimage(Origin::signed(1), hashed([1]))); + assert_noop!( + Preimage::unnote_preimage(Origin::signed(1), hashed([1])), + Error::::Requested + ); + + let h = hashed([1]); + assert!(Preimage::have_preimage(&h)); + assert_eq!(Preimage::get_preimage(&h), Some(vec![1])); + }); +} + +#[test] +fn request_note_order_makes_no_difference() { + let one_way = new_test_ext().execute_with(|| { + assert_ok!(Preimage::request_preimage(Origin::signed(1), hashed([1]))); + assert_ok!(Preimage::note_preimage(Origin::signed(1), vec![1])); + ( + StatusFor::::iter().collect::>(), + PreimageFor::::iter().collect::>(), + ) + }); + new_test_ext().execute_with(|| { + assert_ok!(Preimage::note_preimage(Origin::signed(1), vec![1])); + assert_ok!(Preimage::request_preimage(Origin::signed(1), hashed([1]))); + let other_way = ( + StatusFor::::iter().collect::>(), + PreimageFor::::iter().collect::>(), + ); + assert_eq!(one_way, other_way); + }); +} + +#[test] +fn requested_then_user_noted_preimage_is_free() { + new_test_ext().execute_with(|| { + assert_ok!(Preimage::request_preimage(Origin::signed(1), hashed([1]))); + assert_ok!(Preimage::note_preimage(Origin::signed(2), vec![1])); + assert_eq!(Balances::reserved_balance(2), 0); + assert_eq!(Balances::free_balance(2), 100); + + let h = hashed([1]); + assert!(Preimage::have_preimage(&h)); + assert_eq!(Preimage::get_preimage(&h), Some(vec![1])); + }); +} + +#[test] +fn request_user_note_order_makes_no_difference() { + let one_way = new_test_ext().execute_with(|| { + assert_ok!(Preimage::request_preimage(Origin::signed(1), hashed([1]))); + assert_ok!(Preimage::note_preimage(Origin::signed(2), vec![1])); + ( + StatusFor::::iter().collect::>(), + PreimageFor::::iter().collect::>(), + ) + }); + new_test_ext().execute_with(|| { + assert_ok!(Preimage::note_preimage(Origin::signed(2), vec![1])); + assert_ok!(Preimage::request_preimage(Origin::signed(1), hashed([1]))); + let other_way = ( + StatusFor::::iter().collect::>(), + PreimageFor::::iter().collect::>(), + ); + assert_eq!(one_way, other_way); + }); +} + +#[test] +fn unrequest_preimage_works() { + new_test_ext().execute_with(|| { + assert_ok!(Preimage::request_preimage(Origin::signed(1), hashed([1]))); + assert_ok!(Preimage::request_preimage(Origin::signed(1), hashed([1]))); + assert_ok!(Preimage::note_preimage(Origin::signed(2), vec![1])); + assert_noop!( + Preimage::unrequest_preimage(Origin::signed(1), hashed([2])), + Error::::NotRequested + ); + + assert_ok!(Preimage::unrequest_preimage(Origin::signed(1), hashed([1]))); + assert!(Preimage::have_preimage(&hashed([1]))); + + assert_ok!(Preimage::unrequest_preimage(Origin::signed(1), hashed([1]))); + assert_noop!( + Preimage::unrequest_preimage(Origin::signed(1), hashed([1])), + Error::::NotRequested + ); + }); +} + +#[test] +fn user_noted_then_requested_preimage_is_refunded_once_only() { + new_test_ext().execute_with(|| { + assert_ok!(Preimage::note_preimage(Origin::signed(2), vec![1; 3])); + assert_ok!(Preimage::note_preimage(Origin::signed(2), vec![1])); + assert_ok!(Preimage::request_preimage(Origin::signed(1), hashed([1]))); + assert_ok!(Preimage::unrequest_preimage(Origin::signed(1), hashed([1]))); + // Still have reserve from `vec[1; 3]`. + assert_eq!(Balances::reserved_balance(2), 5); + assert_eq!(Balances::free_balance(2), 95); + }); +} diff --git a/frame/preimage/src/weights.rs b/frame/preimage/src/weights.rs new file mode 100644 index 000000000000..0d7f583727c9 --- /dev/null +++ b/frame/preimage/src/weights.rs @@ -0,0 +1,240 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for pallet_preimage +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 + +// Executed Command: +// ./target/production/substrate +// benchmark +// --chain=dev +// --steps=50 +// --repeat=20 +// --pallet=pallet_preimage +// --extrinsic=* +// --execution=wasm +// --wasm-execution=compiled +// --heap-pages=4096 +// --output=./frame/preimage/src/weights.rs +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use sp_std::marker::PhantomData; + +/// Weight functions needed for pallet_preimage. +pub trait WeightInfo { + fn note_preimage(s: u32, ) -> Weight; + fn note_requested_preimage(s: u32, ) -> Weight; + fn note_no_deposit_preimage(s: u32, ) -> Weight; + fn unnote_preimage() -> Weight; + fn unnote_no_deposit_preimage() -> Weight; + fn request_preimage() -> Weight; + fn request_no_deposit_preimage() -> Weight; + fn request_unnoted_preimage() -> Weight; + fn request_requested_preimage() -> Weight; + fn unrequest_preimage() -> Weight; + fn unrequest_unnoted_preimage() -> Weight; + fn unrequest_multi_referenced_preimage() -> Weight; +} + +/// Weights for pallet_preimage using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + fn note_preimage(s: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:0) + fn note_requested_preimage(s: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:0) + fn note_no_deposit_preimage(s: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Preimage PreimageFor (r:0 w:1) + fn unnote_preimage() -> Weight { + (39_239_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Preimage PreimageFor (r:0 w:1) + fn unnote_no_deposit_preimage() -> Weight { + (24_905_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + fn request_preimage() -> Weight { + (37_451_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + fn request_no_deposit_preimage() -> Weight { + (23_397_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + fn request_unnoted_preimage() -> Weight { + (13_407_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + fn request_requested_preimage() -> Weight { + (4_202_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Preimage PreimageFor (r:0 w:1) + fn unrequest_preimage() -> Weight { + (24_858_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Preimage PreimageFor (r:0 w:1) + fn unrequest_unnoted_preimage() -> Weight { + (13_763_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + fn unrequest_multi_referenced_preimage() -> Weight { + (4_262_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } +} + +// For backwards compatibility and tests +impl WeightInfo for () { + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + fn note_preimage(s: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:0) + fn note_requested_preimage(s: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:0) + fn note_no_deposit_preimage(s: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Preimage PreimageFor (r:0 w:1) + fn unnote_preimage() -> Weight { + (39_239_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Preimage PreimageFor (r:0 w:1) + fn unnote_no_deposit_preimage() -> Weight { + (24_905_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + fn request_preimage() -> Weight { + (37_451_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + fn request_no_deposit_preimage() -> Weight { + (23_397_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + fn request_unnoted_preimage() -> Weight { + (13_407_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + fn request_requested_preimage() -> Weight { + (4_202_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Preimage PreimageFor (r:0 w:1) + fn unrequest_preimage() -> Weight { + (24_858_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Preimage PreimageFor (r:0 w:1) + fn unrequest_unnoted_preimage() -> Weight { + (13_763_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Preimage StatusFor (r:1 w:1) + fn unrequest_multi_referenced_preimage() -> Weight { + (4_262_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } +} diff --git a/frame/proxy/Cargo.toml b/frame/proxy/Cargo.toml index 83db82990d10..bdc39a81e34a 100644 --- a/frame/proxy/Cargo.toml +++ b/frame/proxy/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-proxy" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME proxying pallet" readme = "README.md" @@ -13,18 +13,18 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.2.0", default-features = false, features = ["max-encoded-len"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["max-encoded-len"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-utility = { version = "4.0.0-dev", path = "../utility" } @@ -40,7 +40,8 @@ std = [ "sp-io/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", ] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/proxy/src/benchmarking.rs b/frame/proxy/src/benchmarking.rs index e66f6782c19e..f3098c6ad127 100644 --- a/frame/proxy/src/benchmarking.rs +++ b/frame/proxy/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use super::*; use crate::Pallet as Proxy; -use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite, whitelisted_caller}; +use frame_benchmarking::{account, benchmarks, whitelisted_caller}; use frame_system::RawOrigin; use sp_runtime::traits::Bounded; @@ -33,7 +33,7 @@ fn assert_last_event(generic_event: ::Event) { fn add_proxies(n: u32, maybe_who: Option) -> Result<(), &'static str> { let caller = maybe_who.unwrap_or_else(|| whitelisted_caller()); - T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value() / 2u32.into()); for i in 0..n { Proxy::::add_proxy( RawOrigin::Signed(caller.clone()).into(), @@ -51,12 +51,12 @@ fn add_announcements( maybe_real: Option, ) -> Result<(), &'static str> { let caller = maybe_who.unwrap_or_else(|| account("caller", 0, SEED)); - T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value() / 2u32.into()); let real = if let Some(real) = maybe_real { real } else { let real = account("real", 0, SEED); - T::Currency::make_free_balance_be(&real, BalanceOf::::max_value()); + T::Currency::make_free_balance_be(&real, BalanceOf::::max_value() / 2u32.into()); Proxy::::add_proxy( RawOrigin::Signed(real.clone()).into(), caller.clone(), @@ -80,13 +80,13 @@ benchmarks! { let p in 1 .. (T::MaxProxies::get() - 1).into() => add_proxies::(p, None)?; // In this case the caller is the "target" proxy let caller: T::AccountId = account("target", p - 1, SEED); - T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value() / 2u32.into()); // ... and "real" is the traditional caller. This is not a typo. let real: T::AccountId = whitelisted_caller(); let call: ::Call = frame_system::Call::::remark { remark: vec![] }.into(); }: _(RawOrigin::Signed(caller), real, Some(T::ProxyType::default()), Box::new(call)) verify { - assert_last_event::(Event::ProxyExecuted(Ok(())).into()) + assert_last_event::(Event::ProxyExecuted { result: Ok(()) }.into()) } proxy_announced { @@ -95,7 +95,7 @@ benchmarks! { // In this case the caller is the "target" proxy let caller: T::AccountId = account("anonymous", 0, SEED); let delegate: T::AccountId = account("target", p - 1, SEED); - T::Currency::make_free_balance_be(&delegate, BalanceOf::::max_value()); + T::Currency::make_free_balance_be(&delegate, BalanceOf::::max_value() / 2u32.into()); // ... and "real" is the traditional caller. This is not a typo. let real: T::AccountId = whitelisted_caller(); let call: ::Call = frame_system::Call::::remark { remark: vec![] }.into(); @@ -107,7 +107,7 @@ benchmarks! { add_announcements::(a, Some(delegate.clone()), None)?; }: _(RawOrigin::Signed(caller), delegate, real, Some(T::ProxyType::default()), Box::new(call)) verify { - assert_last_event::(Event::ProxyExecuted(Ok(())).into()) + assert_last_event::(Event::ProxyExecuted { result: Ok(()) }.into()) } remove_announcement { @@ -115,7 +115,7 @@ benchmarks! { let p in 1 .. (T::MaxProxies::get() - 1).into() => add_proxies::(p, None)?; // In this case the caller is the "target" proxy let caller: T::AccountId = account("target", p - 1, SEED); - T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value() / 2u32.into()); // ... and "real" is the traditional caller. This is not a typo. let real: T::AccountId = whitelisted_caller(); let call: ::Call = frame_system::Call::::remark { remark: vec![] }.into(); @@ -136,7 +136,7 @@ benchmarks! { let p in 1 .. (T::MaxProxies::get() - 1).into() => add_proxies::(p, None)?; // In this case the caller is the "target" proxy let caller: T::AccountId = account("target", p - 1, SEED); - T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value() / 2u32.into()); // ... and "real" is the traditional caller. This is not a typo. let real: T::AccountId = whitelisted_caller(); let call: ::Call = frame_system::Call::::remark { remark: vec![] }.into(); @@ -157,7 +157,7 @@ benchmarks! { let p in 1 .. (T::MaxProxies::get() - 1).into() => add_proxies::(p, None)?; // In this case the caller is the "target" proxy let caller: T::AccountId = account("target", p - 1, SEED); - T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value() / 2u32.into()); // ... and "real" is the traditional caller. This is not a typo. let real: T::AccountId = whitelisted_caller(); add_announcements::(a, Some(caller.clone()), None)?; @@ -165,7 +165,7 @@ benchmarks! { let call_hash = T::CallHasher::hash_of(&call); }: _(RawOrigin::Signed(caller.clone()), real.clone(), call_hash) verify { - assert_last_event::(Event::Announced(real, caller, call_hash).into()); + assert_last_event::(Event::Announced { real, proxy: caller, call_hash }.into()); } add_proxy { @@ -216,12 +216,12 @@ benchmarks! { ) verify { let anon_account = Pallet::::anonymous_account(&caller, &T::ProxyType::default(), 0, None); - assert_last_event::(Event::AnonymousCreated( - anon_account, - caller, - T::ProxyType::default(), - 0, - ).into()); + assert_last_event::(Event::AnonymousCreated { + anonymous: anon_account, + who: caller, + proxy_type: T::ProxyType::default(), + disambiguation_index: 0, + }.into()); } kill_anonymous { @@ -245,6 +245,6 @@ benchmarks! { verify { assert!(!Proxies::::contains_key(&anon)); } -} -impl_benchmark_test_suite!(Proxy, crate::tests::new_test_ext(), crate::tests::Test); + impl_benchmark_test_suite!(Proxy, crate::tests::new_test_ext(), crate::tests::Test); +} diff --git a/frame/proxy/src/lib.rs b/frame/proxy/src/lib.rs index b73101fa7348..891315c92aa3 100644 --- a/frame/proxy/src/lib.rs +++ b/frame/proxy/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -45,10 +45,10 @@ use frame_system::{self as system}; use scale_info::TypeInfo; use sp_io::hashing::blake2_256; use sp_runtime::{ - traits::{Dispatchable, Hash, Saturating, Zero}, + traits::{Dispatchable, Hash, Saturating, TrailingZeroInput, Zero}, DispatchResult, }; -use sp_std::{convert::TryInto, prelude::*}; +use sp_std::prelude::*; pub use weights::WeightInfo; pub use pallet::*; @@ -102,7 +102,6 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] - #[pallet::generate_storage_info] pub struct Pallet(_); /// Configuration trait. @@ -327,7 +326,12 @@ pub mod pallet { T::Currency::reserve(&who, deposit)?; Proxies::::insert(&anonymous, (bounded_proxies, deposit)); - Self::deposit_event(Event::AnonymousCreated(anonymous, who, proxy_type, index)); + Self::deposit_event(Event::AnonymousCreated { + anonymous, + who, + proxy_type, + disambiguation_index: index, + }); Ok(()) } @@ -427,7 +431,7 @@ pub mod pallet { }) .map(|d| *deposit = d) })?; - Self::deposit_event(Event::Announced(real, who, call_hash)); + Self::deposit_event(Event::Announced { real, proxy: who, call_hash }); Ok(()) } @@ -547,16 +551,32 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// A proxy was executed correctly, with the given \[result\]. - ProxyExecuted(DispatchResult), + /// A proxy was executed correctly, with the given. + ProxyExecuted { result: DispatchResult }, /// Anonymous account has been created by new proxy with given - /// disambiguation index and proxy type. \[anonymous, who, proxy_type, - /// disambiguation_index\] - AnonymousCreated(T::AccountId, T::AccountId, T::ProxyType, u16), - /// An announcement was placed to make a call in the future. \[real, proxy, call_hash\] - Announced(T::AccountId, T::AccountId, CallHashOf), - /// A proxy was added. \[delegator, delegatee, proxy_type, delay\] - ProxyAdded(T::AccountId, T::AccountId, T::ProxyType, T::BlockNumber), + /// disambiguation index and proxy type. + AnonymousCreated { + anonymous: T::AccountId, + who: T::AccountId, + proxy_type: T::ProxyType, + disambiguation_index: u16, + }, + /// An announcement was placed to make a call in the future. + Announced { real: T::AccountId, proxy: T::AccountId, call_hash: CallHashOf }, + /// A proxy was added. + ProxyAdded { + delegator: T::AccountId, + delegatee: T::AccountId, + proxy_type: T::ProxyType, + delay: T::BlockNumber, + }, + /// A proxy was removed. + ProxyRemoved { + delegator: T::AccountId, + delegatee: T::AccountId, + proxy_type: T::ProxyType, + delay: T::BlockNumber, + }, } /// Old name generated by `decl_event`. @@ -639,7 +659,8 @@ impl Pallet { }); let entropy = (b"modlpy/proxy____", who, height, ext_index, proxy_type, index) .using_encoded(blake2_256); - T::AccountId::decode(&mut &entropy[..]).unwrap_or_default() + Decode::decode(&mut TrailingZeroInput::new(entropy.as_ref())) + .expect("infinite length input; no invalid inputs for type; qed") } /// Register a proxy account for the delegator that is able to make calls on its behalf. @@ -672,12 +693,12 @@ impl Pallet { T::Currency::unreserve(delegator, *deposit - new_deposit); } *deposit = new_deposit; - Self::deposit_event(Event::::ProxyAdded( - delegator.clone(), + Self::deposit_event(Event::::ProxyAdded { + delegator: delegator.clone(), delegatee, proxy_type, delay, - )); + }); Ok(()) }) } @@ -698,7 +719,11 @@ impl Pallet { ) -> DispatchResult { Proxies::::try_mutate_exists(delegator, |x| { let (mut proxies, old_deposit) = x.take().ok_or(Error::::NotFound)?; - let proxy_def = ProxyDefinition { delegate: delegatee, proxy_type, delay }; + let proxy_def = ProxyDefinition { + delegate: delegatee.clone(), + proxy_type: proxy_type.clone(), + delay, + }; let i = proxies.binary_search(&proxy_def).ok().ok_or(Error::::NotFound)?; proxies.remove(i); let new_deposit = Self::deposit(proxies.len() as u32); @@ -710,6 +735,12 @@ impl Pallet { if !proxies.is_empty() { *x = Some((proxies, new_deposit)) } + Self::deposit_event(Event::::ProxyRemoved { + delegator: delegator.clone(), + delegatee, + proxy_type, + delay, + }); Ok(()) }) } @@ -800,6 +831,6 @@ impl Pallet { } }); let e = call.dispatch(origin); - Self::deposit_event(Event::ProxyExecuted(e.map(|_| ()).map_err(|e| e.error))); + Self::deposit_event(Event::ProxyExecuted { result: e.map(|_| ()).map_err(|e| e.error) }); } } diff --git a/frame/proxy/src/tests.rs b/frame/proxy/src/tests.rs index d319ebb1a5ab..a0807f1d3d0b 100644 --- a/frame/proxy/src/tests.rs +++ b/frame/proxy/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,7 +24,10 @@ use super::*; use crate as proxy; use codec::{Decode, Encode}; use frame_support::{ - assert_noop, assert_ok, dispatch::DispatchError, parameter_types, traits::Contains, + assert_noop, assert_ok, + dispatch::DispatchError, + parameter_types, + traits::{ConstU32, ConstU64, Contains}, RuntimeDebug, }; use sp_core::H256; @@ -50,7 +53,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -69,7 +71,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -78,10 +80,9 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } -parameter_types! { - pub const ExistentialDeposit: u64 = 1; -} + impl pallet_balances::Config for Test { type MaxLocks = (); type MaxReserves = (); @@ -89,23 +90,17 @@ impl pallet_balances::Config for Test { type Balance = u64; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } impl pallet_utility::Config for Test { type Event = Event; type Call = Call; + type PalletsOrigin = OriginCaller; type WeightInfo = (); } -parameter_types! { - pub const ProxyDepositBase: u64 = 1; - pub const ProxyDepositFactor: u64 = 1; - pub const MaxProxies: u16 = 4; - pub const MaxPending: u32 = 2; - pub const AnnouncementDepositBase: u64 = 1; - pub const AnnouncementDepositFactor: u64 = 1; -} + #[derive( Copy, Clone, @@ -159,14 +154,14 @@ impl Config for Test { type Call = Call; type Currency = Balances; type ProxyType = ProxyType; - type ProxyDepositBase = ProxyDepositBase; - type ProxyDepositFactor = ProxyDepositFactor; - type MaxProxies = MaxProxies; + type ProxyDepositBase = ConstU64<1>; + type ProxyDepositFactor = ConstU64<1>; + type MaxProxies = ConstU32<4>; type WeightInfo = (); type CallHasher = BlakeTwo256; - type MaxPending = MaxPending; - type AnnouncementDepositBase = AnnouncementDepositBase; - type AnnouncementDepositFactor = AnnouncementDepositFactor; + type MaxPending = ConstU32<2>; + type AnnouncementDepositBase = ConstU64<1>; + type AnnouncementDepositFactor = ConstU64<1>; } use super::{Call as ProxyCall, Event as ProxyEvent}; @@ -174,6 +169,8 @@ use frame_system::Call as SystemCall; use pallet_balances::{Call as BalancesCall, Error as BalancesError, Event as BalancesEvent}; use pallet_utility::{Call as UtilityCall, Event as UtilityEvent}; +type SystemError = frame_system::Error; + pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { @@ -208,7 +205,15 @@ fn call_transfer(dest: u64, value: u64) -> Call { fn announcement_works() { new_test_ext().execute_with(|| { assert_ok!(Proxy::add_proxy(Origin::signed(1), 3, ProxyType::Any, 1)); - System::assert_last_event(ProxyEvent::ProxyAdded(1, 3, ProxyType::Any, 1).into()); + System::assert_last_event( + ProxyEvent::ProxyAdded { + delegator: 1, + delegatee: 3, + proxy_type: ProxyType::Any, + delay: 1, + } + .into(), + ); assert_ok!(Proxy::add_proxy(Origin::signed(2), 3, ProxyType::Any, 1)); assert_eq!(Balances::reserved_balance(3), 0); @@ -329,11 +334,13 @@ fn filtering_works() { let call = Box::new(call_transfer(6, 1)); assert_ok!(Proxy::proxy(Origin::signed(2), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Ok(())).into()); + System::assert_last_event(ProxyEvent::ProxyExecuted { result: Ok(()) }.into()); assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Ok(())).into()); + System::assert_last_event(ProxyEvent::ProxyExecuted { result: Ok(()) }.into()); assert_ok!(Proxy::proxy(Origin::signed(4), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Err(DispatchError::BadOrigin)).into()); + System::assert_last_event( + ProxyEvent::ProxyExecuted { result: Err(SystemError::CallFiltered.into()) }.into(), + ); let derivative_id = Utility::derivative_account_id(1, 0); assert!(Balances::mutate_account(&derivative_id, |a| a.free = 1000).is_ok()); @@ -342,24 +349,31 @@ fn filtering_works() { let call = Box::new(Call::Utility(UtilityCall::as_derivative { index: 0, call: inner.clone() })); assert_ok!(Proxy::proxy(Origin::signed(2), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Ok(())).into()); + System::assert_last_event(ProxyEvent::ProxyExecuted { result: Ok(()) }.into()); assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Err(DispatchError::BadOrigin)).into()); + System::assert_last_event( + ProxyEvent::ProxyExecuted { result: Err(SystemError::CallFiltered.into()) }.into(), + ); assert_ok!(Proxy::proxy(Origin::signed(4), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Err(DispatchError::BadOrigin)).into()); + System::assert_last_event( + ProxyEvent::ProxyExecuted { result: Err(SystemError::CallFiltered.into()) }.into(), + ); let call = Box::new(Call::Utility(UtilityCall::batch { calls: vec![*inner] })); assert_ok!(Proxy::proxy(Origin::signed(2), 1, None, call.clone())); expect_events(vec![ UtilityEvent::BatchCompleted.into(), - ProxyEvent::ProxyExecuted(Ok(())).into(), + ProxyEvent::ProxyExecuted { result: Ok(()) }.into(), ]); assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Err(DispatchError::BadOrigin)).into()); + System::assert_last_event( + ProxyEvent::ProxyExecuted { result: Err(SystemError::CallFiltered.into()) }.into(), + ); assert_ok!(Proxy::proxy(Origin::signed(4), 1, None, call.clone())); expect_events(vec![ - UtilityEvent::BatchInterrupted(0, DispatchError::BadOrigin).into(), - ProxyEvent::ProxyExecuted(Ok(())).into(), + UtilityEvent::BatchInterrupted { index: 0, error: SystemError::CallFiltered.into() } + .into(), + ProxyEvent::ProxyExecuted { result: Ok(()) }.into(), ]); let inner = @@ -368,25 +382,32 @@ fn filtering_works() { assert_ok!(Proxy::proxy(Origin::signed(2), 1, None, call.clone())); expect_events(vec![ UtilityEvent::BatchCompleted.into(), - ProxyEvent::ProxyExecuted(Ok(())).into(), + ProxyEvent::ProxyExecuted { result: Ok(()) }.into(), ]); assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Err(DispatchError::BadOrigin)).into()); + System::assert_last_event( + ProxyEvent::ProxyExecuted { result: Err(SystemError::CallFiltered.into()) }.into(), + ); assert_ok!(Proxy::proxy(Origin::signed(4), 1, None, call.clone())); expect_events(vec![ - UtilityEvent::BatchInterrupted(0, DispatchError::BadOrigin).into(), - ProxyEvent::ProxyExecuted(Ok(())).into(), + UtilityEvent::BatchInterrupted { index: 0, error: SystemError::CallFiltered.into() } + .into(), + ProxyEvent::ProxyExecuted { result: Ok(()) }.into(), ]); let call = Box::new(Call::Proxy(ProxyCall::remove_proxies {})); assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Err(DispatchError::BadOrigin)).into()); + System::assert_last_event( + ProxyEvent::ProxyExecuted { result: Err(SystemError::CallFiltered.into()) }.into(), + ); assert_ok!(Proxy::proxy(Origin::signed(4), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Err(DispatchError::BadOrigin)).into()); + System::assert_last_event( + ProxyEvent::ProxyExecuted { result: Err(SystemError::CallFiltered.into()) }.into(), + ); assert_ok!(Proxy::proxy(Origin::signed(2), 1, None, call.clone())); expect_events(vec![ - BalancesEvent::::Unreserved(1, 5).into(), - ProxyEvent::ProxyExecuted(Ok(())).into(), + BalancesEvent::::Unreserved { who: 1, amount: 5 }.into(), + ProxyEvent::ProxyExecuted { result: Ok(()) }.into(), ]); }); } @@ -415,13 +436,49 @@ fn add_remove_proxies_works() { Error::::NotFound ); assert_ok!(Proxy::remove_proxy(Origin::signed(1), 4, ProxyType::JustUtility, 0)); + System::assert_last_event( + ProxyEvent::ProxyRemoved { + delegator: 1, + delegatee: 4, + proxy_type: ProxyType::JustUtility, + delay: 0, + } + .into(), + ); assert_eq!(Balances::reserved_balance(1), 4); assert_ok!(Proxy::remove_proxy(Origin::signed(1), 3, ProxyType::Any, 0)); assert_eq!(Balances::reserved_balance(1), 3); + System::assert_last_event( + ProxyEvent::ProxyRemoved { + delegator: 1, + delegatee: 3, + proxy_type: ProxyType::Any, + delay: 0, + } + .into(), + ); assert_ok!(Proxy::remove_proxy(Origin::signed(1), 2, ProxyType::Any, 0)); assert_eq!(Balances::reserved_balance(1), 2); + System::assert_last_event( + ProxyEvent::ProxyRemoved { + delegator: 1, + delegatee: 2, + proxy_type: ProxyType::Any, + delay: 0, + } + .into(), + ); assert_ok!(Proxy::remove_proxy(Origin::signed(1), 2, ProxyType::JustTransfer, 0)); assert_eq!(Balances::reserved_balance(1), 0); + System::assert_last_event( + ProxyEvent::ProxyRemoved { + delegator: 1, + delegatee: 2, + proxy_type: ProxyType::JustTransfer, + delay: 0, + } + .into(), + ); assert_noop!( Proxy::add_proxy(Origin::signed(1), 1, ProxyType::Any, 0), Error::::NoSelfProxy @@ -457,20 +514,24 @@ fn proxying_works() { Error::::NotProxy ); assert_ok!(Proxy::proxy(Origin::signed(2), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Ok(())).into()); + System::assert_last_event(ProxyEvent::ProxyExecuted { result: Ok(()) }.into()); assert_eq!(Balances::free_balance(6), 1); let call = Box::new(Call::System(SystemCall::set_code { code: vec![] })); assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Err(DispatchError::BadOrigin)).into()); + System::assert_last_event( + ProxyEvent::ProxyExecuted { result: Err(SystemError::CallFiltered.into()) }.into(), + ); let call = Box::new(Call::Balances(BalancesCall::transfer_keep_alive { dest: 6, value: 1 })); assert_ok!(Call::Proxy(super::Call::new_call_variant_proxy(1, None, call.clone())) .dispatch(Origin::signed(2))); - System::assert_last_event(ProxyEvent::ProxyExecuted(Err(DispatchError::BadOrigin)).into()); + System::assert_last_event( + ProxyEvent::ProxyExecuted { result: Err(SystemError::CallFiltered.into()) }.into(), + ); assert_ok!(Proxy::proxy(Origin::signed(3), 1, None, call.clone())); - System::assert_last_event(ProxyEvent::ProxyExecuted(Ok(())).into()); + System::assert_last_event(ProxyEvent::ProxyExecuted { result: Ok(()) }.into()); assert_eq!(Balances::free_balance(6), 2); }); } @@ -481,7 +542,13 @@ fn anonymous_works() { assert_ok!(Proxy::anonymous(Origin::signed(1), ProxyType::Any, 0, 0)); let anon = Proxy::anonymous_account(&1, &ProxyType::Any, 0, None); System::assert_last_event( - ProxyEvent::AnonymousCreated(anon.clone(), 1, ProxyType::Any, 0).into(), + ProxyEvent::AnonymousCreated { + anonymous: anon.clone(), + who: 1, + proxy_type: ProxyType::Any, + disambiguation_index: 0, + } + .into(), ); // other calls to anonymous allowed as long as they're not exactly the same. @@ -502,7 +569,7 @@ fn anonymous_works() { let call = Box::new(call_transfer(6, 1)); assert_ok!(Balances::transfer(Origin::signed(3), anon, 5)); assert_ok!(Proxy::proxy(Origin::signed(1), anon, None, call)); - System::assert_last_event(ProxyEvent::ProxyExecuted(Ok(())).into()); + System::assert_last_event(ProxyEvent::ProxyExecuted { result: Ok(()) }.into()); assert_eq!(Balances::free_balance(6), 1); let call = Box::new(Call::Proxy(ProxyCall::new_call_variant_kill_anonymous( @@ -514,7 +581,7 @@ fn anonymous_works() { ))); assert_ok!(Proxy::proxy(Origin::signed(2), anon2, None, call.clone())); let de = DispatchError::from(Error::::NoPermission).stripped(); - System::assert_last_event(ProxyEvent::ProxyExecuted(Err(de)).into()); + System::assert_last_event(ProxyEvent::ProxyExecuted { result: Err(de) }.into()); assert_noop!( Proxy::kill_anonymous(Origin::signed(1), 1, ProxyType::Any, 0, 1, 0), Error::::NoPermission diff --git a/frame/proxy/src/weights.rs b/frame/proxy/src/weights.rs index 41aa3034bece..176f8683e0b5 100644 --- a/frame/proxy/src/weights.rs +++ b/frame/proxy/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_proxy //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/proxy/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -62,38 +63,42 @@ pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { // Storage: Proxy Proxies (r:1 w:0) fn proxy(p: u32, ) -> Weight { - (23_213_000 as Weight) - // Standard Error: 2_000 - .saturating_add((153_000 as Weight).saturating_mul(p as Weight)) + (13_323_000 as Weight) + // Standard Error: 1_000 + .saturating_add((114_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) } // Storage: Proxy Proxies (r:1 w:0) // Storage: Proxy Announcements (r:1 w:1) // Storage: System Account (r:1 w:1) fn proxy_announced(a: u32, p: u32, ) -> Weight { - (53_286_000 as Weight) - // Standard Error: 2_000 - .saturating_add((549_000 as Weight).saturating_mul(a as Weight)) + (28_825_000 as Weight) + // Standard Error: 1_000 + .saturating_add((254_000 as Weight).saturating_mul(a as Weight)) // Standard Error: 2_000 - .saturating_add((138_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((120_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Proxy Announcements (r:1 w:1) // Storage: System Account (r:1 w:1) - fn remove_announcement(a: u32, _p: u32, ) -> Weight { - (36_864_000 as Weight) - // Standard Error: 2_000 - .saturating_add((550_000 as Weight).saturating_mul(a as Weight)) + fn remove_announcement(a: u32, p: u32, ) -> Weight { + (20_594_000 as Weight) + // Standard Error: 1_000 + .saturating_add((259_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 1_000 + .saturating_add((15_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Proxy Announcements (r:1 w:1) // Storage: System Account (r:1 w:1) - fn reject_announcement(a: u32, _p: u32, ) -> Weight { - (36_755_000 as Weight) + fn reject_announcement(a: u32, p: u32, ) -> Weight { + (20_316_000 as Weight) + // Standard Error: 1_000 + .saturating_add((265_000 as Weight).saturating_mul(a as Weight)) // Standard Error: 1_000 - .saturating_add((550_000 as Weight).saturating_mul(a as Weight)) + .saturating_add((18_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -101,52 +106,52 @@ impl WeightInfo for SubstrateWeight { // Storage: Proxy Announcements (r:1 w:1) // Storage: System Account (r:1 w:1) fn announce(a: u32, p: u32, ) -> Weight { - (50_765_000 as Weight) + (27_696_000 as Weight) // Standard Error: 2_000 - .saturating_add((547_000 as Weight).saturating_mul(a as Weight)) + .saturating_add((241_000 as Weight).saturating_mul(a as Weight)) // Standard Error: 2_000 - .saturating_add((141_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((111_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Proxy Proxies (r:1 w:1) fn add_proxy(p: u32, ) -> Weight { - (35_556_000 as Weight) - // Standard Error: 3_000 - .saturating_add((211_000 as Weight).saturating_mul(p as Weight)) + (22_849_000 as Weight) + // Standard Error: 2_000 + .saturating_add((140_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Proxy Proxies (r:1 w:1) fn remove_proxy(p: u32, ) -> Weight { - (35_284_000 as Weight) - // Standard Error: 3_000 - .saturating_add((229_000 as Weight).saturating_mul(p as Weight)) + (19_350_000 as Weight) + // Standard Error: 2_000 + .saturating_add((147_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Proxy Proxies (r:1 w:1) fn remove_proxies(p: u32, ) -> Weight { - (34_449_000 as Weight) + (18_878_000 as Weight) // Standard Error: 2_000 - .saturating_add((146_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((112_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: unknown [0x3a65787472696e7369635f696e646578] (r:1 w:0) // Storage: Proxy Proxies (r:1 w:1) fn anonymous(p: u32, ) -> Weight { - (49_149_000 as Weight) + (25_743_000 as Weight) // Standard Error: 2_000 - .saturating_add((15_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((25_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Proxy Proxies (r:1 w:1) fn kill_anonymous(p: u32, ) -> Weight { - (36_399_000 as Weight) + (20_484_000 as Weight) // Standard Error: 2_000 - .saturating_add((152_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((105_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -156,38 +161,42 @@ impl WeightInfo for SubstrateWeight { impl WeightInfo for () { // Storage: Proxy Proxies (r:1 w:0) fn proxy(p: u32, ) -> Weight { - (23_213_000 as Weight) - // Standard Error: 2_000 - .saturating_add((153_000 as Weight).saturating_mul(p as Weight)) + (13_323_000 as Weight) + // Standard Error: 1_000 + .saturating_add((114_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) } // Storage: Proxy Proxies (r:1 w:0) // Storage: Proxy Announcements (r:1 w:1) // Storage: System Account (r:1 w:1) fn proxy_announced(a: u32, p: u32, ) -> Weight { - (53_286_000 as Weight) - // Standard Error: 2_000 - .saturating_add((549_000 as Weight).saturating_mul(a as Weight)) + (28_825_000 as Weight) + // Standard Error: 1_000 + .saturating_add((254_000 as Weight).saturating_mul(a as Weight)) // Standard Error: 2_000 - .saturating_add((138_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((120_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Proxy Announcements (r:1 w:1) // Storage: System Account (r:1 w:1) - fn remove_announcement(a: u32, _p: u32, ) -> Weight { - (36_864_000 as Weight) - // Standard Error: 2_000 - .saturating_add((550_000 as Weight).saturating_mul(a as Weight)) + fn remove_announcement(a: u32, p: u32, ) -> Weight { + (20_594_000 as Weight) + // Standard Error: 1_000 + .saturating_add((259_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 1_000 + .saturating_add((15_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Proxy Announcements (r:1 w:1) // Storage: System Account (r:1 w:1) - fn reject_announcement(a: u32, _p: u32, ) -> Weight { - (36_755_000 as Weight) + fn reject_announcement(a: u32, p: u32, ) -> Weight { + (20_316_000 as Weight) + // Standard Error: 1_000 + .saturating_add((265_000 as Weight).saturating_mul(a as Weight)) // Standard Error: 1_000 - .saturating_add((550_000 as Weight).saturating_mul(a as Weight)) + .saturating_add((18_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -195,52 +204,52 @@ impl WeightInfo for () { // Storage: Proxy Announcements (r:1 w:1) // Storage: System Account (r:1 w:1) fn announce(a: u32, p: u32, ) -> Weight { - (50_765_000 as Weight) + (27_696_000 as Weight) // Standard Error: 2_000 - .saturating_add((547_000 as Weight).saturating_mul(a as Weight)) + .saturating_add((241_000 as Weight).saturating_mul(a as Weight)) // Standard Error: 2_000 - .saturating_add((141_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((111_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Proxy Proxies (r:1 w:1) fn add_proxy(p: u32, ) -> Weight { - (35_556_000 as Weight) - // Standard Error: 3_000 - .saturating_add((211_000 as Weight).saturating_mul(p as Weight)) + (22_849_000 as Weight) + // Standard Error: 2_000 + .saturating_add((140_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Proxy Proxies (r:1 w:1) fn remove_proxy(p: u32, ) -> Weight { - (35_284_000 as Weight) - // Standard Error: 3_000 - .saturating_add((229_000 as Weight).saturating_mul(p as Weight)) + (19_350_000 as Weight) + // Standard Error: 2_000 + .saturating_add((147_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Proxy Proxies (r:1 w:1) fn remove_proxies(p: u32, ) -> Weight { - (34_449_000 as Weight) + (18_878_000 as Weight) // Standard Error: 2_000 - .saturating_add((146_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((112_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: unknown [0x3a65787472696e7369635f696e646578] (r:1 w:0) // Storage: Proxy Proxies (r:1 w:1) fn anonymous(p: u32, ) -> Weight { - (49_149_000 as Weight) + (25_743_000 as Weight) // Standard Error: 2_000 - .saturating_add((15_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((25_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Proxy Proxies (r:1 w:1) fn kill_anonymous(p: u32, ) -> Weight { - (36_399_000 as Weight) + (20_484_000 as Weight) // Standard Error: 2_000 - .saturating_add((152_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((105_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } diff --git a/frame/randomness-collective-flip/Cargo.toml b/frame/randomness-collective-flip/Cargo.toml index 5e8eb6b08287..17d4ff461aa0 100644 --- a/frame/randomness-collective-flip/Cargo.toml +++ b/frame/randomness-collective-flip/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-randomness-collective-flip" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME randomness collective flip pallet" readme = "README.md" @@ -14,17 +14,17 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] safe-mix = { version = "1.0", default-features = false } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } [features] default = ["std"] diff --git a/frame/randomness-collective-flip/README.md b/frame/randomness-collective-flip/README.md index 9885c734d9fa..0730d4abf7cf 100644 --- a/frame/randomness-collective-flip/README.md +++ b/frame/randomness-collective-flip/README.md @@ -20,18 +20,28 @@ the system trait. ### Example - Get random seed for the current block ```rust -use frame_support::{decl_module, dispatch, traits::Randomness}; - -pub trait Config: frame_system::Config {} - -decl_module! { - pub struct Module for enum Call where origin: T::Origin { - #[weight = 0] - pub fn random_module_example(origin) -> dispatch::DispatchResult { - let _random_value = >::random(&b"my context"[..]); - Ok(()) - } - } +use frame_support::traits::Randomness; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config + pallet_randomness_collective_flip::Config {} + + #[pallet::call] + impl Pallet { + #[pallet::weight(0)] + pub fn random_module_example(origin: OriginFor) -> DispatchResult { + let _random_value = >::random(&b"my context"[..]); + Ok(()) + } + } } ``` diff --git a/frame/randomness-collective-flip/src/lib.rs b/frame/randomness-collective-flip/src/lib.rs index 1b1d5cb5cd82..f709578f6941 100644 --- a/frame/randomness-collective-flip/src/lib.rs +++ b/frame/randomness-collective-flip/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,9 +15,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! # Randomness Module +//! # Randomness Pallet //! -//! The Randomness Collective Flip module provides a [`random`](./struct.Module.html#method.random) +//! The Randomness Collective Flip pallet provides a [`random`](./struct.Module.html#method.random) //! function that generates low-influence random values based on the block hashes from the previous //! `81` blocks. Low-influence randomness can be useful when defending against relatively weak //! adversaries. Using this pallet as a randomness source is advisable primarily in low-security @@ -31,7 +31,7 @@ //! //! ### Prerequisites //! -//! Import the Randomness Collective Flip module and derive your module's configuration trait from +//! Import the Randomness Collective Flip pallet and derive your pallet's configuration trait from //! the system trait. //! //! ### Example - Get random seed for the current block @@ -41,9 +41,9 @@ //! //! #[frame_support::pallet] //! pub mod pallet { +//! use super::*; //! use frame_support::pallet_prelude::*; //! use frame_system::pallet_prelude::*; -//! use super::*; //! //! #[pallet::pallet] //! #[pallet::generate_store(pub(super) trait Store)] @@ -71,7 +71,6 @@ use safe_mix::TripletMix; use codec::Encode; use frame_support::traits::Randomness; use sp_runtime::traits::{Hash, Saturating}; -use sp_std::{convert::TryInto, prelude::*}; const RANDOM_MATERIAL_LEN: u32 = 81; @@ -102,9 +101,7 @@ pub mod pallet { let parent_hash = >::parent_hash(); >::mutate(|ref mut values| { - if values.len() < RANDOM_MATERIAL_LEN as usize { - values.push(parent_hash) - } else { + if values.try_push(parent_hash).is_err() { let index = block_number_to_index::(block_number); values[index] = parent_hash; } @@ -119,7 +116,8 @@ pub mod pallet { /// the oldest hash. #[pallet::storage] #[pallet::getter(fn random_material)] - pub(super) type RandomMaterial = StorageValue<_, Vec, ValueQuery>; + pub(super) type RandomMaterial = + StorageValue<_, BoundedVec>, ValueQuery>; } impl Randomness for Pallet { @@ -169,7 +167,7 @@ mod tests { use frame_support::{ parameter_types, - traits::{OnInitialize, Randomness}, + traits::{ConstU32, ConstU64, OnInitialize, Randomness}, }; use frame_system::limits; @@ -188,7 +186,6 @@ mod tests { ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: limits::BlockWeights = limits::BlockWeights ::simple_max(1024); pub BlockLength: limits::BlockLength = limits::BlockLength @@ -210,7 +207,7 @@ mod tests { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -219,6 +216,7 @@ mod tests { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl pallet_randomness_collective_flip::Config for Test {} @@ -239,7 +237,8 @@ mod tests { let mut parent_hash = System::parent_hash(); for i in 1..(blocks + 1) { - System::initialize(&i, &parent_hash, &Default::default(), frame_system::InitKind::Full); + System::reset_events(); + System::initialize(&i, &parent_hash, &Default::default()); CollectiveFlip::on_initialize(i); let header = System::finalize(); diff --git a/frame/recovery/Cargo.toml b/frame/recovery/Cargo.toml index 40a89e9b59f8..0a173fe2c9da 100644 --- a/frame/recovery/Cargo.toml +++ b/frame/recovery/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-recovery" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME account recovery pallet" readme = "README.md" @@ -13,16 +13,16 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } [features] diff --git a/frame/recovery/src/lib.rs b/frame/recovery/src/lib.rs index 797581788077..adc5c0b895c5 100644 --- a/frame/recovery/src/lib.rs +++ b/frame/recovery/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -154,7 +154,7 @@ // Ensure we're `no_std` when compiling for Wasm. #![cfg_attr(not(feature = "std"), no_std)] -use codec::{Decode, Encode}; +use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; use sp_runtime::traits::{CheckedAdd, CheckedMul, Dispatchable, SaturatedConversion}; use sp_std::prelude::*; @@ -163,7 +163,7 @@ use frame_support::{ dispatch::PostDispatchInfo, traits::{BalanceStatus, Currency, ReservableCurrency}, weights::GetDispatchInfo, - RuntimeDebug, + BoundedVec, RuntimeDebug, }; pub use pallet::*; @@ -176,21 +176,23 @@ mod tests; type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; +type FriendsOf = BoundedVec<::AccountId, ::MaxFriends>; + /// An active recovery process. -#[derive(Clone, Eq, PartialEq, Encode, Decode, Default, RuntimeDebug, TypeInfo)] -pub struct ActiveRecovery { +#[derive(Clone, Eq, PartialEq, Encode, Decode, Default, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub struct ActiveRecovery { /// The block number when the recovery process started. created: BlockNumber, /// The amount held in reserve of the `depositor`, /// To be returned once this recovery process is closed. deposit: Balance, /// The friends which have vouched so far. Always sorted. - friends: Vec, + friends: Friends, } /// Configuration for recovering an account. -#[derive(Clone, Eq, PartialEq, Encode, Decode, Default, RuntimeDebug, TypeInfo)] -pub struct RecoveryConfig { +#[derive(Clone, Eq, PartialEq, Encode, Decode, Default, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub struct RecoveryConfig { /// The minimum number of blocks since the start of the recovery process before the account /// can be recovered. delay_period: BlockNumber, @@ -198,7 +200,7 @@ pub struct RecoveryConfig { /// to be returned once this configuration is removed. deposit: Balance, /// The list of friends which can help recover an account. Always sorted. - friends: Vec, + friends: Friends, /// The number of approving friends needed to recover an account. threshold: u16, } @@ -206,8 +208,8 @@ pub struct RecoveryConfig { #[frame_support::pallet] pub mod pallet { use super::*; - use frame_support::{ensure, pallet_prelude::*, traits::Get, Parameter}; - use frame_system::{ensure_root, ensure_signed, pallet_prelude::*}; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; use sp_runtime::ArithmeticError; #[pallet::pallet] @@ -244,8 +246,13 @@ pub mod pallet { type FriendDepositFactor: Get>; /// The maximum amount of friends allowed in a recovery configuration. + /// + /// NOTE: The threshold programmed in this Pallet uses u16, so it does + /// not really make sense to have a limit here greater than u16::MAX. + /// But also, that is a lot more than you should probably set this value + /// to anyway... #[pallet::constant] - type MaxFriends: Get; + type MaxFriends: Get; /// The base amount of currency needed to reserve for starting a recovery. /// @@ -262,22 +269,22 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// A recovery process has been set up for an \[account\]. - RecoveryCreated(T::AccountId), + /// A recovery process has been set up for an account. + RecoveryCreated { account: T::AccountId }, /// A recovery process has been initiated for lost account by rescuer account. - /// \[lost, rescuer\] - RecoveryInitiated(T::AccountId, T::AccountId), + RecoveryInitiated { lost_account: T::AccountId, rescuer_account: T::AccountId }, /// A recovery process for lost account by rescuer account has been vouched for by sender. - /// \[lost, rescuer, sender\] - RecoveryVouched(T::AccountId, T::AccountId, T::AccountId), + RecoveryVouched { + lost_account: T::AccountId, + rescuer_account: T::AccountId, + sender: T::AccountId, + }, /// A recovery process for lost account by rescuer account has been closed. - /// \[lost, rescuer\] - RecoveryClosed(T::AccountId, T::AccountId), + RecoveryClosed { lost_account: T::AccountId, rescuer_account: T::AccountId }, /// Lost account has been successfully recovered by rescuer account. - /// \[lost, rescuer\] - AccountRecovered(T::AccountId, T::AccountId), - /// A recovery process has been removed for an \[account\]. - RecoveryRemoved(T::AccountId), + AccountRecovered { lost_account: T::AccountId, rescuer_account: T::AccountId }, + /// A recovery process has been removed for an account. + RecoveryRemoved { lost_account: T::AccountId }, } #[pallet::error] @@ -323,7 +330,7 @@ pub mod pallet { _, Twox64Concat, T::AccountId, - RecoveryConfig, T::AccountId>, + RecoveryConfig, FriendsOf>, >; /// Active recovery attempts. @@ -338,7 +345,7 @@ pub mod pallet { T::AccountId, Twox64Concat, T::AccountId, - ActiveRecovery, T::AccountId>, + ActiveRecovery, FriendsOf>, >; /// The list of allowed proxy accounts. @@ -409,7 +416,10 @@ pub mod pallet { ensure_root(origin)?; // Create the recovery storage item. >::insert(&rescuer, &lost); - Self::deposit_event(Event::::AccountRecovered(lost, rescuer)); + Self::deposit_event(Event::::AccountRecovered { + lost_account: lost, + rescuer_account: rescuer, + }); Ok(()) } @@ -454,12 +464,12 @@ pub mod pallet { ensure!(threshold >= 1, Error::::ZeroThreshold); ensure!(!friends.is_empty(), Error::::NotEnoughFriends); ensure!(threshold as usize <= friends.len(), Error::::NotEnoughFriends); - let max_friends = T::MaxFriends::get() as usize; - ensure!(friends.len() <= max_friends, Error::::MaxFriends); - ensure!(Self::is_sorted_and_unique(&friends), Error::::NotSorted); + let bounded_friends: FriendsOf = + friends.try_into().map_err(|()| Error::::MaxFriends)?; + ensure!(Self::is_sorted_and_unique(&bounded_friends), Error::::NotSorted); // Total deposit is base fee + number of friends * factor fee let friend_deposit = T::FriendDepositFactor::get() - .checked_mul(&friends.len().saturated_into()) + .checked_mul(&bounded_friends.len().saturated_into()) .ok_or(ArithmeticError::Overflow)?; let total_deposit = T::ConfigDepositBase::get() .checked_add(&friend_deposit) @@ -467,12 +477,16 @@ pub mod pallet { // Reserve the deposit T::Currency::reserve(&who, total_deposit)?; // Create the recovery configuration - let recovery_config = - RecoveryConfig { delay_period, deposit: total_deposit, friends, threshold }; + let recovery_config = RecoveryConfig { + delay_period, + deposit: total_deposit, + friends: bounded_friends, + threshold, + }; // Create the recovery configuration storage item >::insert(&who, recovery_config); - Self::deposit_event(Event::::RecoveryCreated(who)); + Self::deposit_event(Event::::RecoveryCreated { account: who }); Ok(()) } @@ -515,11 +529,14 @@ pub mod pallet { let recovery_status = ActiveRecovery { created: >::block_number(), deposit: recovery_deposit, - friends: vec![], + friends: Default::default(), }; // Create the active recovery storage item >::insert(&account, &who, recovery_status); - Self::deposit_event(Event::::RecoveryInitiated(account, who)); + Self::deposit_event(Event::::RecoveryInitiated { + lost_account: account, + rescuer_account: who, + }); Ok(()) } @@ -564,11 +581,18 @@ pub mod pallet { // Either insert the vouch, or return an error that the user already vouched. match active_recovery.friends.binary_search(&who) { Ok(_pos) => Err(Error::::AlreadyVouched)?, - Err(pos) => active_recovery.friends.insert(pos, who.clone()), + Err(pos) => active_recovery + .friends + .try_insert(pos, who.clone()) + .map_err(|()| Error::::MaxFriends)?, } // Update storage with the latest details >::insert(&lost, &rescuer, active_recovery); - Self::deposit_event(Event::::RecoveryVouched(lost, rescuer, who)); + Self::deposit_event(Event::::RecoveryVouched { + lost_account: lost, + rescuer_account: rescuer, + sender: who, + }); Ok(()) } @@ -617,7 +641,10 @@ pub mod pallet { frame_system::Pallet::::inc_consumers(&who).map_err(|_| Error::::BadState)?; // Create the recovery storage item Proxy::::insert(&who, &account); - Self::deposit_event(Event::::AccountRecovered(account, who)); + Self::deposit_event(Event::::AccountRecovered { + lost_account: account, + rescuer_account: who, + }); Ok(()) } @@ -656,7 +683,10 @@ pub mod pallet { BalanceStatus::Free, ); debug_assert!(res.is_ok()); - Self::deposit_event(Event::::RecoveryClosed(who, rescuer)); + Self::deposit_event(Event::::RecoveryClosed { + lost_account: who, + rescuer_account: rescuer, + }); Ok(()) } @@ -692,7 +722,7 @@ pub mod pallet { // Unreserve the initial deposit for the recovery configuration. T::Currency::unreserve(&who, recovery_config.deposit); - Self::deposit_event(Event::::RecoveryRemoved(who)); + Self::deposit_event(Event::::RecoveryRemoved { lost_account: who }); Ok(()) } diff --git a/frame/recovery/src/mock.rs b/frame/recovery/src/mock.rs index f6d4a6b15943..2088f9eb0937 100644 --- a/frame/recovery/src/mock.rs +++ b/frame/recovery/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,7 @@ use super::*; use crate as recovery; use frame_support::{ parameter_types, - traits::{OnFinalize, OnInitialize}, + traits::{ConstU32, ConstU64, OnFinalize, OnInitialize}, }; use sp_core::H256; use sp_runtime::{ @@ -46,7 +46,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -66,7 +65,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -75,6 +74,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } parameter_types! { @@ -96,7 +96,6 @@ impl pallet_balances::Config for Test { parameter_types! { pub const ConfigDepositBase: u64 = 10; pub const FriendDepositFactor: u64 = 1; - pub const MaxFriends: u16 = 3; pub const RecoveryDeposit: u64 = 10; } @@ -106,7 +105,7 @@ impl Config for Test { type Currency = Balances; type ConfigDepositBase = ConfigDepositBase; type FriendDepositFactor = FriendDepositFactor; - type MaxFriends = MaxFriends; + type MaxFriends = ConstU32<3>; type RecoveryDeposit = RecoveryDeposit; } diff --git a/frame/recovery/src/tests.rs b/frame/recovery/src/tests.rs index fe971319bc97..16fc678d357b 100644 --- a/frame/recovery/src/tests.rs +++ b/frame/recovery/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,7 @@ //! Tests for the module. use super::*; -use frame_support::{assert_noop, assert_ok, traits::Currency}; +use frame_support::{assert_noop, assert_ok, bounded_vec, traits::Currency}; use mock::{ new_test_ext, run_to_block, Balances, BalancesCall, Call, Origin, Recovery, RecoveryCall, Test, }; @@ -201,8 +201,12 @@ fn create_recovery_works() { // Base 10 + 1 per friends = 13 total reserved assert_eq!(Balances::reserved_balance(5), 13); // Recovery configuration is correctly stored - let recovery_config = - RecoveryConfig { delay_period, deposit: 13, friends: friends.clone(), threshold }; + let recovery_config = RecoveryConfig { + delay_period, + deposit: 13, + friends: friends.try_into().unwrap(), + threshold, + }; assert_eq!(Recovery::recovery_config(5), Some(recovery_config)); }); } @@ -254,7 +258,8 @@ fn initiate_recovery_works() { // Deposit is reserved assert_eq!(Balances::reserved_balance(1), 10); // Recovery status object is created correctly - let recovery_status = ActiveRecovery { created: 0, deposit: 10, friends: vec![] }; + let recovery_status = + ActiveRecovery { created: 0, deposit: 10, friends: Default::default() }; assert_eq!(>::get(&5, &1), Some(recovery_status)); // Multiple users can attempt to recover the same account assert_ok!(Recovery::initiate_recovery(Origin::signed(2), 5)); @@ -314,7 +319,8 @@ fn vouch_recovery_works() { assert_ok!(Recovery::vouch_recovery(Origin::signed(4), 5, 1)); assert_ok!(Recovery::vouch_recovery(Origin::signed(3), 5, 1)); // Final recovery status object is updated correctly - let recovery_status = ActiveRecovery { created: 0, deposit: 10, friends: vec![2, 3, 4] }; + let recovery_status = + ActiveRecovery { created: 0, deposit: 10, friends: bounded_vec![2, 3, 4] }; assert_eq!(>::get(&5, &1), Some(recovery_status)); }); } diff --git a/frame/referenda/Cargo.toml b/frame/referenda/Cargo.toml new file mode 100644 index 000000000000..d85503a741f4 --- /dev/null +++ b/frame/referenda/Cargo.toml @@ -0,0 +1,56 @@ +[package] +name = "pallet-referenda" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet for inclusive on-chain decisions" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +serde = { version = "1.0.136", optional = true, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ + "derive", +] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +assert_matches = { version = "1.5", optional = true } + +[dev-dependencies] +sp-core = { version = "6.0.0", path = "../../primitives/core" } +pallet-balances = { version = "4.0.0-dev", path = "../balances" } +pallet-scheduler = { version = "4.0.0-dev", path = "../scheduler" } +pallet-preimage = { version = "4.0.0-dev", path = "../preimage" } +assert_matches = { version = "1.5" } + +[features] +default = ["std"] +std = [ + "serde", + "codec/std", + "scale-info/std", + "sp-std/std", + "sp-io/std", + "frame-benchmarking/std", + "frame-support/std", + "sp-runtime/std", + "frame-system/std", +] +runtime-benchmarks = [ + "frame-benchmarking", + "frame-system/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", + "assert_matches", +] +try-runtime = ["frame-support/try-runtime"] diff --git a/frame/referenda/README.md b/frame/referenda/README.md new file mode 100644 index 000000000000..85031a011303 --- /dev/null +++ b/frame/referenda/README.md @@ -0,0 +1,8 @@ +# Referenda Pallet + +- [`assembly::Config`](https://docs.rs/pallet-assembly/latest/pallet_assembly/trait.Config.html) +- [`Call`](https://docs.rs/pallet-assembly/latest/pallet_assembly/enum.Call.html) + +## Overview + +The Assembly pallet handles the administration of general stakeholder voting. diff --git a/frame/referenda/src/benchmarking.rs b/frame/referenda/src/benchmarking.rs new file mode 100644 index 000000000000..08612e0614ae --- /dev/null +++ b/frame/referenda/src/benchmarking.rs @@ -0,0 +1,520 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Democracy pallet benchmarking. + +use super::*; +use crate::Pallet as Referenda; +use assert_matches::assert_matches; +use frame_benchmarking::{account, benchmarks, whitelist_account}; +use frame_support::{ + assert_ok, + traits::{Currency, EnsureOrigin}, +}; +use frame_system::RawOrigin; +use sp_runtime::traits::{Bounded, Hash}; + +const SEED: u32 = 0; + +#[allow(dead_code)] +fn assert_last_event(generic_event: ::Event) { + frame_system::Pallet::::assert_last_event(generic_event.into()); +} + +fn funded_account(name: &'static str, index: u32) -> T::AccountId { + let caller: T::AccountId = account(name, index, SEED); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + caller +} + +fn create_referendum() -> (T::AccountId, ReferendumIndex) { + let caller = funded_account::("caller", 0); + whitelist_account!(caller); + assert_ok!(Referenda::::submit( + RawOrigin::Signed(caller.clone()).into(), + RawOrigin::Root.into(), + T::Hashing::hash_of(&0), + DispatchTime::After(0u32.into()) + )); + let index = ReferendumCount::::get() - 1; + (caller, index) +} + +fn place_deposit(index: ReferendumIndex) { + let caller = funded_account::("caller", 0); + whitelist_account!(caller); + assert_ok!(Referenda::::place_decision_deposit( + RawOrigin::Signed(caller.clone()).into(), + index, + )); +} + +fn nudge(index: ReferendumIndex) { + assert_ok!(Referenda::::nudge_referendum(RawOrigin::Root.into(), index)); +} + +fn fill_queue( + index: ReferendumIndex, + spaces: u32, + pass_after: u32, +) -> Vec { + // First, create enough other referendums to fill the track. + let mut others = vec![]; + for _ in 0..info::(index).max_deciding { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + others.push(index); + } + + // We will also need enough referenda which are queued and passing, we want `MaxQueued - 1` + // in order to force the maximum amount of work to insert ours into the queue. + for _ in spaces..T::MaxQueued::get() { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + make_passing_after::(index, Perbill::from_percent(pass_after)); + others.push(index); + } + + // Skip to when they can start being decided. + skip_prepare_period::(index); + + // Manually nudge the other referenda first to ensure that they begin. + others.iter().for_each(|&i| nudge::(i)); + + others +} + +fn info(index: ReferendumIndex) -> &'static TrackInfoOf { + let status = Referenda::::ensure_ongoing(index).unwrap(); + T::Tracks::info(status.track).expect("Id value returned from T::Tracks") +} + +fn make_passing_after(index: ReferendumIndex, period_portion: Perbill) { + let turnout = info::(index).min_turnout.threshold(period_portion); + let approval = info::(index).min_approval.threshold(period_portion); + Referenda::::access_poll(index, |status| { + if let PollStatus::Ongoing(tally, ..) = status { + *tally = T::Tally::from_requirements(turnout, approval); + } + }); +} + +fn make_passing(index: ReferendumIndex) { + Referenda::::access_poll(index, |status| { + if let PollStatus::Ongoing(tally, ..) = status { + *tally = T::Tally::unanimity(); + } + }); +} + +fn make_failing(index: ReferendumIndex) { + Referenda::::access_poll(index, |status| { + if let PollStatus::Ongoing(tally, ..) = status { + *tally = T::Tally::default(); + } + }); +} + +fn skip_prepare_period(index: ReferendumIndex) { + let status = Referenda::::ensure_ongoing(index).unwrap(); + let prepare_period_over = status.submitted + info::(index).prepare_period; + frame_system::Pallet::::set_block_number(prepare_period_over); +} + +fn skip_decision_period(index: ReferendumIndex) { + let status = Referenda::::ensure_ongoing(index).unwrap(); + let decision_period_over = status.deciding.unwrap().since + info::(index).decision_period; + frame_system::Pallet::::set_block_number(decision_period_over); +} + +fn skip_confirm_period(index: ReferendumIndex) { + let status = Referenda::::ensure_ongoing(index).unwrap(); + let confirm_period_over = status.deciding.unwrap().confirming.unwrap(); + frame_system::Pallet::::set_block_number(confirm_period_over); +} + +fn skip_timeout_period(index: ReferendumIndex) { + let status = Referenda::::ensure_ongoing(index).unwrap(); + let timeout_period_over = status.submitted + T::UndecidingTimeout::get(); + frame_system::Pallet::::set_block_number(timeout_period_over); +} + +fn alarm_time(index: ReferendumIndex) -> T::BlockNumber { + let status = Referenda::::ensure_ongoing(index).unwrap(); + status.alarm.unwrap().0 +} + +fn is_confirming(index: ReferendumIndex) -> bool { + let status = Referenda::::ensure_ongoing(index).unwrap(); + matches!( + status, + ReferendumStatus { deciding: Some(DecidingStatus { confirming: Some(_), .. }), .. } + ) +} + +fn is_not_confirming(index: ReferendumIndex) -> bool { + let status = Referenda::::ensure_ongoing(index).unwrap(); + matches!( + status, + ReferendumStatus { deciding: Some(DecidingStatus { confirming: None, .. }), .. } + ) +} + +benchmarks! { + submit { + let caller = funded_account::("caller", 0); + whitelist_account!(caller); + }: _( + RawOrigin::Signed(caller), + RawOrigin::Root.into(), + T::Hashing::hash_of(&0), + DispatchTime::After(0u32.into()) + ) verify { + let index = ReferendumCount::::get().checked_sub(1).unwrap(); + assert_matches!(ReferendumInfoFor::::get(index), Some(ReferendumInfo::Ongoing(_))); + } + + place_decision_deposit_preparing { + let (caller, index) = create_referendum::(); + }: place_decision_deposit(RawOrigin::Signed(caller), index) + verify { + assert!(Referenda::::ensure_ongoing(index).unwrap().decision_deposit.is_some()); + } + + place_decision_deposit_queued { + let (caller, index) = create_referendum::(); + fill_queue::(index, 1, 90); + }: place_decision_deposit(RawOrigin::Signed(caller), index) + verify { + let track = Referenda::::ensure_ongoing(index).unwrap().track; + assert_eq!(TrackQueue::::get(&track).len() as u32, T::MaxQueued::get()); + assert_eq!(TrackQueue::::get(&track)[0], (index, 0u32.into())); + } + + place_decision_deposit_not_queued { + let (caller, index) = create_referendum::(); + fill_queue::(index, 0, 90); + }: place_decision_deposit(RawOrigin::Signed(caller), index) + verify { + let track = Referenda::::ensure_ongoing(index).unwrap().track; + assert_eq!(TrackQueue::::get(&track).len() as u32, T::MaxQueued::get()); + assert!(TrackQueue::::get(&track).into_iter().all(|(i, _)| i != index)); + } + + place_decision_deposit_passing { + let (caller, index) = create_referendum::(); + skip_prepare_period::(index); + make_passing::(index); + }: place_decision_deposit(RawOrigin::Signed(caller), index) + verify { + assert!(is_confirming::(index)); + } + + place_decision_deposit_failing { + let (caller, index) = create_referendum::(); + skip_prepare_period::(index); + }: place_decision_deposit(RawOrigin::Signed(caller), index) + verify { + assert!(is_not_confirming::(index)); + } + + refund_decision_deposit { + let (caller, index) = create_referendum::(); + place_deposit::(index); + assert_ok!(Referenda::::cancel(T::CancelOrigin::successful_origin(), index)); + }: _(RawOrigin::Signed(caller), index) + verify { + assert_matches!(ReferendumInfoFor::::get(index), Some(ReferendumInfo::Cancelled(_, _, None))); + } + + cancel { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + }: _(T::CancelOrigin::successful_origin(), index) + verify { + assert_matches!(ReferendumInfoFor::::get(index), Some(ReferendumInfo::Cancelled(..))); + } + + kill { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + }: _(T::KillOrigin::successful_origin(), index) + verify { + assert_matches!(ReferendumInfoFor::::get(index), Some(ReferendumInfo::Killed(..))); + } + + one_fewer_deciding_queue_empty { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + skip_prepare_period::(index); + nudge::(index); + let track = Referenda::::ensure_ongoing(index).unwrap().track; + assert_ok!(Referenda::::cancel(T::CancelOrigin::successful_origin(), index)); + assert_eq!(DecidingCount::::get(&track), 1); + }: one_fewer_deciding(RawOrigin::Root, track.clone()) + verify { + assert_eq!(DecidingCount::::get(&track), 0); + } + + one_fewer_deciding_failing { + let (_caller, index) = create_referendum::(); + // No spaces free in the queue. + let queued = fill_queue::(index, 0, 90); + let track = Referenda::::ensure_ongoing(index).unwrap().track; + assert_ok!(Referenda::::cancel(T::CancelOrigin::successful_origin(), queued[0])); + assert_eq!(TrackQueue::::get(&track).len() as u32, T::MaxQueued::get()); + let deciding_count = DecidingCount::::get(&track); + }: one_fewer_deciding(RawOrigin::Root, track.clone()) + verify { + assert_eq!(DecidingCount::::get(&track), deciding_count); + assert_eq!(TrackQueue::::get(&track).len() as u32, T::MaxQueued::get() - 1); + assert!(queued.into_iter().skip(1).all(|i| Referenda::::ensure_ongoing(i) + .unwrap() + .deciding + .map_or(true, |d| d.confirming.is_none()) + )); + } + + one_fewer_deciding_passing { + let (_caller, index) = create_referendum::(); + // No spaces free in the queue. + let queued = fill_queue::(index, 0, 0); + let track = Referenda::::ensure_ongoing(index).unwrap().track; + assert_ok!(Referenda::::cancel(T::CancelOrigin::successful_origin(), queued[0])); + assert_eq!(TrackQueue::::get(&track).len() as u32, T::MaxQueued::get()); + let deciding_count = DecidingCount::::get(&track); + }: one_fewer_deciding(RawOrigin::Root, track.clone()) + verify { + assert_eq!(DecidingCount::::get(&track), deciding_count); + assert_eq!(TrackQueue::::get(&track).len() as u32, T::MaxQueued::get() - 1); + assert!(queued.into_iter().skip(1).all(|i| Referenda::::ensure_ongoing(i) + .unwrap() + .deciding + .map_or(true, |d| d.confirming.is_some()) + )); + } + + nudge_referendum_requeued_insertion { + // First create our referendum and place the deposit. It will be failing. + let (_caller, index) = create_referendum::(); + place_deposit::(index); + fill_queue::(index, 0, 90); + + // Now nudge ours, with the track now full and the queue full of referenda with votes, + // ours will not be in the queue. + nudge::(index); + let track = Referenda::::ensure_ongoing(index).unwrap().track; + assert!(TrackQueue::::get(&track).into_iter().all(|(i, _)| i != index)); + + // Now alter the voting, so that ours goes into pole-position and shifts others down. + make_passing::(index); + }: nudge_referendum(RawOrigin::Root, index) + verify { + let t = TrackQueue::::get(&track); + assert_eq!(t.len() as u32, T::MaxQueued::get()); + assert_eq!(t[t.len() - 1].0, index); + } + + nudge_referendum_requeued_slide { + // First create our referendum and place the deposit. It will be failing. + let (_caller, index) = create_referendum::(); + place_deposit::(index); + fill_queue::(index, 1, 90); + + // Now nudge ours, with the track now full, ours will be queued, but with no votes, it + // will have the worst position. + nudge::(index); + let track = Referenda::::ensure_ongoing(index).unwrap().track; + assert_eq!(TrackQueue::::get(&track).len() as u32, T::MaxQueued::get()); + assert_eq!(TrackQueue::::get(&track)[0], (index, 0u32.into())); + + // Now alter the voting, so that ours leap-frogs all into the best position. + make_passing::(index); + }: nudge_referendum(RawOrigin::Root, index) + verify { + let t = TrackQueue::::get(&track); + assert_eq!(t.len() as u32, T::MaxQueued::get()); + assert_eq!(t[t.len() - 1].0, index); + } + + nudge_referendum_queued { + // NOTE: worst possible queue situation is with a queue full of passing refs with one slot + // free and this failing. It would result in `QUEUE_SIZE - 1` items being shifted for the + // insertion at the beginning. + + // First create our referendum and place the deposit. It will be failing. + let (_caller, index) = create_referendum::(); + place_deposit::(index); + fill_queue::(index, 1, 0); + + let track = Referenda::::ensure_ongoing(index).unwrap().track; + assert_eq!(TrackQueue::::get(&track).len() as u32, T::MaxQueued::get() - 1); + assert!(TrackQueue::::get(&track).into_iter().all(|(_, v)| v > 0u32.into())); + + // Then nudge ours, with the track now full, ours will be queued. + }: nudge_referendum(RawOrigin::Root, index) + verify { + assert_eq!(TrackQueue::::get(&track).len() as u32, T::MaxQueued::get()); + assert_eq!(TrackQueue::::get(&track)[0], (index, 0u32.into())); + } + + nudge_referendum_not_queued { + // First create our referendum and place the deposit. It will be failing. + let (_caller, index) = create_referendum::(); + place_deposit::(index); + fill_queue::(index, 0, 0); + + let track = Referenda::::ensure_ongoing(index).unwrap().track; + assert_eq!(TrackQueue::::get(&track).len() as u32, T::MaxQueued::get()); + assert!(TrackQueue::::get(&track).into_iter().all(|(_, v)| v > 0u32.into())); + + // Then nudge ours, with the track now full, ours will be queued. + }: nudge_referendum(RawOrigin::Root, index) + verify { + assert_eq!(TrackQueue::::get(&track).len() as u32, T::MaxQueued::get()); + assert!(TrackQueue::::get(&track).into_iter().all(|(i, _)| i != index)); + } + + nudge_referendum_no_deposit { + let (_caller, index) = create_referendum::(); + skip_prepare_period::(index); + }: nudge_referendum(RawOrigin::Root, index) + verify { + let status = Referenda::::ensure_ongoing(index).unwrap(); + assert_matches!(status, ReferendumStatus { deciding: None, .. }); + } + + nudge_referendum_preparing { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + }: nudge_referendum(RawOrigin::Root, index) + verify { + let status = Referenda::::ensure_ongoing(index).unwrap(); + assert_matches!(status, ReferendumStatus { deciding: None, .. }); + } + + nudge_referendum_timed_out { + let (_caller, index) = create_referendum::(); + skip_timeout_period::(index); + }: nudge_referendum(RawOrigin::Root, index) + verify { + let info = ReferendumInfoFor::::get(index).unwrap(); + assert_matches!(info, ReferendumInfo::TimedOut(..)); + } + + nudge_referendum_begin_deciding_failing { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + skip_prepare_period::(index); + }: nudge_referendum(RawOrigin::Root, index) + verify { + assert!(is_not_confirming::(index)); + } + + nudge_referendum_begin_deciding_passing { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + make_passing::(index); + skip_prepare_period::(index); + }: nudge_referendum(RawOrigin::Root, index) + verify { + assert!(is_confirming::(index)); + } + + nudge_referendum_begin_confirming { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + skip_prepare_period::(index); + nudge::(index); + assert!(!is_confirming::(index)); + make_passing::(index); + }: nudge_referendum(RawOrigin::Root, index) + verify { + assert!(is_confirming::(index)); + } + + nudge_referendum_end_confirming { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + skip_prepare_period::(index); + make_passing::(index); + nudge::(index); + assert!(is_confirming::(index)); + make_failing::(index); + }: nudge_referendum(RawOrigin::Root, index) + verify { + assert!(!is_confirming::(index)); + } + + nudge_referendum_continue_not_confirming { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + skip_prepare_period::(index); + nudge::(index); + assert!(!is_confirming::(index)); + let old_alarm = alarm_time::(index); + make_passing_after::(index, Perbill::from_percent(50)); + }: nudge_referendum(RawOrigin::Root, index) + verify { + assert_ne!(old_alarm, alarm_time::(index)); + assert!(!is_confirming::(index)); + } + + nudge_referendum_continue_confirming { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + make_passing::(index); + skip_prepare_period::(index); + nudge::(index); + assert!(is_confirming::(index)); + let old_alarm = alarm_time::(index); + }: nudge_referendum(RawOrigin::Root, index) + verify { + assert!(is_confirming::(index)); + } + + nudge_referendum_approved { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + skip_prepare_period::(index); + make_passing::(index); + nudge::(index); + skip_confirm_period::(index); + }: nudge_referendum(RawOrigin::Root, index) + verify { + let info = ReferendumInfoFor::::get(index).unwrap(); + assert_matches!(info, ReferendumInfo::Approved(..)); + } + + nudge_referendum_rejected { + let (_caller, index) = create_referendum::(); + place_deposit::(index); + skip_prepare_period::(index); + nudge::(index); + skip_decision_period::(index); + }: nudge_referendum(RawOrigin::Root, index) + verify { + let info = ReferendumInfoFor::::get(index).unwrap(); + assert_matches!(info, ReferendumInfo::Rejected(..)); + } + + impl_benchmark_test_suite!( + Referenda, + crate::mock::new_test_ext(), + crate::mock::Test + ); +} diff --git a/frame/referenda/src/branch.rs b/frame/referenda/src/branch.rs new file mode 100644 index 000000000000..f381f5fe5b70 --- /dev/null +++ b/frame/referenda/src/branch.rs @@ -0,0 +1,174 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Helpers for managing the different weights in various algorithmic branches. + +use super::Config; +use crate::weights::WeightInfo; + +/// Branches within the `begin_deciding` function. +pub enum BeginDecidingBranch { + Passing, + Failing, +} + +/// Branches within the `service_referendum` function. +pub enum ServiceBranch { + Fail, + NoDeposit, + Preparing, + Queued, + NotQueued, + RequeuedInsertion, + RequeuedSlide, + BeginDecidingPassing, + BeginDecidingFailing, + BeginConfirming, + ContinueConfirming, + EndConfirming, + ContinueNotConfirming, + Approved, + Rejected, + TimedOut, +} + +impl From for ServiceBranch { + fn from(x: BeginDecidingBranch) -> Self { + use BeginDecidingBranch::*; + use ServiceBranch::*; + match x { + Passing => BeginDecidingPassing, + Failing => BeginDecidingFailing, + } + } +} + +impl ServiceBranch { + /// Return the weight of the `nudge` function when it takes the branch denoted by `self`. + pub fn weight_of_nudge, I: 'static>(self) -> frame_support::weights::Weight { + use ServiceBranch::*; + match self { + NoDeposit => T::WeightInfo::nudge_referendum_no_deposit(), + Preparing => T::WeightInfo::nudge_referendum_preparing(), + Queued => T::WeightInfo::nudge_referendum_queued(), + NotQueued => T::WeightInfo::nudge_referendum_not_queued(), + RequeuedInsertion => T::WeightInfo::nudge_referendum_requeued_insertion(), + RequeuedSlide => T::WeightInfo::nudge_referendum_requeued_slide(), + BeginDecidingPassing => T::WeightInfo::nudge_referendum_begin_deciding_passing(), + BeginDecidingFailing => T::WeightInfo::nudge_referendum_begin_deciding_failing(), + BeginConfirming => T::WeightInfo::nudge_referendum_begin_confirming(), + ContinueConfirming => T::WeightInfo::nudge_referendum_continue_confirming(), + EndConfirming => T::WeightInfo::nudge_referendum_end_confirming(), + ContinueNotConfirming => T::WeightInfo::nudge_referendum_continue_not_confirming(), + Approved => T::WeightInfo::nudge_referendum_approved(), + Rejected => T::WeightInfo::nudge_referendum_rejected(), + TimedOut | Fail => T::WeightInfo::nudge_referendum_timed_out(), + } + } + + /// Return the maximum possible weight of the `nudge` function. + pub fn max_weight_of_nudge, I: 'static>() -> frame_support::weights::Weight { + 0.max(T::WeightInfo::nudge_referendum_no_deposit()) + .max(T::WeightInfo::nudge_referendum_preparing()) + .max(T::WeightInfo::nudge_referendum_queued()) + .max(T::WeightInfo::nudge_referendum_not_queued()) + .max(T::WeightInfo::nudge_referendum_requeued_insertion()) + .max(T::WeightInfo::nudge_referendum_requeued_slide()) + .max(T::WeightInfo::nudge_referendum_begin_deciding_passing()) + .max(T::WeightInfo::nudge_referendum_begin_deciding_failing()) + .max(T::WeightInfo::nudge_referendum_begin_confirming()) + .max(T::WeightInfo::nudge_referendum_continue_confirming()) + .max(T::WeightInfo::nudge_referendum_end_confirming()) + .max(T::WeightInfo::nudge_referendum_continue_not_confirming()) + .max(T::WeightInfo::nudge_referendum_approved()) + .max(T::WeightInfo::nudge_referendum_rejected()) + .max(T::WeightInfo::nudge_referendum_timed_out()) + } + + /// Return the weight of the `place_decision_deposit` function when it takes the branch denoted + /// by `self`. + pub fn weight_of_deposit, I: 'static>( + self, + ) -> Option { + use ServiceBranch::*; + Some(match self { + Preparing => T::WeightInfo::place_decision_deposit_preparing(), + Queued => T::WeightInfo::place_decision_deposit_queued(), + NotQueued => T::WeightInfo::place_decision_deposit_not_queued(), + BeginDecidingPassing => T::WeightInfo::place_decision_deposit_passing(), + BeginDecidingFailing => T::WeightInfo::place_decision_deposit_failing(), + BeginConfirming | + ContinueConfirming | + EndConfirming | + ContinueNotConfirming | + Approved | + Rejected | + RequeuedInsertion | + RequeuedSlide | + TimedOut | + Fail | + NoDeposit => return None, + }) + } + + /// Return the maximum possible weight of the `place_decision_deposit` function. + pub fn max_weight_of_deposit, I: 'static>() -> frame_support::weights::Weight { + 0.max(T::WeightInfo::place_decision_deposit_preparing()) + .max(T::WeightInfo::place_decision_deposit_queued()) + .max(T::WeightInfo::place_decision_deposit_not_queued()) + .max(T::WeightInfo::place_decision_deposit_passing()) + .max(T::WeightInfo::place_decision_deposit_failing()) + } +} + +/// Branches that the `one_fewer_deciding` function may take. +pub enum OneFewerDecidingBranch { + QueueEmpty, + BeginDecidingPassing, + BeginDecidingFailing, +} + +impl From for OneFewerDecidingBranch { + fn from(x: BeginDecidingBranch) -> Self { + use BeginDecidingBranch::*; + use OneFewerDecidingBranch::*; + match x { + Passing => BeginDecidingPassing, + Failing => BeginDecidingFailing, + } + } +} + +impl OneFewerDecidingBranch { + /// Return the weight of the `one_fewer_deciding` function when it takes the branch denoted + /// by `self`. + pub fn weight, I: 'static>(self) -> frame_support::weights::Weight { + use OneFewerDecidingBranch::*; + match self { + QueueEmpty => T::WeightInfo::one_fewer_deciding_queue_empty(), + BeginDecidingPassing => T::WeightInfo::one_fewer_deciding_passing(), + BeginDecidingFailing => T::WeightInfo::one_fewer_deciding_failing(), + } + } + + /// Return the maximum possible weight of the `one_fewer_deciding` function. + pub fn max_weight, I: 'static>() -> frame_support::weights::Weight { + 0.max(T::WeightInfo::one_fewer_deciding_queue_empty()) + .max(T::WeightInfo::one_fewer_deciding_passing()) + .max(T::WeightInfo::one_fewer_deciding_failing()) + } +} diff --git a/frame/referenda/src/lib.rs b/frame/referenda/src/lib.rs new file mode 100644 index 000000000000..067775fd336d --- /dev/null +++ b/frame/referenda/src/lib.rs @@ -0,0 +1,1077 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Referenda Pallet +//! +//! ## Overview +//! +//! A pallet for executing referenda. No voting logic is present here, and the `Polling` and +//! `PollStatus` traits are used to allow the voting logic (likely in a pallet) to be utilized. +//! +//! A referendum is a vote on whether a proposal should be dispatched from a particular origin. The +//! origin is used to determine which one of several _tracks_ that a referendum happens under. +//! Tracks each have their own configuration which governs the voting process and parameters. +//! +//! A referendum's lifecycle has three main stages: Preparation, deciding and conclusion. +//! Referenda are considered "ongoing" immediately after submission until their eventual +//! conclusion, and votes may be cast throughout. +//! +//! In order to progress from preparating to being decided, three things must be in place: +//! - There must have been a *Decision Deposit* placed, an amount determined by the track. Anyone +//! may place this deposit. +//! - A period must have elapsed since submission of the referendum. This period is known as the +//! *Preparation Period* and is determined by the track. +//! - The track must not already be at capacity with referendum being decided. The maximum number of +//! referenda which may be being decided simultaneously is determined by the track. +//! +//! In order to become concluded, one of three things must happen: +//! - The referendum should remain in an unbroken _Passing_ state for a period of time. This +//! is known as the _Confirmation Period_ and is determined by the track. A referendum is considered +//! _Passing_ when there is a sufficiently high turnout and approval, given the amount of time it +//! has been being decided. Generally the threshold for what counts as being "sufficiently high" +//! will reduce over time. The curves setting these thresholds are determined by the track. In this +//! case, the referendum is considered _Approved_ and the proposal is scheduled for dispatch. +//! - The referendum reaches the end of its deciding phase outside not _Passing_. It ends in +//! rejection and the proposal is not dispatched. +//! - The referendum is cancelled. +//! +//! A general time-out is also in place and referenda which exist in preparation for too long may +//! conclude without ever entering into a deciding stage. +//! +//! Once a referendum is concluded, the decision deposit may be refunded. +//! +//! - [`Config`] +//! - [`Call`] + +#![recursion_limit = "256"] +#![cfg_attr(not(feature = "std"), no_std)] + +use codec::{Codec, Encode}; +use frame_support::{ + ensure, + traits::{ + schedule::{ + v2::{Anon as ScheduleAnon, Named as ScheduleNamed}, + DispatchTime, MaybeHashed, + }, + Currency, Get, LockIdentifier, OnUnbalanced, OriginTrait, PollStatus, Polling, + ReservableCurrency, VoteTally, + }, + BoundedVec, +}; +use scale_info::TypeInfo; +use sp_runtime::{ + traits::{AtLeast32BitUnsigned, Dispatchable, One, Saturating, Zero}, + DispatchError, Perbill, +}; +use sp_std::{fmt::Debug, prelude::*}; + +mod branch; +mod types; +pub mod weights; + +use self::branch::{BeginDecidingBranch, OneFewerDecidingBranch, ServiceBranch}; +pub use self::{ + pallet::*, + types::{ + BalanceOf, CallOf, Curve, DecidingStatus, DecidingStatusOf, Deposit, InsertSorted, + NegativeImbalanceOf, PalletsOriginOf, ReferendumIndex, ReferendumInfo, ReferendumInfoOf, + ReferendumStatus, ReferendumStatusOf, ScheduleAddressOf, TallyOf, TrackIdOf, TrackInfo, + TrackInfoOf, TracksInfo, VotesOf, + }, + weights::WeightInfo, +}; + +#[cfg(test)] +mod mock; +#[cfg(test)] +mod tests; + +#[cfg(feature = "runtime-benchmarks")] +pub mod benchmarking; + +const ASSEMBLY_ID: LockIdentifier = *b"assembly"; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config + Sized { + // System level stuff. + type Call: Parameter + Dispatchable + From>; + type Event: From> + IsType<::Event>; + /// Weight information for extrinsics in this pallet. + type WeightInfo: WeightInfo; + /// The Scheduler. + type Scheduler: ScheduleAnon< + Self::BlockNumber, + CallOf, + PalletsOriginOf, + Hash = Self::Hash, + > + ScheduleNamed< + Self::BlockNumber, + CallOf, + PalletsOriginOf, + Hash = Self::Hash, + >; + /// Currency type for this pallet. + type Currency: ReservableCurrency; + // Origins and unbalances. + /// Origin from which any vote may be cancelled. + type CancelOrigin: EnsureOrigin; + /// Origin from which any vote may be killed. + type KillOrigin: EnsureOrigin; + /// Handler for the unbalanced reduction when slashing a preimage deposit. + type Slash: OnUnbalanced>; + /// The counting type for votes. Usually just balance. + type Votes: AtLeast32BitUnsigned + Copy + Parameter + Member; + /// The tallying type. + type Tally: VoteTally + Default + Clone + Codec + Eq + Debug + TypeInfo; + + // Constants + /// The minimum amount to be used as a deposit for a public referendum proposal. + #[pallet::constant] + type SubmissionDeposit: Get>; + + /// Maximum size of the referendum queue for a single track. + #[pallet::constant] + type MaxQueued: Get; + + /// The number of blocks after submission that a referendum must begin being decided by. + /// Once this passes, then anyone may cancel the referendum. + #[pallet::constant] + type UndecidingTimeout: Get; + + /// Quantization level for the referendum wakeup scheduler. A higher number will result in + /// fewer storage reads/writes needed for smaller voters, but also result in delays to the + /// automatic referendum status changes. Explicit servicing instructions are unaffected. + #[pallet::constant] + type AlarmInterval: Get; + + // The other stuff. + /// Information concerning the different referendum tracks. + type Tracks: TracksInfo< + BalanceOf, + Self::BlockNumber, + Origin = ::PalletsOrigin, + >; + } + + /// The next free referendum index, aka the number of referenda started so far. + #[pallet::storage] + pub type ReferendumCount = StorageValue<_, ReferendumIndex, ValueQuery>; + + /// Information concerning any given referendum. + #[pallet::storage] + pub type ReferendumInfoFor, I: 'static = ()> = + StorageMap<_, Blake2_128Concat, ReferendumIndex, ReferendumInfoOf>; + + /// The sorted list of referenda ready to be decided but not yet being decided, ordered by + /// conviction-weighted approvals. + /// + /// This should be empty if `DecidingCount` is less than `TrackInfo::max_deciding`. + #[pallet::storage] + pub type TrackQueue, I: 'static = ()> = StorageMap< + _, + Twox64Concat, + TrackIdOf, + BoundedVec<(ReferendumIndex, T::Votes), T::MaxQueued>, + ValueQuery, + >; + + /// The number of referenda being decided currently. + #[pallet::storage] + pub type DecidingCount, I: 'static = ()> = + StorageMap<_, Twox64Concat, TrackIdOf, u32, ValueQuery>; + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event, I: 'static = ()> { + /// A referendum has being submitted. + Submitted { + /// Index of the referendum. + index: ReferendumIndex, + /// The track (and by extension proposal dispatch origin) of this referendum. + track: TrackIdOf, + /// The hash of the proposal up for referendum. + proposal_hash: T::Hash, + }, + /// The decision deposit has been placed. + DecisionDepositPlaced { + /// Index of the referendum. + index: ReferendumIndex, + /// The account who placed the deposit. + who: T::AccountId, + /// The amount placed by the account. + amount: BalanceOf, + }, + /// The decision deposit has been refunded. + DecisionDepositRefunded { + /// Index of the referendum. + index: ReferendumIndex, + /// The account who placed the deposit. + who: T::AccountId, + /// The amount placed by the account. + amount: BalanceOf, + }, + /// A deposit has been slashaed. + DepositSlashed { + /// The account who placed the deposit. + who: T::AccountId, + /// The amount placed by the account. + amount: BalanceOf, + }, + /// A referendum has moved into the deciding phase. + DecisionStarted { + /// Index of the referendum. + index: ReferendumIndex, + /// The track (and by extension proposal dispatch origin) of this referendum. + track: TrackIdOf, + /// The hash of the proposal up for referendum. + proposal_hash: T::Hash, + /// The current tally of votes in this referendum. + tally: T::Tally, + }, + ConfirmStarted { + /// Index of the referendum. + index: ReferendumIndex, + }, + ConfirmAborted { + /// Index of the referendum. + index: ReferendumIndex, + }, + /// A referendum has ended its confirmation phase and is ready for approval. + Confirmed { + /// Index of the referendum. + index: ReferendumIndex, + /// The final tally of votes in this referendum. + tally: T::Tally, + }, + /// A referendum has been approved and its proposal has been scheduled. + Approved { + /// Index of the referendum. + index: ReferendumIndex, + }, + /// A proposal has been rejected by referendum. + Rejected { + /// Index of the referendum. + index: ReferendumIndex, + /// The final tally of votes in this referendum. + tally: T::Tally, + }, + /// A referendum has been timed out without being decided. + TimedOut { + /// Index of the referendum. + index: ReferendumIndex, + /// The final tally of votes in this referendum. + tally: T::Tally, + }, + /// A referendum has been cancelled. + Cancelled { + /// Index of the referendum. + index: ReferendumIndex, + /// The final tally of votes in this referendum. + tally: T::Tally, + }, + /// A referendum has been killed. + Killed { + /// Index of the referendum. + index: ReferendumIndex, + /// The final tally of votes in this referendum. + tally: T::Tally, + }, + } + + #[pallet::error] + pub enum Error { + /// Referendum is not ongoing. + NotOngoing, + /// Referendum's decision deposit is already paid. + HasDeposit, + /// The track identifier given was invalid. + BadTrack, + /// There are already a full complement of referendums in progress for this track. + Full, + /// The queue of the track is empty. + QueueEmpty, + /// The referendum index provided is invalid in this context. + BadReferendum, + /// There was nothing to do in the advancement. + NothingToDo, + /// No track exists for the proposal origin. + NoTrack, + /// Any deposit cannot be refunded until after the decision is over. + Unfinished, + /// The deposit refunder is not the depositor. + NoPermission, + /// The deposit cannot be refunded since none was made. + NoDeposit, + } + + #[pallet::call] + impl, I: 'static> Pallet { + /// Propose a referendum on a privileged action. + /// + /// - `origin`: must be `Signed` and the account must have `SubmissionDeposit` funds + /// available. + /// - `proposal_origin`: The origin from which the proposal should be executed. + /// - `proposal_hash`: The hash of the proposal preimage. + /// - `enactment_moment`: The moment that the proposal should be enacted. + /// + /// Emits `Submitted`. + #[pallet::weight(T::WeightInfo::submit())] + pub fn submit( + origin: OriginFor, + proposal_origin: PalletsOriginOf, + proposal_hash: T::Hash, + enactment_moment: DispatchTime, + ) -> DispatchResult { + let who = ensure_signed(origin)?; + + let track = + T::Tracks::track_for(&proposal_origin).map_err(|_| Error::::NoTrack)?; + let submission_deposit = Self::take_deposit(who, T::SubmissionDeposit::get())?; + let index = ReferendumCount::::mutate(|x| { + let r = *x; + *x += 1; + r + }); + let now = frame_system::Pallet::::block_number(); + let nudge_call = Call::nudge_referendum { index }; + let status = ReferendumStatus { + track, + origin: proposal_origin, + proposal_hash: proposal_hash.clone(), + enactment: enactment_moment, + submitted: now, + submission_deposit, + decision_deposit: None, + deciding: None, + tally: Default::default(), + in_queue: false, + alarm: Self::set_alarm(nudge_call, now.saturating_add(T::UndecidingTimeout::get())), + }; + ReferendumInfoFor::::insert(index, ReferendumInfo::Ongoing(status)); + + Self::deposit_event(Event::::Submitted { index, track, proposal_hash }); + Ok(()) + } + + /// Post the Decision Deposit for a referendum. + /// + /// - `origin`: must be `Signed` and the account must have funds available for the + /// referendum's track's Decision Deposit. + /// - `index`: The index of the submitted referendum whose Decision Deposit is yet to be + /// posted. + /// + /// Emits `DecisionDepositPlaced`. + #[pallet::weight(ServiceBranch::max_weight_of_deposit::())] + pub fn place_decision_deposit( + origin: OriginFor, + index: ReferendumIndex, + ) -> DispatchResultWithPostInfo { + let who = ensure_signed(origin)?; + let mut status = Self::ensure_ongoing(index)?; + ensure!(status.decision_deposit.is_none(), Error::::HasDeposit); + let track = Self::track(status.track).ok_or(Error::::NoTrack)?; + status.decision_deposit = + Some(Self::take_deposit(who.clone(), track.decision_deposit)?); + let now = frame_system::Pallet::::block_number(); + let (info, _, branch) = Self::service_referendum(now, index, status); + ReferendumInfoFor::::insert(index, info); + let e = + Event::::DecisionDepositPlaced { index, who, amount: track.decision_deposit }; + Self::deposit_event(e); + Ok(branch.weight_of_deposit::().into()) + } + + /// Refund the Decision Deposit for a closed referendum back to the depositor. + /// + /// - `origin`: must be `Signed` or `Root`. + /// - `index`: The index of a closed referendum whose Decision Deposit has not yet been + /// refunded. + /// + /// Emits `DecisionDepositRefunded`. + #[pallet::weight(T::WeightInfo::refund_decision_deposit())] + pub fn refund_decision_deposit( + origin: OriginFor, + index: ReferendumIndex, + ) -> DispatchResult { + ensure_signed_or_root(origin)?; + let mut info = + ReferendumInfoFor::::get(index).ok_or(Error::::BadReferendum)?; + let deposit = info + .take_decision_deposit() + .map_err(|_| Error::::Unfinished)? + .ok_or(Error::::NoDeposit)?; + Self::refund_deposit(Some(deposit.clone())); + ReferendumInfoFor::::insert(index, info); + let e = Event::::DecisionDepositRefunded { + index, + who: deposit.who, + amount: deposit.amount, + }; + Self::deposit_event(e); + Ok(()) + } + + /// Cancel an ongoing referendum. + /// + /// - `origin`: must be the `CancelOrigin`. + /// - `index`: The index of the referendum to be cancelled. + /// + /// Emits `Cancelled`. + #[pallet::weight(T::WeightInfo::cancel())] + pub fn cancel(origin: OriginFor, index: ReferendumIndex) -> DispatchResult { + T::CancelOrigin::ensure_origin(origin)?; + let status = Self::ensure_ongoing(index)?; + if let Some((_, last_alarm)) = status.alarm { + let _ = T::Scheduler::cancel(last_alarm); + } + Self::note_one_fewer_deciding(status.track); + Self::deposit_event(Event::::Cancelled { index, tally: status.tally }); + let info = ReferendumInfo::Cancelled( + frame_system::Pallet::::block_number(), + status.submission_deposit, + status.decision_deposit, + ); + ReferendumInfoFor::::insert(index, info); + Ok(()) + } + + /// Cancel an ongoing referendum and slash the deposits. + /// + /// - `origin`: must be the `KillOrigin`. + /// - `index`: The index of the referendum to be cancelled. + /// + /// Emits `Killed` and `DepositSlashed`. + #[pallet::weight(T::WeightInfo::kill())] + pub fn kill(origin: OriginFor, index: ReferendumIndex) -> DispatchResult { + T::KillOrigin::ensure_origin(origin)?; + let status = Self::ensure_ongoing(index)?; + if let Some((_, last_alarm)) = status.alarm { + let _ = T::Scheduler::cancel(last_alarm); + } + Self::note_one_fewer_deciding(status.track); + Self::deposit_event(Event::::Killed { index, tally: status.tally }); + Self::slash_deposit(Some(status.submission_deposit.clone())); + Self::slash_deposit(status.decision_deposit.clone()); + let info = ReferendumInfo::Killed(frame_system::Pallet::::block_number()); + ReferendumInfoFor::::insert(index, info); + Ok(()) + } + + /// Advance a referendum onto its next logical state. Only used internally. + /// + /// - `origin`: must be `Root`. + /// - `index`: the referendum to be advanced. + #[pallet::weight(ServiceBranch::max_weight_of_nudge::())] + pub fn nudge_referendum( + origin: OriginFor, + index: ReferendumIndex, + ) -> DispatchResultWithPostInfo { + ensure_root(origin)?; + let now = frame_system::Pallet::::block_number(); + let mut status = Self::ensure_ongoing(index)?; + // This is our wake-up, so we can disregard the alarm. + status.alarm = None; + let (info, dirty, branch) = Self::service_referendum(now, index, status); + if dirty { + ReferendumInfoFor::::insert(index, info); + } + Ok(Some(branch.weight_of_nudge::()).into()) + } + + /// Advance a track onto its next logical state. Only used internally. + /// + /// - `origin`: must be `Root`. + /// - `track`: the track to be advanced. + /// + /// Action item for when there is now one fewer referendum in the deciding phase and the + /// `DecidingCount` is not yet updated. This means that we should either: + /// - begin deciding another referendum (and leave `DecidingCount` alone); or + /// - decrement `DecidingCount`. + #[pallet::weight(OneFewerDecidingBranch::max_weight::())] + pub fn one_fewer_deciding( + origin: OriginFor, + track: TrackIdOf, + ) -> DispatchResultWithPostInfo { + ensure_root(origin)?; + let track_info = T::Tracks::info(track).ok_or(Error::::BadTrack)?; + let mut track_queue = TrackQueue::::get(track); + let branch = + if let Some((index, mut status)) = Self::next_for_deciding(&mut track_queue) { + let now = frame_system::Pallet::::block_number(); + let (maybe_alarm, branch) = + Self::begin_deciding(&mut status, index, now, track_info); + if let Some(set_alarm) = maybe_alarm { + Self::ensure_alarm_at(&mut status, index, set_alarm); + } + ReferendumInfoFor::::insert(index, ReferendumInfo::Ongoing(status)); + TrackQueue::::insert(track, track_queue); + branch.into() + } else { + DecidingCount::::mutate(track, |x| x.saturating_dec()); + OneFewerDecidingBranch::QueueEmpty + }; + Ok(Some(branch.weight::()).into()) + } + } +} + +impl, I: 'static> Polling for Pallet { + type Index = ReferendumIndex; + type Votes = VotesOf; + type Moment = T::BlockNumber; + type Class = TrackIdOf; + + fn classes() -> Vec { + T::Tracks::tracks().iter().map(|x| x.0).collect() + } + + fn access_poll( + index: Self::Index, + f: impl FnOnce(PollStatus<&mut T::Tally, T::BlockNumber, TrackIdOf>) -> R, + ) -> R { + match ReferendumInfoFor::::get(index) { + Some(ReferendumInfo::Ongoing(mut status)) => { + let result = f(PollStatus::Ongoing(&mut status.tally, status.track)); + let now = frame_system::Pallet::::block_number(); + Self::ensure_alarm_at(&mut status, index, now + One::one()); + ReferendumInfoFor::::insert(index, ReferendumInfo::Ongoing(status)); + result + }, + Some(ReferendumInfo::Approved(end, ..)) => f(PollStatus::Completed(end, true)), + Some(ReferendumInfo::Rejected(end, ..)) => f(PollStatus::Completed(end, false)), + _ => f(PollStatus::None), + } + } + + fn try_access_poll( + index: Self::Index, + f: impl FnOnce( + PollStatus<&mut T::Tally, T::BlockNumber, TrackIdOf>, + ) -> Result, + ) -> Result { + match ReferendumInfoFor::::get(index) { + Some(ReferendumInfo::Ongoing(mut status)) => { + let result = f(PollStatus::Ongoing(&mut status.tally, status.track))?; + let now = frame_system::Pallet::::block_number(); + Self::ensure_alarm_at(&mut status, index, now + One::one()); + ReferendumInfoFor::::insert(index, ReferendumInfo::Ongoing(status)); + Ok(result) + }, + Some(ReferendumInfo::Approved(end, ..)) => f(PollStatus::Completed(end, true)), + Some(ReferendumInfo::Rejected(end, ..)) => f(PollStatus::Completed(end, false)), + _ => f(PollStatus::None), + } + } + + fn as_ongoing(index: Self::Index) -> Option<(T::Tally, TrackIdOf)> { + Self::ensure_ongoing(index).ok().map(|x| (x.tally, x.track)) + } + + #[cfg(feature = "runtime-benchmarks")] + fn create_ongoing(class: Self::Class) -> Result { + let index = ReferendumCount::::mutate(|x| { + let r = *x; + *x += 1; + r + }); + let now = frame_system::Pallet::::block_number(); + let dummy_account_id = + codec::Decode::decode(&mut sp_runtime::traits::TrailingZeroInput::new(&b"dummy"[..])) + .expect("infinite length input; no invalid inputs for type; qed"); + let mut status = ReferendumStatusOf:: { + track: class, + origin: frame_support::dispatch::RawOrigin::Root.into(), + proposal_hash: ::hash_of(&index), + enactment: DispatchTime::After(Zero::zero()), + submitted: now, + submission_deposit: Deposit { who: dummy_account_id, amount: Zero::zero() }, + decision_deposit: None, + deciding: None, + tally: Default::default(), + in_queue: false, + alarm: None, + }; + Self::ensure_alarm_at(&mut status, index, sp_runtime::traits::Bounded::max_value()); + ReferendumInfoFor::::insert(index, ReferendumInfo::Ongoing(status)); + Ok(index) + } + + #[cfg(feature = "runtime-benchmarks")] + fn end_ongoing(index: Self::Index, approved: bool) -> Result<(), ()> { + let mut status = Self::ensure_ongoing(index).map_err(|_| ())?; + Self::ensure_no_alarm(&mut status); + Self::note_one_fewer_deciding(status.track); + let now = frame_system::Pallet::::block_number(); + let info = if approved { + ReferendumInfo::Approved(now, status.submission_deposit, status.decision_deposit) + } else { + ReferendumInfo::Rejected(now, status.submission_deposit, status.decision_deposit) + }; + ReferendumInfoFor::::insert(index, info); + Ok(()) + } + + #[cfg(feature = "runtime-benchmarks")] + fn max_ongoing() -> (Self::Class, u32) { + let r = T::Tracks::tracks() + .iter() + .max_by_key(|(_, info)| info.max_deciding) + .expect("Always one class"); + (r.0.clone(), r.1.max_deciding) + } +} + +impl, I: 'static> Pallet { + /// Check that referendum `index` is in the `Ongoing` state and return the `ReferendumStatus` + /// value, or `Err` otherwise. + pub fn ensure_ongoing( + index: ReferendumIndex, + ) -> Result, DispatchError> { + match ReferendumInfoFor::::get(index) { + Some(ReferendumInfo::Ongoing(status)) => Ok(status), + _ => Err(Error::::NotOngoing.into()), + } + } + + // Enqueue a proposal from a referendum which has presumably passed. + fn schedule_enactment( + index: ReferendumIndex, + track: &TrackInfoOf, + desired: DispatchTime, + origin: PalletsOriginOf, + call_hash: T::Hash, + ) { + let now = frame_system::Pallet::::block_number(); + let earliest_allowed = now.saturating_add(track.min_enactment_period); + let desired = desired.evaluate(now); + let ok = T::Scheduler::schedule_named( + (ASSEMBLY_ID, "enactment", index).encode(), + DispatchTime::At(desired.max(earliest_allowed)), + None, + 63, + origin, + MaybeHashed::Hash(call_hash), + ) + .is_ok(); + debug_assert!(ok, "LOGIC ERROR: bake_referendum/schedule_named failed"); + } + + /// Set an alarm to dispatch `call` at block number `when`. + fn set_alarm( + call: impl Into>, + when: T::BlockNumber, + ) -> Option<(T::BlockNumber, ScheduleAddressOf)> { + let alarm_interval = T::AlarmInterval::get().max(One::one()); + let when = (when + alarm_interval - One::one()) / alarm_interval * alarm_interval; + let maybe_result = T::Scheduler::schedule( + DispatchTime::At(when), + None, + 128u8, + frame_system::RawOrigin::Root.into(), + MaybeHashed::Value(call.into()), + ) + .ok() + .map(|x| (when, x)); + debug_assert!( + maybe_result.is_some(), + "Unable to schedule a new alarm at #{:?} (now: #{:?})?!", + when, + frame_system::Pallet::::block_number() + ); + maybe_result + } + + /// Mutate a referendum's `status` into the correct deciding state. + /// + /// - `now` is the current block number. + /// - `track` is the track info for the referendum. + /// + /// This will properly set up the `confirming` item. + fn begin_deciding( + status: &mut ReferendumStatusOf, + index: ReferendumIndex, + now: T::BlockNumber, + track: &TrackInfoOf, + ) -> (Option, BeginDecidingBranch) { + let is_passing = Self::is_passing( + &status.tally, + Zero::zero(), + track.decision_period, + &track.min_turnout, + &track.min_approval, + ); + status.in_queue = false; + Self::deposit_event(Event::::DecisionStarted { + index, + tally: status.tally.clone(), + proposal_hash: status.proposal_hash.clone(), + track: status.track.clone(), + }); + let confirming = if is_passing { + Self::deposit_event(Event::::ConfirmStarted { index }); + Some(now.saturating_add(track.confirm_period)) + } else { + None + }; + let deciding_status = DecidingStatus { since: now, confirming }; + let alarm = Self::decision_time(&deciding_status, &status.tally, track); + status.deciding = Some(deciding_status); + let branch = + if is_passing { BeginDecidingBranch::Passing } else { BeginDecidingBranch::Failing }; + (Some(alarm), branch) + } + + /// If it returns `Some`, deciding has begun and it needs waking at the given block number. The + /// second item is the flag for whether it is confirming or not. + /// + /// If `None`, then it is queued and should be nudged automatically as the queue gets drained. + fn ready_for_deciding( + now: T::BlockNumber, + track: &TrackInfoOf, + index: ReferendumIndex, + status: &mut ReferendumStatusOf, + ) -> (Option, ServiceBranch) { + let deciding_count = DecidingCount::::get(status.track); + if deciding_count < track.max_deciding { + // Begin deciding. + DecidingCount::::insert(status.track, deciding_count.saturating_add(1)); + let r = Self::begin_deciding(status, index, now, track); + (r.0, r.1.into()) + } else { + // Add to queue. + let item = (index, status.tally.ayes()); + status.in_queue = true; + TrackQueue::::mutate(status.track, |q| q.insert_sorted_by_key(item, |x| x.1)); + (None, ServiceBranch::Queued) + } + } + + /// Grab the index and status for the referendum which is the highest priority of those for the + /// given track which are ready for being decided. + fn next_for_deciding( + track_queue: &mut BoundedVec<(u32, VotesOf), T::MaxQueued>, + ) -> Option<(ReferendumIndex, ReferendumStatusOf)> { + loop { + let (index, _) = track_queue.pop()?; + match Self::ensure_ongoing(index) { + Ok(s) => return Some((index, s)), + Err(_) => {}, // referendum already timedout or was cancelled. + } + } + } + + /// Schedule a call to `one_fewer_deciding` function via the dispatchable + /// `defer_one_fewer_deciding`. We could theoretically call it immediately (and it would be + /// overall more efficient), however the weights become rather less easy to measure. + fn note_one_fewer_deciding(track: TrackIdOf) { + // Set an alarm call for the next block to nudge the track along. + let now = frame_system::Pallet::::block_number(); + let next_block = now + One::one(); + let alarm_interval = T::AlarmInterval::get().max(One::one()); + let when = (next_block + alarm_interval - One::one()) / alarm_interval * alarm_interval; + + let maybe_result = T::Scheduler::schedule( + DispatchTime::At(when), + None, + 128u8, + frame_system::RawOrigin::Root.into(), + MaybeHashed::Value(Call::one_fewer_deciding { track }.into()), + ); + debug_assert!( + maybe_result.is_ok(), + "Unable to schedule a new alarm at #{:?} (now: #{:?})?!", + when, + now + ); + } + + /// Ensure that a `service_referendum` alarm happens for the referendum `index` at `alarm`. + /// + /// This will do nothing if the alarm is already set. + /// + /// Returns `false` if nothing changed. + fn ensure_alarm_at( + status: &mut ReferendumStatusOf, + index: ReferendumIndex, + alarm: T::BlockNumber, + ) -> bool { + if status.alarm.as_ref().map_or(true, |&(when, _)| when != alarm) { + // Either no alarm or one that was different + Self::ensure_no_alarm(status); + status.alarm = Self::set_alarm(Call::nudge_referendum { index }, alarm); + true + } else { + false + } + } + + /// Advance the state of a referendum, which comes down to: + /// - If it's ready to be decided, start deciding; + /// - If it's not ready to be decided and non-deciding timeout has passed, fail; + /// - If it's ongoing and passing, ensure confirming; if at end of confirmation period, pass. + /// - If it's ongoing and not passing, stop confirning; if it has reached end time, fail. + /// + /// Weight will be a bit different depending on what it does, but it's designed so as not to + /// differ dramatically, especially if `MaxQueue` is kept small. In particular _there are no + /// balance operations in here_. + /// + /// In terms of storage, every call to it is expected to access: + /// - The scheduler, either to insert, remove or alter an entry; + /// - `TrackQueue`, which should be a `BoundedVec` with a low limit (8-16). + /// - `DecidingCount`. + /// + /// Both of the two storage items will only have as many items as there are different tracks, + /// perhaps around 10 and should be whitelisted. + /// + /// The heaviest branch is likely to be when a proposal is placed into, or moved within, the + /// `TrackQueue`. Basically this happens when a referendum is in the deciding queue and receives + /// a vote, or when it moves into the deciding queue. + fn service_referendum( + now: T::BlockNumber, + index: ReferendumIndex, + mut status: ReferendumStatusOf, + ) -> (ReferendumInfoOf, bool, ServiceBranch) { + let mut dirty = false; + // Should it begin being decided? + let track = match Self::track(status.track) { + Some(x) => x, + None => return (ReferendumInfo::Ongoing(status), false, ServiceBranch::Fail), + }; + let timeout = status.submitted + T::UndecidingTimeout::get(); + // Default the alarm to the submission timeout. + let mut alarm = timeout; + let branch; + match &mut status.deciding { + None => { + // Are we already queued for deciding? + if status.in_queue { + // Does our position in the queue need updating? + let ayes = status.tally.ayes(); + let mut queue = TrackQueue::::get(status.track); + let maybe_old_pos = queue.iter().position(|(x, _)| *x == index); + let new_pos = queue.binary_search_by_key(&ayes, |x| x.1).unwrap_or_else(|x| x); + branch = if maybe_old_pos.is_none() && new_pos > 0 { + // Just insert. + let _ = queue.force_insert_keep_right(new_pos, (index, ayes)); + ServiceBranch::RequeuedInsertion + } else if let Some(old_pos) = maybe_old_pos { + // We were in the queue - slide into the correct position. + queue[old_pos].1 = ayes; + queue.slide(old_pos, new_pos); + ServiceBranch::RequeuedSlide + } else { + ServiceBranch::NotQueued + }; + TrackQueue::::insert(status.track, queue); + } else { + // Are we ready for deciding? + branch = if status.decision_deposit.is_some() { + let prepare_end = status.submitted.saturating_add(track.prepare_period); + if now >= prepare_end { + let (maybe_alarm, branch) = + Self::ready_for_deciding(now, &track, index, &mut status); + if let Some(set_alarm) = maybe_alarm { + alarm = alarm.min(set_alarm); + } + dirty = true; + branch + } else { + alarm = alarm.min(prepare_end); + ServiceBranch::Preparing + } + } else { + ServiceBranch::NoDeposit + } + } + // If we didn't move into being decided, then check the timeout. + if status.deciding.is_none() && now >= timeout { + // Too long without being decided - end it. + Self::ensure_no_alarm(&mut status); + Self::deposit_event(Event::::TimedOut { index, tally: status.tally }); + return ( + ReferendumInfo::TimedOut( + now, + status.submission_deposit, + status.decision_deposit, + ), + true, + ServiceBranch::TimedOut, + ) + } + }, + Some(deciding) => { + let is_passing = Self::is_passing( + &status.tally, + now.saturating_sub(deciding.since), + track.decision_period, + &track.min_turnout, + &track.min_approval, + ); + branch = if is_passing { + match deciding.confirming.clone() { + Some(t) if now >= t => { + // Passed! + Self::ensure_no_alarm(&mut status); + Self::note_one_fewer_deciding(status.track); + let (desired, call_hash) = (status.enactment, status.proposal_hash); + Self::schedule_enactment( + index, + track, + desired, + status.origin, + call_hash, + ); + Self::deposit_event(Event::::Confirmed { + index, + tally: status.tally, + }); + return ( + ReferendumInfo::Approved( + now, + status.submission_deposit, + status.decision_deposit, + ), + true, + ServiceBranch::Approved, + ) + }, + Some(_) => ServiceBranch::ContinueConfirming, + None => { + // Start confirming + dirty = true; + deciding.confirming = Some(now.saturating_add(track.confirm_period)); + Self::deposit_event(Event::::ConfirmStarted { index }); + ServiceBranch::BeginConfirming + }, + } + } else { + if now >= deciding.since.saturating_add(track.decision_period) { + // Failed! + Self::ensure_no_alarm(&mut status); + Self::note_one_fewer_deciding(status.track); + Self::deposit_event(Event::::Rejected { index, tally: status.tally }); + return ( + ReferendumInfo::Rejected( + now, + status.submission_deposit, + status.decision_deposit, + ), + true, + ServiceBranch::Rejected, + ) + } + if deciding.confirming.is_some() { + // Stop confirming + dirty = true; + deciding.confirming = None; + Self::deposit_event(Event::::ConfirmAborted { index }); + ServiceBranch::EndConfirming + } else { + ServiceBranch::ContinueNotConfirming + } + }; + alarm = Self::decision_time(&deciding, &status.tally, track); + }, + } + + let dirty_alarm = Self::ensure_alarm_at(&mut status, index, alarm); + (ReferendumInfo::Ongoing(status), dirty_alarm || dirty, branch) + } + + /// Determine the point at which a referendum will be accepted, move into confirmation with the + /// given `tally` or end with rejection (whichever happens sooner). + fn decision_time( + deciding: &DecidingStatusOf, + tally: &T::Tally, + track: &TrackInfoOf, + ) -> T::BlockNumber { + deciding.confirming.unwrap_or_else(|| { + // Set alarm to the point where the current voting would make it pass. + let approval = tally.approval(); + let turnout = tally.turnout(); + let until_approval = track.min_approval.delay(approval); + let until_turnout = track.min_turnout.delay(turnout); + let offset = until_turnout.max(until_approval); + deciding.since.saturating_add(offset * track.decision_period) + }) + } + + /// Cancel the alarm in `status`, if one exists. + fn ensure_no_alarm(status: &mut ReferendumStatusOf) { + if let Some((_, last_alarm)) = status.alarm.take() { + // Incorrect alarm - cancel it. + let _ = T::Scheduler::cancel(last_alarm); + } + } + + /// Reserve a deposit and return the `Deposit` instance. + fn take_deposit( + who: T::AccountId, + amount: BalanceOf, + ) -> Result>, DispatchError> { + T::Currency::reserve(&who, amount)?; + Ok(Deposit { who, amount }) + } + + /// Return a deposit, if `Some`. + fn refund_deposit(deposit: Option>>) { + if let Some(Deposit { who, amount }) = deposit { + T::Currency::unreserve(&who, amount); + } + } + + /// Slash a deposit, if `Some`. + fn slash_deposit(deposit: Option>>) { + if let Some(Deposit { who, amount }) = deposit { + T::Slash::on_unbalanced(T::Currency::slash_reserved(&who, amount).0); + Self::deposit_event(Event::::DepositSlashed { who, amount }); + } + } + + /// Get the track info value for the track `id`. + fn track(id: TrackIdOf) -> Option<&'static TrackInfoOf> { + let tracks = T::Tracks::tracks(); + let index = tracks.binary_search_by_key(&id, |x| x.0).unwrap_or_else(|x| x); + Some(&tracks[index].1) + } + + /// Determine whether the given `tally` would result in a referendum passing at `elapsed` blocks + /// into a total decision `period`, given the two curves for `turnout_needed` and + /// `approval_needed`. + fn is_passing( + tally: &T::Tally, + elapsed: T::BlockNumber, + period: T::BlockNumber, + turnout_needed: &Curve, + approval_needed: &Curve, + ) -> bool { + let x = Perbill::from_rational(elapsed.min(period), period); + turnout_needed.passing(x, tally.turnout()) && approval_needed.passing(x, tally.approval()) + } +} diff --git a/frame/referenda/src/mock.rs b/frame/referenda/src/mock.rs new file mode 100644 index 000000000000..fdd14fdadf04 --- /dev/null +++ b/frame/referenda/src/mock.rs @@ -0,0 +1,460 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The crate's tests. + +use super::*; +use crate as pallet_referenda; +use codec::{Decode, Encode, MaxEncodedLen}; +use frame_support::{ + assert_ok, ord_parameter_types, parameter_types, + traits::{ + ConstU32, ConstU64, Contains, EqualPrivilegeOnly, OnInitialize, OriginTrait, Polling, + PreimageRecipient, SortedMembers, + }, + weights::Weight, +}; +use frame_system::{EnsureRoot, EnsureSignedBy}; +use sp_core::H256; +use sp_runtime::{ + testing::Header, + traits::{BlakeTwo256, Hash, IdentityLookup}, + DispatchResult, Perbill, +}; + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; + +frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system, + Balances: pallet_balances, + Preimage: pallet_preimage, + Scheduler: pallet_scheduler, + Referenda: pallet_referenda, + } +); + +// Test that a fitlered call can be dispatched. +pub struct BaseFilter; +impl Contains for BaseFilter { + fn contains(call: &Call) -> bool { + !matches!(call, &Call::Balances(pallet_balances::Call::set_balance { .. })) + } +} + +parameter_types! { + pub const BlockHashCount: u64 = 250; + pub BlockWeights: frame_system::limits::BlockWeights = + frame_system::limits::BlockWeights::simple_max(1_000_000); +} +impl frame_system::Config for Test { + type BaseCallFilter = BaseFilter; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Call = Call; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = BlockHashCount; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = ConstU32<16>; +} +impl pallet_preimage::Config for Test { + type Event = Event; + type WeightInfo = (); + type Currency = Balances; + type ManagerOrigin = EnsureRoot; + type MaxSize = ConstU32<4096>; + type BaseDeposit = (); + type ByteDeposit = (); +} +parameter_types! { + pub MaximumSchedulerWeight: Weight = 2_000_000_000_000; +} +impl pallet_scheduler::Config for Test { + type Event = Event; + type Origin = Origin; + type PalletsOrigin = OriginCaller; + type Call = Call; + type MaximumWeight = MaximumSchedulerWeight; + type ScheduleOrigin = EnsureRoot; + type MaxScheduledPerBlock = ConstU32<100>; + type WeightInfo = (); + type OriginPrivilegeCmp = EqualPrivilegeOnly; + type PreimageProvider = Preimage; + type NoPreimagePostponement = ConstU64<10>; +} +parameter_types! { + pub const ExistentialDeposit: u64 = 1; + pub const MaxLocks: u32 = 10; +} +impl pallet_balances::Config for Test { + type MaxReserves = (); + type ReserveIdentifier = [u8; 8]; + type MaxLocks = MaxLocks; + type Balance = u64; + type Event = Event; + type DustRemoval = (); + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; + type WeightInfo = (); +} +parameter_types! { + pub static AlarmInterval: u64 = 1; + pub const SubmissionDeposit: u64 = 2; + pub const MaxQueued: u32 = 3; + pub const UndecidingTimeout: u64 = 20; +} +ord_parameter_types! { + pub const One: u64 = 1; + pub const Two: u64 = 2; + pub const Three: u64 = 3; + pub const Four: u64 = 4; + pub const Five: u64 = 5; + pub const Six: u64 = 6; +} +pub struct OneToFive; +impl SortedMembers for OneToFive { + fn sorted_members() -> Vec { + vec![1, 2, 3, 4, 5] + } + #[cfg(feature = "runtime-benchmarks")] + fn add(_m: &u64) {} +} + +pub struct TestTracksInfo; +impl TracksInfo for TestTracksInfo { + type Id = u8; + type Origin = ::PalletsOrigin; + fn tracks() -> &'static [(Self::Id, TrackInfo)] { + static DATA: [(u8, TrackInfo); 2] = [ + ( + 0u8, + TrackInfo { + name: "root", + max_deciding: 1, + decision_deposit: 10, + prepare_period: 4, + decision_period: 4, + confirm_period: 2, + min_enactment_period: 4, + min_approval: Curve::LinearDecreasing { + begin: Perbill::from_percent(100), + delta: Perbill::from_percent(50), + }, + min_turnout: Curve::LinearDecreasing { + begin: Perbill::from_percent(100), + delta: Perbill::from_percent(100), + }, + }, + ), + ( + 1u8, + TrackInfo { + name: "none", + max_deciding: 3, + decision_deposit: 1, + prepare_period: 2, + decision_period: 2, + confirm_period: 1, + min_enactment_period: 2, + min_approval: Curve::LinearDecreasing { + begin: Perbill::from_percent(55), + delta: Perbill::from_percent(5), + }, + min_turnout: Curve::LinearDecreasing { + begin: Perbill::from_percent(10), + delta: Perbill::from_percent(10), + }, + }, + ), + ]; + &DATA[..] + } + fn track_for(id: &Self::Origin) -> Result { + if let Ok(system_origin) = frame_system::RawOrigin::try_from(id.clone()) { + match system_origin { + frame_system::RawOrigin::Root => Ok(0), + frame_system::RawOrigin::None => Ok(1), + _ => Err(()), + } + } else { + Err(()) + } + } +} + +impl Config for Test { + type WeightInfo = (); + type Call = Call; + type Event = Event; + type Scheduler = Scheduler; + type Currency = pallet_balances::Pallet; + type CancelOrigin = EnsureSignedBy; + type KillOrigin = EnsureRoot; + type Slash = (); + type Votes = u32; + type Tally = Tally; + type SubmissionDeposit = SubmissionDeposit; + type MaxQueued = MaxQueued; + type UndecidingTimeout = UndecidingTimeout; + type AlarmInterval = AlarmInterval; + type Tracks = TestTracksInfo; +} + +pub fn new_test_ext() -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let balances = vec![(1, 100), (2, 100), (3, 100), (4, 100), (5, 100), (6, 100)]; + pallet_balances::GenesisConfig:: { balances } + .assimilate_storage(&mut t) + .unwrap(); + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext +} + +/// Execute the function two times, with `true` and with `false`. +#[allow(dead_code)] +pub fn new_test_ext_execute_with_cond(execute: impl FnOnce(bool) -> () + Clone) { + new_test_ext().execute_with(|| (execute.clone())(false)); + new_test_ext().execute_with(|| execute(true)); +} + +#[derive(Encode, Debug, Decode, TypeInfo, Eq, PartialEq, Clone, Default, MaxEncodedLen)] +pub struct Tally { + pub ayes: u32, + pub nays: u32, +} + +impl VoteTally for Tally { + fn ayes(&self) -> u32 { + self.ayes + } + + fn turnout(&self) -> Perbill { + Perbill::from_percent(self.ayes + self.nays) + } + + fn approval(&self) -> Perbill { + Perbill::from_rational(self.ayes, self.ayes + self.nays) + } + + #[cfg(feature = "runtime-benchmarks")] + fn unanimity() -> Self { + Self { ayes: 100, nays: 0 } + } + + #[cfg(feature = "runtime-benchmarks")] + fn from_requirements(turnout: Perbill, approval: Perbill) -> Self { + let turnout = turnout.mul_ceil(100u32); + let ayes = approval.mul_ceil(turnout); + Self { ayes, nays: turnout - ayes } + } +} + +pub fn set_balance_proposal(value: u64) -> Vec { + Call::Balances(pallet_balances::Call::set_balance { who: 42, new_free: value, new_reserved: 0 }) + .encode() +} + +pub fn set_balance_proposal_hash(value: u64) -> H256 { + let c = Call::Balances(pallet_balances::Call::set_balance { + who: 42, + new_free: value, + new_reserved: 0, + }); + >::note_preimage(c.encode().try_into().unwrap()); + BlakeTwo256::hash_of(&c) +} + +#[allow(dead_code)] +pub fn propose_set_balance(who: u64, value: u64, delay: u64) -> DispatchResult { + Referenda::submit( + Origin::signed(who), + frame_system::RawOrigin::Root.into(), + set_balance_proposal_hash(value), + DispatchTime::After(delay), + ) +} + +pub fn next_block() { + System::set_block_number(System::block_number() + 1); + Scheduler::on_initialize(System::block_number()); +} + +pub fn run_to(n: u64) { + while System::block_number() < n { + next_block(); + } +} + +#[allow(dead_code)] +pub fn begin_referendum() -> ReferendumIndex { + System::set_block_number(0); + assert_ok!(propose_set_balance(1, 2, 1)); + run_to(2); + 0 +} + +#[allow(dead_code)] +pub fn tally(r: ReferendumIndex) -> Tally { + Referenda::ensure_ongoing(r).unwrap().tally +} + +pub fn set_tally(index: ReferendumIndex, ayes: u32, nays: u32) { + >::access_poll(index, |status| { + let tally = status.ensure_ongoing().unwrap().0; + tally.ayes = ayes; + tally.nays = nays; + }); +} + +pub fn waiting_since(i: ReferendumIndex) -> u64 { + match ReferendumInfoFor::::get(i).unwrap() { + ReferendumInfo::Ongoing(ReferendumStatus { submitted, deciding: None, .. }) => submitted, + _ => panic!("Not waiting"), + } +} + +pub fn deciding_since(i: ReferendumIndex) -> u64 { + match ReferendumInfoFor::::get(i).unwrap() { + ReferendumInfo::Ongoing(ReferendumStatus { + deciding: Some(DecidingStatus { since, .. }), + .. + }) => since, + _ => panic!("Not deciding"), + } +} + +pub fn deciding_and_failing_since(i: ReferendumIndex) -> u64 { + match ReferendumInfoFor::::get(i).unwrap() { + ReferendumInfo::Ongoing(ReferendumStatus { + deciding: Some(DecidingStatus { since, confirming: None, .. }), + .. + }) => since, + _ => panic!("Not deciding"), + } +} + +pub fn confirming_until(i: ReferendumIndex) -> u64 { + match ReferendumInfoFor::::get(i).unwrap() { + ReferendumInfo::Ongoing(ReferendumStatus { + deciding: Some(DecidingStatus { confirming: Some(until), .. }), + .. + }) => until, + _ => panic!("Not confirming"), + } +} + +pub fn approved_since(i: ReferendumIndex) -> u64 { + match ReferendumInfoFor::::get(i).unwrap() { + ReferendumInfo::Approved(since, ..) => since, + _ => panic!("Not approved"), + } +} + +pub fn rejected_since(i: ReferendumIndex) -> u64 { + match ReferendumInfoFor::::get(i).unwrap() { + ReferendumInfo::Rejected(since, ..) => since, + _ => panic!("Not rejected"), + } +} + +pub fn cancelled_since(i: ReferendumIndex) -> u64 { + match ReferendumInfoFor::::get(i).unwrap() { + ReferendumInfo::Cancelled(since, ..) => since, + _ => panic!("Not cancelled"), + } +} + +pub fn killed_since(i: ReferendumIndex) -> u64 { + match ReferendumInfoFor::::get(i).unwrap() { + ReferendumInfo::Killed(since, ..) => since, + _ => panic!("Not killed"), + } +} + +pub fn timed_out_since(i: ReferendumIndex) -> u64 { + match ReferendumInfoFor::::get(i).unwrap() { + ReferendumInfo::TimedOut(since, ..) => since, + _ => panic!("Not timed out"), + } +} + +fn is_deciding(i: ReferendumIndex) -> bool { + matches!( + ReferendumInfoFor::::get(i), + Some(ReferendumInfo::Ongoing(ReferendumStatus { deciding: Some(_), .. })) + ) +} + +#[derive(Clone, Copy)] +pub enum RefState { + Failing, + Passing, + Confirming { immediate: bool }, +} + +impl RefState { + pub fn create(self) -> ReferendumIndex { + assert_ok!(Referenda::submit( + Origin::signed(1), + frame_support::dispatch::RawOrigin::Root.into(), + set_balance_proposal_hash(1), + DispatchTime::At(10), + )); + assert_ok!(Referenda::place_decision_deposit(Origin::signed(2), 0)); + if matches!(self, RefState::Confirming { immediate: true }) { + set_tally(0, 100, 0); + } + let index = ReferendumCount::::get() - 1; + while !is_deciding(index) { + run_to(System::block_number() + 1); + } + if matches!(self, RefState::Confirming { immediate: false }) { + set_tally(0, 100, 0); + run_to(System::block_number() + 1); + } + if matches!(self, RefState::Confirming { .. }) { + assert_eq!(confirming_until(index), System::block_number() + 2); + } + if matches!(self, RefState::Passing) { + set_tally(0, 100, 99); + run_to(System::block_number() + 1); + } + index + } +} diff --git a/frame/referenda/src/tests.rs b/frame/referenda/src/tests.rs new file mode 100644 index 000000000000..96edd4ce879c --- /dev/null +++ b/frame/referenda/src/tests.rs @@ -0,0 +1,516 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The crate's tests. + +use super::*; +use crate::mock::{RefState::*, *}; +use assert_matches::assert_matches; +use codec::Decode; +use frame_support::{ + assert_noop, assert_ok, + dispatch::{DispatchError::BadOrigin, RawOrigin}, + traits::Contains, +}; +use pallet_balances::Error as BalancesError; + +// TODO: Scheduler should re-use `None` items in its `Agenda`. + +#[test] +fn params_should_work() { + new_test_ext().execute_with(|| { + assert_eq!(ReferendumCount::::get(), 0); + assert_eq!(Balances::free_balance(42), 0); + assert_eq!(Balances::total_issuance(), 600); + }); +} + +#[test] +fn basic_happy_path_works() { + new_test_ext().execute_with(|| { + // #1: submit + assert_ok!(Referenda::submit( + Origin::signed(1), + RawOrigin::Root.into(), + set_balance_proposal_hash(1), + DispatchTime::At(10), + )); + assert_eq!(Balances::reserved_balance(&1), 2); + assert_eq!(ReferendumCount::::get(), 1); + assert_ok!(Referenda::place_decision_deposit(Origin::signed(2), 0)); + run_to(4); + assert_eq!(DecidingCount::::get(0), 0); + run_to(5); + // #5: 4 blocks after submit - vote should now be deciding. + assert_eq!(DecidingCount::::get(0), 1); + run_to(6); + // #6: Lots of ayes. Should now be confirming. + set_tally(0, 100, 0); + run_to(7); + assert_eq!(confirming_until(0), 9); + run_to(9); + // #8: Should be confirmed & ended. + assert_eq!(approved_since(0), 9); + assert_ok!(Referenda::refund_decision_deposit(Origin::signed(2), 0)); + run_to(12); + // #9: Should not yet be enacted. + assert_eq!(Balances::free_balance(&42), 0); + run_to(13); + // #10: Proposal should be executed. + assert_eq!(Balances::free_balance(&42), 1); + }); +} + +#[test] +fn insta_confirm_then_kill_works() { + new_test_ext().execute_with(|| { + let r = Confirming { immediate: true }.create(); + run_to(6); + assert_ok!(Referenda::kill(Origin::root(), r)); + assert_eq!(killed_since(r), 6); + }); +} + +#[test] +fn confirm_then_reconfirm_with_elapsed_trigger_works() { + new_test_ext().execute_with(|| { + let r = Confirming { immediate: false }.create(); + assert_eq!(confirming_until(r), 8); + run_to(7); + set_tally(r, 100, 99); + run_to(8); + assert_eq!(deciding_and_failing_since(r), 5); + run_to(11); + assert_eq!(approved_since(r), 11); + }); +} + +#[test] +fn instaconfirm_then_reconfirm_with_elapsed_trigger_works() { + new_test_ext().execute_with(|| { + let r = Confirming { immediate: true }.create(); + run_to(6); + assert_eq!(confirming_until(r), 7); + set_tally(r, 100, 99); + run_to(7); + assert_eq!(deciding_and_failing_since(r), 5); + run_to(11); + assert_eq!(approved_since(r), 11); + }); +} + +#[test] +fn instaconfirm_then_reconfirm_with_voting_trigger_works() { + new_test_ext().execute_with(|| { + let r = Confirming { immediate: true }.create(); + run_to(6); + assert_eq!(confirming_until(r), 7); + set_tally(r, 100, 99); + run_to(7); + assert_eq!(deciding_and_failing_since(r), 5); + run_to(8); + set_tally(r, 100, 0); + run_to(9); + assert_eq!(confirming_until(r), 11); + run_to(11); + assert_eq!(approved_since(r), 11); + }); +} + +#[test] +fn voting_should_extend_for_late_confirmation() { + new_test_ext().execute_with(|| { + let r = Passing.create(); + run_to(10); + assert_eq!(confirming_until(r), 11); + run_to(11); + assert_eq!(approved_since(r), 11); + }); +} + +#[test] +fn should_instafail_during_extension_confirmation() { + new_test_ext().execute_with(|| { + let r = Passing.create(); + run_to(10); + assert_eq!(confirming_until(r), 11); + // Should insta-fail since it's now past the normal voting time. + set_tally(r, 100, 101); + run_to(11); + assert_eq!(rejected_since(r), 11); + }); +} + +#[test] +fn confirming_then_fail_works() { + new_test_ext().execute_with(|| { + let r = Failing.create(); + // Normally ends at 5 + 4 (voting period) = 9. + assert_eq!(deciding_and_failing_since(r), 5); + set_tally(r, 100, 0); + run_to(6); + assert_eq!(confirming_until(r), 8); + set_tally(r, 100, 101); + run_to(9); + assert_eq!(rejected_since(r), 9); + }); +} + +#[test] +fn queueing_works() { + new_test_ext().execute_with(|| { + // Submit a proposal into a track with a queue len of 1. + assert_ok!(Referenda::submit( + Origin::signed(5), + RawOrigin::Root.into(), + set_balance_proposal_hash(0), + DispatchTime::After(0), + )); + assert_ok!(Referenda::place_decision_deposit(Origin::signed(5), 0)); + + run_to(2); + + // Submit 3 more proposals into the same queue. + for i in 1..=4 { + assert_ok!(Referenda::submit( + Origin::signed(i), + RawOrigin::Root.into(), + set_balance_proposal_hash(i), + DispatchTime::After(0), + )); + assert_ok!(Referenda::place_decision_deposit(Origin::signed(i), i as u32)); + // TODO: decision deposit after some initial votes with a non-highest voted coming + // first. + } + assert_eq!(ReferendumCount::::get(), 5); + + run_to(5); + // One should be being decided. + assert_eq!(DecidingCount::::get(0), 1); + assert_eq!(deciding_and_failing_since(0), 5); + for i in 1..=4 { + assert_eq!(waiting_since(i), 2); + } + + // Vote to set order. + set_tally(1, 1, 10); + set_tally(2, 2, 20); + set_tally(3, 3, 30); + set_tally(4, 100, 0); + println!("Agenda #6: {:?}", pallet_scheduler::Agenda::::get(6)); + run_to(6); + println!("{:?}", Vec::<_>::from(TrackQueue::::get(0))); + + // Cancel the first. + assert_ok!(Referenda::cancel(Origin::signed(4), 0)); + assert_eq!(cancelled_since(0), 6); + + // The other with the most approvals (#4) should be being decided. + run_to(7); + assert_eq!(DecidingCount::::get(0), 1); + assert_eq!(deciding_since(4), 7); + assert_eq!(confirming_until(4), 9); + + // Vote on the remaining two to change order. + println!("Set tally #1"); + set_tally(1, 30, 31); + println!("{:?}", Vec::<_>::from(TrackQueue::::get(0))); + println!("Set tally #2"); + set_tally(2, 20, 20); + println!("{:?}", Vec::<_>::from(TrackQueue::::get(0))); + + // Let confirmation period end. + run_to(9); + + // #4 should have been confirmed. + assert_eq!(approved_since(4), 9); + + // On to the next block to select the new referendum + run_to(10); + // #1 (the one with the most approvals) should now be being decided. + assert_eq!(deciding_since(1), 10); + + // Let it end unsuccessfully. + run_to(14); + assert_eq!(rejected_since(1), 14); + + // Service queue. + run_to(15); + // #2 should now be being decided. It will (barely) pass. + assert_eq!(deciding_and_failing_since(2), 15); + + // #2 moves into confirming at the last moment with a 50% approval. + run_to(19); + assert_eq!(confirming_until(2), 21); + + // #2 gets approved. + run_to(21); + assert_eq!(approved_since(2), 21); + + // The final one has since timed out. + run_to(22); + assert_eq!(timed_out_since(3), 22); + }); +} + +#[test] +fn auto_timeout_should_happen_with_nothing_but_submit() { + new_test_ext().execute_with(|| { + // #1: submit + assert_ok!(Referenda::submit( + Origin::signed(1), + RawOrigin::Root.into(), + set_balance_proposal_hash(1), + DispatchTime::At(20), + )); + run_to(20); + assert_matches!(ReferendumInfoFor::::get(0), Some(ReferendumInfo::Ongoing(..))); + run_to(21); + // #11: Timed out - ended. + assert_matches!( + ReferendumInfoFor::::get(0), + Some(ReferendumInfo::TimedOut(21, _, None)) + ); + }); +} + +#[test] +fn tracks_are_distinguished() { + new_test_ext().execute_with(|| { + assert_ok!(Referenda::submit( + Origin::signed(1), + RawOrigin::Root.into(), + set_balance_proposal_hash(1), + DispatchTime::At(10), + )); + assert_ok!(Referenda::submit( + Origin::signed(2), + RawOrigin::None.into(), + set_balance_proposal_hash(2), + DispatchTime::At(20), + )); + + assert_ok!(Referenda::place_decision_deposit(Origin::signed(3), 0)); + assert_ok!(Referenda::place_decision_deposit(Origin::signed(4), 1)); + + let mut i = ReferendumInfoFor::::iter().collect::>(); + i.sort_by_key(|x| x.0); + assert_eq!( + i, + vec![ + ( + 0, + ReferendumInfo::Ongoing(ReferendumStatus { + track: 0, + origin: OriginCaller::system(RawOrigin::Root), + proposal_hash: set_balance_proposal_hash(1), + enactment: DispatchTime::At(10), + submitted: 1, + submission_deposit: Deposit { who: 1, amount: 2 }, + decision_deposit: Some(Deposit { who: 3, amount: 10 }), + deciding: None, + tally: Tally { ayes: 0, nays: 0 }, + in_queue: false, + alarm: Some((5, (5, 0))), + }) + ), + ( + 1, + ReferendumInfo::Ongoing(ReferendumStatus { + track: 1, + origin: OriginCaller::system(RawOrigin::None), + proposal_hash: set_balance_proposal_hash(2), + enactment: DispatchTime::At(20), + submitted: 1, + submission_deposit: Deposit { who: 2, amount: 2 }, + decision_deposit: Some(Deposit { who: 4, amount: 1 }), + deciding: None, + tally: Tally { ayes: 0, nays: 0 }, + in_queue: false, + alarm: Some((3, (3, 0))), + }) + ), + ] + ); + }); +} + +#[test] +fn submit_errors_work() { + new_test_ext().execute_with(|| { + let h = set_balance_proposal_hash(1); + // No track for Signed origins. + assert_noop!( + Referenda::submit( + Origin::signed(1), + RawOrigin::Signed(2).into(), + h, + DispatchTime::At(10), + ), + Error::::NoTrack + ); + + // No funds for deposit + assert_noop!( + Referenda::submit(Origin::signed(10), RawOrigin::Root.into(), h, DispatchTime::At(10),), + BalancesError::::InsufficientBalance + ); + }); +} + +#[test] +fn decision_deposit_errors_work() { + new_test_ext().execute_with(|| { + let e = Error::::NotOngoing; + assert_noop!(Referenda::place_decision_deposit(Origin::signed(2), 0), e); + + let h = set_balance_proposal_hash(1); + assert_ok!(Referenda::submit( + Origin::signed(1), + RawOrigin::Root.into(), + h, + DispatchTime::At(10), + )); + let e = BalancesError::::InsufficientBalance; + assert_noop!(Referenda::place_decision_deposit(Origin::signed(10), 0), e); + + assert_ok!(Referenda::place_decision_deposit(Origin::signed(2), 0)); + let e = Error::::HasDeposit; + assert_noop!(Referenda::place_decision_deposit(Origin::signed(2), 0), e); + }); +} + +#[test] +fn refund_deposit_works() { + new_test_ext().execute_with(|| { + let e = Error::::BadReferendum; + assert_noop!(Referenda::refund_decision_deposit(Origin::signed(1), 0), e); + + let h = set_balance_proposal_hash(1); + assert_ok!(Referenda::submit( + Origin::signed(1), + RawOrigin::Root.into(), + h, + DispatchTime::At(10), + )); + let e = Error::::NoDeposit; + assert_noop!(Referenda::refund_decision_deposit(Origin::signed(2), 0), e); + + assert_ok!(Referenda::place_decision_deposit(Origin::signed(2), 0)); + let e = Error::::Unfinished; + assert_noop!(Referenda::refund_decision_deposit(Origin::signed(3), 0), e); + + run_to(11); + assert_ok!(Referenda::refund_decision_deposit(Origin::signed(3), 0)); + }); +} + +#[test] +fn cancel_works() { + new_test_ext().execute_with(|| { + let h = set_balance_proposal_hash(1); + assert_ok!(Referenda::submit( + Origin::signed(1), + RawOrigin::Root.into(), + h, + DispatchTime::At(10), + )); + assert_ok!(Referenda::place_decision_deposit(Origin::signed(2), 0)); + + run_to(8); + assert_ok!(Referenda::cancel(Origin::signed(4), 0)); + assert_ok!(Referenda::refund_decision_deposit(Origin::signed(3), 0)); + assert_eq!(cancelled_since(0), 8); + }); +} + +#[test] +fn cancel_errors_works() { + new_test_ext().execute_with(|| { + let h = set_balance_proposal_hash(1); + assert_ok!(Referenda::submit( + Origin::signed(1), + RawOrigin::Root.into(), + h, + DispatchTime::At(10), + )); + assert_ok!(Referenda::place_decision_deposit(Origin::signed(2), 0)); + assert_noop!(Referenda::cancel(Origin::signed(1), 0), BadOrigin); + + run_to(11); + assert_noop!(Referenda::cancel(Origin::signed(4), 0), Error::::NotOngoing); + }); +} + +#[test] +fn kill_works() { + new_test_ext().execute_with(|| { + let h = set_balance_proposal_hash(1); + assert_ok!(Referenda::submit( + Origin::signed(1), + RawOrigin::Root.into(), + h, + DispatchTime::At(10), + )); + assert_ok!(Referenda::place_decision_deposit(Origin::signed(2), 0)); + + run_to(8); + assert_ok!(Referenda::kill(Origin::root(), 0)); + let e = Error::::NoDeposit; + assert_noop!(Referenda::refund_decision_deposit(Origin::signed(3), 0), e); + assert_eq!(killed_since(0), 8); + }); +} + +#[test] +fn kill_errors_works() { + new_test_ext().execute_with(|| { + let h = set_balance_proposal_hash(1); + assert_ok!(Referenda::submit( + Origin::signed(1), + RawOrigin::Root.into(), + h, + DispatchTime::At(10), + )); + assert_ok!(Referenda::place_decision_deposit(Origin::signed(2), 0)); + assert_noop!(Referenda::kill(Origin::signed(4), 0), BadOrigin); + + run_to(11); + assert_noop!(Referenda::kill(Origin::root(), 0), Error::::NotOngoing); + }); +} + +#[test] +fn set_balance_proposal_is_correctly_filtered_out() { + for i in 0..10 { + let call = crate::mock::Call::decode(&mut &set_balance_proposal(i)[..]).unwrap(); + assert!(!::BaseCallFilter::contains(&call)); + } +} + +#[test] +fn curve_handles_all_inputs() { + let test_curve = Curve::LinearDecreasing { begin: Perbill::zero(), delta: Perbill::zero() }; + + let delay = test_curve.delay(Perbill::zero()); + assert_eq!(delay, Perbill::zero()); + + let test_curve = Curve::LinearDecreasing { begin: Perbill::zero(), delta: Perbill::one() }; + + let threshold = test_curve.threshold(Perbill::one()); + assert_eq!(threshold, Perbill::zero()); +} diff --git a/frame/referenda/src/types.rs b/frame/referenda/src/types.rs new file mode 100644 index 000000000000..622075100631 --- /dev/null +++ b/frame/referenda/src/types.rs @@ -0,0 +1,339 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Miscellaneous additional datatypes. + +use super::*; +use codec::{Decode, Encode, EncodeLike, MaxEncodedLen}; +use frame_support::{traits::schedule::Anon, Parameter}; +use scale_info::TypeInfo; +use sp_runtime::RuntimeDebug; +use sp_std::fmt::Debug; + +pub type BalanceOf = + <>::Currency as Currency<::AccountId>>::Balance; +pub type NegativeImbalanceOf = <>::Currency as Currency< + ::AccountId, +>>::NegativeImbalance; +pub type CallOf = >::Call; +pub type VotesOf = >::Votes; +pub type TallyOf = >::Tally; +pub type PalletsOriginOf = <::Origin as OriginTrait>::PalletsOrigin; +pub type ReferendumInfoOf = ReferendumInfo< + TrackIdOf, + PalletsOriginOf, + ::BlockNumber, + ::Hash, + BalanceOf, + TallyOf, + ::AccountId, + ScheduleAddressOf, +>; +pub type ReferendumStatusOf = ReferendumStatus< + TrackIdOf, + PalletsOriginOf, + ::BlockNumber, + ::Hash, + BalanceOf, + TallyOf, + ::AccountId, + ScheduleAddressOf, +>; +pub type DecidingStatusOf = DecidingStatus<::BlockNumber>; +pub type TrackInfoOf = + TrackInfo, ::BlockNumber>; +pub type TrackIdOf = <>::Tracks as TracksInfo< + BalanceOf, + ::BlockNumber, +>>::Id; +pub type ScheduleAddressOf = <>::Scheduler as Anon< + ::BlockNumber, + CallOf, + PalletsOriginOf, +>>::Address; + +/// A referendum index. +pub type ReferendumIndex = u32; + +pub trait InsertSorted { + /// Inserts an item into a sorted series. + /// + /// Returns `true` if it was inserted, `false` if it would belong beyond the bound of the + /// series. + fn insert_sorted_by_key K, K: PartialOrd + Ord>( + &mut self, + t: T, + f: F, + ) -> bool; +} +impl> InsertSorted for BoundedVec { + fn insert_sorted_by_key K, K: PartialOrd + Ord>( + &mut self, + t: T, + mut f: F, + ) -> bool { + let index = self.binary_search_by_key::(&f(&t), f).unwrap_or_else(|x| x); + self.force_insert_keep_right(index, t).is_ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use frame_support::traits::ConstU32; + + #[test] + fn insert_sorted_works() { + let mut b: BoundedVec> = vec![20, 30, 40].try_into().unwrap(); + assert!(b.insert_sorted_by_key(10, |&x| x)); + assert_eq!(&b[..], &[10, 20, 30, 40][..]); + + assert!(b.insert_sorted_by_key(60, |&x| x)); + assert_eq!(&b[..], &[10, 20, 30, 40, 60][..]); + + assert!(b.insert_sorted_by_key(50, |&x| x)); + assert_eq!(&b[..], &[10, 20, 30, 40, 50, 60][..]); + + assert!(!b.insert_sorted_by_key(9, |&x| x)); + assert_eq!(&b[..], &[10, 20, 30, 40, 50, 60][..]); + + assert!(b.insert_sorted_by_key(11, |&x| x)); + assert_eq!(&b[..], &[11, 20, 30, 40, 50, 60][..]); + + assert!(b.insert_sorted_by_key(21, |&x| x)); + assert_eq!(&b[..], &[20, 21, 30, 40, 50, 60][..]); + + assert!(b.insert_sorted_by_key(61, |&x| x)); + assert_eq!(&b[..], &[21, 30, 40, 50, 60, 61][..]); + + assert!(b.insert_sorted_by_key(51, |&x| x)); + assert_eq!(&b[..], &[30, 40, 50, 51, 60, 61][..]); + } +} + +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub struct DecidingStatus { + /// When this referendum began being "decided". If confirming, then the + /// end will actually be delayed until the end of the confirmation period. + pub(crate) since: BlockNumber, + /// If `Some`, then the referendum has entered confirmation stage and will end at + /// the block number as long as it doesn't lose its approval in the meantime. + pub(crate) confirming: Option, +} + +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub struct Deposit { + pub(crate) who: AccountId, + pub(crate) amount: Balance, +} + +#[derive(Clone, Encode, TypeInfo)] +pub struct TrackInfo { + /// Name of this track. TODO was &'static str + pub name: &'static str, + /// A limit for the number of referenda on this track that can be being decided at once. + /// For Root origin this should generally be just one. + pub max_deciding: u32, + /// Amount that must be placed on deposit before a decision can be made. + pub decision_deposit: Balance, + /// Amount of time this must be submitted for before a decision can be made. + pub prepare_period: Moment, + /// Amount of time that a decision may take to be approved prior to cancellation. + pub decision_period: Moment, + /// Amount of time that the approval criteria must hold before it can be approved. + pub confirm_period: Moment, + /// Minimum amount of time that an approved proposal must be in the dispatch queue. + pub min_enactment_period: Moment, + /// Minimum aye votes as percentage of overall conviction-weighted votes needed for + /// approval as a function of time into decision period. + pub min_approval: Curve, + /// Minimum turnout as percentage of overall population that is needed for + /// approval as a function of time into decision period. + pub min_turnout: Curve, +} + +/// Information on the voting tracks. +pub trait TracksInfo { + /// The identifier for a track. + type Id: Copy + Parameter + Ord + PartialOrd + Send + Sync + 'static; + + /// The origin type from which a track is implied. + type Origin; + + /// Return the array of known tracks and their information. + fn tracks() -> &'static [(Self::Id, TrackInfo)]; + + /// Determine the voting track for the given `origin`. + fn track_for(origin: &Self::Origin) -> Result; + + /// Return the track info for track `id`, by default this just looks it up in `Self::tracks()`. + fn info(id: Self::Id) -> Option<&'static TrackInfo> { + Self::tracks().iter().find(|x| &x.0 == &id).map(|x| &x.1) + } +} + +/// Info regarding an ongoing referendum. +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub struct ReferendumStatus< + TrackId: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + Origin: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + Moment: Parameter + Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone + EncodeLike, + Hash: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + Balance: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + Tally: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + AccountId: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + ScheduleAddress: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, +> { + /// The track of this referendum. + pub(crate) track: TrackId, + /// The origin for this referendum. + pub(crate) origin: Origin, + /// The hash of the proposal up for referendum. + pub(crate) proposal_hash: Hash, + /// The time the proposal should be scheduled for enactment. + pub(crate) enactment: DispatchTime, + /// The time of submission. Once `UndecidingTimeout` passes, it may be closed by anyone if it + /// `deciding` is `None`. + pub(crate) submitted: Moment, + /// The deposit reserved for the submission of this referendum. + pub(crate) submission_deposit: Deposit, + /// The deposit reserved for this referendum to be decided. + pub(crate) decision_deposit: Option>, + /// The status of a decision being made. If `None`, it has not entered the deciding period. + pub(crate) deciding: Option>, + /// The current tally of votes in this referendum. + pub(crate) tally: Tally, + /// Whether we have been placed in the queue for being decided or not. + pub(crate) in_queue: bool, + /// The next scheduled wake-up, if `Some`. + pub(crate) alarm: Option<(Moment, ScheduleAddress)>, +} + +/// Info regarding a referendum, present or past. +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub enum ReferendumInfo< + TrackId: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + Origin: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + Moment: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone + EncodeLike, + Hash: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + Balance: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + Tally: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + AccountId: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + ScheduleAddress: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, +> { + /// Referendum has been submitted and is being voted on. + Ongoing( + ReferendumStatus, + ), + /// Referendum finished with approval. Submission deposit is held. + Approved(Moment, Deposit, Option>), + /// Referendum finished with rejection. Submission deposit is held. + Rejected(Moment, Deposit, Option>), + /// Referendum finished with cancelation. Submission deposit is held. + Cancelled(Moment, Deposit, Option>), + /// Referendum finished and was never decided. Submission deposit is held. + TimedOut(Moment, Deposit, Option>), + /// Referendum finished with a kill. + Killed(Moment), +} + +impl< + TrackId: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + Origin: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + Moment: Parameter + Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone + EncodeLike, + Hash: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + Balance: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + Tally: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + AccountId: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + ScheduleAddress: Eq + PartialEq + Debug + Encode + Decode + TypeInfo + Clone, + > ReferendumInfo +{ + /// Take the Decision Deposit from `self`, if there is one. Returns an `Err` if `self` is not + /// in a valid state for the Decision Deposit to be refunded. + pub fn take_decision_deposit(&mut self) -> Result>, ()> { + use ReferendumInfo::*; + match self { + Ongoing(x) if x.decision_deposit.is_none() => Ok(None), + // Cannot refund deposit if Ongoing as this breaks assumptions. + Ongoing(_) => Err(()), + Approved(_, _, d) | Rejected(_, _, d) | TimedOut(_, _, d) | Cancelled(_, _, d) => + Ok(d.take()), + Killed(_) => Ok(None), + } + } +} + +/// Type for describing a curve over the 2-dimensional space of axes between 0-1, as represented +/// by `(Perbill, Perbill)`. +#[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen)] +#[cfg_attr(not(feature = "std"), derive(RuntimeDebug))] +pub enum Curve { + /// Linear curve starting at `(0, begin)`, ending at `(period, begin - delta)`. + LinearDecreasing { begin: Perbill, delta: Perbill }, +} + +impl Curve { + /// Determine the `y` value for the given `x` value. + pub(crate) fn threshold(&self, x: Perbill) -> Perbill { + match self { + Self::LinearDecreasing { begin, delta } => *begin - (*delta * x).min(*begin), + } + } + + /// Determine the smallest `x` value such that `passing` returns `true` when passed along with + /// the given `y` value. + /// + /// ```nocompile + /// let c = Curve::LinearDecreasing { begin: Perbill::one(), delta: Perbill::one() }; + /// // ^^^ Can be any curve. + /// let y = Perbill::from_percent(50); + /// // ^^^ Can be any value. + /// let x = c.delay(y); + /// assert!(c.passing(x, y)); + /// ``` + pub fn delay(&self, y: Perbill) -> Perbill { + match self { + Self::LinearDecreasing { begin, delta } => + if delta.is_zero() { + return *delta + } else { + return (*begin - y.min(*begin)).min(*delta) / *delta + }, + } + } + + /// Return `true` iff the `y` value is greater than the curve at the `x`. + pub fn passing(&self, x: Perbill, y: Perbill) -> bool { + y >= self.threshold(x) + } +} + +#[cfg(feature = "std")] +impl Debug for Curve { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { + match self { + Self::LinearDecreasing { begin, delta } => { + write!( + f, + "Linear[(0%, {}%) -> (100%, {}%)]", + *begin * 100u32, + (*begin - *delta) * 100u32, + ) + }, + } + } +} diff --git a/frame/referenda/src/weights.rs b/frame/referenda/src/weights.rs new file mode 100644 index 000000000000..202901bdd10b --- /dev/null +++ b/frame/referenda/src/weights.rs @@ -0,0 +1,491 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for pallet_referenda +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2021-12-19, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 + +// Executed Command: +// target/release/substrate +// benchmark +// --chain=dev +// --steps=50 +// --repeat=20 +// --pallet=pallet_referenda +// --extrinsic=* +// --execution=wasm +// --wasm-execution=compiled +// --heap-pages=4096 +// --output=./frame/referenda/src/weights.rs +// --template=./.maintain/frame-weight-template.hbs + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use sp_std::marker::PhantomData; + +/// Weight functions needed for pallet_referenda. +pub trait WeightInfo { + fn submit() -> Weight; + fn place_decision_deposit_preparing() -> Weight; + fn place_decision_deposit_queued() -> Weight; + fn place_decision_deposit_not_queued() -> Weight; + fn place_decision_deposit_passing() -> Weight; + fn place_decision_deposit_failing() -> Weight; + fn refund_decision_deposit() -> Weight; + fn cancel() -> Weight; + fn kill() -> Weight; + fn one_fewer_deciding_queue_empty() -> Weight; + fn one_fewer_deciding_failing() -> Weight; + fn one_fewer_deciding_passing() -> Weight; + fn nudge_referendum_requeued_insertion() -> Weight; + fn nudge_referendum_requeued_slide() -> Weight; + fn nudge_referendum_queued() -> Weight; + fn nudge_referendum_not_queued() -> Weight; + fn nudge_referendum_no_deposit() -> Weight; + fn nudge_referendum_preparing() -> Weight; + fn nudge_referendum_timed_out() -> Weight; + fn nudge_referendum_begin_deciding_failing() -> Weight; + fn nudge_referendum_begin_deciding_passing() -> Weight; + fn nudge_referendum_begin_confirming() -> Weight; + fn nudge_referendum_end_confirming() -> Weight; + fn nudge_referendum_continue_not_confirming() -> Weight; + fn nudge_referendum_continue_confirming() -> Weight; + fn nudge_referendum_approved() -> Weight; + fn nudge_referendum_rejected() -> Weight; +} + +/// Weights for pallet_referenda using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + // Storage: Referenda ReferendumCount (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + // Storage: Referenda ReferendumInfoFor (r:0 w:1) + fn submit() -> Weight { + (42_395_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn place_decision_deposit_preparing() -> Weight { + (48_113_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:0) + // Storage: Referenda TrackQueue (r:1 w:1) + fn place_decision_deposit_queued() -> Weight { + (53_624_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:0) + // Storage: Referenda TrackQueue (r:1 w:1) + fn place_decision_deposit_not_queued() -> Weight { + (52_560_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn place_decision_deposit_passing() -> Weight { + (54_067_000 as Weight) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn place_decision_deposit_failing() -> Weight { + (52_457_000 as Weight) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + fn refund_decision_deposit() -> Weight { + (28_504_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn cancel() -> Weight { + (40_425_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn kill() -> Weight { + (65_974_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda TrackQueue (r:1 w:0) + // Storage: Referenda DecidingCount (r:1 w:1) + fn one_fewer_deciding_queue_empty() -> Weight { + (8_904_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Referenda TrackQueue (r:1 w:1) + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn one_fewer_deciding_failing() -> Weight { + (181_387_000 as Weight) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + // Storage: Referenda TrackQueue (r:1 w:1) + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn one_fewer_deciding_passing() -> Weight { + (179_753_000 as Weight) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda TrackQueue (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_requeued_insertion() -> Weight { + (53_592_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda TrackQueue (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_requeued_slide() -> Weight { + (53_173_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:0) + // Storage: Referenda TrackQueue (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_queued() -> Weight { + (55_770_000 as Weight) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:0) + // Storage: Referenda TrackQueue (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_not_queued() -> Weight { + (53_922_000 as Weight) + .saturating_add(T::DbWeight::get().reads(4 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_no_deposit() -> Weight { + (26_906_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_preparing() -> Weight { + (27_943_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + fn nudge_referendum_timed_out() -> Weight { + (20_256_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_begin_deciding_failing() -> Weight { + (30_964_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_begin_deciding_passing() -> Weight { + (31_763_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_begin_confirming() -> Weight { + (28_892_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_end_confirming() -> Weight { + (29_666_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_continue_not_confirming() -> Weight { + (29_740_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_continue_confirming() -> Weight { + (29_661_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + // Storage: Scheduler Lookup (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + fn nudge_referendum_approved() -> Weight { + (55_736_000 as Weight) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) + .saturating_add(T::DbWeight::get().writes(5 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_rejected() -> Weight { + (32_726_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } +} + +// For backwards compatibility and tests +impl WeightInfo for () { + // Storage: Referenda ReferendumCount (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + // Storage: Referenda ReferendumInfoFor (r:0 w:1) + fn submit() -> Weight { + (42_395_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn place_decision_deposit_preparing() -> Weight { + (48_113_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:0) + // Storage: Referenda TrackQueue (r:1 w:1) + fn place_decision_deposit_queued() -> Weight { + (53_624_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:0) + // Storage: Referenda TrackQueue (r:1 w:1) + fn place_decision_deposit_not_queued() -> Weight { + (52_560_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn place_decision_deposit_passing() -> Weight { + (54_067_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn place_decision_deposit_failing() -> Weight { + (52_457_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + fn refund_decision_deposit() -> Weight { + (28_504_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn cancel() -> Weight { + (40_425_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn kill() -> Weight { + (65_974_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda TrackQueue (r:1 w:0) + // Storage: Referenda DecidingCount (r:1 w:1) + fn one_fewer_deciding_queue_empty() -> Weight { + (8_904_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Referenda TrackQueue (r:1 w:1) + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn one_fewer_deciding_failing() -> Weight { + (181_387_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + // Storage: Referenda TrackQueue (r:1 w:1) + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + fn one_fewer_deciding_passing() -> Weight { + (179_753_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda TrackQueue (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_requeued_insertion() -> Weight { + (53_592_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda TrackQueue (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_requeued_slide() -> Weight { + (53_173_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:0) + // Storage: Referenda TrackQueue (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_queued() -> Weight { + (55_770_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:0) + // Storage: Referenda TrackQueue (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_not_queued() -> Weight { + (53_922_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(4 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_no_deposit() -> Weight { + (26_906_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_preparing() -> Weight { + (27_943_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + fn nudge_referendum_timed_out() -> Weight { + (20_256_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_begin_deciding_failing() -> Weight { + (30_964_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Referenda DecidingCount (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_begin_deciding_passing() -> Weight { + (31_763_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_begin_confirming() -> Weight { + (28_892_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_end_confirming() -> Weight { + (29_666_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_continue_not_confirming() -> Weight { + (29_740_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_continue_confirming() -> Weight { + (29_661_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:2 w:2) + // Storage: Scheduler Lookup (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + fn nudge_referendum_approved() -> Weight { + (55_736_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) + } + // Storage: Referenda ReferendumInfoFor (r:1 w:1) + // Storage: Scheduler Agenda (r:1 w:1) + fn nudge_referendum_rejected() -> Weight { + (32_726_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } +} diff --git a/frame/scheduler/Cargo.toml b/frame/scheduler/Cargo.toml index 62b21fe04c9d..9000d1f693cc 100644 --- a/frame/scheduler/Cargo.toml +++ b/frame/scheduler/Cargo.toml @@ -2,45 +2,46 @@ name = "pallet-scheduler" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" -license = "Unlicense" -homepage = "https://substrate.dev" +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" -description = "FRAME example pallet" +description = "FRAME Scheduler pallet" readme = "README.md" [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } -frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } log = { version = "0.4.14", default-features = false } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core", default-features = false } +sp-core = { version = "6.0.0", path = "../../primitives/core", default-features = false } substrate-test-utils = { version = "4.0.0-dev", path = "../../test-utils" } +pallet-preimage = { version = "4.0.0-dev", path = "../preimage" } [features] default = ["std"] +runtime-benchmarks = [ + "frame-benchmarking", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", +] std = [ "codec/std", "scale-info/std", + "log/std", + "sp-std/std", + "sp-io/std", "sp-runtime/std", - "frame-benchmarking/std", - "frame-support/std", "frame-system/std", - "sp-io/std", - "sp-std/std", - "log/std", -] -runtime-benchmarks = [ - "frame-benchmarking", - "frame-support/runtime-benchmarks", - "frame-system/runtime-benchmarks", + "frame-support/std", + "frame-benchmarking/std", ] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/scheduler/src/benchmarking.rs b/frame/scheduler/src/benchmarking.rs index 2c164eaede22..9c97bc597338 100644 --- a/frame/scheduler/src/benchmarking.rs +++ b/frame/scheduler/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,12 +17,14 @@ //! Scheduler pallet benchmarking. -#![cfg(feature = "runtime-benchmarks")] - use super::*; -use frame_benchmarking::{benchmarks, impl_benchmark_test_suite}; -use frame_support::{ensure, traits::OnInitialize}; +use frame_benchmarking::benchmarks; +use frame_support::{ + ensure, + traits::{OnInitialize, PreimageProvider, PreimageRecipient}, +}; use frame_system::RawOrigin; +use sp_runtime::traits::Hash; use sp_std::{prelude::*, vec}; use crate::Pallet as Scheduler; @@ -30,37 +32,184 @@ use frame_system::Pallet as System; const BLOCK_NUMBER: u32 = 2; -// Add `n` named items to the schedule -fn fill_schedule(when: T::BlockNumber, n: u32) -> Result<(), &'static str> { - // Essentially a no-op call. - let call = frame_system::Call::set_storage { items: vec![] }; +/// Add `n` named items to the schedule. +/// +/// For `resolved`: +/// - `None`: aborted (hash without preimage) +/// - `Some(true)`: hash resolves into call if possible, plain call otherwise +/// - `Some(false)`: plain call +fn fill_schedule( + when: T::BlockNumber, + n: u32, + periodic: bool, + named: bool, + resolved: Option, +) -> Result<(), &'static str> { for i in 0..n { // Named schedule is strictly heavier than anonymous - Scheduler::::do_schedule_named( - i.encode(), - DispatchTime::At(when), - // Add periodicity - Some((T::BlockNumber::one(), 100)), - // HARD_DEADLINE priority means it gets executed no matter what - 0, - frame_system::RawOrigin::Root.into(), - call.clone().into(), - )?; + let (call, hash) = call_and_hash::(i); + let call_or_hash = match resolved { + Some(true) => { + T::PreimageProvider::note_preimage(call.encode().try_into().unwrap()); + if T::PreimageProvider::have_preimage(&hash) { + CallOrHashOf::::Hash(hash) + } else { + call.into() + } + }, + Some(false) => call.into(), + None => CallOrHashOf::::Hash(hash), + }; + let period = match periodic { + true => Some(((i + 100).into(), 100)), + false => None, + }; + let t = DispatchTime::At(when); + let origin = frame_system::RawOrigin::Root.into(); + if named { + Scheduler::::do_schedule_named(i.encode(), t, period, 0, origin, call_or_hash)?; + } else { + Scheduler::::do_schedule(t, period, 0, origin, call_or_hash)?; + } } ensure!(Agenda::::get(when).len() == n as usize, "didn't fill schedule"); Ok(()) } +fn call_and_hash(i: u32) -> (::Call, T::Hash) { + // Essentially a no-op call. + let call: ::Call = frame_system::Call::remark { remark: i.encode() }.into(); + let hash = T::Hashing::hash_of(&call); + (call, hash) +} + benchmarks! { + on_initialize_periodic_named_resolved { + let s in 1 .. T::MaxScheduledPerBlock::get(); + let when = BLOCK_NUMBER.into(); + fill_schedule::(when, s, true, true, Some(true))?; + }: { Scheduler::::on_initialize(BLOCK_NUMBER.into()); } + verify { + assert_eq!(System::::event_count(), s * 2); + for i in 0..s { + assert_eq!(Agenda::::get(when + (i + 100).into()).len(), 1 as usize); + } + } + + on_initialize_named_resolved { + let s in 1 .. T::MaxScheduledPerBlock::get(); + let when = BLOCK_NUMBER.into(); + fill_schedule::(when, s, false, true, Some(true))?; + }: { Scheduler::::on_initialize(BLOCK_NUMBER.into()); } + verify { + assert_eq!(System::::event_count(), s * 2); + assert!(Agenda::::iter().count() == 0); + } + + on_initialize_periodic_resolved { + let s in 1 .. T::MaxScheduledPerBlock::get(); + let when = BLOCK_NUMBER.into(); + fill_schedule::(when, s, true, false, Some(true))?; + }: { Scheduler::::on_initialize(BLOCK_NUMBER.into()); } + verify { + assert_eq!(System::::event_count(), s * 2); + for i in 0..s { + assert_eq!(Agenda::::get(when + (i + 100).into()).len(), 1 as usize); + } + } + + on_initialize_resolved { + let s in 1 .. T::MaxScheduledPerBlock::get(); + let when = BLOCK_NUMBER.into(); + fill_schedule::(when, s, false, false, Some(true))?; + }: { Scheduler::::on_initialize(BLOCK_NUMBER.into()); } + verify { + assert_eq!(System::::event_count(), s * 2); + assert!(Agenda::::iter().count() == 0); + } + + on_initialize_named_aborted { + let s in 1 .. T::MaxScheduledPerBlock::get(); + let when = BLOCK_NUMBER.into(); + fill_schedule::(when, s, false, true, None)?; + }: { Scheduler::::on_initialize(BLOCK_NUMBER.into()); } + verify { + assert_eq!(System::::event_count(), 0); + if let Some(delay) = T::NoPreimagePostponement::get() { + assert_eq!(Agenda::::get(when + delay).len(), s as usize); + } else { + assert!(Agenda::::iter().count() == 0); + } + } + + on_initialize_aborted { + let s in 1 .. T::MaxScheduledPerBlock::get(); + let when = BLOCK_NUMBER.into(); + fill_schedule::(when, s, false, false, None)?; + }: { Scheduler::::on_initialize(BLOCK_NUMBER.into()); } + verify { + assert_eq!(System::::event_count(), 0); + if let Some(delay) = T::NoPreimagePostponement::get() { + assert_eq!(Agenda::::get(when + delay).len(), s as usize); + } else { + assert!(Agenda::::iter().count() == 0); + } + } + + on_initialize_periodic_named { + let s in 1 .. T::MaxScheduledPerBlock::get(); + let when = BLOCK_NUMBER.into(); + fill_schedule::(when, s, true, true, Some(false))?; + }: { Scheduler::::on_initialize(BLOCK_NUMBER.into()); } + verify { + assert_eq!(System::::event_count(), s); + for i in 0..s { + assert_eq!(Agenda::::get(when + (i + 100).into()).len(), 1 as usize); + } + } + + on_initialize_periodic { + let s in 1 .. T::MaxScheduledPerBlock::get(); + let when = BLOCK_NUMBER.into(); + fill_schedule::(when, s, true, false, Some(false))?; + }: { Scheduler::::on_initialize(when); } + verify { + assert_eq!(System::::event_count(), s); + for i in 0..s { + assert_eq!(Agenda::::get(when + (i + 100).into()).len(), 1 as usize); + } + } + + on_initialize_named { + let s in 1 .. T::MaxScheduledPerBlock::get(); + let when = BLOCK_NUMBER.into(); + fill_schedule::(when, s, false, true, Some(false))?; + }: { Scheduler::::on_initialize(BLOCK_NUMBER.into()); } + verify { + assert_eq!(System::::event_count(), s); + assert!(Agenda::::iter().count() == 0); + } + + on_initialize { + let s in 1 .. T::MaxScheduledPerBlock::get(); + let when = BLOCK_NUMBER.into(); + fill_schedule::(when, s, false, false, Some(false))?; + }: { Scheduler::::on_initialize(BLOCK_NUMBER.into()); } + verify { + assert_eq!(System::::event_count(), s); + assert!(Agenda::::iter().count() == 0); + } + schedule { let s in 0 .. T::MaxScheduledPerBlock::get(); let when = BLOCK_NUMBER.into(); let periodic = Some((T::BlockNumber::one(), 100)); let priority = 0; // Essentially a no-op call. - let call = Box::new(frame_system::Call::set_storage { items: vec![] }.into()); + let inner_call = frame_system::Call::set_storage { items: vec![] }.into(); + let call = Box::new(CallOrHashOf::::Value(inner_call)); - fill_schedule::(when, s)?; + fill_schedule::(when, s, true, true, Some(false))?; }: _(RawOrigin::Root, when, periodic, priority, call) verify { ensure!( @@ -73,7 +222,7 @@ benchmarks! { let s in 1 .. T::MaxScheduledPerBlock::get(); let when = BLOCK_NUMBER.into(); - fill_schedule::(when, s)?; + fill_schedule::(when, s, true, true, Some(false))?; assert_eq!(Agenda::::get(when).len(), s as usize); }: _(RawOrigin::Root, when, 0) verify { @@ -95,9 +244,10 @@ benchmarks! { let periodic = Some((T::BlockNumber::one(), 100)); let priority = 0; // Essentially a no-op call. - let call = Box::new(frame_system::Call::set_storage { items: vec![] }.into()); + let inner_call = frame_system::Call::set_storage { items: vec![] }.into(); + let call = Box::new(CallOrHashOf::::Value(inner_call)); - fill_schedule::(when, s)?; + fill_schedule::(when, s, true, true, Some(false))?; }: _(RawOrigin::Root, id, when, periodic, priority, call) verify { ensure!( @@ -110,7 +260,7 @@ benchmarks! { let s in 1 .. T::MaxScheduledPerBlock::get(); let when = BLOCK_NUMBER.into(); - fill_schedule::(when, s)?; + fill_schedule::(when, s, true, true, Some(false))?; }: _(RawOrigin::Root, 0.encode()) verify { ensure!( @@ -124,21 +274,5 @@ benchmarks! { ); } - // TODO [#7141]: Make this more complex and flexible so it can be used in automation. - #[extra] - on_initialize { - let s in 0 .. T::MaxScheduledPerBlock::get(); - let when = BLOCK_NUMBER.into(); - fill_schedule::(when, s)?; - }: { Scheduler::::on_initialize(BLOCK_NUMBER.into()); } - verify { - assert_eq!(System::::event_count(), s); - // Next block should have all the schedules again - ensure!( - Agenda::::get(when + T::BlockNumber::one()).len() == s as usize, - "didn't append schedule" - ); - } + impl_benchmark_test_suite!(Scheduler, crate::mock::new_test_ext(), crate::mock::Test); } - -impl_benchmark_test_suite!(Scheduler, crate::tests::new_test_ext(), crate::tests::Test); diff --git a/frame/scheduler/src/lib.rs b/frame/scheduler/src/lib.rs index ca9e15812a76..ec60cedc280b 100644 --- a/frame/scheduler/src/lib.rs +++ b/frame/scheduler/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -50,15 +50,20 @@ // Ensure we're `no_std` when compiling for Wasm. #![cfg_attr(not(feature = "std"), no_std)] +#[cfg(feature = "runtime-benchmarks")] mod benchmarking; +#[cfg(test)] +mod mock; +#[cfg(test)] +mod tests; pub mod weights; use codec::{Codec, Decode, Encode}; use frame_support::{ dispatch::{DispatchError, DispatchResult, Dispatchable, Parameter}, traits::{ - schedule::{self, DispatchTime}, - EnsureOrigin, Get, IsType, OriginTrait, + schedule::{self, DispatchTime, MaybeHashed}, + EnsureOrigin, Get, IsType, OriginTrait, PalletInfoAccess, PrivilegeCmp, StorageVersion, }, weights::{GetDispatchInfo, Weight}, }; @@ -69,7 +74,7 @@ use sp_runtime::{ traits::{BadOrigin, One, Saturating, Zero}, RuntimeDebug, }; -use sp_std::{borrow::Borrow, marker::PhantomData, prelude::*}; +use sp_std::{borrow::Borrow, cmp::Ordering, marker::PhantomData, prelude::*}; pub use weights::WeightInfo; /// Just a simple index for naming period tasks. @@ -77,6 +82,8 @@ pub type PeriodicIndex = u32; /// The location of a scheduled task that can be used to remove it. pub type TaskAddress = (BlockNumber, u32); +pub type CallOrHashOf = MaybeHashed<::Call, ::Hash>; + #[cfg_attr(any(feature = "std", test), derive(PartialEq, Eq))] #[derive(Clone, RuntimeDebug, Encode, Decode)] struct ScheduledV1 { @@ -89,7 +96,7 @@ struct ScheduledV1 { /// Information regarding an item to be executed in the future. #[cfg_attr(any(feature = "std", test), derive(PartialEq, Eq))] #[derive(Clone, RuntimeDebug, Encode, Decode, TypeInfo)] -pub struct ScheduledV2 { +pub struct ScheduledV3 { /// The unique identity for this task, if there is one. maybe_id: Option>, /// This task's priority. @@ -103,33 +110,88 @@ pub struct ScheduledV2 { _phantom: PhantomData, } +use crate::ScheduledV3 as ScheduledV2; + +pub type ScheduledV2Of = ScheduledV3< + ::Call, + ::BlockNumber, + ::PalletsOrigin, + ::AccountId, +>; + +pub type ScheduledV3Of = ScheduledV3< + CallOrHashOf, + ::BlockNumber, + ::PalletsOrigin, + ::AccountId, +>; + +pub type ScheduledOf = ScheduledV3Of; + /// The current version of Scheduled struct. pub type Scheduled = ScheduledV2; -// A value placed in storage that represents the current version of the Scheduler storage. -// This value is used by the `on_runtime_upgrade` logic to determine whether we run -// storage migration logic. -#[derive(Encode, Decode, Clone, Copy, PartialEq, Eq, RuntimeDebug, TypeInfo)] -enum Releases { - V1, - V2, +#[cfg(feature = "runtime-benchmarks")] +mod preimage_provider { + use frame_support::traits::PreimageRecipient; + pub trait PreimageProviderAndMaybeRecipient: PreimageRecipient {} + impl> PreimageProviderAndMaybeRecipient for T {} +} + +#[cfg(not(feature = "runtime-benchmarks"))] +mod preimage_provider { + use frame_support::traits::PreimageProvider; + pub trait PreimageProviderAndMaybeRecipient: PreimageProvider {} + impl> PreimageProviderAndMaybeRecipient for T {} } -impl Default for Releases { - fn default() -> Self { - Releases::V1 +pub use preimage_provider::PreimageProviderAndMaybeRecipient; + +pub(crate) trait MarginalWeightInfo: WeightInfo { + fn item(periodic: bool, named: bool, resolved: Option) -> Weight { + match (periodic, named, resolved) { + (_, false, None) => Self::on_initialize_aborted(2) - Self::on_initialize_aborted(1), + (_, true, None) => + Self::on_initialize_named_aborted(2) - Self::on_initialize_named_aborted(1), + (false, false, Some(false)) => Self::on_initialize(2) - Self::on_initialize(1), + (false, true, Some(false)) => + Self::on_initialize_named(2) - Self::on_initialize_named(1), + (true, false, Some(false)) => + Self::on_initialize_periodic(2) - Self::on_initialize_periodic(1), + (true, true, Some(false)) => + Self::on_initialize_periodic_named(2) - Self::on_initialize_periodic_named(1), + (false, false, Some(true)) => + Self::on_initialize_resolved(2) - Self::on_initialize_resolved(1), + (false, true, Some(true)) => + Self::on_initialize_named_resolved(2) - Self::on_initialize_named_resolved(1), + (true, false, Some(true)) => + Self::on_initialize_periodic_resolved(2) - Self::on_initialize_periodic_resolved(1), + (true, true, Some(true)) => + Self::on_initialize_periodic_named_resolved(2) - + Self::on_initialize_periodic_named_resolved(1), + } } } +impl MarginalWeightInfo for T {} #[frame_support::pallet] pub mod pallet { use super::*; - use frame_support::pallet_prelude::*; + use frame_support::{ + dispatch::PostDispatchInfo, + pallet_prelude::*, + traits::{schedule::LookupError, PreimageProvider}, + }; use frame_system::pallet_prelude::*; + /// The current storage version. + const STORAGE_VERSION: StorageVersion = StorageVersion::new(3); + #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::storage_version(STORAGE_VERSION)] + #[pallet::without_storage_info] pub struct Pallet(_); /// `system::Config` should always be included in our implied traits. @@ -148,7 +210,7 @@ pub mod pallet { /// The aggregated call type. type Call: Parameter - + Dispatchable::Origin> + + Dispatchable::Origin, PostInfo = PostDispatchInfo> + GetDispatchInfo + From>; @@ -160,6 +222,15 @@ pub mod pallet { /// Required origin to schedule or cancel calls. type ScheduleOrigin: EnsureOrigin<::Origin>; + /// Compare the privileges of origins. + /// + /// This will be used when canceling a task, to ensure that the origin that tries + /// to cancel has greater or equal privileges as the origin that created the scheduled task. + /// + /// For simplicity the [`EqualPrivilegeOnly`](frame_support::traits::EqualPrivilegeOnly) can + /// be used. This will only check if two given origins are equal. + type OriginPrivilegeCmp: PrivilegeCmp; + /// The maximum number of scheduled calls in the queue for a single block. /// Not strictly enforced, but used for weight estimation. #[pallet::constant] @@ -167,39 +238,44 @@ pub mod pallet { /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; + + /// The preimage provider with which we look up call hashes to get the call. + type PreimageProvider: PreimageProviderAndMaybeRecipient; + + /// If `Some` then the number of blocks to postpone execution for when the item is delayed. + type NoPreimagePostponement: Get>; } /// Items to be executed, indexed by the block number that they should be executed on. #[pallet::storage] - pub type Agenda = StorageMap< - _, - Twox64Concat, - T::BlockNumber, - Vec::Call, T::BlockNumber, T::PalletsOrigin, T::AccountId>>>, - ValueQuery, - >; + pub type Agenda = + StorageMap<_, Twox64Concat, T::BlockNumber, Vec>>, ValueQuery>; /// Lookup from identity to the block number and index of the task. #[pallet::storage] pub(crate) type Lookup = StorageMap<_, Twox64Concat, Vec, TaskAddress>; - /// Storage version of the pallet. - /// - /// New networks start with last version. - #[pallet::storage] - pub(crate) type StorageVersion = StorageValue<_, Releases, ValueQuery>; - /// Events type. #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// Scheduled some task. \[when, index\] - Scheduled(T::BlockNumber, u32), - /// Canceled some task. \[when, index\] - Canceled(T::BlockNumber, u32), - /// Dispatched some task. \[task, id, result\] - Dispatched(TaskAddress, Option>, DispatchResult), + /// Scheduled some task. + Scheduled { when: T::BlockNumber, index: u32 }, + /// Canceled some task. + Canceled { when: T::BlockNumber, index: u32 }, + /// Dispatched some task. + Dispatched { + task: TaskAddress, + id: Option>, + result: DispatchResult, + }, + /// The call for the provided hash was not found so the task has been aborted. + CallLookupFailed { + task: TaskAddress, + id: Option>, + error: LookupError, + }, } #[pallet::error] @@ -214,43 +290,18 @@ pub mod pallet { RescheduleNoChange, } - #[pallet::genesis_config] - pub struct GenesisConfig; - - #[cfg(feature = "std")] - impl Default for GenesisConfig { - fn default() -> Self { - Self - } - } - - #[pallet::genesis_build] - impl GenesisBuild for GenesisConfig { - fn build(&self) { - StorageVersion::::put(Releases::V2); - } - } - #[pallet::hooks] impl Hooks> for Pallet { /// Execute the scheduled calls - /// - /// # - /// - S = Number of already scheduled calls - /// - N = Named scheduled calls - /// - P = Periodic Calls - /// - Base Weight: 9.243 + 23.45 * S µs - /// - DB Weight: - /// - Read: Agenda + Lookup * N + Agenda(Future) * P - /// - Write: Agenda + Lookup * N + Agenda(future) * P - /// # fn on_initialize(now: T::BlockNumber) -> Weight { let limit = T::MaximumWeight::get(); + let mut queued = Agenda::::take(now) .into_iter() .enumerate() - .filter_map(|(index, s)| s.map(|inner| (index as u32, inner))) + .filter_map(|(index, s)| Some((index as u32, s?))) .collect::>(); + if queued.len() as u32 > T::MaxScheduledPerBlock::get() { log::warn!( target: "runtime::scheduler", @@ -258,84 +309,111 @@ pub mod pallet { expected from the runtime configuration. An update might be needed." ); } - queued.sort_by_key(|(_, s)| s.priority); - let base_weight: Weight = T::DbWeight::get().reads_writes(1, 2); // Agenda + Agenda(next) - let mut total_weight: Weight = 0; - queued - .into_iter() - .enumerate() - .scan(base_weight, |cumulative_weight, (order, (index, s))| { - *cumulative_weight = - cumulative_weight.saturating_add(s.call.get_dispatch_info().weight); - - let origin = - <::Origin as From>::from(s.origin.clone()) - .into(); - - if ensure_signed(origin).is_ok() { - // AccountData for inner call origin accountdata. - *cumulative_weight = - cumulative_weight.saturating_add(T::DbWeight::get().reads_writes(1, 1)); - } - if s.maybe_id.is_some() { - // Remove/Modify Lookup - *cumulative_weight = - cumulative_weight.saturating_add(T::DbWeight::get().writes(1)); - } - if s.maybe_periodic.is_some() { - // Read/Write Agenda for future block - *cumulative_weight = - cumulative_weight.saturating_add(T::DbWeight::get().reads_writes(1, 1)); - } + queued.sort_by_key(|(_, s)| s.priority); - Some((order, index, *cumulative_weight, s)) - }) - .filter_map(|(order, index, cumulative_weight, mut s)| { - // We allow a scheduled call if any is true: - // - It's priority is `HARD_DEADLINE` - // - It does not push the weight past the limit. - // - It is the first item in the schedule - if s.priority <= schedule::HARD_DEADLINE || - cumulative_weight <= limit || - order == 0 - { - let r = s.call.clone().dispatch(s.origin.clone().into()); - let maybe_id = s.maybe_id.clone(); - if let &Some((period, count)) = &s.maybe_periodic { - if count > 1 { - s.maybe_periodic = Some((period, count - 1)); - } else { - s.maybe_periodic = None; - } - let next = now + period; - // If scheduled is named, place it's information in `Lookup` - if let Some(ref id) = s.maybe_id { - let next_index = Agenda::::decode_len(now + period).unwrap_or(0); - Lookup::::insert(id, (next, next_index as u32)); - } - Agenda::::append(next, Some(s)); - } else { + let next = now + One::one(); + + let mut total_weight: Weight = T::WeightInfo::on_initialize(0); + for (order, (index, mut s)) in queued.into_iter().enumerate() { + let named = if let Some(ref id) = s.maybe_id { + Lookup::::remove(id); + true + } else { + false + }; + + let (call, maybe_completed) = s.call.resolved::(); + s.call = call; + + let resolved = if let Some(completed) = maybe_completed { + T::PreimageProvider::unrequest_preimage(&completed); + true + } else { + false + }; + + let call = match s.call.as_value().cloned() { + Some(c) => c, + None => { + // Preimage not available - postpone until some block. + total_weight.saturating_accrue(T::WeightInfo::item(false, named, None)); + if let Some(delay) = T::NoPreimagePostponement::get() { + let until = now.saturating_add(delay); if let Some(ref id) = s.maybe_id { - Lookup::::remove(id); + let index = Agenda::::decode_len(until).unwrap_or(0); + Lookup::::insert(id, (until, index as u32)); } + Agenda::::append(until, Some(s)); } - Self::deposit_event(Event::Dispatched( - (now, index), - maybe_id, - r.map(|_| ()).map_err(|e| e.error), - )); - total_weight = cumulative_weight; - None - } else { - Some(Some(s)) + continue + }, + }; + + let periodic = s.maybe_periodic.is_some(); + let call_weight = call.get_dispatch_info().weight; + let mut item_weight = T::WeightInfo::item(periodic, named, Some(resolved)); + let origin = + <::Origin as From>::from(s.origin.clone()) + .into(); + if ensure_signed(origin).is_ok() { + // Weights of Signed dispatches expect their signing account to be whitelisted. + item_weight.saturating_accrue(T::DbWeight::get().reads_writes(1, 1)); + } + + // We allow a scheduled call if any is true: + // - It's priority is `HARD_DEADLINE` + // - It does not push the weight past the limit. + // - It is the first item in the schedule + let hard_deadline = s.priority <= schedule::HARD_DEADLINE; + let test_weight = + total_weight.saturating_add(call_weight).saturating_add(item_weight); + if !hard_deadline && order > 0 && test_weight > limit { + // Cannot be scheduled this block - postpone until next. + total_weight.saturating_accrue(T::WeightInfo::item(false, named, None)); + if let Some(ref id) = s.maybe_id { + // NOTE: We could reasonably not do this (in which case there would be one + // block where the named and delayed item could not be referenced by name), + // but we will do it anyway since it should be mostly free in terms of + // weight and it is slightly cleaner. + let index = Agenda::::decode_len(next).unwrap_or(0); + Lookup::::insert(id, (next, index as u32)); } - }) - .for_each(|unused| { - let next = now + One::one(); - Agenda::::append(next, unused); + Agenda::::append(next, Some(s)); + continue + } + + let dispatch_origin = s.origin.clone().into(); + let (maybe_actual_call_weight, result) = match call.dispatch(dispatch_origin) { + Ok(post_info) => (post_info.actual_weight, Ok(())), + Err(error_and_info) => + (error_and_info.post_info.actual_weight, Err(error_and_info.error)), + }; + let actual_call_weight = maybe_actual_call_weight.unwrap_or(call_weight); + total_weight.saturating_accrue(item_weight); + total_weight.saturating_accrue(actual_call_weight); + + Self::deposit_event(Event::Dispatched { + task: (now, index), + id: s.maybe_id.clone(), + result, }); + if let &Some((period, count)) = &s.maybe_periodic { + if count > 1 { + s.maybe_periodic = Some((period, count - 1)); + } else { + s.maybe_periodic = None; + } + let wake = now + period; + // If scheduled is named, place its information in `Lookup` + if let Some(ref id) = s.maybe_id { + let wake_index = Agenda::::decode_len(wake).unwrap_or(0); + Lookup::::insert(id, (wake, wake_index as u32)); + } + Agenda::::append(wake, Some(s)); + } + } total_weight } } @@ -343,22 +421,13 @@ pub mod pallet { #[pallet::call] impl Pallet { /// Anonymously schedule a task. - /// - /// # - /// - S = Number of already scheduled calls - /// - Base Weight: 22.29 + .126 * S µs - /// - DB Weight: - /// - Read: Agenda - /// - Write: Agenda - /// - Will use base weight of 25 which should be good for up to 30 scheduled calls - /// # #[pallet::weight(::WeightInfo::schedule(T::MaxScheduledPerBlock::get()))] pub fn schedule( origin: OriginFor, when: T::BlockNumber, maybe_periodic: Option>, priority: schedule::Priority, - call: Box<::Call>, + call: Box>, ) -> DispatchResult { T::ScheduleOrigin::ensure_origin(origin.clone())?; let origin = ::Origin::from(origin); @@ -373,15 +442,6 @@ pub mod pallet { } /// Cancel an anonymously scheduled task. - /// - /// # - /// - S = Number of already scheduled calls - /// - Base Weight: 22.15 + 2.869 * S µs - /// - DB Weight: - /// - Read: Agenda - /// - Write: Agenda, Lookup - /// - Will use base weight of 100 which should be good for up to 30 scheduled calls - /// # #[pallet::weight(::WeightInfo::cancel(T::MaxScheduledPerBlock::get()))] pub fn cancel(origin: OriginFor, when: T::BlockNumber, index: u32) -> DispatchResult { T::ScheduleOrigin::ensure_origin(origin.clone())?; @@ -391,15 +451,6 @@ pub mod pallet { } /// Schedule a named task. - /// - /// # - /// - S = Number of already scheduled calls - /// - Base Weight: 29.6 + .159 * S µs - /// - DB Weight: - /// - Read: Agenda, Lookup - /// - Write: Agenda, Lookup - /// - Will use base weight of 35 which should be good for more than 30 scheduled calls - /// # #[pallet::weight(::WeightInfo::schedule_named(T::MaxScheduledPerBlock::get()))] pub fn schedule_named( origin: OriginFor, @@ -407,7 +458,7 @@ pub mod pallet { when: T::BlockNumber, maybe_periodic: Option>, priority: schedule::Priority, - call: Box<::Call>, + call: Box>, ) -> DispatchResult { T::ScheduleOrigin::ensure_origin(origin.clone())?; let origin = ::Origin::from(origin); @@ -423,15 +474,6 @@ pub mod pallet { } /// Cancel a named scheduled task. - /// - /// # - /// - S = Number of already scheduled calls - /// - Base Weight: 24.91 + 2.907 * S µs - /// - DB Weight: - /// - Read: Agenda, Lookup - /// - Write: Agenda, Lookup - /// - Will use base weight of 100 which should be good for up to 30 scheduled calls - /// # #[pallet::weight(::WeightInfo::cancel_named(T::MaxScheduledPerBlock::get()))] pub fn cancel_named(origin: OriginFor, id: Vec) -> DispatchResult { T::ScheduleOrigin::ensure_origin(origin.clone())?; @@ -451,7 +493,7 @@ pub mod pallet { after: T::BlockNumber, maybe_periodic: Option>, priority: schedule::Priority, - call: Box<::Call>, + call: Box>, ) -> DispatchResult { T::ScheduleOrigin::ensure_origin(origin.clone())?; let origin = ::Origin::from(origin); @@ -477,7 +519,7 @@ pub mod pallet { after: T::BlockNumber, maybe_periodic: Option>, priority: schedule::Priority, - call: Box<::Call>, + call: Box>, ) -> DispatchResult { T::ScheduleOrigin::ensure_origin(origin.clone())?; let origin = ::Origin::from(origin); @@ -495,24 +537,24 @@ pub mod pallet { } impl Pallet { - /// Migrate storage format from V1 to V2. - /// Return true if migration is performed. - pub fn migrate_v1_to_t2() -> bool { - if StorageVersion::::get() == Releases::V1 { - StorageVersion::::put(Releases::V2); - - Agenda::::translate::< - Vec::Call, T::BlockNumber>>>, - _, - >(|_, agenda| { + /// Migrate storage format from V1 to V3. + /// + /// Returns the weight consumed by this migration. + pub fn migrate_v1_to_v3() -> Weight { + let mut weight = T::DbWeight::get().reads_writes(1, 1); + + Agenda::::translate::::Call, T::BlockNumber>>>, _>( + |_, agenda| { Some( agenda .into_iter() .map(|schedule| { - schedule.map(|schedule| ScheduledV2 { + weight.saturating_accrue(T::DbWeight::get().reads_writes(1, 1)); + + schedule.map(|schedule| ScheduledV3 { maybe_id: schedule.maybe_id, priority: schedule.priority, - call: schedule.call, + call: schedule.call.into(), maybe_periodic: schedule.maybe_periodic, origin: system::RawOrigin::Root.into(), _phantom: Default::default(), @@ -520,18 +562,76 @@ impl Pallet { }) .collect::>(), ) - }); + }, + ); - true - } else { - false + frame_support::storage::migration::remove_storage_prefix( + Self::name().as_bytes(), + b"StorageVersion", + &[], + ); + + StorageVersion::new(3).put::(); + + weight + T::DbWeight::get().writes(2) + } + + /// Migrate storage format from V2 to V3. + /// + /// Returns the weight consumed by this migration. + pub fn migrate_v2_to_v3() -> Weight { + let mut weight = T::DbWeight::get().reads_writes(1, 1); + + Agenda::::translate::>>, _>(|_, agenda| { + Some( + agenda + .into_iter() + .map(|schedule| { + weight.saturating_accrue(T::DbWeight::get().reads_writes(1, 1)); + schedule.map(|schedule| ScheduledV3 { + maybe_id: schedule.maybe_id, + priority: schedule.priority, + call: schedule.call.into(), + maybe_periodic: schedule.maybe_periodic, + origin: schedule.origin, + _phantom: Default::default(), + }) + }) + .collect::>(), + ) + }); + + frame_support::storage::migration::remove_storage_prefix( + Self::name().as_bytes(), + b"StorageVersion", + &[], + ); + + StorageVersion::new(3).put::(); + + weight + T::DbWeight::get().writes(2) + } + + #[cfg(feature = "try-runtime")] + pub fn pre_migrate_to_v3() -> Result<(), &'static str> { + Ok(()) + } + + #[cfg(feature = "try-runtime")] + pub fn post_migrate_to_v3() -> Result<(), &'static str> { + use frame_support::dispatch::GetStorageVersion; + + assert!(Self::current_storage_version() == 3); + for k in Agenda::::iter_keys() { + let _ = Agenda::::try_get(k).map_err(|()| "Invalid item in Agenda")?; } + Ok(()) } /// Helper to migrate scheduler when the pallet origin type has changed. pub fn migrate_origin + codec::Decode>() { Agenda::::translate::< - Vec::Call, T::BlockNumber, OldOrigin, T::AccountId>>>, + Vec, T::BlockNumber, OldOrigin, T::AccountId>>>, _, >(|_, agenda| { Some( @@ -574,9 +674,10 @@ impl Pallet { maybe_periodic: Option>, priority: schedule::Priority, origin: T::PalletsOrigin, - call: ::Call, + call: CallOrHashOf, ) -> Result, DispatchError> { let when = Self::resolve_time(when)?; + call.ensure_requested::(); // sanitize maybe_periodic let maybe_periodic = maybe_periodic @@ -593,14 +694,7 @@ impl Pallet { }); Agenda::::append(when, s); let index = Agenda::::decode_len(when).unwrap_or(1) as u32 - 1; - if index > T::MaxScheduledPerBlock::get() { - log::warn!( - target: "runtime::scheduler", - "Warning: There are more items queued in the Scheduler than \ - expected from the runtime configuration. An update might be needed.", - ); - } - Self::deposit_event(Event::Scheduled(when, index)); + Self::deposit_event(Event::Scheduled { when, index }); Ok((when, index)) } @@ -614,7 +708,10 @@ impl Pallet { Ok(None), |s| -> Result>, DispatchError> { if let (Some(ref o), Some(ref s)) = (origin, s.borrow()) { - if *o != s.origin { + if matches!( + T::OriginPrivilegeCmp::cmp_privilege(o, &s.origin), + Some(Ordering::Less) | None + ) { return Err(BadOrigin.into()) } }; @@ -623,10 +720,11 @@ impl Pallet { ) })?; if let Some(s) = scheduled { + s.call.ensure_unrequested::(); if let Some(id) = s.maybe_id { Lookup::::remove(id); } - Self::deposit_event(Event::Canceled(when, index)); + Self::deposit_event(Event::Canceled { when, index }); Ok(()) } else { Err(Error::::NotFound)? @@ -651,8 +749,8 @@ impl Pallet { })?; let new_index = Agenda::::decode_len(new_time).unwrap_or(1) as u32 - 1; - Self::deposit_event(Event::Canceled(when, index)); - Self::deposit_event(Event::Scheduled(new_time, new_index)); + Self::deposit_event(Event::Canceled { when, index }); + Self::deposit_event(Event::Scheduled { when: new_time, index: new_index }); Ok((new_time, new_index)) } @@ -663,7 +761,7 @@ impl Pallet { maybe_periodic: Option>, priority: schedule::Priority, origin: T::PalletsOrigin, - call: ::Call, + call: CallOrHashOf, ) -> Result, DispatchError> { // ensure id it is unique if Lookup::::contains_key(&id) { @@ -672,6 +770,8 @@ impl Pallet { let when = Self::resolve_time(when)?; + call.ensure_requested::(); + // sanitize maybe_periodic let maybe_periodic = maybe_periodic .filter(|p| p.1 > 1 && !p.0.is_zero()) @@ -688,16 +788,9 @@ impl Pallet { }; Agenda::::append(when, Some(s)); let index = Agenda::::decode_len(when).unwrap_or(1) as u32 - 1; - if index > T::MaxScheduledPerBlock::get() { - log::warn!( - target: "runtime::scheduler", - "Warning: There are more items queued in the Scheduler than \ - expected from the runtime configuration. An update might be needed.", - ); - } let address = (when, index); Lookup::::insert(&id, &address); - Self::deposit_event(Event::Scheduled(when, index)); + Self::deposit_event(Event::Scheduled { when, index }); Ok(address) } @@ -709,15 +802,19 @@ impl Pallet { Agenda::::try_mutate(when, |agenda| -> DispatchResult { if let Some(s) = agenda.get_mut(i) { if let (Some(ref o), Some(ref s)) = (origin, s.borrow()) { - if *o != s.origin { + if matches!( + T::OriginPrivilegeCmp::cmp_privilege(o, &s.origin), + Some(Ordering::Less) | None + ) { return Err(BadOrigin.into()) } + s.call.ensure_unrequested::(); } *s = None; } Ok(()) })?; - Self::deposit_event(Event::Canceled(when, index)); + Self::deposit_event(Event::Canceled { when, index }); Ok(()) } else { Err(Error::::NotFound)? @@ -749,8 +846,8 @@ impl Pallet { })?; let new_index = Agenda::::decode_len(new_time).unwrap_or(1) as u32 - 1; - Self::deposit_event(Event::Canceled(when, index)); - Self::deposit_event(Event::Scheduled(new_time, new_index)); + Self::deposit_event(Event::Canceled { when, index }); + Self::deposit_event(Event::Scheduled { when: new_time, index: new_index }); *lookup = Some((new_time, new_index)); @@ -760,17 +857,18 @@ impl Pallet { } } -impl schedule::Anon::Call, T::PalletsOrigin> +impl schedule::v2::Anon::Call, T::PalletsOrigin> for Pallet { type Address = TaskAddress; + type Hash = T::Hash; fn schedule( when: DispatchTime, maybe_periodic: Option>, priority: schedule::Priority, origin: T::PalletsOrigin, - call: ::Call, + call: CallOrHashOf, ) -> Result { Self::do_schedule(when, maybe_periodic, priority, origin, call) } @@ -791,10 +889,11 @@ impl schedule::Anon::Call, T::PalletsOr } } -impl schedule::Named::Call, T::PalletsOrigin> +impl schedule::v2::Named::Call, T::PalletsOrigin> for Pallet { type Address = TaskAddress; + type Hash = T::Hash; fn schedule_named( id: Vec, @@ -802,7 +901,7 @@ impl schedule::Named::Call, T::PalletsO maybe_periodic: Option>, priority: schedule::Priority, origin: T::PalletsOrigin, - call: ::Call, + call: CallOrHashOf, ) -> Result { Self::do_schedule_named(id, when, maybe_periodic, priority, origin, call).map_err(|_| ()) } @@ -824,1020 +923,3 @@ impl schedule::Named::Call, T::PalletsO .ok_or(()) } } - -#[cfg(test)] -mod tests { - use super::*; - - use crate as scheduler; - use frame_support::{ - assert_err, assert_noop, assert_ok, ord_parameter_types, parameter_types, - traits::{Contains, OnFinalize, OnInitialize}, - weights::constants::RocksDbWeight, - Hashable, - }; - use frame_system::{EnsureOneOf, EnsureRoot, EnsureSignedBy}; - use sp_core::H256; - use sp_runtime::{ - testing::Header, - traits::{BlakeTwo256, IdentityLookup}, - Perbill, - }; - use substrate_test_utils::assert_eq_uvec; - - // Logger module to track execution. - #[frame_support::pallet] - pub mod logger { - use super::*; - use frame_support::pallet_prelude::*; - use frame_system::pallet_prelude::*; - use std::cell::RefCell; - - thread_local! { - static LOG: RefCell> = RefCell::new(Vec::new()); - } - pub fn log() -> Vec<(OriginCaller, u32)> { - LOG.with(|log| log.borrow().clone()) - } - - #[pallet::pallet] - #[pallet::generate_store(pub(super) trait Store)] - pub struct Pallet(PhantomData); - - #[pallet::hooks] - impl Hooks> for Pallet {} - - #[pallet::config] - pub trait Config: frame_system::Config { - type Event: From> + IsType<::Event>; - } - - #[pallet::event] - #[pallet::generate_deposit(pub(super) fn deposit_event)] - pub enum Event { - Logged(u32, Weight), - } - - #[pallet::call] - impl Pallet - where - ::Origin: OriginTrait, - { - #[pallet::weight(*weight)] - pub fn log(origin: OriginFor, i: u32, weight: Weight) -> DispatchResult { - Self::deposit_event(Event::Logged(i, weight)); - LOG.with(|log| { - log.borrow_mut().push((origin.caller().clone(), i)); - }); - Ok(()) - } - - #[pallet::weight(*weight)] - pub fn log_without_filter( - origin: OriginFor, - i: u32, - weight: Weight, - ) -> DispatchResult { - Self::deposit_event(Event::Logged(i, weight)); - LOG.with(|log| { - log.borrow_mut().push((origin.caller().clone(), i)); - }); - Ok(()) - } - } - } - - type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; - type Block = frame_system::mocking::MockBlock; - - frame_support::construct_runtime!( - pub enum Test where - Block = Block, - NodeBlock = Block, - UncheckedExtrinsic = UncheckedExtrinsic, - { - System: frame_system::{Pallet, Call, Config, Storage, Event}, - Logger: logger::{Pallet, Call, Event}, - Scheduler: scheduler::{Pallet, Call, Storage, Event}, - } - ); - - // Scheduler must dispatch with root and no filter, this tests base filter is indeed not used. - pub struct BaseFilter; - impl Contains for BaseFilter { - fn contains(call: &Call) -> bool { - !matches!(call, Call::Logger(LoggerCall::log { .. })) - } - } - - parameter_types! { - pub const BlockHashCount: u64 = 250; - pub BlockWeights: frame_system::limits::BlockWeights = - frame_system::limits::BlockWeights::simple_max(2_000_000_000_000); - } - impl system::Config for Test { - type BaseCallFilter = BaseFilter; - type BlockWeights = (); - type BlockLength = (); - type DbWeight = RocksDbWeight; - type Origin = Origin; - type Call = Call; - type Index = u64; - type BlockNumber = u64; - type Hash = H256; - type Hashing = BlakeTwo256; - type AccountId = u64; - type Lookup = IdentityLookup; - type Header = Header; - type Event = Event; - type BlockHashCount = BlockHashCount; - type Version = (); - type PalletInfo = PalletInfo; - type AccountData = (); - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - type OnSetCode = (); - } - impl logger::Config for Test { - type Event = Event; - } - parameter_types! { - pub MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * BlockWeights::get().max_block; - pub const MaxScheduledPerBlock: u32 = 10; - } - ord_parameter_types! { - pub const One: u64 = 1; - } - - impl Config for Test { - type Event = Event; - type Origin = Origin; - type PalletsOrigin = OriginCaller; - type Call = Call; - type MaximumWeight = MaximumSchedulerWeight; - type ScheduleOrigin = EnsureOneOf, EnsureSignedBy>; - type MaxScheduledPerBlock = MaxScheduledPerBlock; - type WeightInfo = (); - } - - pub type LoggerCall = logger::Call; - - pub fn new_test_ext() -> sp_io::TestExternalities { - let t = system::GenesisConfig::default().build_storage::().unwrap(); - t.into() - } - - fn run_to_block(n: u64) { - while System::block_number() < n { - Scheduler::on_finalize(System::block_number()); - System::set_block_number(System::block_number() + 1); - Scheduler::on_initialize(System::block_number()); - } - } - - fn root() -> OriginCaller { - system::RawOrigin::Root.into() - } - - #[test] - fn basic_scheduling_works() { - new_test_ext().execute_with(|| { - let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); - assert!(!::BaseCallFilter::contains(&call)); - assert_ok!(Scheduler::do_schedule(DispatchTime::At(4), None, 127, root(), call)); - run_to_block(3); - assert!(logger::log().is_empty()); - run_to_block(4); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - run_to_block(100); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - }); - } - - #[test] - fn schedule_after_works() { - new_test_ext().execute_with(|| { - run_to_block(2); - let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); - assert!(!::BaseCallFilter::contains(&call)); - // This will schedule the call 3 blocks after the next block... so block 3 + 3 = 6 - assert_ok!(Scheduler::do_schedule(DispatchTime::After(3), None, 127, root(), call)); - run_to_block(5); - assert!(logger::log().is_empty()); - run_to_block(6); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - run_to_block(100); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - }); - } - - #[test] - fn schedule_after_zero_works() { - new_test_ext().execute_with(|| { - run_to_block(2); - let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); - assert!(!::BaseCallFilter::contains(&call)); - assert_ok!(Scheduler::do_schedule(DispatchTime::After(0), None, 127, root(), call)); - // Will trigger on the next block. - run_to_block(3); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - run_to_block(100); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - }); - } - - #[test] - fn periodic_scheduling_works() { - new_test_ext().execute_with(|| { - // at #4, every 3 blocks, 3 times. - assert_ok!(Scheduler::do_schedule( - DispatchTime::At(4), - Some((3, 3)), - 127, - root(), - Call::Logger(logger::Call::log { i: 42, weight: 1000 }) - )); - run_to_block(3); - assert!(logger::log().is_empty()); - run_to_block(4); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - run_to_block(6); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - run_to_block(7); - assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32)]); - run_to_block(9); - assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32)]); - run_to_block(10); - assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32), (root(), 42u32)]); - run_to_block(100); - assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32), (root(), 42u32)]); - }); - } - - #[test] - fn reschedule_works() { - new_test_ext().execute_with(|| { - let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); - assert!(!::BaseCallFilter::contains(&call)); - assert_eq!( - Scheduler::do_schedule(DispatchTime::At(4), None, 127, root(), call).unwrap(), - (4, 0) - ); - - run_to_block(3); - assert!(logger::log().is_empty()); - - assert_eq!(Scheduler::do_reschedule((4, 0), DispatchTime::At(6)).unwrap(), (6, 0)); - - assert_noop!( - Scheduler::do_reschedule((6, 0), DispatchTime::At(6)), - Error::::RescheduleNoChange - ); - - run_to_block(4); - assert!(logger::log().is_empty()); - - run_to_block(6); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - - run_to_block(100); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - }); - } - - #[test] - fn reschedule_named_works() { - new_test_ext().execute_with(|| { - let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); - assert!(!::BaseCallFilter::contains(&call)); - assert_eq!( - Scheduler::do_schedule_named( - 1u32.encode(), - DispatchTime::At(4), - None, - 127, - root(), - call - ) - .unwrap(), - (4, 0) - ); - - run_to_block(3); - assert!(logger::log().is_empty()); - - assert_eq!( - Scheduler::do_reschedule_named(1u32.encode(), DispatchTime::At(6)).unwrap(), - (6, 0) - ); - - assert_noop!( - Scheduler::do_reschedule_named(1u32.encode(), DispatchTime::At(6)), - Error::::RescheduleNoChange - ); - - run_to_block(4); - assert!(logger::log().is_empty()); - - run_to_block(6); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - - run_to_block(100); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - }); - } - - #[test] - fn reschedule_named_perodic_works() { - new_test_ext().execute_with(|| { - let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); - assert!(!::BaseCallFilter::contains(&call)); - assert_eq!( - Scheduler::do_schedule_named( - 1u32.encode(), - DispatchTime::At(4), - Some((3, 3)), - 127, - root(), - call - ) - .unwrap(), - (4, 0) - ); - - run_to_block(3); - assert!(logger::log().is_empty()); - - assert_eq!( - Scheduler::do_reschedule_named(1u32.encode(), DispatchTime::At(5)).unwrap(), - (5, 0) - ); - assert_eq!( - Scheduler::do_reschedule_named(1u32.encode(), DispatchTime::At(6)).unwrap(), - (6, 0) - ); - - run_to_block(5); - assert!(logger::log().is_empty()); - - run_to_block(6); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - - assert_eq!( - Scheduler::do_reschedule_named(1u32.encode(), DispatchTime::At(10)).unwrap(), - (10, 0) - ); - - run_to_block(9); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - - run_to_block(10); - assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32)]); - - run_to_block(13); - assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32), (root(), 42u32)]); - - run_to_block(100); - assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32), (root(), 42u32)]); - }); - } - - #[test] - fn cancel_named_scheduling_works_with_normal_cancel() { - new_test_ext().execute_with(|| { - // at #4. - Scheduler::do_schedule_named( - 1u32.encode(), - DispatchTime::At(4), - None, - 127, - root(), - Call::Logger(LoggerCall::log { i: 69, weight: 1000 }), - ) - .unwrap(); - let i = Scheduler::do_schedule( - DispatchTime::At(4), - None, - 127, - root(), - Call::Logger(LoggerCall::log { i: 42, weight: 1000 }), - ) - .unwrap(); - run_to_block(3); - assert!(logger::log().is_empty()); - assert_ok!(Scheduler::do_cancel_named(None, 1u32.encode())); - assert_ok!(Scheduler::do_cancel(None, i)); - run_to_block(100); - assert!(logger::log().is_empty()); - }); - } - - #[test] - fn cancel_named_periodic_scheduling_works() { - new_test_ext().execute_with(|| { - // at #4, every 3 blocks, 3 times. - Scheduler::do_schedule_named( - 1u32.encode(), - DispatchTime::At(4), - Some((3, 3)), - 127, - root(), - Call::Logger(LoggerCall::log { i: 42, weight: 1000 }), - ) - .unwrap(); - // same id results in error. - assert!(Scheduler::do_schedule_named( - 1u32.encode(), - DispatchTime::At(4), - None, - 127, - root(), - Call::Logger(LoggerCall::log { i: 69, weight: 1000 }) - ) - .is_err()); - // different id is ok. - Scheduler::do_schedule_named( - 2u32.encode(), - DispatchTime::At(8), - None, - 127, - root(), - Call::Logger(LoggerCall::log { i: 69, weight: 1000 }), - ) - .unwrap(); - run_to_block(3); - assert!(logger::log().is_empty()); - run_to_block(4); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - run_to_block(6); - assert_ok!(Scheduler::do_cancel_named(None, 1u32.encode())); - run_to_block(100); - assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 69u32)]); - }); - } - - #[test] - fn scheduler_respects_weight_limits() { - new_test_ext().execute_with(|| { - assert_ok!(Scheduler::do_schedule( - DispatchTime::At(4), - None, - 127, - root(), - Call::Logger(LoggerCall::log { i: 42, weight: MaximumSchedulerWeight::get() / 2 }) - )); - assert_ok!(Scheduler::do_schedule( - DispatchTime::At(4), - None, - 127, - root(), - Call::Logger(LoggerCall::log { i: 69, weight: MaximumSchedulerWeight::get() / 2 }) - )); - // 69 and 42 do not fit together - run_to_block(4); - assert_eq!(logger::log(), vec![(root(), 42u32)]); - run_to_block(5); - assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 69u32)]); - }); - } - - #[test] - fn scheduler_respects_hard_deadlines_more() { - new_test_ext().execute_with(|| { - assert_ok!(Scheduler::do_schedule( - DispatchTime::At(4), - None, - 0, - root(), - Call::Logger(LoggerCall::log { i: 42, weight: MaximumSchedulerWeight::get() / 2 }) - )); - assert_ok!(Scheduler::do_schedule( - DispatchTime::At(4), - None, - 0, - root(), - Call::Logger(LoggerCall::log { i: 69, weight: MaximumSchedulerWeight::get() / 2 }) - )); - // With base weights, 69 and 42 should not fit together, but do because of hard - // deadlines - run_to_block(4); - assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 69u32)]); - }); - } - - #[test] - fn scheduler_respects_priority_ordering() { - new_test_ext().execute_with(|| { - assert_ok!(Scheduler::do_schedule( - DispatchTime::At(4), - None, - 1, - root(), - Call::Logger(LoggerCall::log { i: 42, weight: MaximumSchedulerWeight::get() / 2 }) - )); - assert_ok!(Scheduler::do_schedule( - DispatchTime::At(4), - None, - 0, - root(), - Call::Logger(LoggerCall::log { i: 69, weight: MaximumSchedulerWeight::get() / 2 }) - )); - run_to_block(4); - assert_eq!(logger::log(), vec![(root(), 69u32), (root(), 42u32)]); - }); - } - - #[test] - fn scheduler_respects_priority_ordering_with_soft_deadlines() { - new_test_ext().execute_with(|| { - assert_ok!(Scheduler::do_schedule( - DispatchTime::At(4), - None, - 255, - root(), - Call::Logger(LoggerCall::log { i: 42, weight: MaximumSchedulerWeight::get() / 3 }) - )); - assert_ok!(Scheduler::do_schedule( - DispatchTime::At(4), - None, - 127, - root(), - Call::Logger(LoggerCall::log { i: 69, weight: MaximumSchedulerWeight::get() / 2 }) - )); - assert_ok!(Scheduler::do_schedule( - DispatchTime::At(4), - None, - 126, - root(), - Call::Logger(LoggerCall::log { - i: 2600, - weight: MaximumSchedulerWeight::get() / 2 - }) - )); - - // 2600 does not fit with 69 or 42, but has higher priority, so will go through - run_to_block(4); - assert_eq!(logger::log(), vec![(root(), 2600u32)]); - // 69 and 42 fit together - run_to_block(5); - assert_eq!(logger::log(), vec![(root(), 2600u32), (root(), 69u32), (root(), 42u32)]); - }); - } - - #[test] - fn on_initialize_weight_is_correct() { - new_test_ext().execute_with(|| { - let base_weight: Weight = - ::DbWeight::get().reads_writes(1, 2); - let base_multiplier = 0; - let named_multiplier = ::DbWeight::get().writes(1); - let periodic_multiplier = - ::DbWeight::get().reads_writes(1, 1); - - // Named - assert_ok!(Scheduler::do_schedule_named( - 1u32.encode(), - DispatchTime::At(1), - None, - 255, - root(), - Call::Logger(LoggerCall::log { i: 3, weight: MaximumSchedulerWeight::get() / 3 }) - )); - // Anon Periodic - assert_ok!(Scheduler::do_schedule( - DispatchTime::At(1), - Some((1000, 3)), - 128, - root(), - Call::Logger(LoggerCall::log { i: 42, weight: MaximumSchedulerWeight::get() / 3 }) - )); - // Anon - assert_ok!(Scheduler::do_schedule( - DispatchTime::At(1), - None, - 127, - root(), - Call::Logger(LoggerCall::log { i: 69, weight: MaximumSchedulerWeight::get() / 2 }) - )); - // Named Periodic - assert_ok!(Scheduler::do_schedule_named( - 2u32.encode(), - DispatchTime::At(1), - Some((1000, 3)), - 126, - root(), - Call::Logger(LoggerCall::log { - i: 2600, - weight: MaximumSchedulerWeight::get() / 2 - }) - )); - - // Will include the named periodic only - let actual_weight = Scheduler::on_initialize(1); - let call_weight = MaximumSchedulerWeight::get() / 2; - assert_eq!( - actual_weight, - call_weight + - base_weight + base_multiplier + - named_multiplier + periodic_multiplier - ); - assert_eq!(logger::log(), vec![(root(), 2600u32)]); - - // Will include anon and anon periodic - let actual_weight = Scheduler::on_initialize(2); - let call_weight = MaximumSchedulerWeight::get() / 2 + MaximumSchedulerWeight::get() / 3; - assert_eq!( - actual_weight, - call_weight + base_weight + base_multiplier * 2 + periodic_multiplier - ); - assert_eq!(logger::log(), vec![(root(), 2600u32), (root(), 69u32), (root(), 42u32)]); - - // Will include named only - let actual_weight = Scheduler::on_initialize(3); - let call_weight = MaximumSchedulerWeight::get() / 3; - assert_eq!( - actual_weight, - call_weight + base_weight + base_multiplier + named_multiplier - ); - assert_eq!( - logger::log(), - vec![(root(), 2600u32), (root(), 69u32), (root(), 42u32), (root(), 3u32)] - ); - - // Will contain none - let actual_weight = Scheduler::on_initialize(4); - assert_eq!(actual_weight, 0); - }); - } - - #[test] - fn root_calls_works() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Logger(LoggerCall::log { i: 69, weight: 1000 })); - let call2 = Box::new(Call::Logger(LoggerCall::log { i: 42, weight: 1000 })); - assert_ok!(Scheduler::schedule_named( - Origin::root(), - 1u32.encode(), - 4, - None, - 127, - call - )); - assert_ok!(Scheduler::schedule(Origin::root(), 4, None, 127, call2)); - run_to_block(3); - // Scheduled calls are in the agenda. - assert_eq!(Agenda::::get(4).len(), 2); - assert!(logger::log().is_empty()); - assert_ok!(Scheduler::cancel_named(Origin::root(), 1u32.encode())); - assert_ok!(Scheduler::cancel(Origin::root(), 4, 1)); - // Scheduled calls are made NONE, so should not effect state - run_to_block(100); - assert!(logger::log().is_empty()); - }); - } - - #[test] - fn fails_to_schedule_task_in_the_past() { - new_test_ext().execute_with(|| { - run_to_block(3); - - let call = Box::new(Call::Logger(LoggerCall::log { i: 69, weight: 1000 })); - let call2 = Box::new(Call::Logger(LoggerCall::log { i: 42, weight: 1000 })); - - assert_err!( - Scheduler::schedule_named(Origin::root(), 1u32.encode(), 2, None, 127, call), - Error::::TargetBlockNumberInPast, - ); - - assert_err!( - Scheduler::schedule(Origin::root(), 2, None, 127, call2.clone()), - Error::::TargetBlockNumberInPast, - ); - - assert_err!( - Scheduler::schedule(Origin::root(), 3, None, 127, call2), - Error::::TargetBlockNumberInPast, - ); - }); - } - - #[test] - fn should_use_orign() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Logger(LoggerCall::log { i: 69, weight: 1000 })); - let call2 = Box::new(Call::Logger(LoggerCall::log { i: 42, weight: 1000 })); - assert_ok!(Scheduler::schedule_named( - system::RawOrigin::Signed(1).into(), - 1u32.encode(), - 4, - None, - 127, - call - )); - assert_ok!(Scheduler::schedule( - system::RawOrigin::Signed(1).into(), - 4, - None, - 127, - call2 - )); - run_to_block(3); - // Scheduled calls are in the agenda. - assert_eq!(Agenda::::get(4).len(), 2); - assert!(logger::log().is_empty()); - assert_ok!(Scheduler::cancel_named(system::RawOrigin::Signed(1).into(), 1u32.encode())); - assert_ok!(Scheduler::cancel(system::RawOrigin::Signed(1).into(), 4, 1)); - // Scheduled calls are made NONE, so should not effect state - run_to_block(100); - assert!(logger::log().is_empty()); - }); - } - - #[test] - fn should_check_orign() { - new_test_ext().execute_with(|| { - let call = Box::new(Call::Logger(LoggerCall::log { i: 69, weight: 1000 })); - let call2 = Box::new(Call::Logger(LoggerCall::log { i: 42, weight: 1000 })); - assert_noop!( - Scheduler::schedule_named( - system::RawOrigin::Signed(2).into(), - 1u32.encode(), - 4, - None, - 127, - call - ), - BadOrigin - ); - assert_noop!( - Scheduler::schedule(system::RawOrigin::Signed(2).into(), 4, None, 127, call2), - BadOrigin - ); - }); - } - - #[test] - fn should_check_orign_for_cancel() { - new_test_ext().execute_with(|| { - let call = - Box::new(Call::Logger(LoggerCall::log_without_filter { i: 69, weight: 1000 })); - let call2 = - Box::new(Call::Logger(LoggerCall::log_without_filter { i: 42, weight: 1000 })); - assert_ok!(Scheduler::schedule_named( - system::RawOrigin::Signed(1).into(), - 1u32.encode(), - 4, - None, - 127, - call - )); - assert_ok!(Scheduler::schedule( - system::RawOrigin::Signed(1).into(), - 4, - None, - 127, - call2 - )); - run_to_block(3); - // Scheduled calls are in the agenda. - assert_eq!(Agenda::::get(4).len(), 2); - assert!(logger::log().is_empty()); - assert_noop!( - Scheduler::cancel_named(system::RawOrigin::Signed(2).into(), 1u32.encode()), - BadOrigin - ); - assert_noop!(Scheduler::cancel(system::RawOrigin::Signed(2).into(), 4, 1), BadOrigin); - assert_noop!( - Scheduler::cancel_named(system::RawOrigin::Root.into(), 1u32.encode()), - BadOrigin - ); - assert_noop!(Scheduler::cancel(system::RawOrigin::Root.into(), 4, 1), BadOrigin); - run_to_block(5); - assert_eq!( - logger::log(), - vec![ - (system::RawOrigin::Signed(1).into(), 69u32), - (system::RawOrigin::Signed(1).into(), 42u32) - ] - ); - }); - } - - #[test] - fn migration_to_v2_works() { - new_test_ext().execute_with(|| { - for i in 0..3u64 { - let k = i.twox_64_concat(); - let old = vec![ - Some(ScheduledV1 { - maybe_id: None, - priority: i as u8 + 10, - call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }), - maybe_periodic: None, - }), - None, - Some(ScheduledV1 { - maybe_id: Some(b"test".to_vec()), - priority: 123, - call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }), - maybe_periodic: Some((456u64, 10)), - }), - ]; - frame_support::migration::put_storage_value(b"Scheduler", b"Agenda", &k, old); - } - - assert_eq!(StorageVersion::::get(), Releases::V1); - - assert!(Scheduler::migrate_v1_to_t2()); - - assert_eq_uvec!( - Agenda::::iter().collect::>(), - vec![ - ( - 0, - vec![ - Some(ScheduledV2 { - maybe_id: None, - priority: 10, - call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }), - maybe_periodic: None, - origin: root(), - _phantom: PhantomData::::default(), - }), - None, - Some(ScheduledV2 { - maybe_id: Some(b"test".to_vec()), - priority: 123, - call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }), - maybe_periodic: Some((456u64, 10)), - origin: root(), - _phantom: PhantomData::::default(), - }), - ] - ), - ( - 1, - vec![ - Some(ScheduledV2 { - maybe_id: None, - priority: 11, - call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }), - maybe_periodic: None, - origin: root(), - _phantom: PhantomData::::default(), - }), - None, - Some(ScheduledV2 { - maybe_id: Some(b"test".to_vec()), - priority: 123, - call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }), - maybe_periodic: Some((456u64, 10)), - origin: root(), - _phantom: PhantomData::::default(), - }), - ] - ), - ( - 2, - vec![ - Some(ScheduledV2 { - maybe_id: None, - priority: 12, - call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }), - maybe_periodic: None, - origin: root(), - _phantom: PhantomData::::default(), - }), - None, - Some(ScheduledV2 { - maybe_id: Some(b"test".to_vec()), - priority: 123, - call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }), - maybe_periodic: Some((456u64, 10)), - origin: root(), - _phantom: PhantomData::::default(), - }), - ] - ) - ] - ); - - assert_eq!(StorageVersion::::get(), Releases::V2); - }); - } - - #[test] - fn test_migrate_origin() { - new_test_ext().execute_with(|| { - for i in 0..3u64 { - let k = i.twox_64_concat(); - let old: Vec>> = vec![ - Some(Scheduled { - maybe_id: None, - priority: i as u8 + 10, - call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }), - origin: 3u32, - maybe_periodic: None, - _phantom: Default::default(), - }), - None, - Some(Scheduled { - maybe_id: Some(b"test".to_vec()), - priority: 123, - origin: 2u32, - call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }), - maybe_periodic: Some((456u64, 10)), - _phantom: Default::default(), - }), - ]; - frame_support::migration::put_storage_value(b"Scheduler", b"Agenda", &k, old); - } - - impl Into for u32 { - fn into(self) -> OriginCaller { - match self { - 3u32 => system::RawOrigin::Root.into(), - 2u32 => system::RawOrigin::None.into(), - _ => unreachable!("test make no use of it"), - } - } - } - - Scheduler::migrate_origin::(); - - assert_eq_uvec!( - Agenda::::iter().collect::>(), - vec![ - ( - 0, - vec![ - Some(ScheduledV2::<_, _, OriginCaller, u64> { - maybe_id: None, - priority: 10, - call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }), - maybe_periodic: None, - origin: system::RawOrigin::Root.into(), - _phantom: PhantomData::::default(), - }), - None, - Some(ScheduledV2 { - maybe_id: Some(b"test".to_vec()), - priority: 123, - call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }), - maybe_periodic: Some((456u64, 10)), - origin: system::RawOrigin::None.into(), - _phantom: PhantomData::::default(), - }), - ] - ), - ( - 1, - vec![ - Some(ScheduledV2 { - maybe_id: None, - priority: 11, - call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }), - maybe_periodic: None, - origin: system::RawOrigin::Root.into(), - _phantom: PhantomData::::default(), - }), - None, - Some(ScheduledV2 { - maybe_id: Some(b"test".to_vec()), - priority: 123, - call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }), - maybe_periodic: Some((456u64, 10)), - origin: system::RawOrigin::None.into(), - _phantom: PhantomData::::default(), - }), - ] - ), - ( - 2, - vec![ - Some(ScheduledV2 { - maybe_id: None, - priority: 12, - call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }), - maybe_periodic: None, - origin: system::RawOrigin::Root.into(), - _phantom: PhantomData::::default(), - }), - None, - Some(ScheduledV2 { - maybe_id: Some(b"test".to_vec()), - priority: 123, - call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }), - maybe_periodic: Some((456u64, 10)), - origin: system::RawOrigin::None.into(), - _phantom: PhantomData::::default(), - }), - ] - ) - ] - ); - }); - } -} diff --git a/frame/scheduler/src/mock.rs b/frame/scheduler/src/mock.rs new file mode 100644 index 000000000000..ecd04c3e48b5 --- /dev/null +++ b/frame/scheduler/src/mock.rs @@ -0,0 +1,202 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Scheduler test environment. + +use super::*; + +use crate as scheduler; +use frame_support::{ + ord_parameter_types, parameter_types, + traits::{ + ConstU32, ConstU64, Contains, EnsureOneOf, EqualPrivilegeOnly, OnFinalize, OnInitialize, + }, + weights::constants::RocksDbWeight, +}; +use frame_system::{EnsureRoot, EnsureSignedBy}; +use sp_core::H256; +use sp_runtime::{ + testing::Header, + traits::{BlakeTwo256, IdentityLookup}, + Perbill, +}; + +// Logger module to track execution. +#[frame_support::pallet] +pub mod logger { + use super::{OriginCaller, OriginTrait}; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + use std::cell::RefCell; + + thread_local! { + static LOG: RefCell> = RefCell::new(Vec::new()); + } + pub fn log() -> Vec<(OriginCaller, u32)> { + LOG.with(|log| log.borrow().clone()) + } + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(PhantomData); + + #[pallet::hooks] + impl Hooks> for Pallet {} + + #[pallet::config] + pub trait Config: frame_system::Config { + type Event: From> + IsType<::Event>; + } + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + Logged(u32, Weight), + } + + #[pallet::call] + impl Pallet + where + ::Origin: OriginTrait, + { + #[pallet::weight(*weight)] + pub fn log(origin: OriginFor, i: u32, weight: Weight) -> DispatchResult { + Self::deposit_event(Event::Logged(i, weight)); + LOG.with(|log| { + log.borrow_mut().push((origin.caller().clone(), i)); + }); + Ok(()) + } + + #[pallet::weight(*weight)] + pub fn log_without_filter(origin: OriginFor, i: u32, weight: Weight) -> DispatchResult { + Self::deposit_event(Event::Logged(i, weight)); + LOG.with(|log| { + log.borrow_mut().push((origin.caller().clone(), i)); + }); + Ok(()) + } + } +} + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; + +frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + Logger: logger::{Pallet, Call, Event}, + Scheduler: scheduler::{Pallet, Call, Storage, Event}, + Preimage: pallet_preimage::{Pallet, Call, Storage, Event}, + } +); + +// Scheduler must dispatch with root and no filter, this tests base filter is indeed not used. +pub struct BaseFilter; +impl Contains for BaseFilter { + fn contains(call: &Call) -> bool { + !matches!(call, Call::Logger(LoggerCall::log { .. })) + } +} + +parameter_types! { + pub BlockWeights: frame_system::limits::BlockWeights = + frame_system::limits::BlockWeights::simple_max(2_000_000_000_000); +} +impl system::Config for Test { + type BaseCallFilter = BaseFilter; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = RocksDbWeight; + type Origin = Origin; + type Call = Call; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = ConstU64<250>; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = ConstU32<16>; +} +impl logger::Config for Test { + type Event = Event; +} +parameter_types! { + pub MaximumSchedulerWeight: Weight = Perbill::from_percent(80) * BlockWeights::get().max_block; + pub const NoPreimagePostponement: Option = Some(2); +} +ord_parameter_types! { + pub const One: u64 = 1; +} + +impl pallet_preimage::Config for Test { + type Event = Event; + type WeightInfo = (); + type Currency = (); + type ManagerOrigin = EnsureRoot; + type MaxSize = ConstU32<1024>; + type BaseDeposit = (); + type ByteDeposit = (); +} + +impl Config for Test { + type Event = Event; + type Origin = Origin; + type PalletsOrigin = OriginCaller; + type Call = Call; + type MaximumWeight = MaximumSchedulerWeight; + type ScheduleOrigin = EnsureOneOf, EnsureSignedBy>; + type MaxScheduledPerBlock = ConstU32<10>; + type WeightInfo = (); + type OriginPrivilegeCmp = EqualPrivilegeOnly; + type PreimageProvider = Preimage; + type NoPreimagePostponement = NoPreimagePostponement; +} + +pub type LoggerCall = logger::Call; + +pub fn new_test_ext() -> sp_io::TestExternalities { + let t = system::GenesisConfig::default().build_storage::().unwrap(); + t.into() +} + +pub fn run_to_block(n: u64) { + while System::block_number() < n { + Scheduler::on_finalize(System::block_number()); + System::set_block_number(System::block_number() + 1); + Scheduler::on_initialize(System::block_number()); + } +} + +pub fn root() -> OriginCaller { + system::RawOrigin::Root.into() +} diff --git a/frame/scheduler/src/tests.rs b/frame/scheduler/src/tests.rs new file mode 100644 index 000000000000..d2a795cb19fa --- /dev/null +++ b/frame/scheduler/src/tests.rs @@ -0,0 +1,899 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Scheduler tests. + +use super::*; +use crate::mock::{logger, new_test_ext, root, run_to_block, Call, LoggerCall, Scheduler, Test, *}; +use frame_support::{ + assert_err, assert_noop, assert_ok, + traits::{Contains, GetStorageVersion, OnInitialize, PreimageProvider}, + Hashable, +}; +use sp_runtime::traits::Hash; +use substrate_test_utils::assert_eq_uvec; + +#[test] +fn basic_scheduling_works() { + new_test_ext().execute_with(|| { + let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); + assert!(!::BaseCallFilter::contains(&call)); + assert_ok!(Scheduler::do_schedule(DispatchTime::At(4), None, 127, root(), call.into())); + run_to_block(3); + assert!(logger::log().is_empty()); + run_to_block(4); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + run_to_block(100); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + }); +} + +#[test] +fn scheduling_with_preimages_works() { + new_test_ext().execute_with(|| { + let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); + let hash = ::Hashing::hash_of(&call); + let hashed = MaybeHashed::Hash(hash.clone()); + assert_ok!(Preimage::note_preimage(Origin::signed(0), call.encode())); + assert_ok!(Scheduler::do_schedule(DispatchTime::At(4), None, 127, root(), hashed)); + assert!(Preimage::preimage_requested(&hash)); + run_to_block(3); + assert!(logger::log().is_empty()); + run_to_block(4); + assert!(!Preimage::have_preimage(&hash)); + assert!(!Preimage::preimage_requested(&hash)); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + run_to_block(100); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + }); +} + +#[test] +fn scheduling_with_preimage_postpones_correctly() { + new_test_ext().execute_with(|| { + let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); + let hash = ::Hashing::hash_of(&call); + let hashed = MaybeHashed::Hash(hash.clone()); + + assert_ok!(Scheduler::do_schedule(DispatchTime::At(4), None, 127, root(), hashed)); + assert!(Preimage::preimage_requested(&hash)); + + run_to_block(4); + // #4 empty due to no preimage + assert!(logger::log().is_empty()); + + // Register preimage. + assert_ok!(Preimage::note_preimage(Origin::signed(0), call.encode())); + + run_to_block(5); + // #5 empty since postponement is 2 blocks. + assert!(logger::log().is_empty()); + + run_to_block(6); + // #6 is good. + assert_eq!(logger::log(), vec![(root(), 42u32)]); + assert!(!Preimage::have_preimage(&hash)); + assert!(!Preimage::preimage_requested(&hash)); + + run_to_block(100); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + }); +} + +#[test] +fn schedule_after_works() { + new_test_ext().execute_with(|| { + run_to_block(2); + let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); + assert!(!::BaseCallFilter::contains(&call)); + // This will schedule the call 3 blocks after the next block... so block 3 + 3 = 6 + assert_ok!(Scheduler::do_schedule(DispatchTime::After(3), None, 127, root(), call.into())); + run_to_block(5); + assert!(logger::log().is_empty()); + run_to_block(6); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + run_to_block(100); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + }); +} + +#[test] +fn schedule_after_zero_works() { + new_test_ext().execute_with(|| { + run_to_block(2); + let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); + assert!(!::BaseCallFilter::contains(&call)); + assert_ok!(Scheduler::do_schedule(DispatchTime::After(0), None, 127, root(), call.into())); + // Will trigger on the next block. + run_to_block(3); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + run_to_block(100); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + }); +} + +#[test] +fn periodic_scheduling_works() { + new_test_ext().execute_with(|| { + // at #4, every 3 blocks, 3 times. + assert_ok!(Scheduler::do_schedule( + DispatchTime::At(4), + Some((3, 3)), + 127, + root(), + Call::Logger(logger::Call::log { i: 42, weight: 1000 }).into() + )); + run_to_block(3); + assert!(logger::log().is_empty()); + run_to_block(4); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + run_to_block(6); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + run_to_block(7); + assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32)]); + run_to_block(9); + assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32)]); + run_to_block(10); + assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32), (root(), 42u32)]); + run_to_block(100); + assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32), (root(), 42u32)]); + }); +} + +#[test] +fn reschedule_works() { + new_test_ext().execute_with(|| { + let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); + assert!(!::BaseCallFilter::contains(&call)); + assert_eq!( + Scheduler::do_schedule(DispatchTime::At(4), None, 127, root(), call.into()).unwrap(), + (4, 0) + ); + + run_to_block(3); + assert!(logger::log().is_empty()); + + assert_eq!(Scheduler::do_reschedule((4, 0), DispatchTime::At(6)).unwrap(), (6, 0)); + + assert_noop!( + Scheduler::do_reschedule((6, 0), DispatchTime::At(6)), + Error::::RescheduleNoChange + ); + + run_to_block(4); + assert!(logger::log().is_empty()); + + run_to_block(6); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + + run_to_block(100); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + }); +} + +#[test] +fn reschedule_named_works() { + new_test_ext().execute_with(|| { + let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); + assert!(!::BaseCallFilter::contains(&call)); + assert_eq!( + Scheduler::do_schedule_named( + 1u32.encode(), + DispatchTime::At(4), + None, + 127, + root(), + call.into(), + ) + .unwrap(), + (4, 0) + ); + + run_to_block(3); + assert!(logger::log().is_empty()); + + assert_eq!( + Scheduler::do_reschedule_named(1u32.encode(), DispatchTime::At(6)).unwrap(), + (6, 0) + ); + + assert_noop!( + Scheduler::do_reschedule_named(1u32.encode(), DispatchTime::At(6)), + Error::::RescheduleNoChange + ); + + run_to_block(4); + assert!(logger::log().is_empty()); + + run_to_block(6); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + + run_to_block(100); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + }); +} + +#[test] +fn reschedule_named_perodic_works() { + new_test_ext().execute_with(|| { + let call = Call::Logger(LoggerCall::log { i: 42, weight: 1000 }); + assert!(!::BaseCallFilter::contains(&call)); + assert_eq!( + Scheduler::do_schedule_named( + 1u32.encode(), + DispatchTime::At(4), + Some((3, 3)), + 127, + root(), + call.into(), + ) + .unwrap(), + (4, 0) + ); + + run_to_block(3); + assert!(logger::log().is_empty()); + + assert_eq!( + Scheduler::do_reschedule_named(1u32.encode(), DispatchTime::At(5)).unwrap(), + (5, 0) + ); + assert_eq!( + Scheduler::do_reschedule_named(1u32.encode(), DispatchTime::At(6)).unwrap(), + (6, 0) + ); + + run_to_block(5); + assert!(logger::log().is_empty()); + + run_to_block(6); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + + assert_eq!( + Scheduler::do_reschedule_named(1u32.encode(), DispatchTime::At(10)).unwrap(), + (10, 0) + ); + + run_to_block(9); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + + run_to_block(10); + assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32)]); + + run_to_block(13); + assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32), (root(), 42u32)]); + + run_to_block(100); + assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 42u32), (root(), 42u32)]); + }); +} + +#[test] +fn cancel_named_scheduling_works_with_normal_cancel() { + new_test_ext().execute_with(|| { + // at #4. + Scheduler::do_schedule_named( + 1u32.encode(), + DispatchTime::At(4), + None, + 127, + root(), + Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into(), + ) + .unwrap(); + let i = Scheduler::do_schedule( + DispatchTime::At(4), + None, + 127, + root(), + Call::Logger(LoggerCall::log { i: 42, weight: 1000 }).into(), + ) + .unwrap(); + run_to_block(3); + assert!(logger::log().is_empty()); + assert_ok!(Scheduler::do_cancel_named(None, 1u32.encode())); + assert_ok!(Scheduler::do_cancel(None, i)); + run_to_block(100); + assert!(logger::log().is_empty()); + }); +} + +#[test] +fn cancel_named_periodic_scheduling_works() { + new_test_ext().execute_with(|| { + // at #4, every 3 blocks, 3 times. + Scheduler::do_schedule_named( + 1u32.encode(), + DispatchTime::At(4), + Some((3, 3)), + 127, + root(), + Call::Logger(LoggerCall::log { i: 42, weight: 1000 }).into(), + ) + .unwrap(); + // same id results in error. + assert!(Scheduler::do_schedule_named( + 1u32.encode(), + DispatchTime::At(4), + None, + 127, + root(), + Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into(), + ) + .is_err()); + // different id is ok. + Scheduler::do_schedule_named( + 2u32.encode(), + DispatchTime::At(8), + None, + 127, + root(), + Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into(), + ) + .unwrap(); + run_to_block(3); + assert!(logger::log().is_empty()); + run_to_block(4); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + run_to_block(6); + assert_ok!(Scheduler::do_cancel_named(None, 1u32.encode())); + run_to_block(100); + assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 69u32)]); + }); +} + +#[test] +fn scheduler_respects_weight_limits() { + new_test_ext().execute_with(|| { + assert_ok!(Scheduler::do_schedule( + DispatchTime::At(4), + None, + 127, + root(), + Call::Logger(LoggerCall::log { i: 42, weight: MaximumSchedulerWeight::get() / 2 }) + .into(), + )); + assert_ok!(Scheduler::do_schedule( + DispatchTime::At(4), + None, + 127, + root(), + Call::Logger(LoggerCall::log { i: 69, weight: MaximumSchedulerWeight::get() / 2 }) + .into(), + )); + // 69 and 42 do not fit together + run_to_block(4); + assert_eq!(logger::log(), vec![(root(), 42u32)]); + run_to_block(5); + assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 69u32)]); + }); +} + +#[test] +fn scheduler_respects_hard_deadlines_more() { + new_test_ext().execute_with(|| { + assert_ok!(Scheduler::do_schedule( + DispatchTime::At(4), + None, + 0, + root(), + Call::Logger(LoggerCall::log { i: 42, weight: MaximumSchedulerWeight::get() / 2 }) + .into(), + )); + assert_ok!(Scheduler::do_schedule( + DispatchTime::At(4), + None, + 0, + root(), + Call::Logger(LoggerCall::log { i: 69, weight: MaximumSchedulerWeight::get() / 2 }) + .into(), + )); + // With base weights, 69 and 42 should not fit together, but do because of hard + // deadlines + run_to_block(4); + assert_eq!(logger::log(), vec![(root(), 42u32), (root(), 69u32)]); + }); +} + +#[test] +fn scheduler_respects_priority_ordering() { + new_test_ext().execute_with(|| { + assert_ok!(Scheduler::do_schedule( + DispatchTime::At(4), + None, + 1, + root(), + Call::Logger(LoggerCall::log { i: 42, weight: MaximumSchedulerWeight::get() / 2 }) + .into(), + )); + assert_ok!(Scheduler::do_schedule( + DispatchTime::At(4), + None, + 0, + root(), + Call::Logger(LoggerCall::log { i: 69, weight: MaximumSchedulerWeight::get() / 2 }) + .into(), + )); + run_to_block(4); + assert_eq!(logger::log(), vec![(root(), 69u32), (root(), 42u32)]); + }); +} + +#[test] +fn scheduler_respects_priority_ordering_with_soft_deadlines() { + new_test_ext().execute_with(|| { + let max_weight = MaximumSchedulerWeight::get() - <() as WeightInfo>::on_initialize(0); + let item_weight = + <() as WeightInfo>::on_initialize(1) - <() as WeightInfo>::on_initialize(0); + assert_ok!(Scheduler::do_schedule( + DispatchTime::At(4), + None, + 255, + root(), + Call::Logger(LoggerCall::log { i: 42, weight: max_weight / 2 - item_weight }).into(), + )); + assert_ok!(Scheduler::do_schedule( + DispatchTime::At(4), + None, + 127, + root(), + Call::Logger(LoggerCall::log { i: 69, weight: max_weight / 2 - item_weight }).into(), + )); + assert_ok!(Scheduler::do_schedule( + DispatchTime::At(4), + None, + 126, + root(), + Call::Logger(LoggerCall::log { i: 2600, weight: max_weight / 2 - item_weight + 1 }) + .into(), + )); + + // 2600 does not fit with 69 or 42, but has higher priority, so will go through + run_to_block(4); + assert_eq!(logger::log(), vec![(root(), 2600u32)]); + // 69 and 42 fit together + run_to_block(5); + assert_eq!(logger::log(), vec![(root(), 2600u32), (root(), 69u32), (root(), 42u32)]); + }); +} + +#[test] +fn on_initialize_weight_is_correct() { + new_test_ext().execute_with(|| { + let base_weight = <() as WeightInfo>::on_initialize(0); + let call_weight = MaximumSchedulerWeight::get() / 4; + + // Named + assert_ok!(Scheduler::do_schedule_named( + 1u32.encode(), + DispatchTime::At(3), + None, + 255, + root(), + Call::Logger(LoggerCall::log { i: 3, weight: call_weight + 1 }).into(), + )); + // Anon Periodic + assert_ok!(Scheduler::do_schedule( + DispatchTime::At(2), + Some((1000, 3)), + 128, + root(), + Call::Logger(LoggerCall::log { i: 42, weight: call_weight + 2 }).into(), + )); + // Anon + assert_ok!(Scheduler::do_schedule( + DispatchTime::At(2), + None, + 127, + root(), + Call::Logger(LoggerCall::log { i: 69, weight: call_weight + 3 }).into(), + )); + // Named Periodic + assert_ok!(Scheduler::do_schedule_named( + 2u32.encode(), + DispatchTime::At(1), + Some((1000, 3)), + 126, + root(), + Call::Logger(LoggerCall::log { i: 2600, weight: call_weight + 4 }).into(), + )); + + // Will include the named periodic only + let actual_weight = Scheduler::on_initialize(1); + assert_eq!( + actual_weight, + base_weight + + call_weight + 4 + <() as MarginalWeightInfo>::item(true, true, Some(false)) + ); + assert_eq!(logger::log(), vec![(root(), 2600u32)]); + + // Will include anon and anon periodic + let actual_weight = Scheduler::on_initialize(2); + assert_eq!( + actual_weight, + base_weight + + call_weight + 2 + <() as MarginalWeightInfo>::item(false, false, Some(false)) + + call_weight + 3 + <() as MarginalWeightInfo>::item(true, false, Some(false)) + ); + assert_eq!(logger::log(), vec![(root(), 2600u32), (root(), 69u32), (root(), 42u32)]); + + // Will include named only + let actual_weight = Scheduler::on_initialize(3); + assert_eq!( + actual_weight, + base_weight + + call_weight + 1 + <() as MarginalWeightInfo>::item(false, true, Some(false)) + ); + assert_eq!( + logger::log(), + vec![(root(), 2600u32), (root(), 69u32), (root(), 42u32), (root(), 3u32)] + ); + + // Will contain none + let actual_weight = Scheduler::on_initialize(4); + assert_eq!(actual_weight, base_weight); + }); +} + +#[test] +fn root_calls_works() { + new_test_ext().execute_with(|| { + let call = Box::new(Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into()); + let call2 = Box::new(Call::Logger(LoggerCall::log { i: 42, weight: 1000 }).into()); + assert_ok!(Scheduler::schedule_named(Origin::root(), 1u32.encode(), 4, None, 127, call,)); + assert_ok!(Scheduler::schedule(Origin::root(), 4, None, 127, call2)); + run_to_block(3); + // Scheduled calls are in the agenda. + assert_eq!(Agenda::::get(4).len(), 2); + assert!(logger::log().is_empty()); + assert_ok!(Scheduler::cancel_named(Origin::root(), 1u32.encode())); + assert_ok!(Scheduler::cancel(Origin::root(), 4, 1)); + // Scheduled calls are made NONE, so should not effect state + run_to_block(100); + assert!(logger::log().is_empty()); + }); +} + +#[test] +fn fails_to_schedule_task_in_the_past() { + new_test_ext().execute_with(|| { + run_to_block(3); + + let call1 = Box::new(Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into()); + let call2 = Box::new(Call::Logger(LoggerCall::log { i: 42, weight: 1000 }).into()); + let call3 = Box::new(Call::Logger(LoggerCall::log { i: 42, weight: 1000 }).into()); + + assert_err!( + Scheduler::schedule_named(Origin::root(), 1u32.encode(), 2, None, 127, call1), + Error::::TargetBlockNumberInPast, + ); + + assert_err!( + Scheduler::schedule(Origin::root(), 2, None, 127, call2), + Error::::TargetBlockNumberInPast, + ); + + assert_err!( + Scheduler::schedule(Origin::root(), 3, None, 127, call3), + Error::::TargetBlockNumberInPast, + ); + }); +} + +#[test] +fn should_use_orign() { + new_test_ext().execute_with(|| { + let call = Box::new(Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into()); + let call2 = Box::new(Call::Logger(LoggerCall::log { i: 42, weight: 1000 }).into()); + assert_ok!(Scheduler::schedule_named( + system::RawOrigin::Signed(1).into(), + 1u32.encode(), + 4, + None, + 127, + call, + )); + assert_ok!(Scheduler::schedule(system::RawOrigin::Signed(1).into(), 4, None, 127, call2,)); + run_to_block(3); + // Scheduled calls are in the agenda. + assert_eq!(Agenda::::get(4).len(), 2); + assert!(logger::log().is_empty()); + assert_ok!(Scheduler::cancel_named(system::RawOrigin::Signed(1).into(), 1u32.encode())); + assert_ok!(Scheduler::cancel(system::RawOrigin::Signed(1).into(), 4, 1)); + // Scheduled calls are made NONE, so should not effect state + run_to_block(100); + assert!(logger::log().is_empty()); + }); +} + +#[test] +fn should_check_orign() { + new_test_ext().execute_with(|| { + let call = Box::new(Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into()); + let call2 = Box::new(Call::Logger(LoggerCall::log { i: 42, weight: 1000 }).into()); + assert_noop!( + Scheduler::schedule_named( + system::RawOrigin::Signed(2).into(), + 1u32.encode(), + 4, + None, + 127, + call + ), + BadOrigin + ); + assert_noop!( + Scheduler::schedule(system::RawOrigin::Signed(2).into(), 4, None, 127, call2), + BadOrigin + ); + }); +} + +#[test] +fn should_check_orign_for_cancel() { + new_test_ext().execute_with(|| { + let call = + Box::new(Call::Logger(LoggerCall::log_without_filter { i: 69, weight: 1000 }).into()); + let call2 = + Box::new(Call::Logger(LoggerCall::log_without_filter { i: 42, weight: 1000 }).into()); + assert_ok!(Scheduler::schedule_named( + system::RawOrigin::Signed(1).into(), + 1u32.encode(), + 4, + None, + 127, + call, + )); + assert_ok!(Scheduler::schedule(system::RawOrigin::Signed(1).into(), 4, None, 127, call2,)); + run_to_block(3); + // Scheduled calls are in the agenda. + assert_eq!(Agenda::::get(4).len(), 2); + assert!(logger::log().is_empty()); + assert_noop!( + Scheduler::cancel_named(system::RawOrigin::Signed(2).into(), 1u32.encode()), + BadOrigin + ); + assert_noop!(Scheduler::cancel(system::RawOrigin::Signed(2).into(), 4, 1), BadOrigin); + assert_noop!( + Scheduler::cancel_named(system::RawOrigin::Root.into(), 1u32.encode()), + BadOrigin + ); + assert_noop!(Scheduler::cancel(system::RawOrigin::Root.into(), 4, 1), BadOrigin); + run_to_block(5); + assert_eq!( + logger::log(), + vec![ + (system::RawOrigin::Signed(1).into(), 69u32), + (system::RawOrigin::Signed(1).into(), 42u32) + ] + ); + }); +} + +#[test] +fn migration_to_v3_works() { + new_test_ext().execute_with(|| { + for i in 0..3u64 { + let k = i.twox_64_concat(); + let old = vec![ + Some(ScheduledV1 { + maybe_id: None, + priority: i as u8 + 10, + call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }), + maybe_periodic: None, + }), + None, + Some(ScheduledV1 { + maybe_id: Some(b"test".to_vec()), + priority: 123, + call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }), + maybe_periodic: Some((456u64, 10)), + }), + ]; + frame_support::migration::put_storage_value(b"Scheduler", b"Agenda", &k, old); + } + + Scheduler::migrate_v1_to_v3(); + + assert_eq_uvec!( + Agenda::::iter().collect::>(), + vec![ + ( + 0, + vec![ + Some(ScheduledV3Of:: { + maybe_id: None, + priority: 10, + call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }).into(), + maybe_periodic: None, + origin: root(), + _phantom: PhantomData::::default(), + }), + None, + Some(ScheduledV3Of:: { + maybe_id: Some(b"test".to_vec()), + priority: 123, + call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into(), + maybe_periodic: Some((456u64, 10)), + origin: root(), + _phantom: PhantomData::::default(), + }), + ] + ), + ( + 1, + vec![ + Some(ScheduledV3Of:: { + maybe_id: None, + priority: 11, + call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }).into(), + maybe_periodic: None, + origin: root(), + _phantom: PhantomData::::default(), + }), + None, + Some(ScheduledV3Of:: { + maybe_id: Some(b"test".to_vec()), + priority: 123, + call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into(), + maybe_periodic: Some((456u64, 10)), + origin: root(), + _phantom: PhantomData::::default(), + }), + ] + ), + ( + 2, + vec![ + Some(ScheduledV3Of:: { + maybe_id: None, + priority: 12, + call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }).into(), + maybe_periodic: None, + origin: root(), + _phantom: PhantomData::::default(), + }), + None, + Some(ScheduledV3Of:: { + maybe_id: Some(b"test".to_vec()), + priority: 123, + call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into(), + maybe_periodic: Some((456u64, 10)), + origin: root(), + _phantom: PhantomData::::default(), + }), + ] + ) + ] + ); + + assert_eq!(Scheduler::current_storage_version(), 3); + }); +} + +#[test] +fn test_migrate_origin() { + new_test_ext().execute_with(|| { + for i in 0..3u64 { + let k = i.twox_64_concat(); + let old: Vec, u64, u32, u64>>> = vec![ + Some(Scheduled { + maybe_id: None, + priority: i as u8 + 10, + call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }).into(), + origin: 3u32, + maybe_periodic: None, + _phantom: Default::default(), + }), + None, + Some(Scheduled { + maybe_id: Some(b"test".to_vec()), + priority: 123, + origin: 2u32, + call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into(), + maybe_periodic: Some((456u64, 10)), + _phantom: Default::default(), + }), + ]; + frame_support::migration::put_storage_value(b"Scheduler", b"Agenda", &k, old); + } + + impl Into for u32 { + fn into(self) -> OriginCaller { + match self { + 3u32 => system::RawOrigin::Root.into(), + 2u32 => system::RawOrigin::None.into(), + _ => unreachable!("test make no use of it"), + } + } + } + + Scheduler::migrate_origin::(); + + assert_eq_uvec!( + Agenda::::iter().collect::>(), + vec![ + ( + 0, + vec![ + Some(ScheduledV2::, u64, OriginCaller, u64> { + maybe_id: None, + priority: 10, + call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }).into(), + maybe_periodic: None, + origin: system::RawOrigin::Root.into(), + _phantom: PhantomData::::default(), + }), + None, + Some(ScheduledV2 { + maybe_id: Some(b"test".to_vec()), + priority: 123, + call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into(), + maybe_periodic: Some((456u64, 10)), + origin: system::RawOrigin::None.into(), + _phantom: PhantomData::::default(), + }), + ] + ), + ( + 1, + vec![ + Some(ScheduledV2 { + maybe_id: None, + priority: 11, + call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }).into(), + maybe_periodic: None, + origin: system::RawOrigin::Root.into(), + _phantom: PhantomData::::default(), + }), + None, + Some(ScheduledV2 { + maybe_id: Some(b"test".to_vec()), + priority: 123, + call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into(), + maybe_periodic: Some((456u64, 10)), + origin: system::RawOrigin::None.into(), + _phantom: PhantomData::::default(), + }), + ] + ), + ( + 2, + vec![ + Some(ScheduledV2 { + maybe_id: None, + priority: 12, + call: Call::Logger(LoggerCall::log { i: 96, weight: 100 }).into(), + maybe_periodic: None, + origin: system::RawOrigin::Root.into(), + _phantom: PhantomData::::default(), + }), + None, + Some(ScheduledV2 { + maybe_id: Some(b"test".to_vec()), + priority: 123, + call: Call::Logger(LoggerCall::log { i: 69, weight: 1000 }).into(), + maybe_periodic: Some((456u64, 10)), + origin: system::RawOrigin::None.into(), + _phantom: PhantomData::::default(), + }), + ] + ) + ] + ); + }); +} diff --git a/frame/scheduler/src/weights.rs b/frame/scheduler/src/weights.rs index d83aefdc453a..dd00b8dfe6cd 100644 --- a/frame/scheduler/src/weights.rs +++ b/frame/scheduler/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_scheduler //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/scheduler/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -45,6 +46,16 @@ use sp_std::marker::PhantomData; /// Weight functions needed for pallet_scheduler. pub trait WeightInfo { + fn on_initialize_periodic_named_resolved(s: u32, ) -> Weight; + fn on_initialize_named_resolved(s: u32, ) -> Weight; + fn on_initialize_periodic_resolved(s: u32, ) -> Weight; + fn on_initialize_resolved(s: u32, ) -> Weight; + fn on_initialize_named_aborted(s: u32, ) -> Weight; + fn on_initialize_aborted(s: u32, ) -> Weight; + fn on_initialize_periodic_named(s: u32, ) -> Weight; + fn on_initialize_periodic(s: u32, ) -> Weight; + fn on_initialize_named(s: u32, ) -> Weight; + fn on_initialize(s: u32, ) -> Weight; fn schedule(s: u32, ) -> Weight; fn cancel(s: u32, ) -> Weight; fn schedule_named(s: u32, ) -> Weight; @@ -54,38 +65,149 @@ pub trait WeightInfo { /// Weights for pallet_scheduler using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { + // Storage: Scheduler Agenda (r:2 w:2) + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Scheduler Lookup (r:0 w:1) + fn on_initialize_periodic_named_resolved(s: u32, ) -> Weight { + (11_587_000 as Weight) + // Standard Error: 17_000 + .saturating_add((17_428_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(s as Weight))) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + .saturating_add(T::DbWeight::get().writes((4 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:1 w:1) + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Scheduler Lookup (r:0 w:1) + fn on_initialize_named_resolved(s: u32, ) -> Weight { + (8_965_000 as Weight) + // Standard Error: 11_000 + .saturating_add((13_410_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().reads((2 as Weight).saturating_mul(s as Weight))) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:2 w:2) + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + fn on_initialize_periodic_resolved(s: u32, ) -> Weight { + (8_654_000 as Weight) + // Standard Error: 17_000 + .saturating_add((14_990_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(s as Weight))) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:1 w:1) + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + fn on_initialize_resolved(s: u32, ) -> Weight { + (9_303_000 as Weight) + // Standard Error: 10_000 + .saturating_add((12_244_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().reads((2 as Weight).saturating_mul(s as Weight))) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + .saturating_add(T::DbWeight::get().writes((2 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:2 w:2) + // Storage: Preimage PreimageFor (r:1 w:0) + // Storage: Scheduler Lookup (r:0 w:1) + fn on_initialize_named_aborted(s: u32, ) -> Weight { + (7_506_000 as Weight) + // Standard Error: 3_000 + .saturating_add((5_208_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:2 w:2) + // Storage: Preimage PreimageFor (r:1 w:0) + fn on_initialize_aborted(s: u32, ) -> Weight { + (8_046_000 as Weight) + // Standard Error: 3_000 + .saturating_add((2_914_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Scheduler Agenda (r:2 w:2) + // Storage: Scheduler Lookup (r:0 w:1) + fn on_initialize_periodic_named(s: u32, ) -> Weight { + (13_704_000 as Weight) + // Standard Error: 4_000 + .saturating_add((8_186_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + .saturating_add(T::DbWeight::get().writes((2 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:2 w:2) + fn on_initialize_periodic(s: u32, ) -> Weight { + (12_668_000 as Weight) + // Standard Error: 5_000 + .saturating_add((5_868_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:1 w:1) + // Storage: Scheduler Lookup (r:0 w:1) + fn on_initialize_named(s: u32, ) -> Weight { + (13_946_000 as Weight) + // Standard Error: 4_000 + .saturating_add((4_367_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:1 w:1) + fn on_initialize(s: u32, ) -> Weight { + (13_151_000 as Weight) + // Standard Error: 4_000 + .saturating_add((3_455_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } // Storage: Scheduler Agenda (r:1 w:1) fn schedule(s: u32, ) -> Weight { - (24_730_000 as Weight) + (14_040_000 as Weight) // Standard Error: 1_000 - .saturating_add((77_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((89_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Scheduler Agenda (r:1 w:1) // Storage: Scheduler Lookup (r:0 w:1) fn cancel(s: u32, ) -> Weight { - (23_272_000 as Weight) - // Standard Error: 4_000 - .saturating_add((1_261_000 as Weight).saturating_mul(s as Weight)) + (14_376_000 as Weight) + // Standard Error: 1_000 + .saturating_add((576_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Scheduler Lookup (r:1 w:1) // Storage: Scheduler Agenda (r:1 w:1) fn schedule_named(s: u32, ) -> Weight { - (30_971_000 as Weight) + (16_806_000 as Weight) // Standard Error: 1_000 - .saturating_add((96_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((102_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Scheduler Lookup (r:1 w:1) // Storage: Scheduler Agenda (r:1 w:1) fn cancel_named(s: u32, ) -> Weight { - (25_778_000 as Weight) - // Standard Error: 4_000 - .saturating_add((1_270_000 as Weight).saturating_mul(s as Weight)) + (15_852_000 as Weight) + // Standard Error: 2_000 + .saturating_add((590_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -93,38 +215,149 @@ impl WeightInfo for SubstrateWeight { // For backwards compatibility and tests impl WeightInfo for () { + // Storage: Scheduler Agenda (r:2 w:2) + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Scheduler Lookup (r:0 w:1) + fn on_initialize_periodic_named_resolved(s: u32, ) -> Weight { + (11_587_000 as Weight) + // Standard Error: 17_000 + .saturating_add((17_428_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(s as Weight))) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes((4 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:1 w:1) + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Scheduler Lookup (r:0 w:1) + fn on_initialize_named_resolved(s: u32, ) -> Weight { + (8_965_000 as Weight) + // Standard Error: 11_000 + .saturating_add((13_410_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().reads((2 as Weight).saturating_mul(s as Weight))) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:2 w:2) + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + fn on_initialize_periodic_resolved(s: u32, ) -> Weight { + (8_654_000 as Weight) + // Standard Error: 17_000 + .saturating_add((14_990_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(s as Weight))) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:1 w:1) + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + fn on_initialize_resolved(s: u32, ) -> Weight { + (9_303_000 as Weight) + // Standard Error: 10_000 + .saturating_add((12_244_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().reads((2 as Weight).saturating_mul(s as Weight))) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes((2 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:2 w:2) + // Storage: Preimage PreimageFor (r:1 w:0) + // Storage: Scheduler Lookup (r:0 w:1) + fn on_initialize_named_aborted(s: u32, ) -> Weight { + (7_506_000 as Weight) + // Standard Error: 3_000 + .saturating_add((5_208_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:2 w:2) + // Storage: Preimage PreimageFor (r:1 w:0) + fn on_initialize_aborted(s: u32, ) -> Weight { + (8_046_000 as Weight) + // Standard Error: 3_000 + .saturating_add((2_914_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Scheduler Agenda (r:2 w:2) + // Storage: Scheduler Lookup (r:0 w:1) + fn on_initialize_periodic_named(s: u32, ) -> Weight { + (13_704_000 as Weight) + // Standard Error: 4_000 + .saturating_add((8_186_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes((2 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:2 w:2) + fn on_initialize_periodic(s: u32, ) -> Weight { + (12_668_000 as Weight) + // Standard Error: 5_000 + .saturating_add((5_868_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:1 w:1) + // Storage: Scheduler Lookup (r:0 w:1) + fn on_initialize_named(s: u32, ) -> Weight { + (13_946_000 as Weight) + // Standard Error: 4_000 + .saturating_add((4_367_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) + } + // Storage: Scheduler Agenda (r:1 w:1) + fn on_initialize(s: u32, ) -> Weight { + (13_151_000 as Weight) + // Standard Error: 4_000 + .saturating_add((3_455_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } // Storage: Scheduler Agenda (r:1 w:1) fn schedule(s: u32, ) -> Weight { - (24_730_000 as Weight) + (14_040_000 as Weight) // Standard Error: 1_000 - .saturating_add((77_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((89_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Scheduler Agenda (r:1 w:1) // Storage: Scheduler Lookup (r:0 w:1) fn cancel(s: u32, ) -> Weight { - (23_272_000 as Weight) - // Standard Error: 4_000 - .saturating_add((1_261_000 as Weight).saturating_mul(s as Weight)) + (14_376_000 as Weight) + // Standard Error: 1_000 + .saturating_add((576_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Scheduler Lookup (r:1 w:1) // Storage: Scheduler Agenda (r:1 w:1) fn schedule_named(s: u32, ) -> Weight { - (30_971_000 as Weight) + (16_806_000 as Weight) // Standard Error: 1_000 - .saturating_add((96_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((102_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Scheduler Lookup (r:1 w:1) // Storage: Scheduler Agenda (r:1 w:1) fn cancel_named(s: u32, ) -> Weight { - (25_778_000 as Weight) - // Standard Error: 4_000 - .saturating_add((1_270_000 as Weight).saturating_mul(s as Weight)) + (15_852_000 as Weight) + // Standard Error: 2_000 + .saturating_add((590_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } diff --git a/frame/scored-pool/Cargo.toml b/frame/scored-pool/Cargo.toml index 9d5f156c175d..73fd25158ca8 100644 --- a/frame/scored-pool/Cargo.toml +++ b/frame/scored-pool/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-scored-pool" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for scored pools" readme = "README.md" @@ -13,17 +13,17 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] pallet-balances = { version = "4.0.0-dev", path = "../balances" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } [features] default = ["std"] diff --git a/frame/scored-pool/README.md b/frame/scored-pool/README.md index bf20124edf52..56c6af916ecd 100644 --- a/frame/scored-pool/README.md +++ b/frame/scored-pool/README.md @@ -37,26 +37,33 @@ by the next highest scoring candidate in the pool, if available. ## Usage ```rust -use frame_support::{decl_module, dispatch}; -use frame_system::ensure_signed; use pallet_scored_pool::{self as scored_pool}; -pub trait Config: scored_pool::Config {} - -decl_module! { - pub struct Module for enum Call where origin: T::Origin { - #[weight = 0] - pub fn candidate(origin) -> dispatch::DispatchResult { - let who = ensure_signed(origin)?; - - let _ = >::submit_candidacy( - T::Origin::from(Some(who.clone()).into()) - ); - Ok(()) - } - } +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config + scored_pool::Config {} + + #[pallet::call] + impl Pallet { + #[pallet::weight(0)] + pub fn candidate(origin: OriginFor) -> DispatchResult { + let who = ensure_signed(origin)?; + + let _ = >::submit_candidacy( + T::Origin::from(Some(who.clone()).into()) + ); + Ok(()) + } + } } - ``` ## Dependencies diff --git a/frame/scored-pool/src/lib.rs b/frame/scored-pool/src/lib.rs index a5cdb6274f99..3f0674d720ef 100644 --- a/frame/scored-pool/src/lib.rs +++ b/frame/scored-pool/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -54,16 +54,24 @@ //! ## Usage //! //! ``` -//! use frame_support::{decl_module, dispatch}; -//! use frame_system::ensure_signed; //! use pallet_scored_pool::{self as scored_pool}; //! -//! pub trait Config: scored_pool::Config {} +//! #[frame_support::pallet] +//! pub mod pallet { +//! use super::*; +//! use frame_support::pallet_prelude::*; +//! use frame_system::pallet_prelude::*; //! -//! decl_module! { -//! pub struct Module for enum Call where origin: T::Origin { -//! #[weight = 0] -//! pub fn candidate(origin) -> dispatch::DispatchResult { +//! #[pallet::pallet] +//! pub struct Pallet(_); +//! +//! #[pallet::config] +//! pub trait Config: frame_system::Config + scored_pool::Config {} +//! +//! #[pallet::call] +//! impl Pallet { +//! #[pallet::weight(0)] +//! pub fn candidate(origin: OriginFor) -> DispatchResult { //! let who = ensure_signed(origin)?; //! //! let _ = >::submit_candidacy( @@ -116,12 +124,12 @@ enum ChangeReceiver { #[frame_support::pallet] pub mod pallet { use super::*; - use frame_support::{pallet_prelude::*, traits::EnsureOrigin, weights::Weight}; - use frame_system::{ensure_root, ensure_signed, pallet_prelude::*}; - use sp_runtime::traits::MaybeSerializeDeserialize; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); #[pallet::config] diff --git a/frame/scored-pool/src/mock.rs b/frame/scored-pool/src/mock.rs index 5c5425ae2bdd..4fef5385eb2c 100644 --- a/frame/scored-pool/src/mock.rs +++ b/frame/scored-pool/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,10 @@ use super::*; use crate as pallet_scored_pool; -use frame_support::{ord_parameter_types, parameter_types, traits::GenesisBuild}; +use frame_support::{ + ord_parameter_types, parameter_types, + traits::{ConstU32, ConstU64, GenesisBuild}, +}; use frame_system::EnsureSignedBy; use sp_core::H256; use sp_runtime::{ @@ -46,9 +49,6 @@ frame_support::construct_runtime!( parameter_types! { pub const CandidateDeposit: u64 = 25; - pub const Period: u64 = 4; - pub const BlockHashCount: u64 = 250; - pub const ExistentialDeposit: u64 = 1; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -72,7 +72,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -81,6 +81,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl pallet_balances::Config for Test { @@ -90,7 +91,7 @@ impl pallet_balances::Config for Test { type Balance = u64; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } @@ -129,7 +130,7 @@ impl Config for Test { type MembershipChanged = TestChangeMembers; type Currency = Balances; type CandidateDeposit = CandidateDeposit; - type Period = Period; + type Period = ConstU64<4>; type Score = u64; type ScoreOrigin = EnsureSignedBy; } @@ -152,7 +153,6 @@ pub fn new_test_ext() -> sp_io::TestExternalities { pallet_scored_pool::GenesisConfig:: { pool: vec![(5, None), (10, Some(1)), (20, Some(2)), (31, Some(2)), (40, Some(3))], member_count: 2, - ..Default::default() } .assimilate_storage(&mut t) .unwrap(); diff --git a/frame/scored-pool/src/tests.rs b/frame/scored-pool/src/tests.rs index 0503e308e76a..7b431160ddfe 100644 --- a/frame/scored-pool/src/tests.rs +++ b/frame/scored-pool/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/session/Cargo.toml b/frame/session/Cargo.toml index 3d2de5339543..28f2df47027b 100644 --- a/frame/session/Cargo.toml +++ b/frame/session/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-session" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME sessions pallet" readme = "README.md" @@ -14,18 +14,18 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] log = { version = "0.4.0", default-features = false } -impl-trait-for-tuples = "0.2.1" +impl-trait-for-tuples = "0.2.2" -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-session = { version = "4.0.0-dev", default-features = false, path = "../../primitives/session" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } -sp-trie = { version = "4.0.0-dev", default-features = false, path = "../../primitives/trie", optional = true } +sp-trie = { version = "6.0.0", default-features = false, path = "../../primitives/trie", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } diff --git a/frame/session/benchmarking/Cargo.toml b/frame/session/benchmarking/Cargo.toml index a24d4a1173ab..b00d1335d22a 100644 --- a/frame/session/benchmarking/Cargo.toml +++ b/frame/session/benchmarking/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-session-benchmarking" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME sessions pallet benchmarking" readme = "README.md" @@ -15,8 +15,8 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] rand = { version = "0.7.2", default-features = false } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } sp-session = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/session" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../../benchmarking" } @@ -26,10 +26,10 @@ pallet-session = { version = "4.0.0-dev", default-features = false, path = "../. pallet-staking = { version = "4.0.0-dev", default-features = false, features = ["runtime-benchmarks"], path = "../../staking" } [dev-dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", features = ["derive"] } -scale-info = "1.0" -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../../primitives/io" } +codec = { package = "parity-scale-codec", version = "3.0.0", features = ["derive"] } +scale-info = "2.0.1" +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../../primitives/io" } pallet-balances = { version = "4.0.0-dev", path = "../../balances" } pallet-timestamp = { version = "4.0.0-dev", path = "../../timestamp" } pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../../staking/reward-curve" } diff --git a/frame/session/benchmarking/src/lib.rs b/frame/session/benchmarking/src/lib.rs index c0131957c873..265c35cbe490 100644 --- a/frame/session/benchmarking/src/lib.rs +++ b/frame/session/benchmarking/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,20 +22,20 @@ mod mock; +use sp_runtime::traits::{One, StaticLookup, TrailingZeroInput}; use sp_std::{prelude::*, vec}; -use frame_benchmarking::{benchmarks, impl_benchmark_test_suite}; +use frame_benchmarking::benchmarks; use frame_support::{ codec::Decode, - traits::{KeyOwnerProofSystem, OnInitialize}, + traits::{Get, KeyOwnerProofSystem, OnInitialize}, }; use frame_system::RawOrigin; -use pallet_session::{historical::Module as Historical, Pallet as Session, *}; +use pallet_session::{historical::Pallet as Historical, Pallet as Session, *}; use pallet_staking::{ benchmarking::create_validator_with_nominators, testing_utils::create_validators, RewardDestination, }; -use sp_runtime::traits::{One, StaticLookup}; const MAX_VALIDATORS: u32 = 1000; @@ -53,15 +53,16 @@ impl OnInitialize for Pallet { benchmarks! { set_keys { - let n = ::MAX_NOMINATIONS; + let n = ::MaxNominations::get(); let (v_stash, _) = create_validator_with_nominators::( n, - ::MAX_NOMINATIONS, + ::MaxNominations::get(), false, RewardDestination::Staked, )?; let v_controller = pallet_staking::Pallet::::bonded(&v_stash).ok_or("not stash")?; - let keys = T::Keys::default(); + + let keys = T::Keys::decode(&mut TrailingZeroInput::zeroes()).unwrap(); let proof: Vec = vec![0,1,2,3]; // Whitelist controller account from further DB operations. let v_controller_key = frame_system::Account::::hashed_key_for(&v_controller); @@ -69,15 +70,15 @@ benchmarks! { }: _(RawOrigin::Signed(v_controller), keys, proof) purge_keys { - let n = ::MAX_NOMINATIONS; + let n = ::MaxNominations::get(); let (v_stash, _) = create_validator_with_nominators::( n, - ::MAX_NOMINATIONS, + ::MaxNominations::get(), false, RewardDestination::Staked )?; let v_controller = pallet_staking::Pallet::::bonded(&v_stash).ok_or("not stash")?; - let keys = T::Keys::default(); + let keys = T::Keys::decode(&mut TrailingZeroInput::zeroes()).unwrap(); let proof: Vec = vec![0,1,2,3]; Session::::set_keys(RawOrigin::Signed(v_controller.clone()).into(), keys, proof)?; // Whitelist controller account from further DB operations. @@ -115,6 +116,8 @@ benchmarks! { verify { assert!(Historical::::check_proof(key, key_owner_proof2).is_some()); } + + impl_benchmark_test_suite!(Pallet, crate::mock::new_test_ext(), crate::mock::Test, extra = false); } /// Sets up the benchmark for checking a membership proof. It creates the given @@ -161,5 +164,3 @@ fn check_membership_proof_setup( (key, Historical::::prove(key).unwrap()) } - -impl_benchmark_test_suite!(Pallet, crate::mock::new_test_ext(), crate::mock::Test, extra = false); diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs index 4d3a1a2d8689..24b42b3e9f4b 100644 --- a/frame/session/benchmarking/src/mock.rs +++ b/frame/session/benchmarking/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,8 +19,11 @@ #![cfg(test)] -use frame_election_provider_support::onchain; -use frame_support::parameter_types; +use frame_election_provider_support::{onchain, SequentialPhragmen}; +use frame_support::{ + parameter_types, + traits::{ConstU32, ConstU64}, +}; use sp_runtime::traits::IdentityLookup; type AccountId = u64; @@ -68,6 +71,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } parameter_types! { pub const ExistentialDeposit: Balance = 10; @@ -84,13 +88,10 @@ impl pallet_balances::Config for Test { type WeightInfo = (); } -parameter_types! { - pub const MinimumPeriod: u64 = 5; -} impl pallet_timestamp::Config for Test { type Moment = u64; type OnTimestampSet = (); - type MinimumPeriod = MinimumPeriod; + type MinimumPeriod = ConstU64<5>; type WeightInfo = (); } impl pallet_session::historical::Config for Test { @@ -117,7 +118,7 @@ impl pallet_session::SessionHandler for TestSessionHandler { ) { } - fn on_disabled(_: usize) {} + fn on_disabled(_: u32) {} } impl pallet_session::Config for Test { @@ -129,7 +130,6 @@ impl pallet_session::Config for Test { type Event = Event; type ValidatorId = AccountId; type ValidatorIdOf = pallet_staking::StashOf; - type DisabledValidatorsThreshold = (); type WeightInfo = (); } pallet_staking_reward_curve::build! { @@ -144,8 +144,6 @@ pallet_staking_reward_curve::build! { } parameter_types! { pub const RewardCurve: &'static sp_runtime::curve::PiecewiseLinear<'static> = &I_NPOS; - pub const MaxNominatorRewardedPerValidator: u32 = 64; - pub const UnsignedPriority: u64 = 1 << 20; } pub type Extrinsic = sp_runtime::testing::TestXt; @@ -158,13 +156,15 @@ where type Extrinsic = Extrinsic; } -impl onchain::Config for Test { - type Accuracy = sp_runtime::Perbill; +pub struct OnChainSeqPhragmen; +impl onchain::ExecutionConfig for OnChainSeqPhragmen { + type System = Test; + type Solver = SequentialPhragmen; type DataProvider = Staking; } impl pallet_staking::Config for Test { - const MAX_NOMINATIONS: u32 = 16; + type MaxNominations = ConstU32<16>; type Currency = Balances; type UnixTime = pallet_timestamp::Pallet; type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; @@ -179,10 +179,13 @@ impl pallet_staking::Config for Test { type SessionInterface = Self; type EraPayout = pallet_staking::ConvertCurve; type NextNewSession = Session; - type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; - type ElectionProvider = onchain::OnChainSequentialPhragmen; + type MaxNominatorRewardedPerValidator = ConstU32<64>; + type OffendingValidatorsThreshold = (); + type ElectionProvider = onchain::UnboundedExecution; type GenesisElectionProvider = Self::ElectionProvider; - type SortedListProvider = pallet_staking::UseNominatorsMap; + type MaxUnlockingChunks = ConstU32<32>; + type VoterList = pallet_staking::UseNominatorsAndValidatorsMap; + type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig; type WeightInfo = (); } diff --git a/frame/session/src/historical/mod.rs b/frame/session/src/historical/mod.rs index 0801b2aca170..76cefeb7b053 100644 --- a/frame/session/src/historical/mod.rs +++ b/frame/session/src/historical/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,62 +26,74 @@ //! These roots and proofs of inclusion can be generated at any time during the current session. //! Afterwards, the proofs can be fed to a consensus module when reporting misbehavior. -use super::{Pallet as SessionModule, SessionIndex}; +pub mod offchain; +pub mod onchain; +mod shared; + use codec::{Decode, Encode}; -use frame_support::{ - decl_module, decl_storage, print, - traits::{ValidatorSet, ValidatorSetWithIdentification}, - Parameter, -}; use sp_runtime::{ traits::{Convert, OpaqueKeys}, KeyTypeId, }; use sp_session::{MembershipProof, ValidatorCount}; +use sp_staking::SessionIndex; use sp_std::prelude::*; use sp_trie::{ - trie_types::{TrieDB, TrieDBMut}, + trie_types::{TrieDB, TrieDBMutV0}, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, }; -pub mod offchain; -pub mod onchain; -mod shared; +use frame_support::{ + print, + traits::{KeyOwnerProofSystem, StorageVersion, ValidatorSet, ValidatorSetWithIdentification}, + Parameter, +}; -/// Config necessary for the historical module. -pub trait Config: super::Config { - /// Full identification of the validator. - type FullIdentification: Parameter; - - /// A conversion from validator ID to full identification. - /// - /// This should contain any references to economic actors associated with the - /// validator, since they may be outdated by the time this is queried from a - /// historical trie. - /// - /// It must return the identification for the current session index. - type FullIdentificationOf: Convert>; -} +use crate::{self as pallet_session, Pallet as Session}; + +pub use pallet::*; -decl_storage! { - trait Store for Module as Session { - /// Mapping from historical session indices to session-data root hash and validator count. - HistoricalSessions get(fn historical_root): - map hasher(twox_64_concat) SessionIndex => Option<(T::Hash, ValidatorCount)>; - /// The range of historical sessions we store. [first, last) - StoredRange: Option<(SessionIndex, SessionIndex)>; - /// Deprecated. - CachedObsolete: - map hasher(twox_64_concat) SessionIndex - => Option>; +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + + /// The current storage version. + const STORAGE_VERSION: StorageVersion = StorageVersion::new(1); + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + #[pallet::storage_version(STORAGE_VERSION)] + pub struct Pallet(_); + + /// Config necessary for the historical pallet. + #[pallet::config] + pub trait Config: pallet_session::Config + frame_system::Config { + /// Full identification of the validator. + type FullIdentification: Parameter; + + /// A conversion from validator ID to full identification. + /// + /// This should contain any references to economic actors associated with the + /// validator, since they may be outdated by the time this is queried from a + /// historical trie. + /// + /// It must return the identification for the current session index. + type FullIdentificationOf: Convert>; } -} -decl_module! { - pub struct Module for enum Call where origin: T::Origin {} + /// Mapping from historical session indices to session-data root hash and validator count. + #[pallet::storage] + #[pallet::getter(fn historical_root)] + pub type HistoricalSessions = + StorageMap<_, Twox64Concat, SessionIndex, (T::Hash, ValidatorCount), OptionQuery>; + + /// The range of historical sessions we store. [first, last) + #[pallet::storage] + pub type StoredRange = StorageValue<_, (SessionIndex, SessionIndex), OptionQuery>; } -impl Module { +impl Pallet { /// Prune historical stored session roots up to (but not including) /// `up_to`. pub fn prune_up_to(up_to: SessionIndex) { @@ -109,7 +121,7 @@ impl Module { } } -impl ValidatorSet for Module { +impl ValidatorSet for Pallet { type ValidatorId = T::ValidatorId; type ValidatorIdOf = T::ValidatorIdOf; @@ -122,7 +134,7 @@ impl ValidatorSet for Module { } } -impl ValidatorSetWithIdentification for Module { +impl ValidatorSetWithIdentification for Pallet { type Identification = T::FullIdentification; type IdentificationOf = T::FullIdentificationOf; } @@ -130,7 +142,7 @@ impl ValidatorSetWithIdentification for Module { /// Specialization of the crate-level `SessionManager` which returns the set of full identification /// when creating a new session. pub trait SessionManager: - crate::SessionManager + pallet_session::SessionManager { /// If there was a validator set change, its returns the set of new validators along with their /// full identifications. @@ -150,7 +162,7 @@ pub struct NoteHistoricalRoot(sp_std::marker::PhantomData<(T, I)>); impl> NoteHistoricalRoot { fn do_new_session(new_index: SessionIndex, is_genesis: bool) -> Option> { - StoredRange::mutate(|range| { + >::mutate(|range| { range.get_or_insert_with(|| (new_index, new_index)).1 = new_index + 1; }); @@ -183,7 +195,7 @@ impl> NoteHi } } -impl crate::SessionManager for NoteHistoricalRoot +impl pallet_session::SessionManager for NoteHistoricalRoot where I: SessionManager, { @@ -207,7 +219,7 @@ where /// A tuple of the validator's ID and their full identification. pub type IdentificationTuple = - (::ValidatorId, ::FullIdentification); + (::ValidatorId, ::FullIdentification); /// A trie instance for checking and generating proofs. pub struct ProvingTrie { @@ -224,10 +236,10 @@ impl ProvingTrie { let mut root = Default::default(); { - let mut trie = TrieDBMut::new(&mut db, &mut root); + let mut trie = TrieDBMutV0::new(&mut db, &mut root); for (i, (validator, full_id)) in validators.into_iter().enumerate() { let i = i as u32; - let keys = match >::load_keys(&validator) { + let keys = match >::load_keys(&validator) { None => continue, Some(k) => k, }; @@ -304,15 +316,13 @@ impl ProvingTrie { } } -impl> frame_support::traits::KeyOwnerProofSystem<(KeyTypeId, D)> - for Module -{ +impl> KeyOwnerProofSystem<(KeyTypeId, D)> for Pallet { type Proof = MembershipProof; type IdentificationTuple = IdentificationTuple; fn prove(key: (KeyTypeId, D)) -> Option { - let session = >::current_index(); - let validators = >::validators() + let session = >::current_index(); + let validators = >::validators() .into_iter() .filter_map(|validator| { T::FullIdentificationOf::convert(validator.clone()) @@ -335,10 +345,10 @@ impl> frame_support::traits::KeyOwnerProofSystem<(KeyT fn check_proof(key: (KeyTypeId, D), proof: Self::Proof) -> Option> { let (id, data) = key; - if proof.session == >::current_index() { - >::key_owner(id, data.as_ref()).and_then(|owner| { + if proof.session == >::current_index() { + >::key_owner(id, data.as_ref()).and_then(|owner| { T::FullIdentificationOf::convert(owner.clone()).and_then(move |id| { - let count = >::validators().len() as ValidatorCount; + let count = >::validators().len() as ValidatorCount; if count != proof.validator_count { return None @@ -374,7 +384,7 @@ pub(crate) mod tests { BasicExternalities, }; - type Historical = Module; + type Historical = Pallet; pub(crate) fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); @@ -386,7 +396,9 @@ pub(crate) mod tests { frame_system::Pallet::::inc_providers(k); } }); - crate::GenesisConfig:: { keys }.assimilate_storage(&mut t).unwrap(); + pallet_session::GenesisConfig:: { keys } + .assimilate_storage(&mut t) + .unwrap(); sp_io::TestExternalities::new(t) } @@ -436,27 +448,27 @@ pub(crate) mod tests { Session::on_initialize(i); } - assert_eq!(StoredRange::get(), Some((0, 100))); + assert_eq!(>::get(), Some((0, 100))); for i in 0..100 { assert!(Historical::historical_root(i).is_some()) } Historical::prune_up_to(10); - assert_eq!(StoredRange::get(), Some((10, 100))); + assert_eq!(>::get(), Some((10, 100))); Historical::prune_up_to(9); - assert_eq!(StoredRange::get(), Some((10, 100))); + assert_eq!(>::get(), Some((10, 100))); for i in 10..100 { assert!(Historical::historical_root(i).is_some()) } Historical::prune_up_to(99); - assert_eq!(StoredRange::get(), Some((99, 100))); + assert_eq!(>::get(), Some((99, 100))); Historical::prune_up_to(100); - assert_eq!(StoredRange::get(), None); + assert_eq!(>::get(), None); for i in 99..199u64 { set_next_validators(vec![i]); @@ -466,14 +478,14 @@ pub(crate) mod tests { Session::on_initialize(i); } - assert_eq!(StoredRange::get(), Some((100, 200))); + assert_eq!(>::get(), Some((100, 200))); for i in 100..200 { assert!(Historical::historical_root(i).is_some()) } Historical::prune_up_to(9999); - assert_eq!(StoredRange::get(), None); + assert_eq!(>::get(), None); for i in 100..200 { assert!(Historical::historical_root(i).is_none()) diff --git a/frame/session/src/historical/offchain.rs b/frame/session/src/historical/offchain.rs index b646ecc2764f..95813d0a7027 100644 --- a/frame/session/src/historical/offchain.rs +++ b/frame/session/src/historical/offchain.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -140,7 +140,7 @@ pub fn keep_newest(n_to_keep: usize) { mod tests { use super::*; use crate::{ - historical::{onchain, Module}, + historical::{onchain, Pallet}, mock::{force_new_session, set_next_validators, Session, System, Test, NEXT_VALIDATORS}, }; @@ -156,7 +156,7 @@ mod tests { BasicExternalities, }; - type Historical = Module; + type Historical = Pallet; pub fn new_test_ext() -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default() diff --git a/frame/session/src/historical/onchain.rs b/frame/session/src/historical/onchain.rs index c80817c28d72..c7160e2fcf53 100644 --- a/frame/session/src/historical/onchain.rs +++ b/frame/session/src/historical/onchain.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/session/src/historical/shared.rs b/frame/session/src/historical/shared.rs index 182e9ecacee1..9e19b9df6d78 100644 --- a/frame/session/src/historical/shared.rs +++ b/frame/session/src/historical/shared.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/session/src/lib.rs b/frame/session/src/lib.rs index 2742d302ce43..4cf793a9b473 100644 --- a/frame/session/src/lib.rs +++ b/frame/session/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -108,23 +108,13 @@ #[cfg(feature = "historical")] pub mod historical; +pub mod migrations; #[cfg(test)] mod mock; #[cfg(test)] mod tests; pub mod weights; -use sp_runtime::{ - traits::{AtLeast32BitUnsigned, Convert, Member, One, OpaqueKeys, Zero}, - ConsensusEngineId, KeyTypeId, Perbill, Permill, RuntimeAppPublic, -}; -use sp_staking::SessionIndex; -use sp_std::{ - marker::PhantomData, - ops::{Rem, Sub}, - prelude::*, -}; - use frame_support::{ codec::{Decode, MaxEncodedLen}, dispatch::{DispatchError, DispatchResult}, @@ -136,6 +126,16 @@ use frame_support::{ weights::Weight, Parameter, }; +use sp_runtime::{ + traits::{AtLeast32BitUnsigned, Convert, Member, One, OpaqueKeys, Zero}, + ConsensusEngineId, KeyTypeId, Permill, RuntimeAppPublic, +}; +use sp_staking::SessionIndex; +use sp_std::{ + marker::PhantomData, + ops::{Rem, Sub}, + prelude::*, +}; pub use pallet::*; pub use weights::WeightInfo; @@ -298,7 +298,7 @@ pub trait SessionHandler { fn on_before_session_ending() {} /// A validator got disabled. Act accordingly until a new session begins. - fn on_disabled(validator_index: usize); + fn on_disabled(validator_index: u32); } #[impl_trait_for_tuples::impl_for_tuples(1, 30)] @@ -312,8 +312,10 @@ impl SessionHandler for Tuple { for_tuples!( #( let our_keys: Box> = Box::new(validators.iter() - .map(|k| (&k.0, k.1.get::(::ID) - .unwrap_or_default()))); + .filter_map(|k| + k.1.get::(::ID).map(|k1| (&k.0, k1)) + ) + ); Tuple::on_genesis_session(our_keys); )* @@ -328,11 +330,13 @@ impl SessionHandler for Tuple { for_tuples!( #( let our_keys: Box> = Box::new(validators.iter() - .map(|k| (&k.0, k.1.get::(::ID) - .unwrap_or_default()))); + .filter_map(|k| + k.1.get::(::ID).map(|k1| (&k.0, k1)) + )); let queued_keys: Box> = Box::new(queued_validators.iter() - .map(|k| (&k.0, k.1.get::(::ID) - .unwrap_or_default()))); + .filter_map(|k| + k.1.get::(::ID).map(|k1| (&k.0, k1)) + )); Tuple::on_new_session(changed, our_keys, queued_keys); )* ) @@ -342,7 +346,7 @@ impl SessionHandler for Tuple { for_tuples!( #( Tuple::on_before_session_ending(); )* ) } - fn on_disabled(i: usize) { + fn on_disabled(i: u32) { for_tuples!( #( Tuple::on_disabled(i); )* ) } } @@ -354,7 +358,7 @@ impl SessionHandler for TestSessionHandler { fn on_genesis_session(_: &[(AId, Ks)]) {} fn on_new_session(_: bool, _: &[(AId, Ks)], _: &[(AId, Ks)]) {} fn on_before_session_ending() {} - fn on_disabled(_: usize) {} + fn on_disabled(_: u32) {} } #[frame_support::pallet] @@ -369,6 +373,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] #[pallet::storage_version(STORAGE_VERSION)] + #[pallet::without_storage_info] pub struct Pallet(_); #[pallet::config] @@ -377,7 +382,11 @@ pub mod pallet { type Event: From + IsType<::Event>; /// A stable ID for a validator. - type ValidatorId: Member + Parameter + MaybeSerializeDeserialize + MaxEncodedLen; + type ValidatorId: Member + + Parameter + + MaybeSerializeDeserialize + + MaxEncodedLen + + TryFrom; /// A conversion from account ID to validator ID. /// @@ -399,13 +408,7 @@ pub mod pallet { type SessionHandler: SessionHandler; /// The keys. - type Keys: OpaqueKeys + Member + Parameter + Default + MaybeSerializeDeserialize; - - /// The fraction of validators set that is safe to be disabled. - /// - /// After the threshold is reached `disabled` method starts to return true, - /// which in combination with `pallet_staking` forces a new era. - type DisabledValidatorsThreshold: Get; + type Keys: OpaqueKeys + Member + Parameter + MaybeSerializeDeserialize; /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; @@ -446,7 +449,7 @@ pub mod pallet { for (account, val, keys) in self.keys.iter().cloned() { >::inner_set_keys(&val, keys) .expect("genesis config must not contain duplicates; qed"); - if frame_system::Pallet::::inc_consumers(&account).is_err() { + if frame_system::Pallet::::inc_consumers_without_limit(&account).is_err() { // This will leak a provider reference, however it only happens once (at // genesis) so it's really not a big deal and we assume that the user wants to // do this since it's the only way a non-endowed account can contain a session @@ -478,13 +481,18 @@ pub mod pallet { let queued_keys: Vec<_> = initial_validators_1 .iter() .cloned() - .map(|v| (v.clone(), >::load_keys(&v).unwrap_or_default())) + .map(|v| { + ( + v.clone(), + Pallet::::load_keys(&v).expect("Validator in session 1 missing keys!"), + ) + }) .collect(); // Tell everyone about the genesis session keys T::SessionHandler::on_genesis_session::(&queued_keys); - >::put(initial_validators_0); + Validators::::put(initial_validators_0); >::put(queued_keys); T::SessionManager::start_session(0); @@ -514,7 +522,9 @@ pub mod pallet { /// Indices of disabled validators. /// - /// The set is cleared when `on_session_ending` returns a new set of identities. + /// The vec is always kept sorted so that we can find whether a given validator is + /// disabled using binary search. It gets cleared when `on_session_ending` returns + /// a new set of identities. #[pallet::storage] #[pallet::getter(fn disabled_validators)] pub type DisabledValidators = StorageValue<_, Vec, ValueQuery>; @@ -532,9 +542,9 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// New session has happened. Note that the argument is the \[session_index\], not the + /// New session has happened. Note that the argument is the session index, not the /// block number as the type might suggest. - NewSession(SessionIndex), + NewSession { session_index: SessionIndex }, } /// Old name generated by `decl_event`. @@ -599,9 +609,13 @@ pub mod pallet { } /// Removes any session key(s) of the function caller. + /// /// This doesn't take effect until the next session. /// - /// The dispatch origin of this function must be signed. + /// The dispatch origin of this function must be Signed and the account must be either be + /// convertible to a validator ID using the chain's typical addressing system (this usually + /// means being a controller account) or directly convertible into a validator ID (which + /// usually means being a stash account). /// /// # /// - Complexity: `O(1)` in number of key types. Actual cost depends on the number of length @@ -637,7 +651,7 @@ impl Pallet { let session_keys = >::get(); let validators = session_keys.iter().map(|(validator, _)| validator.clone()).collect::>(); - >::put(&validators); + Validators::::put(&validators); if changed { // reset disabled validators @@ -659,7 +673,7 @@ impl Pallet { // same as before, as underlying economic conditions may have changed. (validators, true) } else { - (>::get(), false) + (Validators::::get(), false) }; // Queue next session keys. @@ -685,10 +699,10 @@ impl Pallet { }; let queued_amalgamated = next_validators .into_iter() - .map(|a| { - let k = Self::load_keys(&a).unwrap_or_default(); + .filter_map(|a| { + let k = Self::load_keys(&a)?; check_next_changed(&k); - (a, k) + Some((a, k)) }) .collect::>(); @@ -699,48 +713,40 @@ impl Pallet { >::put(next_changed); // Record that this happened. - Self::deposit_event(Event::NewSession(session_index)); + Self::deposit_event(Event::NewSession { session_index }); // Tell everyone about the new session keys. T::SessionHandler::on_new_session::(changed, &session_keys, &queued_amalgamated); } - /// Disable the validator of index `i`. - /// - /// Returns `true` if this causes a `DisabledValidatorsThreshold` of validators - /// to be already disabled. - pub fn disable_index(i: usize) -> bool { - let (fire_event, threshold_reached) = >::mutate(|disabled| { - let i = i as u32; + /// Disable the validator of index `i`, returns `false` if the validator was already disabled. + pub fn disable_index(i: u32) -> bool { + if i >= Validators::::decode_len().unwrap_or(0) as u32 { + return false + } + + >::mutate(|disabled| { if let Err(index) = disabled.binary_search(&i) { - let count = >::decode_len().unwrap_or(0) as u32; - let threshold = T::DisabledValidatorsThreshold::get() * count; disabled.insert(index, i); - (true, disabled.len() as u32 > threshold) - } else { - (false, false) + T::SessionHandler::on_disabled(i); + return true } - }); - - if fire_event { - T::SessionHandler::on_disabled(i); - } - threshold_reached + false + }) } /// Disable the validator identified by `c`. (If using with the staking pallet, /// this would be their *stash* account.) /// - /// Returns `Ok(true)` if more than `DisabledValidatorsThreshold` validators in current - /// session is already disabled. - /// If used with the staking pallet it allows to force a new era in such case. - pub fn disable(c: &T::ValidatorId) -> sp_std::result::Result { + /// Returns `false` either if the validator could not be found or it was already + /// disabled. + pub fn disable(c: &T::ValidatorId) -> bool { Self::validators() .iter() .position(|i| i == c) - .map(Self::disable_index) - .ok_or(()) + .map(|i| Self::disable_index(i as u32)) + .unwrap_or(false) } /// Upgrade the key type from some old type to a new type. Supports adding @@ -853,6 +859,10 @@ impl Pallet { fn do_purge_keys(account: &T::AccountId) -> DispatchResult { let who = T::ValidatorIdOf::convert(account.clone()) + // `purge_keys` may not have a controller-stash pair any more. If so then we expect the + // stash account to be passed in directly and convert that to a `ValidatorId` using the + // `TryFrom` trait if supported. + .or_else(|| T::ValidatorId::try_from(account.clone()).ok()) .ok_or(Error::::NoAssociatedValidatorId)?; let old_keys = Self::take_keys(&who).ok_or(Error::::NoKeys)?; diff --git a/frame/session/src/migrations/mod.rs b/frame/session/src/migrations/mod.rs new file mode 100644 index 000000000000..5981e87b677c --- /dev/null +++ b/frame/session/src/migrations/mod.rs @@ -0,0 +1,24 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Version 1. +/// +/// In version 0 session historical pallet uses `Session` for storage module prefix. +/// In version 1 it uses its name as configured in `construct_runtime`. +/// This migration moves session historical pallet storages from old prefix to new prefix. +#[cfg(feature = "historical")] +pub mod v1; diff --git a/frame/session/src/migrations/v1.rs b/frame/session/src/migrations/v1.rs new file mode 100644 index 000000000000..3c687ea7d9d6 --- /dev/null +++ b/frame/session/src/migrations/v1.rs @@ -0,0 +1,194 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sp_io::hashing::twox_128; +use sp_std::str; + +use frame_support::{ + storage::{generator::StorageValue, StoragePrefixedMap}, + traits::{ + Get, GetStorageVersion, PalletInfoAccess, StorageVersion, + STORAGE_VERSION_STORAGE_KEY_POSTFIX, + }, + weights::Weight, +}; + +use crate::historical as pallet_session_historical; + +const OLD_PREFIX: &str = "Session"; + +/// Migrate the entire storage of this pallet to a new prefix. +/// +/// This new prefix must be the same as the one set in construct_runtime. +/// +/// The migration will look into the storage version in order not to trigger a migration on an up +/// to date storage. Thus the on chain storage version must be less than 1 in order to trigger the +/// migration. +pub fn migrate( +) -> Weight { + let new_pallet_name =

::name(); + + if new_pallet_name == OLD_PREFIX { + log::info!( + target: "runtime::session_historical", + "New pallet name is equal to the old prefix. No migration needs to be done.", + ); + return 0 + } + + let on_chain_storage_version =

::on_chain_storage_version(); + log::info!( + target: "runtime::session_historical", + "Running migration to v1 for session_historical with storage version {:?}", + on_chain_storage_version, + ); + + if on_chain_storage_version < 1 { + let storage_prefix = pallet_session_historical::HistoricalSessions::::storage_prefix(); + frame_support::storage::migration::move_storage_from_pallet( + storage_prefix, + OLD_PREFIX.as_bytes(), + new_pallet_name.as_bytes(), + ); + log_migration("migration", storage_prefix, OLD_PREFIX, new_pallet_name); + + let storage_prefix = pallet_session_historical::StoredRange::::storage_prefix(); + frame_support::storage::migration::move_storage_from_pallet( + storage_prefix, + OLD_PREFIX.as_bytes(), + new_pallet_name.as_bytes(), + ); + log_migration("migration", storage_prefix, OLD_PREFIX, new_pallet_name); + + StorageVersion::new(1).put::

(); + ::BlockWeights::get().max_block + } else { + log::warn!( + target: "runtime::session_historical", + "Attempted to apply migration to v1 but failed because storage version is {:?}", + on_chain_storage_version, + ); + 0 + } +} + +/// Some checks prior to migration. This can be linked to +/// `frame_support::traits::OnRuntimeUpgrade::pre_upgrade` for further testing. +/// +/// Panics if anything goes wrong. +pub fn pre_migrate< + T: pallet_session_historical::Config, + P: GetStorageVersion + PalletInfoAccess, +>() { + let new_pallet_name =

::name(); + + let storage_prefix_historical_sessions = + pallet_session_historical::HistoricalSessions::::storage_prefix(); + let storage_prefix_stored_range = pallet_session_historical::StoredRange::::storage_prefix(); + + log_migration("pre-migration", storage_prefix_historical_sessions, OLD_PREFIX, new_pallet_name); + log_migration("pre-migration", storage_prefix_stored_range, OLD_PREFIX, new_pallet_name); + + if new_pallet_name == OLD_PREFIX { + return + } + + let new_pallet_prefix = twox_128(new_pallet_name.as_bytes()); + let storage_version_key = twox_128(STORAGE_VERSION_STORAGE_KEY_POSTFIX); + + let mut new_pallet_prefix_iter = frame_support::storage::KeyPrefixIterator::new( + new_pallet_prefix.to_vec(), + new_pallet_prefix.to_vec(), + |key| Ok(key.to_vec()), + ); + + // Ensure nothing except the storage_version_key is stored in the new prefix. + assert!(new_pallet_prefix_iter.all(|key| key == storage_version_key)); + + assert!(

::on_chain_storage_version() < 1); +} + +/// Some checks for after migration. This can be linked to +/// `frame_support::traits::OnRuntimeUpgrade::post_upgrade` for further testing. +/// +/// Panics if anything goes wrong. +pub fn post_migrate< + T: pallet_session_historical::Config, + P: GetStorageVersion + PalletInfoAccess, +>() { + let new_pallet_name =

::name(); + + let storage_prefix_historical_sessions = + pallet_session_historical::HistoricalSessions::::storage_prefix(); + let storage_prefix_stored_range = pallet_session_historical::StoredRange::::storage_prefix(); + + log_migration( + "post-migration", + storage_prefix_historical_sessions, + OLD_PREFIX, + new_pallet_name, + ); + log_migration("post-migration", storage_prefix_stored_range, OLD_PREFIX, new_pallet_name); + + if new_pallet_name == OLD_PREFIX { + return + } + + // Assert that no `HistoricalSessions` and `StoredRange` storages remains at the old prefix. + let old_pallet_prefix = twox_128(OLD_PREFIX.as_bytes()); + let old_historical_sessions_key = + [&old_pallet_prefix, &twox_128(storage_prefix_historical_sessions)[..]].concat(); + let old_historical_sessions_key_iter = frame_support::storage::KeyPrefixIterator::new( + old_historical_sessions_key.to_vec(), + old_historical_sessions_key.to_vec(), + |_| Ok(()), + ); + assert_eq!(old_historical_sessions_key_iter.count(), 0); + + let old_stored_range_key = + [&old_pallet_prefix, &twox_128(storage_prefix_stored_range)[..]].concat(); + let old_stored_range_key_iter = frame_support::storage::KeyPrefixIterator::new( + old_stored_range_key.to_vec(), + old_stored_range_key.to_vec(), + |_| Ok(()), + ); + assert_eq!(old_stored_range_key_iter.count(), 0); + + // Assert that the `HistoricalSessions` and `StoredRange` storages (if they exist) have been + // moved to the new prefix. + // NOTE: storage_version_key is already in the new prefix. + let new_pallet_prefix = twox_128(new_pallet_name.as_bytes()); + let new_pallet_prefix_iter = frame_support::storage::KeyPrefixIterator::new( + new_pallet_prefix.to_vec(), + new_pallet_prefix.to_vec(), + |_| Ok(()), + ); + assert!(new_pallet_prefix_iter.count() >= 1); + + assert_eq!(

::on_chain_storage_version(), 1); +} + +fn log_migration(stage: &str, storage_prefix: &[u8], old_pallet_name: &str, new_pallet_name: &str) { + log::info!( + target: "runtime::session_historical", + "{} prefix of storage '{}': '{}' ==> '{}'", + stage, + str::from_utf8(storage_prefix).unwrap_or(""), + old_pallet_name, + new_pallet_name, + ); +} diff --git a/frame/session/src/mock.rs b/frame/session/src/mock.rs index c6b5f6444811..7c6cc02c9e78 100644 --- a/frame/session/src/mock.rs +++ b/frame/session/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,18 +22,21 @@ use crate as pallet_session; #[cfg(feature = "historical")] use crate::historical as pallet_session_historical; -use std::cell::RefCell; +use std::{cell::RefCell, collections::BTreeMap}; use sp_core::{crypto::key_types::DUMMY, H256}; use sp_runtime::{ impl_opaque_keys, testing::{Header, UintAuthorityId}, - traits::{BlakeTwo256, ConvertInto, IdentityLookup}, - Perbill, + traits::{BlakeTwo256, IdentityLookup}, }; use sp_staking::SessionIndex; -use frame_support::{parameter_types, traits::GenesisBuild, BasicExternalities}; +use frame_support::{ + parameter_types, + traits::{ConstU32, ConstU64, GenesisBuild}, + BasicExternalities, +}; impl_opaque_keys! { pub struct MockSessionKeys { @@ -112,6 +115,7 @@ thread_local! { pub static DISABLED: RefCell = RefCell::new(false); // Stores if `on_before_session_end` was called pub static BEFORE_SESSION_END_CALLED: RefCell = RefCell::new(false); + pub static VALIDATOR_ACCOUNTS: RefCell> = RefCell::new(BTreeMap::new()); } pub struct TestShouldEndSession; @@ -144,7 +148,7 @@ impl SessionHandler for TestSessionHandler { .collect() }); } - fn on_disabled(_validator_index: usize) { + fn on_disabled(_validator_index: u32) { DISABLED.with(|l| *l.borrow_mut() = true) } fn on_before_session_ending() { @@ -226,12 +230,14 @@ pub fn new_test_ext() -> sp_io::TestExternalities { pallet_session::GenesisConfig:: { keys } .assimilate_storage(&mut t) .unwrap(); + NEXT_VALIDATORS.with(|l| { + let v = l.borrow().iter().map(|&i| (i, i)).collect(); + VALIDATOR_ACCOUNTS.with(|m| *m.borrow_mut() = v); + }); sp_io::TestExternalities::new(t) } parameter_types! { - pub const MinimumPeriod: u64 = 5; - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -251,7 +257,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -260,17 +266,26 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl pallet_timestamp::Config for Test { type Moment = u64; type OnTimestampSet = (); - type MinimumPeriod = MinimumPeriod; + type MinimumPeriod = ConstU64<5>; type WeightInfo = (); } -parameter_types! { - pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(33); +pub struct TestValidatorIdOf; +impl TestValidatorIdOf { + pub fn set(v: BTreeMap) { + VALIDATOR_ACCOUNTS.with(|m| *m.borrow_mut() = v); + } +} +impl Convert> for TestValidatorIdOf { + fn convert(x: u64) -> Option { + VALIDATOR_ACCOUNTS.with(|m| m.borrow().get(&x).cloned()) + } } impl Config for Test { @@ -281,10 +296,9 @@ impl Config for Test { type SessionManager = TestSessionManager; type SessionHandler = TestSessionHandler; type ValidatorId = u64; - type ValidatorIdOf = ConvertInto; + type ValidatorIdOf = TestValidatorIdOf; type Keys = MockSessionKeys; type Event = Event; - type DisabledValidatorsThreshold = DisabledValidatorsThreshold; type NextSessionRotation = (); type WeightInfo = (); } diff --git a/frame/session/src/tests.rs b/frame/session/src/tests.rs index 47152042d204..4f2108a9a438 100644 --- a/frame/session/src/tests.rs +++ b/frame/session/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use super::*; use crate::mock::{ authorities, before_session_end_called, force_new_session, new_test_ext, reset_before_session_end_called, session_changed, set_next_validators, set_session_length, - Origin, PreUpgradeMockSessionKeys, Session, System, Test, SESSION_CHANGED, + Origin, PreUpgradeMockSessionKeys, Session, System, Test, TestValidatorIdOf, SESSION_CHANGED, TEST_SESSION_CHANGED, }; @@ -29,7 +29,10 @@ use codec::Decode; use sp_core::crypto::key_types::DUMMY; use sp_runtime::testing::UintAuthorityId; -use frame_support::{assert_noop, assert_ok, traits::OnInitialize}; +use frame_support::{ + assert_noop, assert_ok, + traits::{ConstU64, OnInitialize}, +}; fn initialize_block(block: u64) { SESSION_CHANGED.with(|l| *l.borrow_mut() = false); @@ -72,11 +75,35 @@ fn keys_cleared_on_kill() { }) } +#[test] +fn purge_keys_works_for_stash_id() { + let mut ext = new_test_ext(); + ext.execute_with(|| { + assert_eq!(Session::validators(), vec![1, 2, 3]); + TestValidatorIdOf::set(vec![(10, 1), (20, 2), (3, 3)].into_iter().collect()); + assert_eq!(Session::load_keys(&1), Some(UintAuthorityId(1).into())); + assert_eq!(Session::load_keys(&2), Some(UintAuthorityId(2).into())); + + let id = DUMMY; + assert_eq!(Session::key_owner(id, UintAuthorityId(1).get_raw(id)), Some(1)); + + assert_ok!(Session::purge_keys(Origin::signed(10))); + assert_ok!(Session::purge_keys(Origin::signed(2))); + + assert_eq!(Session::load_keys(&10), None); + assert_eq!(Session::load_keys(&20), None); + assert_eq!(Session::key_owner(id, UintAuthorityId(10).get_raw(id)), None); + assert_eq!(Session::key_owner(id, UintAuthorityId(20).get_raw(id)), None); + }) +} + #[test] fn authorities_should_track_validators() { reset_before_session_end_called(); new_test_ext().execute_with(|| { + TestValidatorIdOf::set(vec![(1, 1), (2, 2), (3, 3), (4, 4)].into_iter().collect()); + set_next_validators(vec![1, 2]); force_new_session(); initialize_block(1); @@ -187,6 +214,8 @@ fn session_change_should_work() { #[test] fn duplicates_are_not_allowed() { new_test_ext().execute_with(|| { + TestValidatorIdOf::set(vec![(1, 1), (2, 2), (3, 3), (4, 4)].into_iter().collect()); + System::set_block_number(1); Session::on_initialize(1); assert_noop!( @@ -205,6 +234,7 @@ fn session_changed_flag_works() { reset_before_session_end_called(); new_test_ext().execute_with(|| { + TestValidatorIdOf::set(vec![(1, 1), (2, 2), (3, 3), (69, 69)].into_iter().collect()); TEST_SESSION_CHANGED.with(|l| *l.borrow_mut() = true); force_new_session(); @@ -264,12 +294,7 @@ fn session_changed_flag_works() { #[test] fn periodic_session_works() { - frame_support::parameter_types! { - const Period: u64 = 10; - const Offset: u64 = 3; - } - - type P = PeriodicSessions; + type P = PeriodicSessions, ConstU64<3>>; // make sure that offset phase behaves correctly for i in 0u64..3 { @@ -338,7 +363,7 @@ fn session_keys_generate_output_works_as_set_keys_input() { } #[test] -fn return_true_if_more_than_third_is_disabled() { +fn disable_index_returns_false_if_already_disabled() { new_test_ext().execute_with(|| { set_next_validators(vec![1, 2, 3, 4, 5, 6, 7]); force_new_session(); @@ -347,10 +372,9 @@ fn return_true_if_more_than_third_is_disabled() { force_new_session(); initialize_block(2); + assert_eq!(Session::disable_index(0), true); assert_eq!(Session::disable_index(0), false); - assert_eq!(Session::disable_index(1), false); - assert_eq!(Session::disable_index(2), true); - assert_eq!(Session::disable_index(3), true); + assert_eq!(Session::disable_index(1), true); }); } @@ -427,3 +451,30 @@ fn upgrade_keys() { } }) } + +#[cfg(feature = "historical")] +#[test] +fn test_migration_v1() { + use crate::{ + historical::{HistoricalSessions, StoredRange}, + mock::Historical, + }; + use frame_support::traits::PalletInfoAccess; + + new_test_ext().execute_with(|| { + assert!(>::iter_values().count() > 0); + assert!(>::exists()); + + let old_pallet = "Session"; + let new_pallet = ::name(); + frame_support::storage::migration::move_pallet( + new_pallet.as_bytes(), + old_pallet.as_bytes(), + ); + StorageVersion::new(0).put::(); + + crate::migrations::v1::pre_migrate::(); + crate::migrations::v1::migrate::(); + crate::migrations::v1::post_migrate::(); + }); +} diff --git a/frame/session/src/weights.rs b/frame/session/src/weights.rs index 64e7ac19ea7a..39276fb0b17e 100644 --- a/frame/session/src/weights.rs +++ b/frame/session/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_session //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/session/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -56,7 +57,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Session NextKeys (r:1 w:1) // Storage: Session KeyOwner (r:4 w:4) fn set_keys() -> Weight { - (64_427_000 as Weight) + (42_131_000 as Weight) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } @@ -64,7 +65,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Session NextKeys (r:1 w:1) // Storage: Session KeyOwner (r:0 w:4) fn purge_keys() -> Weight { - (42_497_000 as Weight) + (32_374_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } @@ -76,7 +77,7 @@ impl WeightInfo for () { // Storage: Session NextKeys (r:1 w:1) // Storage: Session KeyOwner (r:4 w:4) fn set_keys() -> Weight { - (64_427_000 as Weight) + (42_131_000 as Weight) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } @@ -84,7 +85,7 @@ impl WeightInfo for () { // Storage: Session NextKeys (r:1 w:1) // Storage: Session KeyOwner (r:0 w:4) fn purge_keys() -> Weight { - (42_497_000 as Weight) + (32_374_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } diff --git a/frame/society/Cargo.toml b/frame/society/Cargo.toml index 942b2844195f..1fd9693d0d00 100644 --- a/frame/society/Cargo.toml +++ b/frame/society/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-society" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME society pallet" readme = "README.md" @@ -13,17 +13,17 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } rand_chacha = { version = "0.2", default-features = false } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-io ={ version = "4.0.0-dev", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-io ={ version = "6.0.0", path = "../../primitives/io" } frame-support-test = { version = "3.0.0", path = "../support/test" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } diff --git a/frame/society/src/lib.rs b/frame/society/src/lib.rs index 83b1c4203722..a9f83094fd49 100644 --- a/frame/society/src/lib.rs +++ b/frame/society/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -371,6 +371,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); #[pallet::config] @@ -475,42 +476,41 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event, I: 'static = ()> { - /// The society is founded by the given identity. \[founder\] - Founded(T::AccountId), + /// The society is founded by the given identity. + Founded { founder: T::AccountId }, /// A membership bid just happened. The given account is the candidate's ID and their offer - /// is the second. \[candidate_id, offer\] - Bid(T::AccountId, BalanceOf), + /// is the second. + Bid { candidate_id: T::AccountId, offer: BalanceOf }, /// A membership bid just happened by vouching. The given account is the candidate's ID and - /// their offer is the second. The vouching party is the third. \[candidate_id, offer, - /// vouching\] - Vouch(T::AccountId, BalanceOf, T::AccountId), - /// A \[candidate\] was dropped (due to an excess of bids in the system). - AutoUnbid(T::AccountId), - /// A \[candidate\] was dropped (by their request). - Unbid(T::AccountId), - /// A \[candidate\] was dropped (by request of who vouched for them). - Unvouch(T::AccountId), + /// their offer is the second. The vouching party is the third. + Vouch { candidate_id: T::AccountId, offer: BalanceOf, vouching: T::AccountId }, + /// A candidate was dropped (due to an excess of bids in the system). + AutoUnbid { candidate: T::AccountId }, + /// A candidate was dropped (by their request). + Unbid { candidate: T::AccountId }, + /// A candidate was dropped (by request of who vouched for them). + Unvouch { candidate: T::AccountId }, /// A group of candidates have been inducted. The batch's primary is the first value, the - /// batch in full is the second. \[primary, candidates\] - Inducted(T::AccountId, Vec), - /// A suspended member has been judged. \[who, judged\] - SuspendedMemberJudgement(T::AccountId, bool), - /// A \[candidate\] has been suspended - CandidateSuspended(T::AccountId), - /// A \[member\] has been suspended - MemberSuspended(T::AccountId), - /// A \[member\] has been challenged - Challenged(T::AccountId), - /// A vote has been placed \[candidate, voter, vote\] - Vote(T::AccountId, T::AccountId, bool), - /// A vote has been placed for a defending member \[voter, vote\] - DefenderVote(T::AccountId, bool), + /// batch in full is the second. + Inducted { primary: T::AccountId, candidates: Vec }, + /// A suspended member has been judged. + SuspendedMemberJudgement { who: T::AccountId, judged: bool }, + /// A candidate has been suspended + CandidateSuspended { candidate: T::AccountId }, + /// A member has been suspended + MemberSuspended { member: T::AccountId }, + /// A member has been challenged + Challenged { member: T::AccountId }, + /// A vote has been placed + Vote { candidate: T::AccountId, voter: T::AccountId, vote: bool }, + /// A vote has been placed for a defending member + DefenderVote { voter: T::AccountId, vote: bool }, /// A new \[max\] member count has been set - NewMaxMembers(u32), - /// Society is unfounded. \[founder\] - Unfounded(T::AccountId), - /// Some funds were deposited into the society account. \[value\] - Deposit(BalanceOf), + NewMaxMembers { max: u32 }, + /// Society is unfounded. + Unfounded { founder: T::AccountId }, + /// Some funds were deposited into the society account. + Deposit { value: BalanceOf }, } /// Old name generated by `decl_event`. @@ -729,7 +729,7 @@ pub mod pallet { T::Currency::reserve(&who, deposit)?; Self::put_bid(bids, &who, value.clone(), BidKind::Deposit(deposit)); - Self::deposit_event(Event::::Bid(who, value)); + Self::deposit_event(Event::::Bid { candidate_id: who, offer: value }); Ok(()) } @@ -771,7 +771,7 @@ pub mod pallet { >::remove(&voucher); }, } - Self::deposit_event(Event::::Unbid(who)); + Self::deposit_event(Event::::Unbid { candidate: who }); Ok(()) } else { Err(Error::::BadPosition)? @@ -849,7 +849,11 @@ pub mod pallet { >::insert(&voucher, VouchingStatus::Vouching); Self::put_bid(bids, &who, value.clone(), BidKind::Vouch(voucher.clone(), tip)); - Self::deposit_event(Event::::Vouch(who, value, voucher)); + Self::deposit_event(Event::::Vouch { + candidate_id: who, + offer: value, + vouching: voucher, + }); Ok(()) } @@ -884,7 +888,7 @@ pub mod pallet { b[pos].kind.check_voucher(&voucher)?; >::remove(&voucher); let who = b.remove(pos).who; - Self::deposit_event(Event::::Unvouch(who)); + Self::deposit_event(Event::::Unvouch { candidate: who }); Ok(()) } else { Err(Error::::BadPosition)? @@ -927,7 +931,7 @@ pub mod pallet { let vote = if approve { Vote::Approve } else { Vote::Reject }; >::insert(&candidate, &voter, vote); - Self::deposit_event(Event::::Vote(candidate, voter, approve)); + Self::deposit_event(Event::::Vote { candidate, voter, vote: approve }); Ok(()) } @@ -956,7 +960,7 @@ pub mod pallet { let vote = if approve { Vote::Approve } else { Vote::Reject }; >::insert(&voter, vote); - Self::deposit_event(Event::::DefenderVote(voter, approve)); + Self::deposit_event(Event::::DefenderVote { voter, vote: approve }); Ok(()) } @@ -1039,7 +1043,7 @@ pub mod pallet { >::put(&founder); >::put(&founder); Rules::::put(T::Hashing::hash(&rules)); - Self::deposit_event(Event::::Founded(founder)); + Self::deposit_event(Event::::Founded { founder }); Ok(()) } @@ -1068,7 +1072,7 @@ pub mod pallet { Rules::::kill(); Candidates::::kill(); SuspendedCandidates::::remove_all(None); - Self::deposit_event(Event::::Unfounded(founder)); + Self::deposit_event(Event::::Unfounded { founder }); Ok(()) } @@ -1125,14 +1129,14 @@ pub mod pallet { if let Some(pos) = bids.iter().position(|b| b.kind.check_voucher(&who).is_ok()) { // Remove the bid, and emit an event let vouched = bids.remove(pos).who; - Self::deposit_event(Event::::Unvouch(vouched)); + Self::deposit_event(Event::::Unvouch { candidate: vouched }); } ); } } >::remove(&who); - Self::deposit_event(Event::::SuspendedMemberJudgement(who, forgive)); + Self::deposit_event(Event::::SuspendedMemberJudgement { who, judged: forgive }); Ok(()) } @@ -1253,7 +1257,7 @@ pub mod pallet { ensure_root(origin)?; ensure!(max > 1, Error::::MaxMembers); MaxMembers::::put(max); - Self::deposit_event(Event::::NewMaxMembers(max)); + Self::deposit_event(Event::::NewMaxMembers { max }); Ok(()) } } @@ -1287,7 +1291,7 @@ fn pick_item<'a, R: RngCore, T>(rng: &mut R, items: &'a [T]) -> Option<&'a T> { } /// Pick a new PRN, in the range [0, `max`] (inclusive). -fn pick_usize<'a, R: RngCore>(rng: &mut R, max: usize) -> usize { +fn pick_usize(rng: &mut R, max: usize) -> usize { (rng.next_u32() % (max as u32 + 1)) as usize } @@ -1313,9 +1317,9 @@ impl, I: 'static> Pallet { // Skip ahead to the suggested position .skip(pos) // Keep skipping ahead until the position changes - .skip_while(|(_, x)| x.value <= bids[pos].value) // Get the element when things changed - .next(); + .find(|(_, x)| x.value > bids[pos].value); + // If the element is not at the end of the list, insert the new element // in the spot. if let Some((p, _)) = different_bid { @@ -1339,7 +1343,7 @@ impl, I: 'static> Pallet { >::remove(&voucher); }, } - Self::deposit_event(Event::::AutoUnbid(popped)); + Self::deposit_event(Event::::AutoUnbid { candidate: popped }); } >::put(bids); @@ -1348,7 +1352,7 @@ impl, I: 'static> Pallet { /// Check a user is a bid. fn is_bid(bids: &Vec>>, who: &T::AccountId) -> bool { // Bids are ordered by `value`, so we cannot binary search for a user. - bids.iter().find(|bid| bid.who == *who).is_some() + bids.iter().any(|bid| bid.who == *who) } /// Check a user is a candidate. @@ -1504,7 +1508,7 @@ impl, I: 'static> Pallet { } else { // Suspend Candidate >::insert(&candidate, (value, kind)); - Self::deposit_event(Event::::CandidateSuspended(candidate)); + Self::deposit_event(Event::::CandidateSuspended { candidate }); None } }) @@ -1573,7 +1577,7 @@ impl, I: 'static> Pallet { >::put(&primary); T::MembershipChanged::change_members_sorted(&accounts, &[], &members); - Self::deposit_event(Event::::Inducted(primary, accounts)); + Self::deposit_event(Event::::Inducted { primary, candidates: accounts }); } // Bump the pot by at most PeriodSpend, but less if there's not very much left in our @@ -1638,7 +1642,7 @@ impl, I: 'static> Pallet { if Self::remove_member(&who).is_ok() { >::insert(who, true); >::remove(who); - Self::deposit_event(Event::::MemberSuspended(who.clone())); + Self::deposit_event(Event::::MemberSuspended { member: who.clone() }); } } @@ -1716,7 +1720,7 @@ impl, I: 'static> Pallet { let chosen = pick_item(&mut rng, &members[1..members.len() - 1]) .expect("exited if members empty; qed"); >::put(&chosen); - Self::deposit_event(Event::::Challenged(chosen.clone())); + Self::deposit_event(Event::::Challenged { member: chosen.clone() }); } else { >::kill(); } @@ -1820,6 +1824,6 @@ impl, I: 'static> OnUnbalanced> for Palle // Must resolve into existing but better to be safe. let _ = T::Currency::resolve_creating(&Self::account_id(), amount); - Self::deposit_event(Event::::Deposit(numeric_amount)); + Self::deposit_event(Event::::Deposit { value: numeric_amount }); } } diff --git a/frame/society/src/mock.rs b/frame/society/src/mock.rs index 9356c083f233..04ea705eed55 100644 --- a/frame/society/src/mock.rs +++ b/frame/society/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,10 @@ use super::*; use crate as pallet_society; -use frame_support::{ord_parameter_types, parameter_types}; +use frame_support::{ + ord_parameter_types, parameter_types, + traits::{ConstU32, ConstU64}, +}; use frame_support_test::TestRandomness; use frame_system::EnsureSignedBy; use sp_core::H256; @@ -45,16 +48,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const CandidateDeposit: u64 = 25; - pub const WrongSideDeduction: u64 = 2; - pub const MaxStrikes: u32 = 2; - pub const RotationPeriod: u64 = 4; - pub const PeriodSpend: u64 = 1000; - pub const MaxLockDuration: u64 = 100; - pub const ChallengePeriod: u64 = 8; - pub const BlockHashCount: u64 = 250; - pub const ExistentialDeposit: u64 = 1; - pub const MaxCandidateIntake: u32 = 10; pub const SocietyPalletId: PalletId = PalletId(*b"py/socie"); pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); @@ -80,7 +73,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type OnNewAccount = (); @@ -89,6 +82,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl pallet_balances::Config for Test { @@ -98,7 +92,7 @@ impl pallet_balances::Config for Test { type Balance = u64; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } @@ -107,17 +101,17 @@ impl Config for Test { type Event = Event; type Currency = pallet_balances::Pallet; type Randomness = TestRandomness; - type CandidateDeposit = CandidateDeposit; - type WrongSideDeduction = WrongSideDeduction; - type MaxStrikes = MaxStrikes; - type PeriodSpend = PeriodSpend; + type CandidateDeposit = ConstU64<25>; + type WrongSideDeduction = ConstU64<2>; + type MaxStrikes = ConstU32<2>; + type PeriodSpend = ConstU64<1000>; type MembershipChanged = (); - type RotationPeriod = RotationPeriod; - type MaxLockDuration = MaxLockDuration; + type RotationPeriod = ConstU64<4>; + type MaxLockDuration = ConstU64<100>; type FounderSetOrigin = EnsureSignedBy; type SuspensionJudgementOrigin = EnsureSignedBy; - type ChallengePeriod = ChallengePeriod; - type MaxCandidateIntake = MaxCandidateIntake; + type ChallengePeriod = ConstU64<8>; + type MaxCandidateIntake = ConstU32<10>; type PalletId = SocietyPalletId; } diff --git a/frame/society/src/tests.rs b/frame/society/src/tests.rs index 9f8e32dea508..d394ddc9011b 100644 --- a/frame/society/src/tests.rs +++ b/frame/society/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 70637bcd7726..073d959e2d1d 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-staking" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet staking" readme = "README.md" @@ -13,14 +13,14 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -serde = { version = "1.0.126", optional = true } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +serde = { version = "1.0.136", optional = true } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } @@ -28,7 +28,7 @@ pallet-session = { version = "4.0.0-dev", default-features = false, features = [ "historical", ], path = "../session" } pallet-authorship = { version = "4.0.0-dev", default-features = false, path = "../authorship" } -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../../primitives/application-crypto" } frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } log = { version = "0.4.14", default-features = false } @@ -37,8 +37,8 @@ frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = " rand_chacha = { version = "0.2", default-features = false, optional = true } [dev-dependencies] -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } sp-npos-elections = { version = "4.0.0-dev", path = "../../primitives/npos-elections" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } @@ -68,7 +68,7 @@ std = [ "frame-election-provider-support/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-election-provider-support/runtime-benchmarks", "rand_chacha", ] diff --git a/frame/staking/README.md b/frame/staking/README.md index 072353b1a586..bbd5bd18f6e8 100644 --- a/frame/staking/README.md +++ b/frame/staking/README.md @@ -133,19 +133,27 @@ The Staking module contains many public storage items and (im)mutable functions. ### Example: Rewarding a validator by id. ```rust -use frame_support::{decl_module, dispatch}; -use frame_system::ensure_signed; use pallet_staking::{self as staking}; -pub trait Config: staking::Config {} +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; -decl_module! { - pub struct Module for enum Call where origin: T::Origin { + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config + staking::Config {} + + #[pallet::call] + impl Pallet { /// Reward a validator. - #[weight = 0] - pub fn reward_myself(origin) -> dispatch::DispatchResult { + #[pallet::weight(0)] + pub fn reward_myself(origin: OriginFor) -> DispatchResult { let reported = ensure_signed(origin)?; - >::reward_by_ids(vec![(reported, 10)]); + >::reward_by_ids(vec![(reported, 10)]); Ok(()) } } diff --git a/frame/staking/fuzzer/src/mock.rs b/frame/staking/fuzzer/src/mock.rs deleted file mode 100644 index 921e0d3b48d7..000000000000 --- a/frame/staking/fuzzer/src/mock.rs +++ /dev/null @@ -1,202 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Mock file for staking fuzzing. - -use frame_support::parameter_types; - -type AccountId = u64; -type AccountIndex = u32; -type BlockNumber = u64; -type Balance = u64; - -type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; -type Block = frame_system::mocking::MockBlock; - -frame_support::construct_runtime!( - pub enum Test where - Block = Block, - NodeBlock = Block, - UncheckedExtrinsic = UncheckedExtrinsic, - { - System: frame_system::{Pallet, Call, Config, Storage, Event}, - Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, - Staking: pallet_staking::{Pallet, Call, Config, Storage, Event, ValidateUnsigned}, - Indices: pallet_indices::{Pallet, Call, Storage, Config, Event}, - Session: pallet_session::{Pallet, Call, Storage, Event, Config}, - } -); - -impl frame_system::Config for Test { - type BaseCallFilter = frame_support::traits::Everything; - type BlockWeights = (); - type BlockLength = (); - type DbWeight = (); - type Origin = Origin; - type Index = AccountIndex; - type BlockNumber = BlockNumber; - type Call = Call; - type Hash = sp_core::H256; - type Hashing = ::sp_runtime::traits::BlakeTwo256; - type AccountId = AccountId; - type Lookup = Indices; - type Header = sp_runtime::testing::Header; - type Event = Event; - type BlockHashCount = (); - type Version = (); - type PalletInfo = PalletInfo; - type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - type OnSetCode = (); -} -parameter_types! { - pub const ExistentialDeposit: Balance = 10; -} -impl pallet_balances::Config for Test { - type MaxLocks = (); - type MaxReserves = (); - type ReserveIdentifier = [u8; 8]; - type Balance = Balance; - type Event = Event; - type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; - type AccountStore = System; - type WeightInfo = (); -} -impl pallet_indices::Config for Test { - type AccountIndex = AccountIndex; - type Event = Event; - type Currency = Balances; - type Deposit = (); - type WeightInfo = (); -} -parameter_types! { - pub const MinimumPeriod: u64 = 5; -} -impl pallet_timestamp::Config for Test { - type Moment = u64; - type OnTimestampSet = (); - type MinimumPeriod = MinimumPeriod; - type WeightInfo = (); -} -impl pallet_session::historical::Config for Test { - type FullIdentification = pallet_staking::Exposure; - type FullIdentificationOf = pallet_staking::ExposureOf; -} - -sp_runtime::impl_opaque_keys! { - pub struct SessionKeys { - pub foo: sp_runtime::testing::UintAuthorityId, - } -} - -pub struct TestSessionHandler; -impl pallet_session::SessionHandler for TestSessionHandler { - const KEY_TYPE_IDS: &'static [sp_runtime::KeyTypeId] = &[]; - - fn on_genesis_session(_validators: &[(AccountId, Ks)]) {} - - fn on_new_session( - _: bool, - _: &[(AccountId, Ks)], - _: &[(AccountId, Ks)], - ) {} - - fn on_disabled(_: usize) {} -} - -impl pallet_session::Config for Test { - type SessionManager = pallet_session::historical::NoteHistoricalRoot; - type Keys = SessionKeys; - type ShouldEndSession = pallet_session::PeriodicSessions<(), ()>; - type NextSessionRotation = pallet_session::PeriodicSessions<(), ()>; - type SessionHandler = TestSessionHandler; - type Event = Event; - type ValidatorId = AccountId; - type ValidatorIdOf = pallet_staking::StashOf; - type DisabledValidatorsThreshold = (); - type WeightInfo = (); -} -pallet_staking_reward_curve::build! { - const I_NPOS: sp_runtime::curve::PiecewiseLinear<'static> = curve!( - min_inflation: 0_025_000, - max_inflation: 0_100_000, - ideal_stake: 0_500_000, - falloff: 0_050_000, - max_piece_count: 40, - test_precision: 0_005_000, - ); -} -parameter_types! { - pub const RewardCurve: &'static sp_runtime::curve::PiecewiseLinear<'static> = &I_NPOS; - pub const MaxNominatorRewardedPerValidator: u32 = 64; - pub const MaxIterations: u32 = 20; -} - -pub type Extrinsic = sp_runtime::testing::TestXt; - -impl frame_system::offchain::SendTransactionTypes for Test -where - Call: From, -{ - type OverarchingCall = Call; - type Extrinsic = Extrinsic; -} - -pub struct MockElectionProvider; -impl frame_election_provider_support::ElectionProvider - for MockElectionProvider -{ - type Error = (); - type DataProvider = pallet_staking::Module; - - fn elect() -> Result< - (sp_npos_elections::Supports, frame_support::weights::Weight), - Self::Error - > { - Err(()) - } -} - -impl pallet_staking::Config for Test { - type Currency = Balances; - type UnixTime = pallet_timestamp::Pallet; - type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; - type RewardRemainder = (); - type Event = Event; - type Slash = (); - type Reward = (); - type SessionsPerEra = (); - type SlashDeferDuration = (); - type SlashCancelOrigin = frame_system::EnsureRoot; - type BondingDuration = (); - type SessionInterface = Self; - type EraPayout = pallet_staking::ConvertCurve; - type NextNewSession = Session; - type ElectionLookahead = (); - type Call = Call; - type MaxIterations = MaxIterations; - type MinSolutionScoreBump = (); - type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; - type UnsignedPriority = (); - type OffchainSolutionWeightLimit = (); - type WeightInfo = (); - type ElectionProvider = MockElectionProvider; -} diff --git a/frame/staking/reward-curve/Cargo.toml b/frame/staking/reward-curve/Cargo.toml index 4cbc2473cb52..d53fb72b0e08 100644 --- a/frame/staking/reward-curve/Cargo.toml +++ b/frame/staking/reward-curve/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-staking-reward-curve" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Reward Curve for FRAME staking pallet" @@ -15,10 +15,10 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -syn = { version = "1.0.58", features = ["full", "visit"] } -quote = "1.0.3" -proc-macro2 = "1.0.29" -proc-macro-crate = "1.0.0" +syn = { version = "1.0.82", features = ["full", "visit"] } +quote = "1.0.10" +proc-macro2 = "1.0.36" +proc-macro-crate = "1.1.3" [dev-dependencies] -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } diff --git a/frame/staking/reward-curve/src/lib.rs b/frame/staking/reward-curve/src/lib.rs index 06e35d11350e..e66f6fde3759 100644 --- a/frame/staking/reward-curve/src/lib.rs +++ b/frame/staking/reward-curve/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,7 +24,6 @@ use proc_macro::TokenStream; use proc_macro2::{Span, TokenStream as TokenStream2}; use proc_macro_crate::{crate_name, FoundCrate}; use quote::{quote, ToTokens}; -use std::convert::TryInto; use syn::parse::{Parse, ParseStream}; /// Accepts a number of expressions to create a instance of PiecewiseLinear which represents the diff --git a/frame/staking/reward-curve/src/log.rs b/frame/staking/reward-curve/src/log.rs index c196aaaa31a9..248a1e3c36a6 100644 --- a/frame/staking/reward-curve/src/log.rs +++ b/frame/staking/reward-curve/src/log.rs @@ -1,5 +1,3 @@ -use std::convert::TryInto; - /// Simple u32 power of 2 function - simply uses a bit shift macro_rules! pow2 { ($n:expr) => { diff --git a/frame/staking/reward-curve/tests/test.rs b/frame/staking/reward-curve/tests/test.rs index fda7df145d0f..aa19b1782453 100644 --- a/frame/staking/reward-curve/tests/test.rs +++ b/frame/staking/reward-curve/tests/test.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/staking/reward-fn/Cargo.toml b/frame/staking/reward-fn/Cargo.toml index 076e05bf2a61..9a768becae83 100644 --- a/frame/staking/reward-fn/Cargo.toml +++ b/frame/staking/reward-fn/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-staking-reward-fn" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Reward function for FRAME staking pallet" @@ -14,7 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"] [lib] [dependencies] -sp-arithmetic = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/arithmetic" } +sp-arithmetic = { version = "5.0.0", default-features = false, path = "../../../primitives/arithmetic" } log = { version = "0.4.14", default-features = false } [features] diff --git a/frame/staking/reward-fn/src/lib.rs b/frame/staking/reward-fn/src/lib.rs index dd5e629b3984..cb0b660bf544 100644 --- a/frame/staking/reward-fn/src/lib.rs +++ b/frame/staking/reward-fn/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,6 @@ //! Useful function for inflation for nominated proof of stake. -use core::convert::TryFrom; use sp_arithmetic::{ biguint::BigUint, traits::{SaturatedConversion, Zero}, diff --git a/frame/staking/reward-fn/tests/test.rs b/frame/staking/reward-fn/tests/test.rs index dc5b661c4098..a79137716fd2 100644 --- a/frame/staking/reward-fn/tests/test.rs +++ b/frame/staking/reward-fn/tests/test.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index fe60d516e144..983f1bd54deb 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,9 +18,10 @@ //! Staking pallet benchmarking. use super::*; -use crate::Pallet as Staking; +use crate::{ConfigOp, Pallet as Staking}; use testing_utils::*; +use codec::Decode; use frame_election_provider_support::SortedListProvider; use frame_support::{ dispatch::UnfilteredDispatchable, @@ -28,7 +29,7 @@ use frame_support::{ traits::{Currency, CurrencyToVote, Get, Imbalance}, }; use sp_runtime::{ - traits::{StaticLookup, Zero}, + traits::{Bounded, One, StaticLookup, TrailingZeroInput, Zero}, Perbill, Percent, }; use sp_staking::SessionIndex; @@ -38,14 +39,14 @@ pub use frame_benchmarking::{ account, benchmarks, impl_benchmark_test_suite, whitelist_account, whitelisted_caller, }; use frame_system::RawOrigin; -use sp_runtime::traits::{Bounded, One}; const SEED: u32 = 0; const MAX_SPANS: u32 = 100; -const MAX_VALIDATORS: u32 = 1000; -const MAX_NOMINATORS: u32 = 1000; const MAX_SLASHES: u32 = 1000; +type MaxValidators = <::BenchmarkingConfig as BenchmarkingConfig>::MaxValidators; +type MaxNominators = <::BenchmarkingConfig as BenchmarkingConfig>::MaxNominators; + // Add slashing spans to a user account. Not relevant for actual use, only to benchmark // read and write operations. fn add_slashing_spans(who: &T::AccountId, spans: u32) { @@ -112,8 +113,8 @@ pub fn create_validator_with_nominators( assert_eq!(new_validators.len(), 1); assert_eq!(new_validators[0], v_stash, "Our validator was not selected!"); - assert_ne!(CounterForValidators::::get(), 0); - assert_ne!(CounterForNominators::::get(), 0); + assert_ne!(Validators::::count(), 0); + assert_ne!(Nominators::::count(), 0); // Give Era Points let reward = EraRewardPoints:: { @@ -154,8 +155,8 @@ impl ListScenario { /// - the destination bag has at least one node, which will need its next pointer updated. /// /// NOTE: while this scenario specifically targets a worst case for the bags-list, it should - /// also elicit a worst case for other known `SortedListProvider` implementations; although - /// this may not be true against unknown `SortedListProvider` implementations. + /// also elicit a worst case for other known `VoterList` implementations; although + /// this may not be true against unknown `VoterList` implementations. fn new(origin_weight: BalanceOf, is_increase: bool) -> Result { ensure!(!origin_weight.is_zero(), "origin weight must be greater than 0"); @@ -188,7 +189,7 @@ impl ListScenario { // find a destination weight that will trigger the worst case scenario let dest_weight_as_vote = - T::SortedListProvider::weight_update_worst_case(&origin_stash1, is_increase); + T::VoterList::score_update_worst_case(&origin_stash1, is_increase); let total_issuance = T::Currency::total_issuance(); @@ -256,7 +257,6 @@ benchmarks! { } unbond { - use sp_std::convert::TryFrom; // clean up any existing state. clear_validators_and_nominators::(); @@ -316,7 +316,7 @@ benchmarks! { let scenario = ListScenario::::new(origin_weight, true)?; let controller = scenario.origin_controller1.clone(); let stash = scenario.origin_stash1.clone(); - assert!(T::SortedListProvider::contains(&stash)); + assert!(T::VoterList::contains(&stash)); let ed = T::Currency::minimum_balance(); let mut ledger = Ledger::::get(&controller).unwrap(); @@ -328,44 +328,40 @@ benchmarks! { }: withdraw_unbonded(RawOrigin::Signed(controller.clone()), s) verify { assert!(!Ledger::::contains_key(controller)); - assert!(!T::SortedListProvider::contains(&stash)); + assert!(!T::VoterList::contains(&stash)); } validate { - // clean up any existing state. - clear_validators_and_nominators::(); - - let origin_weight = MinNominatorBond::::get().max(T::Currency::minimum_balance()); - - // setup a worst case scenario where the user calling validate was formerly a nominator so - // they must be removed from the list. - let scenario = ListScenario::::new(origin_weight, true)?; - let controller = scenario.origin_controller1.clone(); - let stash = scenario.origin_stash1.clone(); - assert!(T::SortedListProvider::contains(&stash)); + let (stash, controller) = create_stash_controller::( + T::MaxNominations::get() - 1, + 100, + Default::default(), + )?; + // because it is chilled. + assert!(!T::VoterList::contains(&stash)); let prefs = ValidatorPrefs::default(); whitelist_account!(controller); }: _(RawOrigin::Signed(controller), prefs) verify { assert!(Validators::::contains_key(&stash)); - assert!(!T::SortedListProvider::contains(&stash)); + assert!(T::VoterList::contains(&stash)); } kick { // scenario: we want to kick `k` nominators from nominating us (we are a validator). // we'll assume that `k` is under 128 for the purposes of determining the slope. - // each nominator should have `T::MAX_NOMINATIONS` validators nominated, and our validator + // each nominator should have `T::MaxNominations::get()` validators nominated, and our validator // should be somewhere in there. let k in 1 .. 128; - // these are the other validators; there are `T::MAX_NOMINATIONS - 1` of them, so - // there are a total of `T::MAX_NOMINATIONS` validators in the system. - let rest_of_validators = create_validators_with_seed::(T::MAX_NOMINATIONS - 1, 100, 415)?; + // these are the other validators; there are `T::MaxNominations::get() - 1` of them, so + // there are a total of `T::MaxNominations::get()` validators in the system. + let rest_of_validators = create_validators_with_seed::(T::MaxNominations::get() - 1, 100, 415)?; // this is the validator that will be kicking. let (stash, controller) = create_stash_controller::( - T::MAX_NOMINATIONS - 1, + T::MaxNominations::get() - 1, 100, Default::default(), )?; @@ -380,7 +376,7 @@ benchmarks! { for i in 0 .. k { // create a nominator stash. let (n_stash, n_controller) = create_stash_controller::( - T::MAX_NOMINATIONS + i, + T::MaxNominations::get() + i, 100, Default::default(), )?; @@ -415,9 +411,9 @@ benchmarks! { } } - // Worst case scenario, T::MAX_NOMINATIONS + // Worst case scenario, T::MaxNominations::get() nominate { - let n in 1 .. T::MAX_NOMINATIONS; + let n in 1 .. T::MaxNominations::get(); // clean up any existing state. clear_validators_and_nominators::(); @@ -428,20 +424,20 @@ benchmarks! { // we are just doing an insert into the origin position. let scenario = ListScenario::::new(origin_weight, true)?; let (stash, controller) = create_stash_controller_with_balance::( - SEED + T::MAX_NOMINATIONS + 1, // make sure the account does not conflict with others + SEED + T::MaxNominations::get() + 1, // make sure the account does not conflict with others origin_weight, Default::default(), ).unwrap(); assert!(!Nominators::::contains_key(&stash)); - assert!(!T::SortedListProvider::contains(&stash)); + assert!(!T::VoterList::contains(&stash)); let validators = create_validators::(n, 100).unwrap(); whitelist_account!(controller); }: _(RawOrigin::Signed(controller), validators) verify { assert!(Nominators::::contains_key(&stash)); - assert!(T::SortedListProvider::contains(&stash)) + assert!(T::VoterList::contains(&stash)) } chill { @@ -455,12 +451,12 @@ benchmarks! { let scenario = ListScenario::::new(origin_weight, true)?; let controller = scenario.origin_controller1.clone(); let stash = scenario.origin_stash1.clone(); - assert!(T::SortedListProvider::contains(&stash)); + assert!(T::VoterList::contains(&stash)); whitelist_account!(controller); }: _(RawOrigin::Signed(controller)) verify { - assert!(!T::SortedListProvider::contains(&stash)); + assert!(!T::VoterList::contains(&stash)); } set_payee { @@ -483,7 +479,7 @@ benchmarks! { } set_validator_count { - let validator_count = MAX_VALIDATORS; + let validator_count = MaxValidators::::get(); }: _(RawOrigin::Root, validator_count) verify { assert_eq!(ValidatorCount::::get(), validator_count); @@ -500,7 +496,7 @@ benchmarks! { // Worst case scenario, the list of invulnerables is very long. set_invulnerables { - let v in 0 .. MAX_VALIDATORS; + let v in 0 .. MaxValidators::::get(); let mut invulnerables = Vec::new(); for i in 0 .. v { invulnerables.push(account("invulnerable", i, SEED)); @@ -523,21 +519,22 @@ benchmarks! { let scenario = ListScenario::::new(origin_weight, true)?; let controller = scenario.origin_controller1.clone(); let stash = scenario.origin_stash1.clone(); - assert!(T::SortedListProvider::contains(&stash)); + assert!(T::VoterList::contains(&stash)); add_slashing_spans::(&stash, s); }: _(RawOrigin::Root, stash.clone(), s) verify { assert!(!Ledger::::contains_key(&controller)); - assert!(!T::SortedListProvider::contains(&stash)); + assert!(!T::VoterList::contains(&stash)); } cancel_deferred_slash { let s in 1 .. MAX_SLASHES; let mut unapplied_slashes = Vec::new(); let era = EraIndex::one(); + let dummy = || T::AccountId::decode(&mut TrailingZeroInput::zeroes()).unwrap(); for _ in 0 .. MAX_SLASHES { - unapplied_slashes.push(UnappliedSlash::>::default()); + unapplied_slashes.push(UnappliedSlash::>::default_from(dummy())); } UnappliedSlashes::::insert(era, &unapplied_slashes); @@ -617,7 +614,7 @@ benchmarks! { } rebond { - let l in 1 .. MAX_UNLOCKING_CHUNKS as u32; + let l in 1 .. MaxUnlockingChunks::get() as u32; // clean up any existing state. clear_validators_and_nominators::(); @@ -651,7 +648,7 @@ benchmarks! { let mut staking_ledger = Ledger::::get(controller.clone()).unwrap(); for _ in 0 .. l { - staking_ledger.unlocking.push(unlock_chunk.clone()) + staking_ledger.unlocking.try_push(unlock_chunk.clone()).unwrap() } Ledger::::insert(controller.clone(), staking_ledger.clone()); let original_bonded: BalanceOf = staking_ledger.active; @@ -668,10 +665,11 @@ benchmarks! { let e in 1 .. 100; HistoryDepth::::put(e); CurrentEra::::put(e); + let dummy = || -> T::AccountId { codec::Decode::decode(&mut TrailingZeroInput::zeroes()).unwrap() }; for i in 0 .. e { - >::insert(i, T::AccountId::default(), Exposure::>::default()); - >::insert(i, T::AccountId::default(), Exposure::>::default()); - >::insert(i, T::AccountId::default(), ValidatorPrefs::default()); + >::insert(i, dummy(), Exposure::>::default()); + >::insert(i, dummy(), Exposure::>::default()); + >::insert(i, dummy(), ValidatorPrefs::default()); >::insert(i, BalanceOf::::one()); >::insert(i, EraRewardPoints::::default()); >::insert(i, BalanceOf::::one()); @@ -696,16 +694,23 @@ benchmarks! { let stash = scenario.origin_stash1.clone(); add_slashing_spans::(&stash, s); - T::Currency::make_free_balance_be(&stash, T::Currency::minimum_balance()); + let l = StakingLedger { + stash: stash.clone(), + active: T::Currency::minimum_balance() - One::one(), + total: T::Currency::minimum_balance() - One::one(), + unlocking: Default::default(), + claimed_rewards: vec![], + }; + Ledger::::insert(&controller, l); assert!(Bonded::::contains_key(&stash)); - assert!(T::SortedListProvider::contains(&stash)); + assert!(T::VoterList::contains(&stash)); whitelist_account!(controller); }: _(RawOrigin::Signed(controller), stash.clone(), s) verify { assert!(!Bonded::::contains_key(&stash)); - assert!(!T::SortedListProvider::contains(&stash)); + assert!(!T::VoterList::contains(&stash)); } new_era { @@ -715,7 +720,7 @@ benchmarks! { create_validators_with_nominators_for_era::( v, n, - ::MAX_NOMINATIONS as usize, + ::MaxNominations::get() as usize, false, None, )?; @@ -733,7 +738,7 @@ benchmarks! { create_validators_with_nominators_for_era::( v, n, - ::MAX_NOMINATIONS as usize, + ::MaxNominations::get() as usize, false, None, )?; @@ -779,7 +784,7 @@ benchmarks! { #[extra] do_slash { - let l in 1 .. MAX_UNLOCKING_CHUNKS as u32; + let l in 1 .. MaxUnlockingChunks::get() as u32; let (stash, controller) = create_stash_controller::(0, 100, Default::default())?; let mut staking_ledger = Ledger::::get(controller.clone()).unwrap(); let unlock_chunk = UnlockChunk::> { @@ -787,7 +792,7 @@ benchmarks! { era: EraIndex::zero(), }; for _ in 0 .. l { - staking_ledger.unlocking.push(unlock_chunk.clone()) + staking_ledger.unlocking.try_push(unlock_chunk.clone()).unwrap(); } Ledger::::insert(controller, staking_ledger); let slash_amount = T::Currency::minimum_balance() * 10u32.into(); @@ -806,14 +811,14 @@ benchmarks! { get_npos_voters { // number of validator intention. - let v in (MAX_VALIDATORS / 2) .. MAX_VALIDATORS; + let v in (MaxValidators::::get() / 2) .. MaxValidators::::get(); // number of nominator intention. - let n in (MAX_NOMINATORS / 2) .. MAX_NOMINATORS; + let n in (MaxNominators::::get() / 2) .. MaxNominators::::get(); // total number of slashing spans. Assigned to validators randomly. let s in 1 .. 20; let validators = create_validators_with_nominators_for_era::( - v, n, T::MAX_NOMINATIONS as usize, false, None + v, n, T::MaxNominations::get() as usize, false, None )? .into_iter() .map(|v| T::Lookup::lookup(v).unwrap()) @@ -831,33 +836,52 @@ benchmarks! { get_npos_targets { // number of validator intention. - let v in (MAX_VALIDATORS / 2) .. MAX_VALIDATORS; + let v in (MaxValidators::::get() / 2) .. MaxValidators::::get(); // number of nominator intention. - let n = MAX_NOMINATORS; + let n = MaxNominators::::get(); let _ = create_validators_with_nominators_for_era::( - v, n, T::MAX_NOMINATIONS as usize, false, None + v, n, T::MaxNominations::get() as usize, false, None )?; }: { let targets = >::get_npos_targets(); assert_eq!(targets.len() as u32, v); } - set_staking_limits { - // This function always does the same thing... just write to 4 storage items. - }: _( + set_staking_configs_all_set { + }: set_staking_configs( RawOrigin::Root, - BalanceOf::::max_value(), - BalanceOf::::max_value(), - Some(u32::MAX), - Some(u32::MAX), - Some(Percent::max_value()) + ConfigOp::Set(BalanceOf::::max_value()), + ConfigOp::Set(BalanceOf::::max_value()), + ConfigOp::Set(u32::MAX), + ConfigOp::Set(u32::MAX), + ConfigOp::Set(Percent::max_value()), + ConfigOp::Set(Perbill::max_value()) ) verify { assert_eq!(MinNominatorBond::::get(), BalanceOf::::max_value()); assert_eq!(MinValidatorBond::::get(), BalanceOf::::max_value()); assert_eq!(MaxNominatorsCount::::get(), Some(u32::MAX)); assert_eq!(MaxValidatorsCount::::get(), Some(u32::MAX)); assert_eq!(ChillThreshold::::get(), Some(Percent::from_percent(100))); + assert_eq!(MinCommission::::get(), Perbill::from_percent(100)); + } + + set_staking_configs_all_remove { + }: set_staking_configs( + RawOrigin::Root, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove + ) verify { + assert!(!MinNominatorBond::::exists()); + assert!(!MinValidatorBond::::exists()); + assert!(!MaxNominatorsCount::::exists()); + assert!(!MaxValidatorsCount::::exists()); + assert!(!ChillThreshold::::exists()); + assert!(!MinCommission::::exists()); } chill_other { @@ -871,22 +895,59 @@ benchmarks! { let scenario = ListScenario::::new(origin_weight, true)?; let controller = scenario.origin_controller1.clone(); let stash = scenario.origin_stash1.clone(); - assert!(T::SortedListProvider::contains(&stash)); + assert!(T::VoterList::contains(&stash)); - Staking::::set_staking_limits( + Staking::::set_staking_configs( RawOrigin::Root.into(), - BalanceOf::::max_value(), - BalanceOf::::max_value(), - Some(0), - Some(0), - Some(Percent::from_percent(0)) + ConfigOp::Set(BalanceOf::::max_value()), + ConfigOp::Set(BalanceOf::::max_value()), + ConfigOp::Set(0), + ConfigOp::Set(0), + ConfigOp::Set(Percent::from_percent(0)), + ConfigOp::Set(Zero::zero()), )?; let caller = whitelisted_caller(); }: _(RawOrigin::Signed(caller), controller.clone()) verify { - assert!(!T::SortedListProvider::contains(&stash)); + assert!(!T::VoterList::contains(&stash)); + } + + force_apply_min_commission { + // Clean up any existing state + clear_validators_and_nominators::(); + + // Create a validator with a commission of 50% + let (stash, controller) = + create_stash_controller::(1, 1, RewardDestination::Staked)?; + let validator_prefs = + ValidatorPrefs { commission: Perbill::from_percent(50), ..Default::default() }; + Staking::::validate(RawOrigin::Signed(controller).into(), validator_prefs)?; + + // Sanity check + assert_eq!( + Validators::::get(&stash), + ValidatorPrefs { commission: Perbill::from_percent(50), ..Default::default() } + ); + + // Set the min commission to 75% + MinCommission::::set(Perbill::from_percent(75)); + let caller = whitelisted_caller(); + }: _(RawOrigin::Signed(caller), stash.clone()) + verify { + // The validators commission has been bumped to 75% + assert_eq!( + Validators::::get(&stash), + ValidatorPrefs { commission: Perbill::from_percent(75), ..Default::default() } + ); } + + impl_benchmark_test_suite!( + Staking, + crate::mock::ExtBuilder::default().has_stakers(true), + crate::mock::Test, + exec_name = build_and_execute + ); } #[cfg(test)] @@ -904,7 +965,7 @@ mod tests { create_validators_with_nominators_for_era::( v, n, - ::MAX_NOMINATIONS as usize, + ::MaxNominations::get() as usize, false, None, ) @@ -913,8 +974,8 @@ mod tests { let count_validators = Validators::::iter().count(); let count_nominators = Nominators::::iter().count(); - assert_eq!(count_validators, CounterForValidators::::get() as usize); - assert_eq!(count_nominators, CounterForNominators::::get() as usize); + assert_eq!(count_validators, Validators::::count() as usize); + assert_eq!(count_nominators, Nominators::::count() as usize); assert_eq!(count_validators, v as usize); assert_eq!(count_nominators, n as usize); @@ -928,7 +989,7 @@ mod tests { let (validator_stash, nominators) = create_validator_with_nominators::( n, - ::MaxNominatorRewardedPerValidator::get() as u32, + ::MaxNominatorRewardedPerValidator::get(), false, RewardDestination::Staked, ) @@ -953,7 +1014,7 @@ mod tests { let (validator_stash, _nominators) = create_validator_with_nominators::( n, - ::MaxNominatorRewardedPerValidator::get() as u32, + ::MaxNominatorRewardedPerValidator::get(), false, RewardDestination::Staked, ) @@ -1001,10 +1062,3 @@ mod tests { }); } } - -impl_benchmark_test_suite!( - Staking, - crate::mock::ExtBuilder::default().has_stakers(true), - crate::mock::Test, - exec_name = build_and_execute -); diff --git a/frame/staking/src/inflation.rs b/frame/staking/src/inflation.rs index 8e44a8c5482e..c7519683c75d 100644 --- a/frame/staking/src/inflation.rs +++ b/frame/staking/src/inflation.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index d8e72e267ea9..872ed2e1af40 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -157,17 +157,25 @@ //! ### Example: Rewarding a validator by id. //! //! ``` -//! use frame_support::{decl_module, dispatch}; -//! use frame_system::ensure_signed; //! use pallet_staking::{self as staking}; //! -//! pub trait Config: staking::Config {} +//! #[frame_support::pallet] +//! pub mod pallet { +//! use super::*; +//! use frame_support::pallet_prelude::*; +//! use frame_system::pallet_prelude::*; //! -//! decl_module! { -//! pub struct Module for enum Call where origin: T::Origin { +//! #[pallet::pallet] +//! pub struct Pallet(_); +//! +//! #[pallet::config] +//! pub trait Config: frame_system::Config + staking::Config {} +//! +//! #[pallet::call] +//! impl Pallet { //! /// Reward a validator. -//! #[weight = 0] -//! pub fn reward_myself(origin) -> dispatch::DispatchResult { +//! #[pallet::weight(0)] +//! pub fn reward_myself(origin: OriginFor) -> DispatchResult { //! let reported = ensure_signed(origin)?; //! >::reward_by_ids(vec![(reported, 10)]); //! Ok(()) @@ -213,16 +221,16 @@ //! //! The validator and its nominator split their reward as following: //! -//! The validator can declare an amount, named -//! [`commission`](ValidatorPrefs::commission), that does not get shared -//! with the nominators at each reward payout through its -//! [`ValidatorPrefs`]. This value gets deducted from the total reward -//! that is paid to the validator and its nominators. The remaining portion is split among the -//! validator and all of the nominators that nominated the validator, proportional to the value -//! staked behind this validator (_i.e._ dividing the -//! [`own`](Exposure::own) or -//! [`others`](Exposure::others) by -//! [`total`](Exposure::total) in [`Exposure`]). +//! The validator can declare an amount, named [`commission`](ValidatorPrefs::commission), that does +//! not get shared with the nominators at each reward payout through its [`ValidatorPrefs`]. This +//! value gets deducted from the total reward that is paid to the validator and its nominators. The +//! remaining portion is split pro rata among the validator and the top +//! [`Config::MaxNominatorRewardedPerValidator`] nominators that nominated the validator, +//! proportional to the value staked behind the validator (_i.e._ dividing the +//! [`own`](Exposure::own) or [`others`](Exposure::others) by [`total`](Exposure::total) in +//! [`Exposure`]). Note that the pro rata division of rewards uses the total exposure behind the +//! validator, *not* just the exposure of the validator and the top +//! [`Config::MaxNominatorRewardedPerValidator`] nominators. //! //! All entities who receive a reward have the option to choose their reward destination through the //! [`Payee`] storage item (see @@ -272,6 +280,7 @@ //! validators is stored in the Session pallet's `Validators` at the end of each era. #![cfg_attr(not(feature = "std"), no_std)] +#![recursion_limit = "256"] #[cfg(feature = "runtime-benchmarks")] pub mod benchmarking; @@ -292,8 +301,10 @@ mod pallet; use codec::{Decode, Encode, HasCompact}; use frame_support::{ + parameter_types, traits::{Currency, Get}, weights::Weight, + BoundedVec, EqNoBound, PartialEqNoBound, RuntimeDebugNoBound, }; use scale_info::TypeInfo; use sp_runtime::{ @@ -303,9 +314,9 @@ use sp_runtime::{ }; use sp_staking::{ offence::{Offence, OffenceError, ReportOffence}, - SessionIndex, + EraIndex, SessionIndex, }; -use sp_std::{collections::btree_map::BTreeMap, convert::From, prelude::*}; +use sp_std::{collections::btree_map::BTreeMap, prelude::*}; pub use weights::WeightInfo; pub use pallet::{pallet::*, *}; @@ -323,9 +334,6 @@ macro_rules! log { }; } -/// Counter for the number of eras that have passed. -pub type EraIndex = u32; - /// Counter for the number of "reward" points earned by a given validator. pub type RewardPoint = u32; @@ -340,6 +348,10 @@ type NegativeImbalanceOf = <::Currency as Currency< ::AccountId, >>::NegativeImbalance; +parameter_types! { + pub MaxUnlockingChunks: u32 = 32; +} + /// Information regarding the active era (era in used in session). #[derive(Encode, Decode, RuntimeDebug, TypeInfo)] pub struct ActiveEraInfo { @@ -355,7 +367,7 @@ pub struct ActiveEraInfo { /// Reward points of an era. Used to split era total payout between validators. /// /// This points will be used to reward validators and their respective nominators. -#[derive(PartialEq, Encode, Decode, Default, RuntimeDebug, TypeInfo)] +#[derive(PartialEq, Encode, Decode, RuntimeDebug, TypeInfo)] pub struct EraRewardPoints { /// Total number of points. Equals the sum of reward points for each validator. total: RewardPoint, @@ -363,9 +375,15 @@ pub struct EraRewardPoints { individual: BTreeMap, } +impl Default for EraRewardPoints { + fn default() -> Self { + EraRewardPoints { total: Default::default(), individual: BTreeMap::new() } + } +} + /// Indicates the initial status of the staker. #[derive(RuntimeDebug, TypeInfo)] -#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize, Clone))] pub enum StakerStatus { /// Chilling. Idle, @@ -397,7 +415,7 @@ impl Default for RewardDestination { } /// Preference of what happens regarding validation. -#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, Default)] pub struct ValidatorPrefs { /// Reward that validator takes up-front; only the rest is split between themselves and /// nominators. @@ -409,12 +427,6 @@ pub struct ValidatorPrefs { pub blocked: bool, } -impl Default for ValidatorPrefs { - fn default() -> Self { - ValidatorPrefs { commission: Default::default(), blocked: false } - } -} - /// Just a Balance/BlockNumber tuple to encode when a chunk of funds will be unlocked. #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] pub struct UnlockChunk { @@ -427,7 +439,6 @@ pub struct UnlockChunk { } /// The ledger of a (bonded) stash. -#[cfg_attr(feature = "runtime-benchmarks", derive(Default))] #[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] pub struct StakingLedger { /// The stash account whose balance is actually locked and at stake. @@ -440,22 +451,34 @@ pub struct StakingLedger { /// rounds. #[codec(compact)] pub active: Balance, - /// Any balance that is becoming free, which may eventually be transferred out - /// of the stash (assuming it doesn't get slashed first). - pub unlocking: Vec>, + /// Any balance that is becoming free, which may eventually be transferred out of the stash + /// (assuming it doesn't get slashed first). It is assumed that this will be treated as a first + /// in, first out queue where the new (higher value) eras get pushed on the back. + pub unlocking: BoundedVec, MaxUnlockingChunks>, /// List of eras for which the stakers behind a validator have claimed rewards. Only updated /// for validators. pub claimed_rewards: Vec, } -impl +impl StakingLedger { + /// Initializes the default object using the given `validator`. + pub fn default_from(stash: AccountId) -> Self { + Self { + stash, + total: Zero::zero(), + active: Zero::zero(), + unlocking: Default::default(), + claimed_rewards: vec![], + } + } + /// Remove entries from `unlocking` that are sufficiently old and reduce the /// total by the sum of their balances. fn consolidate_unlocked(self, current_era: EraIndex) -> Self { let mut total = self.total; - let unlocking = self + let unlocking: BoundedVec<_, _> = self .unlocking .into_iter() .filter(|chunk| { @@ -466,7 +489,11 @@ impl false } }) - .collect(); + .collect::>() + .try_into() + .expect( + "filtering items from a bounded vec always leaves length less than bounds. qed", + ); Self { stash: self.stash, @@ -558,10 +585,12 @@ where } /// A record of the nominations made by a specific account. -#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] -pub struct Nominations { +#[derive(PartialEqNoBound, EqNoBound, Clone, Encode, Decode, RuntimeDebugNoBound, TypeInfo)] +#[codec(mel_bound())] +#[scale_info(skip_type_params(T))] +pub struct Nominations { /// The targets of nomination. - pub targets: Vec, + pub targets: BoundedVec, /// The era the nominations were submitted. /// /// Except for initial nominations which are considered submitted at era 0. @@ -584,9 +613,7 @@ pub struct IndividualExposure { } /// A snapshot of the stake backing a single validator in the system. -#[derive( - PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, Default, RuntimeDebug, TypeInfo, -)] +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] pub struct Exposure { /// The total balance backing this validator. #[codec(compact)] @@ -598,9 +625,15 @@ pub struct Exposure { pub others: Vec>, } +impl Default for Exposure { + fn default() -> Self { + Self { total: Default::default(), own: Default::default(), others: vec![] } + } +} + /// A pending slash record. The value of the slash has been computed but not applied yet, /// rather deferred for several eras. -#[derive(Encode, Decode, Default, RuntimeDebug, TypeInfo)] +#[derive(Encode, Decode, RuntimeDebug, TypeInfo)] pub struct UnappliedSlash { /// The stash ID of the offending validator. validator: AccountId, @@ -614,16 +647,26 @@ pub struct UnappliedSlash { payout: Balance, } +impl UnappliedSlash { + /// Initializes the default object using the given `validator`. + pub fn default_from(validator: AccountId) -> Self { + Self { + validator, + own: Zero::zero(), + others: vec![], + reporters: vec![], + payout: Zero::zero(), + } + } +} + /// Means for interacting with a specialized version of the `session` trait. /// /// This is needed because `Staking` sets the `ValidatorIdOf` of the `pallet_session::Config` -pub trait SessionInterface: frame_system::Config { - /// Disable a given validator by stash ID. - /// - /// Returns `true` if new era should be forced at the end of this session. - /// This allows preventing a situation where there is too many validators - /// disabled and block production stalls. - fn disable_validator(validator: &AccountId) -> Result; +pub trait SessionInterface { + /// Disable the validator at the given index, returns `false` if the validator was already + /// disabled or the index is out of bounds. + fn disable_validator(validator_index: u32) -> bool; /// Get the validators from session. fn validators() -> Vec; /// Prune historical session tries up to but not including the given index. @@ -644,8 +687,8 @@ where Option<::AccountId>, >, { - fn disable_validator(validator: &::AccountId) -> Result { - >::disable(validator) + fn disable_validator(validator_index: u32) -> bool { + >::disable_index(validator_index) } fn validators() -> Vec<::AccountId> { @@ -737,7 +780,8 @@ enum Releases { V5_0_0, // blockable validators. V6_0_0, // removal of all storage associated with offchain phragmen. V7_0_0, // keep track of number of nominators / validators in map - V8_0_0, // populate `SortedListProvider`. + V8_0_0, // populate `VoterList`. + V9_0_0, // inject validators into `VoterList` as well. } impl Default for Releases { @@ -801,3 +845,23 @@ where R::is_known_offence(offenders, time_slot) } } + +/// Configurations of the benchmarking of the pallet. +pub trait BenchmarkingConfig { + /// The maximum number of validators to use. + type MaxValidators: Get; + /// The maximum number of nominators to use. + type MaxNominators: Get; +} + +/// A mock benchmarking config for pallet-staking. +/// +/// Should only be used for testing. +#[cfg(feature = "std")] +pub struct TestBenchmarkingConfig; + +#[cfg(feature = "std")] +impl BenchmarkingConfig for TestBenchmarkingConfig { + type MaxValidators = frame_support::traits::ConstU32<100>; + type MaxNominators = frame_support::traits::ConstU32<100>; +} diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs index 7064f06dd12c..96c905f4e594 100644 --- a/frame/staking/src/migrations.rs +++ b/frame/staking/src/migrations.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,13 +17,83 @@ //! Storage migrations for the Staking pallet. use super::*; +use frame_election_provider_support::SortedListProvider; +use frame_support::traits::OnRuntimeUpgrade; + +pub mod v9 { + use super::*; + + /// Migration implementation that injects all validators into sorted list. + /// + /// This is only useful for chains that started their `VoterList` just based on nominators. + pub struct InjectValidatorsIntoVoterList(sp_std::marker::PhantomData); + impl OnRuntimeUpgrade for InjectValidatorsIntoVoterList { + fn on_runtime_upgrade() -> Weight { + if StorageVersion::::get() == Releases::V8_0_0 { + let prev_count = T::VoterList::count(); + let weight_of_cached = Pallet::::weight_of_fn(); + for (v, _) in Validators::::iter() { + let weight = weight_of_cached(&v); + let _ = T::VoterList::on_insert(v.clone(), weight).map_err(|err| { + log!(warn, "failed to insert {:?} into VoterList: {:?}", v, err) + }); + } + + log!( + info, + "injected a total of {} new voters, prev count: {} next count: {}, updating to version 9", + Validators::::count(), + prev_count, + T::VoterList::count(), + ); + + StorageVersion::::put(crate::Releases::V9_0_0); + T::BlockWeights::get().max_block + } else { + log!( + warn, + "InjectValidatorsIntoVoterList being executed on the wrong storage \ + version, expected Releases::V8_0_0" + ); + T::DbWeight::get().reads(1) + } + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result<(), &'static str> { + use frame_support::traits::OnRuntimeUpgradeHelpersExt; + frame_support::ensure!( + StorageVersion::::get() == crate::Releases::V8_0_0, + "must upgrade linearly" + ); + + let prev_count = T::VoterList::count(); + Self::set_temp_storage(prev_count, "prev"); + Ok(()) + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade() -> Result<(), &'static str> { + use frame_support::traits::OnRuntimeUpgradeHelpersExt; + let post_count = T::VoterList::count(); + let prev_count = Self::get_temp_storage::("prev").unwrap(); + let validators = Validators::::count(); + assert!(post_count == prev_count + validators); + + frame_support::ensure!( + StorageVersion::::get() == crate::Releases::V9_0_0, + "must upgrade " + ); + Ok(()) + } + } +} pub mod v8 { + use crate::{Config, Nominators, Pallet, StorageVersion, Weight}; use frame_election_provider_support::SortedListProvider; use frame_support::traits::Get; - use crate::{Config, Nominators, Pallet, StorageVersion, Weight}; - #[cfg(feature = "try-runtime")] pub fn pre_migrate() -> Result<(), &'static str> { frame_support::ensure!( @@ -35,16 +105,16 @@ pub mod v8 { Ok(()) } - /// Migration to sorted [`SortedListProvider`]. + /// Migration to sorted `VoterList`. pub fn migrate() -> Weight { if StorageVersion::::get() == crate::Releases::V7_0_0 { crate::log!(info, "migrating staking to Releases::V8_0_0"); - let migrated = T::SortedListProvider::regenerate( + let migrated = T::VoterList::unsafe_regenerate( Nominators::::iter().map(|(id, _)| id), Pallet::::weight_of_fn(), ); - debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); + debug_assert_eq!(T::VoterList::sanity_check(), Ok(())); StorageVersion::::put(crate::Releases::V8_0_0); crate::log!( @@ -61,8 +131,7 @@ pub mod v8 { #[cfg(feature = "try-runtime")] pub fn post_migrate() -> Result<(), &'static str> { - T::SortedListProvider::sanity_check() - .map_err(|_| "SortedListProvider is not in a sane state.")?; + T::VoterList::sanity_check().map_err(|_| "VoterList is not in a sane state.")?; crate::log!(info, "👜 staking bags-list migration passes POST migrate checks ✅",); Ok(()) } @@ -70,10 +139,22 @@ pub mod v8 { pub mod v7 { use super::*; + use frame_support::generate_storage_alias; + + generate_storage_alias!(Staking, CounterForValidators => Value); + generate_storage_alias!(Staking, CounterForNominators => Value); pub fn pre_migrate() -> Result<(), &'static str> { - assert!(CounterForValidators::::get().is_zero(), "CounterForValidators already set."); - assert!(CounterForNominators::::get().is_zero(), "CounterForNominators already set."); + assert!( + CounterForValidators::get().unwrap().is_zero(), + "CounterForValidators already set." + ); + assert!( + CounterForNominators::get().unwrap().is_zero(), + "CounterForNominators already set." + ); + assert!(Validators::::count().is_zero(), "Validators already set."); + assert!(Nominators::::count().is_zero(), "Nominators already set."); assert!(StorageVersion::::get() == Releases::V6_0_0); Ok(()) } @@ -83,8 +164,8 @@ pub mod v7 { let validator_count = Validators::::iter().count() as u32; let nominator_count = Nominators::::iter().count() as u32; - CounterForValidators::::put(validator_count); - CounterForNominators::::put(nominator_count); + CounterForValidators::put(validator_count); + CounterForNominators::put(nominator_count); StorageVersion::::put(Releases::V7_0_0); log!(info, "Completed staking migration to Releases::V7_0_0"); diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 06c9be9c01e1..bb90aded852e 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,14 @@ //! Test utilities use crate::{self as pallet_staking, *}; -use frame_election_provider_support::{onchain, SortedListProvider}; +use frame_election_provider_support::{ + onchain, SequentialPhragmen, SortedListProvider, VoteWeight, +}; use frame_support::{ assert_ok, parameter_types, traits::{ - Currency, FindAuthor, GenesisBuild, Get, Hooks, Imbalance, OnUnbalanced, OneSessionHandler, + ConstU32, ConstU64, Currency, FindAuthor, GenesisBuild, Get, Hooks, Imbalance, + OnUnbalanced, OneSessionHandler, }, weights::constants::RocksDbWeight, }; @@ -33,8 +36,8 @@ use sp_runtime::{ testing::{Header, TestXt, UintAuthorityId}, traits::{IdentityLookup, Zero}, }; -use sp_staking::offence::{OffenceDetails, OnOffenceHandler}; -use std::{cell::RefCell, collections::HashSet}; +use sp_staking::offence::{DisableStrategy, OffenceDetails, OnOffenceHandler}; +use std::cell::RefCell; pub const INIT_TIMESTAMP: u64 = 30_000; pub const BLOCK_TIME: u64 = 1000; @@ -45,10 +48,6 @@ pub(crate) type AccountIndex = u64; pub(crate) type BlockNumber = u64; pub(crate) type Balance = u128; -thread_local! { - static SESSION: RefCell<(Vec, HashSet)> = RefCell::new(Default::default()); -} - /// Another session handler struct to test on_disabled. pub struct OtherSessionHandler; impl OneSessionHandler for OtherSessionHandler { @@ -61,23 +60,14 @@ impl OneSessionHandler for OtherSessionHandler { { } - fn on_new_session<'a, I: 'a>(_: bool, validators: I, _: I) + fn on_new_session<'a, I: 'a>(_: bool, _: I, _: I) where I: Iterator, AccountId: 'a, { - SESSION.with(|x| { - *x.borrow_mut() = (validators.map(|x| x.0.clone()).collect(), HashSet::new()) - }); } - fn on_disabled(validator_index: usize) { - SESSION.with(|d| { - let mut d = d.borrow_mut(); - let value = d.0[validator_index]; - d.1.insert(value); - }) - } + fn on_disabled(_validator_index: u32) {} } impl sp_runtime::BoundToRuntimeAppPublic for OtherSessionHandler { @@ -86,7 +76,12 @@ impl sp_runtime::BoundToRuntimeAppPublic for OtherSessionHandler { pub fn is_disabled(controller: AccountId) -> bool { let stash = Staking::ledger(&controller).unwrap().stash; - SESSION.with(|d| d.borrow().1.contains(&stash)) + let validator_index = match Session::validators().iter().position(|v| *v == stash) { + Some(index) => index as u32, + None => return false, + }; + + Session::disabled_validators().contains(&validator_index) } type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; @@ -104,6 +99,7 @@ frame_support::construct_runtime!( Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, Staking: pallet_staking::{Pallet, Call, Config, Storage, Event}, Session: pallet_session::{Pallet, Call, Storage, Event, Config}, + Historical: pallet_session::historical::{Pallet, Storage}, BagsList: pallet_bags_list::{Pallet, Call, Storage, Event}, } ); @@ -120,12 +116,10 @@ impl FindAuthor for Author11 { } parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max( frame_support::weights::constants::WEIGHT_PER_SECOND * 2 ); - pub const MaxLocks: u32 = 1024; pub static SessionsPerEra: SessionIndex = 3; pub static ExistentialDeposit: Balance = 1; pub static SlashDeferDuration: EraIndex = 0; @@ -148,7 +142,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = frame_support::traits::ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -157,9 +151,10 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } impl pallet_balances::Config for Test { - type MaxLocks = MaxLocks; + type MaxLocks = frame_support::traits::ConstU32<1024>; type MaxReserves = (); type ReserveIdentifier = [u8; 8]; type Balance = Balance; @@ -169,10 +164,7 @@ impl pallet_balances::Config for Test { type AccountStore = System; type WeightInfo = (); } -parameter_types! { - pub const UncleGenerations: u64 = 0; - pub const DisabledValidatorsThreshold: Perbill = Perbill::from_percent(25); -} + sp_runtime::impl_opaque_keys! { pub struct SessionKeys { pub other: OtherSessionHandler, @@ -186,7 +178,6 @@ impl pallet_session::Config for Test { type Event = Event; type ValidatorId = AccountId; type ValidatorIdOf = crate::StashOf; - type DisabledValidatorsThreshold = DisabledValidatorsThreshold; type NextSessionRotation = pallet_session::PeriodicSessions; type WeightInfo = (); } @@ -197,19 +188,18 @@ impl pallet_session::historical::Config for Test { } impl pallet_authorship::Config for Test { type FindAuthor = Author11; - type UncleGenerations = UncleGenerations; + type UncleGenerations = ConstU64<0>; type FilterUncle = (); type EventHandler = Pallet; } -parameter_types! { - pub const MinimumPeriod: u64 = 5; -} + impl pallet_timestamp::Config for Test { type Moment = u64; type OnTimestampSet = (); - type MinimumPeriod = MinimumPeriod; + type MinimumPeriod = ConstU64<5>; type WeightInfo = (); } + pallet_staking_reward_curve::build! { const I_NPOS: PiecewiseLinear<'static> = curve!( min_inflation: 0_025_000, @@ -223,7 +213,7 @@ pallet_staking_reward_curve::build! { parameter_types! { pub const BondingDuration: EraIndex = 3; pub const RewardCurve: &'static PiecewiseLinear<'static> = &I_NPOS; - pub const MaxNominatorRewardedPerValidator: u32 = 64; + pub const OffendingValidatorsThreshold: Perbill = Perbill::from_percent(75); } thread_local! { @@ -246,22 +236,26 @@ const THRESHOLDS: [sp_npos_elections::VoteWeight; 9] = parameter_types! { pub static BagThresholds: &'static [sp_npos_elections::VoteWeight] = &THRESHOLDS; + pub static MaxNominations: u32 = 16; } impl pallet_bags_list::Config for Test { type Event = Event; type WeightInfo = (); - type VoteWeightProvider = Staking; + type ScoreProvider = Staking; type BagThresholds = BagThresholds; + type Score = VoteWeight; } -impl onchain::Config for Test { - type Accuracy = Perbill; +pub struct OnChainSeqPhragmen; +impl onchain::ExecutionConfig for OnChainSeqPhragmen { + type System = Test; + type Solver = SequentialPhragmen; type DataProvider = Staking; } impl crate::pallet::pallet::Config for Test { - const MAX_NOMINATIONS: u32 = 16; + type MaxNominations = MaxNominations; type Currency = Balances; type UnixTime = Timestamp; type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; @@ -276,12 +270,15 @@ impl crate::pallet::pallet::Config for Test { type SessionInterface = Self; type EraPayout = ConvertCurve; type NextNewSession = Session; - type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; - type ElectionProvider = onchain::OnChainSequentialPhragmen; + type MaxNominatorRewardedPerValidator = ConstU32<64>; + type OffendingValidatorsThreshold = OffendingValidatorsThreshold; + type ElectionProvider = onchain::UnboundedExecution; type GenesisElectionProvider = Self::ElectionProvider; + // NOTE: consider a macro and use `UseNominatorsAndValidatorsMap` as well. + type VoterList = BagsList; + type MaxUnlockingChunks = ConstU32<32>; + type BenchmarkingConfig = TestBenchmarkingConfig; type WeightInfo = (); - // NOTE: consider a macro and use `UseNominatorsMap` as well. - type SortedListProvider = BagsList; } impl frame_system::offchain::SendTransactionTypes for Test @@ -485,7 +482,7 @@ impl ExtBuilder { } let _ = pallet_staking::GenesisConfig:: { - stakers, + stakers: stakers.clone(), validator_count: self.validator_count, minimum_validator_count: self.minimum_validator_count, invulnerables: self.invulnerables, @@ -498,22 +495,21 @@ impl ExtBuilder { let _ = pallet_session::GenesisConfig:: { keys: if self.has_stakers { - // genesis election will overwrite this, no worries. - Default::default() + // set the keys for the first session. + stakers + .into_iter() + .map(|(id, ..)| (id, id, SessionKeys { other: id.into() })) + .collect() } else { // set some dummy validators in genesis. (0..self.validator_count as u64) - .map(|x| (x, x, SessionKeys { other: UintAuthorityId(x as u64) })) + .map(|id| (id, id, SessionKeys { other: id.into() })) .collect() }, } .assimilate_storage(&mut storage); let mut ext = sp_io::TestExternalities::from(storage); - ext.execute_with(|| { - let validators = Session::validators(); - SESSION.with(|x| *x.borrow_mut() = (validators.clone(), HashSet::new())); - }); if self.initialize_first_session { // We consider all test to start after timestamp is initialized This must be ensured by @@ -544,14 +540,14 @@ fn post_conditions() { } fn check_count() { - let nominator_count = Nominators::::iter().count() as u32; + let nominator_count = Nominators::::iter_keys().count() as u32; let validator_count = Validators::::iter().count() as u32; - assert_eq!(nominator_count, CounterForNominators::::get()); - assert_eq!(validator_count, CounterForValidators::::get()); + assert_eq!(nominator_count, Nominators::::count()); + assert_eq!(validator_count, Validators::::count()); - // the voters that the `SortedListProvider` list is storing for us. - let external_voters = ::SortedListProvider::count(); - assert_eq!(external_voters, nominator_count); + // the voters that the `VoterList` list is storing for us. + let external_voters = ::VoterList::count(); + assert_eq!(external_voters, nominator_count + validator_count); } fn check_ledgers() { @@ -653,6 +649,7 @@ pub(crate) fn bond(stash: AccountId, ctrl: AccountId, val: Balance) { pub(crate) fn bond_validator(stash: AccountId, ctrl: AccountId, val: Balance) { bond(stash, ctrl, val); assert_ok!(Staking::validate(Origin::signed(ctrl), ValidatorPrefs::default())); + assert_ok!(Session::set_keys(Origin::signed(ctrl), SessionKeys { other: ctrl.into() }, vec![])); } pub(crate) fn bond_nominator( @@ -776,11 +773,12 @@ pub(crate) fn on_offence_in_era( >], slash_fraction: &[Perbill], era: EraIndex, + disable_strategy: DisableStrategy, ) { let bonded_eras = crate::BondedEras::::get(); for &(bonded_era, start_session) in bonded_eras.iter() { if bonded_era == era { - let _ = Staking::on_offence(offenders, slash_fraction, start_session); + let _ = Staking::on_offence(offenders, slash_fraction, start_session, disable_strategy); return } else if bonded_era > era { break @@ -792,6 +790,7 @@ pub(crate) fn on_offence_in_era( offenders, slash_fraction, Staking::eras_start_session_index(era).unwrap(), + disable_strategy, ); } else { panic!("cannot slash in era {}", era); @@ -806,7 +805,7 @@ pub(crate) fn on_offence_now( slash_fraction: &[Perbill], ) { let now = Staking::active_era().unwrap().index; - on_offence_in_era(offenders, slash_fraction, now) + on_offence_in_era(offenders, slash_fraction, now, DisableStrategy::WhenSlashed) } pub(crate) fn add_slash(who: &AccountId) { diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 3ae520872f27..90f19c6badd8 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,14 +18,14 @@ //! Implementations for the Staking FRAME Pallet. use frame_election_provider_support::{ - data_provider, ElectionDataProvider, ElectionProvider, SortedListProvider, Supports, - VoteWeight, VoteWeightProvider, + data_provider, ElectionDataProvider, ElectionProvider, ScoreProvider, SortedListProvider, + Supports, VoteWeight, VoterOf, }; use frame_support::{ pallet_prelude::*, traits::{ - Currency, CurrencyToVote, EstimateNextNewSession, Get, Imbalance, LockableCurrency, - OnUnbalanced, UnixTime, WithdrawReasons, + Currency, CurrencyToVote, Defensive, EstimateNextNewSession, Get, Imbalance, + LockableCurrency, OnUnbalanced, UnixTime, WithdrawReasons, }, weights::{Weight, WithPostDispatchInfo}, }; @@ -36,19 +36,27 @@ use sp_runtime::{ Perbill, }; use sp_staking::{ - offence::{OffenceDetails, OnOffenceHandler}, - SessionIndex, + offence::{DisableStrategy, OffenceDetails, OnOffenceHandler}, + EraIndex, SessionIndex, }; use sp_std::{collections::btree_map::BTreeMap, prelude::*}; use crate::{ - log, slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraIndex, EraPayout, Exposure, - ExposureOf, Forcing, IndividualExposure, Nominations, PositiveImbalanceOf, RewardDestination, + log, slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraPayout, Exposure, ExposureOf, + Forcing, IndividualExposure, Nominations, PositiveImbalanceOf, RewardDestination, SessionInterface, StakingLedger, ValidatorPrefs, }; use super::{pallet::*, STAKING_ID}; +/// The maximum number of iterations that we do whilst iterating over `T::VoterList` in +/// `get_npos_voters`. +/// +/// In most cases, if we want n items, we iterate exactly n times. In rare cases, if a voter is +/// invalid (for any reason) the iteration continues. With this constant, we iterate at most 2 * n +/// times and then give up. +const NPOS_MAX_ITERATIONS_COEFFICIENT: u32 = 2; + impl Pallet { /// The total balance that can be slashed from a stash account as of right now. pub fn slashable_balance_of(stash: &T::AccountId) -> BalanceOf { @@ -224,7 +232,7 @@ impl Pallet { let dest = Self::payee(stash); match dest { RewardDestination::Controller => Self::bonded(stash) - .and_then(|controller| Some(T::Currency::deposit_creating(&controller, amount))), + .map(|controller| T::Currency::deposit_creating(&controller, amount)), RewardDestination::Stash => T::Currency::deposit_into_existing(stash, amount).ok(), RewardDestination::Staked => Self::bonded(stash) .and_then(|c| Self::ledger(&c).map(|l| (c, l))) @@ -302,6 +310,13 @@ impl Pallet { Self::start_era(start_session); } } + + // disable all offending validators that have been disabled for the whole era + for (index, disabled) in >::get() { + if disabled { + T::SessionInterface::disable_validator(index); + } + } } /// End a session potentially ending an era. @@ -374,6 +389,9 @@ impl Pallet { // Set ending era reward. >::insert(&active_era.index, validator_payout); T::RewardRemainder::on_unbalanced(T::Currency::issue(rest)); + + // Clear offending validators. + >::kill(); } } @@ -639,73 +657,77 @@ impl Pallet { /// Get all of the voters that are eligible for the npos election. /// - /// `maybe_max_len` can imposes a cap on the number of voters returned; First all the validator - /// are included in no particular order, then remainder is taken from the nominators, as - /// returned by [`Config::SortedListProvider`]. - /// - /// This will use nominators, and all the validators will inject a self vote. + /// `maybe_max_len` can imposes a cap on the number of voters returned; /// /// This function is self-weighing as [`DispatchClass::Mandatory`]. /// /// ### Slashing /// - /// All nominations that have been submitted before the last non-zero slash of the validator are - /// auto-chilled, but still count towards the limit imposed by `maybe_max_len`. - pub fn get_npos_voters( - maybe_max_len: Option, - ) -> Vec<(T::AccountId, VoteWeight, Vec)> { + /// All votes that have been submitted before the last non-zero slash of the corresponding + /// target are *auto-chilled*, but still count towards the limit imposed by `maybe_max_len`. + pub fn get_npos_voters(maybe_max_len: Option) -> Vec> { let max_allowed_len = { - let nominator_count = CounterForNominators::::get() as usize; - let validator_count = CounterForValidators::::get() as usize; - let all_voter_count = validator_count.saturating_add(nominator_count); + let all_voter_count = T::VoterList::count() as usize; maybe_max_len.unwrap_or(all_voter_count).min(all_voter_count) }; let mut all_voters = Vec::<_>::with_capacity(max_allowed_len); - // first, grab all validators in no particular order, capped by the maximum allowed length. - let mut validators_taken = 0u32; - for (validator, _) in >::iter().take(max_allowed_len) { - // Append self vote. - let self_vote = - (validator.clone(), Self::weight_of(&validator), vec![validator.clone()]); - all_voters.push(self_vote); - validators_taken.saturating_inc(); - } - - // .. and grab whatever we have left from nominators. - let nominators_quota = (max_allowed_len as u32).saturating_sub(validators_taken); + // cache a few things. + let weight_of = Self::weight_of_fn(); let slashing_spans = >::iter().collect::>(); - // track the count of nominators added to `all_voters + let mut voters_seen = 0u32; + let mut validators_taken = 0u32; let mut nominators_taken = 0u32; - // track every nominator iterated over, but not necessarily added to `all_voters` - let mut nominators_seen = 0u32; - - let mut nominators_iter = T::SortedListProvider::iter(); - while nominators_taken < nominators_quota && nominators_seen < nominators_quota * 2 { - let nominator = match nominators_iter.next() { - Some(nominator) => { - nominators_seen.saturating_inc(); - nominator + + let mut sorted_voters = T::VoterList::iter(); + while all_voters.len() < max_allowed_len && + voters_seen < (NPOS_MAX_ITERATIONS_COEFFICIENT * max_allowed_len as u32) + { + let voter = match sorted_voters.next() { + Some(voter) => { + voters_seen.saturating_inc(); + voter }, None => break, }; if let Some(Nominations { submitted_in, mut targets, suppressed: _ }) = - >::get(&nominator) + >::get(&voter) { + // if this voter is a nominator: targets.retain(|stash| { slashing_spans .get(stash) .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) }); if !targets.len().is_zero() { - all_voters.push((nominator.clone(), Self::weight_of(&nominator), targets)); + all_voters.push((voter.clone(), weight_of(&voter), targets)); nominators_taken.saturating_inc(); } + } else if Validators::::contains_key(&voter) { + // if this voter is a validator: + let self_vote = ( + voter.clone(), + weight_of(&voter), + vec![voter.clone()] + .try_into() + .expect("`MaxVotesPerVoter` must be greater than or equal to 1"), + ); + all_voters.push(self_vote); + validators_taken.saturating_inc(); } else { - log!(error, "invalid item in `SortedListProvider`: {:?}", nominator) + // this can only happen if: 1. there a bug in the bags-list (or whatever is the + // sorted list) logic and the state of the two pallets is no longer compatible, or + // because the nominators is not decodable since they have more nomination than + // `T::MaxNominations`. The latter can rarely happen, and is not really an emergency + // or bug if it does. + log!( + warn, + "DEFENSIVE: invalid item in `VoterList`: {:?}, this nominator probably has too many nominations now", + voter + ) } } @@ -725,6 +747,7 @@ impl Pallet { validators_taken, nominators_taken ); + all_voters } @@ -746,82 +769,99 @@ impl Pallet { } /// This function will add a nominator to the `Nominators` storage map, - /// [`SortedListProvider`] and keep track of the `CounterForNominators`. + /// and `VoterList`. /// /// If the nominator already exists, their nominations will be updated. /// /// NOTE: you must ALWAYS use this function to add nominator or update their targets. Any access - /// to `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// to `Nominators` or `VoterList` outside of this function is almost certainly /// wrong. - pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { + pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { if !Nominators::::contains_key(who) { - // maybe update the counter. - CounterForNominators::::mutate(|x| x.saturating_inc()); - - // maybe update sorted list. Error checking is defensive-only - this should never fail. - if T::SortedListProvider::on_insert(who.clone(), Self::weight_of(who)).is_err() { - log!(warn, "attempt to insert duplicate nominator ({:#?})", who); - debug_assert!(false, "attempt to insert duplicate nominator"); - }; - - debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); + // maybe update sorted list. + let _ = T::VoterList::on_insert(who.clone(), Self::weight_of(who)) + .defensive_unwrap_or_default(); } - Nominators::::insert(who, nominations); + + debug_assert_eq!( + Nominators::::count() + Validators::::count(), + T::VoterList::count() + ); + debug_assert_eq!(T::VoterList::sanity_check(), Ok(())); } /// This function will remove a nominator from the `Nominators` storage map, - /// [`SortedListProvider`] and keep track of the `CounterForNominators`. + /// and `VoterList`. /// /// Returns true if `who` was removed from `Nominators`, otherwise false. /// /// NOTE: you must ALWAYS use this function to remove a nominator from the system. Any access to - /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// `Nominators` or `VoterList` outside of this function is almost certainly /// wrong. pub fn do_remove_nominator(who: &T::AccountId) -> bool { - if Nominators::::contains_key(who) { + let outcome = if Nominators::::contains_key(who) { Nominators::::remove(who); - CounterForNominators::::mutate(|x| x.saturating_dec()); - T::SortedListProvider::on_remove(who); - debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); - debug_assert_eq!(CounterForNominators::::get(), T::SortedListProvider::count()); + T::VoterList::on_remove(who); true } else { false - } + }; + + debug_assert_eq!(T::VoterList::sanity_check(), Ok(())); + debug_assert_eq!( + Nominators::::count() + Validators::::count(), + T::VoterList::count() + ); + + outcome } - /// This function will add a validator to the `Validators` storage map, and keep track of the - /// `CounterForValidators`. + /// This function will add a validator to the `Validators` storage map. /// /// If the validator already exists, their preferences will be updated. /// /// NOTE: you must ALWAYS use this function to add a validator to the system. Any access to - /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly + /// `Validators` or `VoterList` outside of this function is almost certainly /// wrong. pub fn do_add_validator(who: &T::AccountId, prefs: ValidatorPrefs) { if !Validators::::contains_key(who) { - CounterForValidators::::mutate(|x| x.saturating_inc()) + // maybe update sorted list. + let _ = T::VoterList::on_insert(who.clone(), Self::weight_of(who)) + .defensive_unwrap_or_default(); } Validators::::insert(who, prefs); + + debug_assert_eq!( + Nominators::::count() + Validators::::count(), + T::VoterList::count() + ); + debug_assert_eq!(T::VoterList::sanity_check(), Ok(())); } - /// This function will remove a validator from the `Validators` storage map, - /// and keep track of the `CounterForValidators`. + /// This function will remove a validator from the `Validators` storage map. /// /// Returns true if `who` was removed from `Validators`, otherwise false. /// /// NOTE: you must ALWAYS use this function to remove a validator from the system. Any access to - /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly + /// `Validators` or `VoterList` outside of this function is almost certainly /// wrong. pub fn do_remove_validator(who: &T::AccountId) -> bool { - if Validators::::contains_key(who) { + let outcome = if Validators::::contains_key(who) { Validators::::remove(who); - CounterForValidators::::mutate(|x| x.saturating_dec()); + T::VoterList::on_remove(who); true } else { false - } + }; + + debug_assert_eq!(T::VoterList::sanity_check(), Ok(())); + debug_assert_eq!( + Nominators::::count() + Validators::::count(), + T::VoterList::count() + ); + + outcome } /// Register some amount of weight directly with the system pallet. @@ -835,25 +875,17 @@ impl Pallet { } } -impl ElectionDataProvider> for Pallet { - const MAXIMUM_VOTES_PER_VOTER: u32 = T::MAX_NOMINATIONS; +impl ElectionDataProvider for Pallet { + type AccountId = T::AccountId; + type BlockNumber = BlockNumberFor; + type MaxVotesPerVoter = T::MaxNominations; fn desired_targets() -> data_provider::Result { Self::register_weight(T::DbWeight::get().reads(1)); Ok(Self::validator_count()) } - fn voters( - maybe_max_len: Option, - ) -> data_provider::Result)>> { - debug_assert!(>::iter().count() as u32 == CounterForNominators::::get()); - debug_assert!(>::iter().count() as u32 == CounterForValidators::::get()); - debug_assert_eq!( - CounterForNominators::::get(), - T::SortedListProvider::count(), - "voter_count must be accurate", - ); - + fn electing_voters(maybe_max_len: Option) -> data_provider::Result>> { // This can never fail -- if `maybe_max_len` is `Some(_)` we handle it. let voters = Self::get_npos_voters(maybe_max_len); debug_assert!(maybe_max_len.map_or(true, |max| voters.len() <= max)); @@ -861,8 +893,8 @@ impl ElectionDataProvider> for Pallet Ok(voters) } - fn targets(maybe_max_len: Option) -> data_provider::Result> { - let target_count = CounterForValidators::::get(); + fn electable_targets(maybe_max_len: Option) -> data_provider::Result> { + let target_count = Validators::::count(); // We can't handle this case yet -- return an error. if maybe_max_len.map_or(false, |max_len| target_count > max_len as u32) { @@ -906,8 +938,11 @@ impl ElectionDataProvider> for Pallet } #[cfg(feature = "runtime-benchmarks")] - fn add_voter(voter: T::AccountId, weight: VoteWeight, targets: Vec) { - use sp_std::convert::TryFrom; + fn add_voter( + voter: T::AccountId, + weight: VoteWeight, + targets: BoundedVec, + ) { let stake = >::try_from(weight).unwrap_or_else(|_| { panic!("cannot convert a VoteWeight into BalanceOf, benchmark needs reconfiguring.") }); @@ -918,10 +953,11 @@ impl ElectionDataProvider> for Pallet stash: voter.clone(), active: stake, total: stake, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }, ); + Self::do_add_nominator(&voter, Nominations { targets, submitted_in: 0, suppressed: false }); } @@ -935,7 +971,7 @@ impl ElectionDataProvider> for Pallet stash: target.clone(), active: stake, total: stake, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }, ); @@ -949,20 +985,18 @@ impl ElectionDataProvider> for Pallet fn clear() { >::remove_all(None); >::remove_all(None); - >::remove_all(None); - >::remove_all(None); - >::kill(); - >::kill(); - let _ = T::SortedListProvider::clear(None); + >::remove_all(); + >::remove_all(); + + T::VoterList::unsafe_clear(); } #[cfg(feature = "runtime-benchmarks")] fn put_snapshot( - voters: Vec<(T::AccountId, VoteWeight, Vec)>, + voters: Vec>, targets: Vec, target_stake: Option, ) { - use sp_std::convert::TryFrom; targets.into_iter().for_each(|v| { let stake: BalanceOf = target_stake .and_then(|w| >::try_from(w).ok()) @@ -974,7 +1008,7 @@ impl ElectionDataProvider> for Pallet stash: v.clone(), active: stake, total: stake, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }, ); @@ -995,13 +1029,13 @@ impl ElectionDataProvider> for Pallet stash: v.clone(), active: stake, total: stake, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }, ); Self::do_add_nominator( &v, - Nominations { targets: t, submitted_in: 0, suppressed: false }, + Nominations { targets: t.try_into().unwrap(), submitted_in: 0, suppressed: false }, ); }); } @@ -1091,8 +1125,13 @@ where fn note_author(author: T::AccountId) { Self::reward_by_ids(vec![(author, 20)]) } - fn note_uncle(author: T::AccountId, _age: T::BlockNumber) { - Self::reward_by_ids(vec![(>::author(), 2), (author, 1)]) + fn note_uncle(uncle_author: T::AccountId, _age: T::BlockNumber) { + // defensive-only: block author must exist. + if let Some(block_author) = >::author() { + Self::reward_by_ids(vec![(block_author, 2), (uncle_author, 1)]) + } else { + crate::log!(warn, "block author not set, this should never happen"); + } } } @@ -1120,6 +1159,7 @@ where >], slash_fraction: &[Perbill], slash_session: SessionIndex, + disable_strategy: DisableStrategy, ) -> Weight { let reward_proportion = SlashRewardFraction::::get(); let mut consumed_weight: Weight = 0; @@ -1154,7 +1194,7 @@ where add_db_reads_writes(1, 0); // Reverse because it's more likely to find reports from recent eras. - match eras.iter().rev().filter(|&&(_, ref sesh)| sesh <= &slash_session).next() { + match eras.iter().rev().find(|&&(_, ref sesh)| sesh <= &slash_session) { Some(&(ref slash_era, _)) => *slash_era, // Before bonding period. defensive - should be filtered out. None => return consumed_weight, @@ -1189,6 +1229,7 @@ where window_start, now: active_era, reward_proportion, + disable_strategy, }); if let Some(mut unapplied) = unapplied { @@ -1228,19 +1269,24 @@ where } } -impl VoteWeightProvider for Pallet { - fn vote_weight(who: &T::AccountId) -> VoteWeight { +impl ScoreProvider for Pallet { + type Score = VoteWeight; + + fn score(who: &T::AccountId) -> Self::Score { Self::weight_of(who) } #[cfg(feature = "runtime-benchmarks")] - fn set_vote_weight_of(who: &T::AccountId, weight: VoteWeight) { + fn set_score_of(who: &T::AccountId, weight: Self::Score) { // this will clearly results in an inconsistent state, but it should not matter for a // benchmark. - use sp_std::convert::TryInto; let active: BalanceOf = weight.try_into().map_err(|_| ()).unwrap(); - let mut ledger = Self::ledger(who).unwrap_or_default(); + let mut ledger = match Self::ledger(who) { + None => StakingLedger::default_from(who.clone()), + Some(l) => l, + }; ledger.active = active; + >::insert(who, ledger); >::insert(who, who); @@ -1257,33 +1303,54 @@ impl VoteWeightProvider for Pallet { /// A simple voter list implementation that does not require any additional pallets. Note, this /// does not provided nominators in sorted ordered. If you desire nominators in a sorted order take /// a look at [`pallet-bags-list]. -pub struct UseNominatorsMap(sp_std::marker::PhantomData); -impl SortedListProvider for UseNominatorsMap { +pub struct UseNominatorsAndValidatorsMap(sp_std::marker::PhantomData); +impl SortedListProvider for UseNominatorsAndValidatorsMap { type Error = (); + type Score = VoteWeight; - /// Returns iterator over voter list, which can have `take` called on it. fn iter() -> Box> { - Box::new(Nominators::::iter().map(|(n, _)| n)) + Box::new( + Validators::::iter() + .map(|(v, _)| v) + .chain(Nominators::::iter().map(|(n, _)| n)), + ) + } + fn iter_from( + start: &T::AccountId, + ) -> Result>, Self::Error> { + if Validators::::contains_key(start) { + let start_key = Validators::::hashed_key_for(start); + Ok(Box::new( + Validators::::iter_from(start_key) + .map(|(n, _)| n) + .chain(Nominators::::iter().map(|(x, _)| x)), + )) + } else if Nominators::::contains_key(start) { + let start_key = Nominators::::hashed_key_for(start); + Ok(Box::new(Nominators::::iter_from(start_key).map(|(n, _)| n))) + } else { + Err(()) + } } fn count() -> u32 { - CounterForNominators::::get() + Nominators::::count().saturating_add(Validators::::count()) } fn contains(id: &T::AccountId) -> bool { - Nominators::::contains_key(id) + Nominators::::contains_key(id) || Validators::::contains_key(id) } - fn on_insert(_: T::AccountId, _weight: VoteWeight) -> Result<(), Self::Error> { + fn on_insert(_: T::AccountId, _weight: Self::Score) -> Result<(), Self::Error> { // nothing to do on insert. Ok(()) } - fn on_update(_: &T::AccountId, _weight: VoteWeight) { + fn on_update(_: &T::AccountId, _weight: Self::Score) { // nothing to do on update. } fn on_remove(_: &T::AccountId) { // nothing to do on remove. } - fn regenerate( + fn unsafe_regenerate( _: impl IntoIterator, - _: Box VoteWeight>, + _: Box Self::Score>, ) -> u32 { // nothing to do upon regenerate. 0 @@ -1291,13 +1358,11 @@ impl SortedListProvider for UseNominatorsMap { fn sanity_check() -> Result<(), &'static str> { Ok(()) } - fn clear(maybe_count: Option) -> u32 { - Nominators::::remove_all(maybe_count); - if let Some(count) = maybe_count { - CounterForNominators::::mutate(|noms| *noms - count); - count - } else { - CounterForNominators::::take() - } + + fn unsafe_clear() { + // NOTE: Caller must ensure this doesn't lead to too many storage accesses. This is a + // condition of SortedListProvider::unsafe_clear. + Nominators::::remove_all(); + Validators::::remove_all(); } } diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index dad958ccaea2..fa8c453c2b0f 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,12 +17,13 @@ //! Staking FRAME Pallet. -use frame_election_provider_support::SortedListProvider; +use frame_election_provider_support::{SortedListProvider, VoteWeight}; use frame_support::{ + dispatch::Codec, pallet_prelude::*, traits::{ - Currency, CurrencyToVote, EnsureOrigin, EstimateNextNewSession, Get, LockIdentifier, - LockableCurrency, OnUnbalanced, UnixTime, + Currency, CurrencyToVote, DefensiveSaturating, EnsureOrigin, EstimateNextNewSession, Get, + LockIdentifier, LockableCurrency, OnUnbalanced, UnixTime, }, weights::Weight, }; @@ -31,31 +32,46 @@ use sp_runtime::{ traits::{CheckedSub, SaturatedConversion, StaticLookup, Zero}, DispatchError, Perbill, Percent, }; -use sp_staking::SessionIndex; -use sp_std::{convert::From, prelude::*, result}; +use sp_staking::{EraIndex, SessionIndex}; +use sp_std::{cmp::max, prelude::*}; mod impls; pub use impls::*; use crate::{ - log, migrations, slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraIndex, EraPayout, - EraRewardPoints, Exposure, Forcing, NegativeImbalanceOf, Nominations, PositiveImbalanceOf, - Releases, RewardDestination, SessionInterface, StakingLedger, UnappliedSlash, UnlockChunk, + slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraPayout, EraRewardPoints, Exposure, + Forcing, MaxUnlockingChunks, NegativeImbalanceOf, Nominations, PositiveImbalanceOf, Releases, + RewardDestination, SessionInterface, StakingLedger, UnappliedSlash, UnlockChunk, ValidatorPrefs, }; -pub const MAX_UNLOCKING_CHUNKS: usize = 32; const STAKING_ID: LockIdentifier = *b"staking "; #[frame_support::pallet] pub mod pallet { + use frame_election_provider_support::ElectionDataProvider; + + use crate::BenchmarkingConfig; + use super::*; #[pallet::pallet] #[pallet::generate_store(pub(crate) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); + /// Possible operations on the configuration values of this pallet. + #[derive(TypeInfo, Debug, Clone, Encode, Decode, PartialEq)] + pub enum ConfigOp { + /// Don't change. + Noop, + /// Set the given value. + Set(T), + /// Remove from storage. + Remove, + } + #[pallet::config] pub trait Config: frame_system::Config + SendTransactionTypes> { /// The staking balance. @@ -69,28 +85,30 @@ pub mod pallet { /// Convert a balance into a number used for election calculation. This must fit into a /// `u64` but is allowed to be sensibly lossy. The `u64` is used to communicate with the - /// [`sp_npos_elections`] crate which accepts u64 numbers and does operations in 128. + /// [`frame_election_provider_support`] crate which accepts u64 numbers and does operations + /// in 128. /// Consequently, the backward convert is used convert the u128s from sp-elections back to a /// [`BalanceOf`]. type CurrencyToVote: CurrencyToVote>; /// Something that provides the election functionality. type ElectionProvider: frame_election_provider_support::ElectionProvider< - Self::AccountId, - Self::BlockNumber, + AccountId = Self::AccountId, + BlockNumber = Self::BlockNumber, // we only accept an election provider that has staking as data provider. DataProvider = Pallet, >; /// Something that provides the election functionality at genesis. type GenesisElectionProvider: frame_election_provider_support::ElectionProvider< - Self::AccountId, - Self::BlockNumber, + AccountId = Self::AccountId, + BlockNumber = Self::BlockNumber, DataProvider = Pallet, >; /// Maximum number of nominations per nominator. - const MAX_NOMINATIONS: u32; + #[pallet::constant] + type MaxNominations: Get; /// Tokens have been minted and are unused for validator-reward. /// See [Era payout](./index.html#era-payout). @@ -141,24 +159,29 @@ pub mod pallet { #[pallet::constant] type MaxNominatorRewardedPerValidator: Get; - /// Something that can provide a sorted list of voters in a somewhat sorted way. The - /// original use case for this was designed with [`pallet_bags_list::Pallet`] in mind. If - /// the bags-list is not desired, [`impls::UseNominatorsMap`] is likely the desired option. - type SortedListProvider: SortedListProvider; + /// The fraction of the validator set that is safe to be offending. + /// After the threshold is reached a new era will be forced. + type OffendingValidatorsThreshold: Get; + + /// Something that provides a best-effort sorted list of voters aka electing nominators, + /// used for NPoS election. + /// + /// The changes to nominators are reported to this. Moreover, each validator's self-vote is + /// also reported as one independent vote. + type VoterList: SortedListProvider; + + /// The maximum number of `unlocking` chunks a [`StakingLedger`] can have. Effectively + /// determines how many unique eras a staker may be unbonding in. + #[pallet::constant] + type MaxUnlockingChunks: Get; + + /// Some parameters of the benchmarking. + type BenchmarkingConfig: BenchmarkingConfig; /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; } - #[pallet::extra_constants] - impl Pallet { - // TODO: rename to snake case after https://github.com/paritytech/substrate/issues/8826 fixed. - #[allow(non_snake_case)] - fn MaxNominations() -> u32 { - T::MAX_NOMINATIONS - } - } - #[pallet::type_value] pub(crate) fn HistoryDepthOnEmpty() -> u32 { 84u32 @@ -205,6 +228,12 @@ pub mod pallet { #[pallet::storage] pub type MinValidatorBond = StorageValue<_, BalanceOf, ValueQuery>; + /// The minimum amount of commission that validators can set. + /// + /// If set to `0`, no limit exists. + #[pallet::storage] + pub type MinCommission = StorageValue<_, Perbill, ValueQuery>; + /// Map from all (unlocked) "controller" accounts to the info regarding the staking. #[pallet::storage] #[pallet::getter(fn ledger)] @@ -218,16 +247,10 @@ pub mod pallet { StorageMap<_, Twox64Concat, T::AccountId, RewardDestination, ValueQuery>; /// The map from (wannabe) validator stash key to the preferences of that validator. - /// - /// When updating this storage item, you must also update the `CounterForValidators`. #[pallet::storage] #[pallet::getter(fn validators)] pub type Validators = - StorageMap<_, Twox64Concat, T::AccountId, ValidatorPrefs, ValueQuery>; - - /// A tracker to keep count of the number of items in the `Validators` map. - #[pallet::storage] - pub type CounterForValidators = StorageValue<_, u32, ValueQuery>; + CountedStorageMap<_, Twox64Concat, T::AccountId, ValidatorPrefs, ValueQuery>; /// The maximum validator count before we stop allowing new validators to join. /// @@ -235,17 +258,26 @@ pub mod pallet { #[pallet::storage] pub type MaxValidatorsCount = StorageValue<_, u32, OptionQuery>; - /// The map from nominator stash key to the set of stash keys of all validators to nominate. + /// The map from nominator stash key to their nomination preferences, namely the validators that + /// they wish to support. + /// + /// Note that the keys of this storage map might become non-decodable in case the + /// [`Config::MaxNominations`] configuration is decreased. In this rare case, these nominators + /// are still existent in storage, their key is correct and retrievable (i.e. `contains_key` + /// indicates that they exist), but their value cannot be decoded. Therefore, the non-decodable + /// nominators will effectively not-exist, until they re-submit their preferences such that it + /// is within the bounds of the newly set `Config::MaxNominations`. /// - /// When updating this storage item, you must also update the `CounterForNominators`. + /// This implies that `::iter_keys().count()` and `::iter().count()` might return different + /// values for this map. Moreover, the main `::count()` is aligned with the former, namely the + /// number of keys that exist. + /// + /// Lastly, if any of the nominators become non-decodable, they can be chilled immediately via + /// [`Call::chill_other`] dispatchable by anyone. #[pallet::storage] #[pallet::getter(fn nominators)] pub type Nominators = - StorageMap<_, Twox64Concat, T::AccountId, Nominations>; - - /// A tracker to keep count of the number of items in the `Nominators` map. - #[pallet::storage] - pub type CounterForNominators = StorageValue<_, u32, ValueQuery>; + CountedStorageMap<_, Twox64Concat, T::AccountId, Nominations>; /// The maximum nominator count before we stop allowing new validators to join. /// @@ -437,6 +469,19 @@ pub mod pallet { #[pallet::getter(fn current_planned_session)] pub type CurrentPlannedSession = StorageValue<_, SessionIndex, ValueQuery>; + /// Indices of validators that have offended in the active era and whether they are currently + /// disabled. + /// + /// This value should be a superset of disabled validators since not all offences lead to the + /// validator being disabled (if there was no slash). This is needed to track the percentage of + /// validators that have offended in the current era, ensuring a new era is forced if + /// `OffendingValidatorsThreshold` is reached. The vec is always kept sorted so that we can find + /// whether a given validator has previously offended using binary search. It gets cleared when + /// the era ends. + #[pallet::storage] + #[pallet::getter(fn offending_validators)] + pub type OffendingValidators = StorageValue<_, Vec<(u32, bool)>, ValueQuery>; + /// True if network has been upgraded to this version. /// Storage version of the pallet. /// @@ -463,6 +508,8 @@ pub mod pallet { Vec<(T::AccountId, T::AccountId, BalanceOf, crate::StakerStatus)>, pub min_nominator_bond: BalanceOf, pub min_validator_bond: BalanceOf, + pub max_validator_count: Option, + pub max_nominator_count: Option, } #[cfg(feature = "std")] @@ -479,6 +526,8 @@ pub mod pallet { stakers: Default::default(), min_nominator_bond: Default::default(), min_validator_bond: Default::default(), + max_validator_count: None, + max_nominator_count: None, } } } @@ -496,9 +545,15 @@ pub mod pallet { StorageVersion::::put(Releases::V7_0_0); MinNominatorBond::::put(self.min_nominator_bond); MinValidatorBond::::put(self.min_validator_bond); + if let Some(x) = self.max_validator_count { + MaxValidatorsCount::::put(x); + } + if let Some(x) = self.max_nominator_count { + MaxNominatorsCount::::put(x); + } for &(ref stash, ref controller, balance, ref status) in &self.stakers { - log!( + crate::log!( trace, "inserting genesis staker: {:?} => {:?} => {:?}", stash, @@ -528,10 +583,10 @@ pub mod pallet { }); } - // all voters are reported to the `SortedListProvider`. + // all voters are reported to the `VoterList`. assert_eq!( - T::SortedListProvider::count(), - CounterForNominators::::get(), + T::VoterList::count(), + Nominators::::count() + Validators::::count(), "not all genesis stakers were inserted into sorted list provider, something is wrong." ); } @@ -591,7 +646,9 @@ pub mod pallet { DuplicateIndex, /// Slash record index out of bounds. InvalidSlashIndex, - /// Can not bond with value less than minimum required. + /// Cannot have a validator or nominator role, with value less than the minimum defined by + /// governance (see `MinValidatorBond` and `MinNominatorBond`). If unbonding is the + /// intention, `chill` first to remove one's role as validator/nominator. InsufficientBond, /// Can not schedule more unlock chunks. NoMoreChunks, @@ -625,27 +682,12 @@ pub mod pallet { /// There are too many validators in the system. Governance needs to adjust the staking /// settings to keep things safe for the runtime. TooManyValidators, + /// Commission is too low. Must be at least `MinCommission`. + CommissionTooLow, } #[pallet::hooks] impl Hooks> for Pallet { - fn on_runtime_upgrade() -> Weight { - if StorageVersion::::get() == Releases::V6_0_0 { - migrations::v7::migrate::() - } else { - T::DbWeight::get().reads(1) - } - } - - #[cfg(feature = "try-runtime")] - fn pre_upgrade() -> Result<(), &'static str> { - if StorageVersion::::get() == Releases::V6_0_0 { - migrations::v7::pre_migrate::() - } else { - Ok(()) - } - } - fn on_initialize(_now: BlockNumberFor) -> Weight { // just return the weight of the on_finalize. T::DbWeight::get().reads(1) @@ -666,6 +708,14 @@ pub mod pallet { } fn integrity_test() { + // ensure that we funnel the correct value to the `DataProvider::MaxVotesPerVoter`; + assert_eq!( + T::MaxNominations::get(), + ::MaxVotesPerVoter::get() + ); + // and that MaxNominations is always greater than 1, since we count on this. + assert!(!T::MaxNominations::get().is_zero()); + sp_std::if_std! { sp_io::TestExternalities::new_empty().execute_with(|| assert!( @@ -740,7 +790,7 @@ pub mod pallet { stash, total: value, active: value, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: (last_reward_era..current_era).collect(), }; Self::update_ledger(&controller, &item); @@ -786,9 +836,9 @@ pub mod pallet { // NOTE: ledger must be updated prior to calling `Self::weight_of`. Self::update_ledger(&controller, &ledger); // update this staker in the sorted list, if they exist in it. - if T::SortedListProvider::contains(&stash) { - T::SortedListProvider::on_update(&stash, Self::weight_of(&ledger.stash)); - debug_assert_eq!(T::SortedListProvider::sanity_check(), Ok(())); + if T::VoterList::contains(&stash) { + T::VoterList::on_update(&stash, Self::weight_of(&ledger.stash)); + debug_assert_eq!(T::VoterList::sanity_check(), Ok(())); } Self::deposit_event(Event::::Bonded(stash.clone(), extra)); @@ -805,7 +855,7 @@ pub mod pallet { /// Once the unlock period is done, you can call `withdraw_unbonded` to actually move /// the funds out of management ready for transfer. /// - /// No more than a limited number of unlocking chunks (see `MAX_UNLOCKING_CHUNKS`) + /// No more than a limited number of unlocking chunks (see `MaxUnlockingChunks`) /// can co-exists at the same time. In that case, [`Call::withdraw_unbonded`] need /// to be called first to remove some of the chunks (if possible). /// @@ -822,7 +872,10 @@ pub mod pallet { ) -> DispatchResult { let controller = ensure_signed(origin)?; let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - ensure!(ledger.unlocking.len() < MAX_UNLOCKING_CHUNKS, Error::::NoMoreChunks,); + ensure!( + ledger.unlocking.len() < MaxUnlockingChunks::get() as usize, + Error::::NoMoreChunks, + ); let mut value = value.min(ledger.active); @@ -849,13 +902,25 @@ pub mod pallet { // Note: in case there is no current era it is fine to bond one era more. let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); - ledger.unlocking.push(UnlockChunk { value, era }); + if let Some(mut chunk) = + ledger.unlocking.last_mut().filter(|chunk| chunk.era == era) + { + // To keep the chunk count down, we only keep one chunk per era. Since + // `unlocking` is a FiFo queue, if a chunk exists for `era` we know that it will + // be the last one. + chunk.value = chunk.value.defensive_saturating_add(value) + } else { + ledger + .unlocking + .try_push(UnlockChunk { value, era }) + .map_err(|_| Error::::NoMoreChunks)?; + }; // NOTE: ledger must be updated prior to calling `Self::weight_of`. Self::update_ledger(&controller, &ledger); // update this staker in the sorted list, if they exist in it. - if T::SortedListProvider::contains(&ledger.stash) { - T::SortedListProvider::on_update(&ledger.stash, Self::weight_of(&ledger.stash)); + if T::VoterList::contains(&ledger.stash) { + T::VoterList::on_update(&ledger.stash, Self::weight_of(&ledger.stash)); } Self::deposit_event(Event::::Unbonded(ledger.stash, value)); @@ -930,9 +995,13 @@ pub mod pallet { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + ensure!(ledger.active >= MinValidatorBond::::get(), Error::::InsufficientBond); let stash = &ledger.stash; + // ensure their commission is correct. + ensure!(prefs.commission >= MinCommission::::get(), Error::::CommissionTooLow); + // Only check limits if they are not already a validator. if !Validators::::contains_key(stash) { // If this error is reached, we need to adjust the `MinValidatorBond` and start @@ -940,7 +1009,7 @@ pub mod pallet { // the runtime. if let Some(max_validators) = MaxValidatorsCount::::get() { ensure!( - CounterForValidators::::get() < max_validators, + Validators::::count() < max_validators, Error::::TooManyValidators ); } @@ -959,7 +1028,7 @@ pub mod pallet { /// /// # /// - The transaction's complexity is proportional to the size of `targets` (N) - /// which is capped at CompactAssignments::LIMIT (MAX_NOMINATIONS). + /// which is capped at CompactAssignments::LIMIT (T::MaxNominations). /// - Both the reads and writes follow a similar pattern. /// # #[pallet::weight(T::WeightInfo::nominate(targets.len() as u32))] @@ -980,18 +1049,18 @@ pub mod pallet { // the runtime. if let Some(max_nominators) = MaxNominatorsCount::::get() { ensure!( - CounterForNominators::::get() < max_nominators, + Nominators::::count() < max_nominators, Error::::TooManyNominators ); } } ensure!(!targets.is_empty(), Error::::EmptyTargets); - ensure!(targets.len() <= T::MAX_NOMINATIONS as usize, Error::::TooManyTargets); + ensure!(targets.len() <= T::MaxNominations::get() as usize, Error::::TooManyTargets); - let old = Nominators::::get(stash).map_or_else(Vec::new, |x| x.targets); + let old = Nominators::::get(stash).map_or_else(Vec::new, |x| x.targets.into_inner()); - let targets = targets + let targets: BoundedVec<_, _> = targets .into_iter() .map(|t| T::Lookup::lookup(t).map_err(DispatchError::from)) .map(|n| { @@ -1003,11 +1072,13 @@ pub mod pallet { } }) }) - .collect::, _>>()?; + .collect::, _>>()? + .try_into() + .map_err(|_| Error::::TooManyNominators)?; let nominations = Nominations { targets, - // Initial nominations are considered submitted at era 0. See `Nominations` doc + // Initial nominations are considered submitted at era 0. See `Nominations` doc. submitted_in: Self::current_era().unwrap_or(0), suppressed: false, }; @@ -1197,11 +1268,6 @@ pub mod pallet { /// Set the validators who cannot be slashed (if any). /// /// The dispatch origin must be Root. - /// - /// # - /// - O(V) - /// - Write: Invulnerables - /// # #[pallet::weight(T::WeightInfo::set_invulnerables(invulnerables.len() as u32))] pub fn set_invulnerables( origin: OriginFor, @@ -1215,13 +1281,6 @@ pub mod pallet { /// Force a current staker to become completely unstaked, immediately. /// /// The dispatch origin must be Root. - /// - /// # - /// O(S) where S is the number of slashing spans to be removed - /// Reads: Bonded, Slashing Spans, Account, Locks - /// Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, - /// Account, Locks Writes Each: SpanSlash * S - /// # #[pallet::weight(T::WeightInfo::force_unstake(*num_slashing_spans))] pub fn force_unstake( origin: OriginFor, @@ -1247,11 +1306,6 @@ pub mod pallet { /// The election process starts multiple blocks before the end of the era. /// If this is called just before a new era is triggered, the election process may not /// have enough blocks to get a result. - /// - /// # - /// - Weight: O(1) - /// - Write: ForceEra - /// # #[pallet::weight(T::WeightInfo::force_new_era_always())] pub fn force_new_era_always(origin: OriginFor) -> DispatchResult { ensure_root(origin)?; @@ -1264,14 +1318,6 @@ pub mod pallet { /// Can be called by the `T::SlashCancelOrigin`. /// /// Parameters: era and indices of the slashes for that era to kill. - /// - /// # - /// Complexity: O(U + S) - /// with U unapplied slashes weighted with U=1000 - /// and S is the number of slash indices to be canceled. - /// - Read: Unapplied Slashes - /// - Write: Unapplied Slashes - /// # #[pallet::weight(T::WeightInfo::cancel_deferred_slash(slash_indices.len() as u32))] pub fn cancel_deferred_slash( origin: OriginFor, @@ -1335,10 +1381,10 @@ pub mod pallet { /// /// # /// - Time complexity: O(L), where L is unlocking chunks - /// - Bounded by `MAX_UNLOCKING_CHUNKS`. + /// - Bounded by `MaxUnlockingChunks`. /// - Storage changes: Can't increase storage, only decrease it. /// # - #[pallet::weight(T::WeightInfo::rebond(MAX_UNLOCKING_CHUNKS as u32))] + #[pallet::weight(T::WeightInfo::rebond(MaxUnlockingChunks::get() as u32))] pub fn rebond( origin: OriginFor, #[pallet::compact] value: BalanceOf, @@ -1356,8 +1402,8 @@ pub mod pallet { // NOTE: ledger must be updated prior to calling `Self::weight_of`. Self::update_ledger(&controller, &ledger); - if T::SortedListProvider::contains(&ledger.stash) { - T::SortedListProvider::on_update(&ledger.stash, Self::weight_of(&ledger.stash)); + if T::VoterList::contains(&ledger.stash) { + T::VoterList::on_update(&ledger.stash, Self::weight_of(&ledger.stash)); } let removed_chunks = 1u32 // for the case where the last iterated chunk is not removed @@ -1408,33 +1454,37 @@ pub mod pallet { Ok(()) } - /// Remove all data structure concerning a staker/stash once its balance is at the minimum. - /// This is essentially equivalent to `withdraw_unbonded` except it can be called by anyone - /// and the target `stash` must have no funds left beyond the ED. + /// Remove all data structures concerning a staker/stash once it is at a state where it can + /// be considered `dust` in the staking system. The requirements are: /// - /// This can be called from any origin. + /// 1. the `total_balance` of the stash is below existential deposit. + /// 2. or, the `ledger.total` of the stash is below existential deposit. /// - /// - `stash`: The stash account to reap. Its balance must be zero. + /// The former can happen in cases like a slash; the latter when a fully unbonded account + /// is still receiving staking rewards in `RewardDestination::Staked`. /// - /// # - /// Complexity: O(S) where S is the number of slashing spans on the account. - /// DB Weight: - /// - Reads: Stash Account, Bonded, Slashing Spans, Locks - /// - Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, - /// Stash Account, Locks - /// - Writes Each: SpanSlash * S - /// # + /// It can be called by anyone, as long as `stash` meets the above requirements. + /// + /// Refunds the transaction fees upon successful execution. #[pallet::weight(T::WeightInfo::reap_stash(*num_slashing_spans))] pub fn reap_stash( - _origin: OriginFor, + origin: OriginFor, stash: T::AccountId, num_slashing_spans: u32, - ) -> DispatchResult { - let at_minimum = T::Currency::total_balance(&stash) == T::Currency::minimum_balance(); - ensure!(at_minimum, Error::::FundedTarget); + ) -> DispatchResultWithPostInfo { + let _ = ensure_signed(origin)?; + + let ed = T::Currency::minimum_balance(); + let reapable = T::Currency::total_balance(&stash) < ed || + Self::ledger(Self::bonded(stash.clone()).ok_or(Error::::NotStash)?) + .map(|l| l.total) + .unwrap_or_default() < ed; + ensure!(reapable, Error::::FundedTarget); + Self::kill_stash(&stash, num_slashing_spans)?; T::Currency::remove_lock(STAKING_ID, &stash); - Ok(()) + + Ok(Pays::No.into()) } /// Remove the given nominations from the calling validator. @@ -1479,7 +1529,7 @@ pub mod pallet { Ok(()) } - /// Update the various staking limits this pallet. + /// Update the various staking configurations . /// /// * `min_nominator_bond`: The minimum active bond needed to be a nominator. /// * `min_validator_bond`: The minimum active bond needed to be a validator. @@ -1487,26 +1537,48 @@ pub mod pallet { /// set to `None`, no limit is enforced. /// * `max_validator_count`: The max number of users who can be a validator at once. When /// set to `None`, no limit is enforced. + /// * `chill_threshold`: The ratio of `max_nominator_count` or `max_validator_count` which + /// should be filled in order for the `chill_other` transaction to work. + /// * `min_commission`: The minimum amount of commission that each validators must maintain. + /// This is checked only upon calling `validate`. Existing validators are not affected. /// /// Origin must be Root to call this function. /// /// NOTE: Existing nominators and validators will not be affected by this update. /// to kick people under the new limits, `chill_other` should be called. - #[pallet::weight(T::WeightInfo::set_staking_limits())] - pub fn set_staking_limits( + // We assume the worst case for this call is either: all items are set or all items are + // removed. + #[pallet::weight(max( + T::WeightInfo::set_staking_configs_all_set(), + T::WeightInfo::set_staking_configs_all_remove() + ))] + pub fn set_staking_configs( origin: OriginFor, - min_nominator_bond: BalanceOf, - min_validator_bond: BalanceOf, - max_nominator_count: Option, - max_validator_count: Option, - threshold: Option, + min_nominator_bond: ConfigOp>, + min_validator_bond: ConfigOp>, + max_nominator_count: ConfigOp, + max_validator_count: ConfigOp, + chill_threshold: ConfigOp, + min_commission: ConfigOp, ) -> DispatchResult { ensure_root(origin)?; - MinNominatorBond::::set(min_nominator_bond); - MinValidatorBond::::set(min_validator_bond); - MaxNominatorsCount::::set(max_nominator_count); - MaxValidatorsCount::::set(max_validator_count); - ChillThreshold::::set(threshold); + + macro_rules! config_op_exp { + ($storage:ty, $op:ident) => { + match $op { + ConfigOp::Noop => (), + ConfigOp::Set(v) => <$storage>::put(v), + ConfigOp::Remove => <$storage>::kill(), + } + }; + } + + config_op_exp!(MinNominatorBond, min_nominator_bond); + config_op_exp!(MinValidatorBond, min_validator_bond); + config_op_exp!(MaxNominatorsCount, max_nominator_count); + config_op_exp!(MaxValidatorsCount, max_validator_count); + config_op_exp!(ChillThreshold, chill_threshold); + config_op_exp!(MinCommission, min_commission); Ok(()) } @@ -1521,6 +1593,11 @@ pub mod pallet { /// /// If the caller is different than the controller being targeted, the following conditions /// must be met: + /// + /// * `controller` must belong to a nominator who has become non-decodable, + /// + /// Or: + /// /// * A `ChillThreshold` must be set and checked which defines how close to the max /// nominators or validators we must reach before users can start chilling one-another. /// * A `MaxNominatorCount` and `MaxValidatorCount` must be set which is used to determine @@ -1539,6 +1616,11 @@ pub mod pallet { let stash = ledger.stash; // In order for one user to chill another user, the following conditions must be met: + // + // * `controller` belongs to a nominator who has become non-decodable, + // + // Or + // // * A `ChillThreshold` is set which defines how close to the max nominators or // validators we must reach before users can start chilling one-another. // * A `MaxNominatorCount` and `MaxValidatorCount` which is used to determine how close @@ -1548,12 +1630,18 @@ pub mod pallet { // threshold bond required. // // Otherwise, if caller is the same as the controller, this is just like `chill`. + + if Nominators::::contains_key(&stash) && Nominators::::get(&stash).is_none() { + Self::chill_stash(&stash); + return Ok(()) + } + if caller != controller { let threshold = ChillThreshold::::get().ok_or(Error::::CannotChillOther)?; let min_active_bond = if Nominators::::contains_key(&stash) { let max_nominator_count = MaxNominatorsCount::::get().ok_or(Error::::CannotChillOther)?; - let current_nominator_count = CounterForNominators::::get(); + let current_nominator_count = Nominators::::count(); ensure!( threshold * max_nominator_count < current_nominator_count, Error::::CannotChillOther @@ -1562,7 +1650,7 @@ pub mod pallet { } else if Validators::::contains_key(&stash) { let max_validator_count = MaxValidatorsCount::::get().ok_or(Error::::CannotChillOther)?; - let current_validator_count = CounterForValidators::::get(); + let current_validator_count = Validators::::count(); ensure!( threshold * max_validator_count < current_validator_count, Error::::CannotChillOther @@ -1578,6 +1666,28 @@ pub mod pallet { Self::chill_stash(&stash); Ok(()) } + + /// Force a validator to have at least the minimum commission. This will not affect a + /// validator who already has a commission greater than or equal to the minimum. Any account + /// can call this. + #[pallet::weight(T::WeightInfo::force_apply_min_commission())] + pub fn force_apply_min_commission( + origin: OriginFor, + validator_stash: T::AccountId, + ) -> DispatchResult { + ensure_signed(origin)?; + let min_commission = MinCommission::::get(); + Validators::::try_mutate_exists(validator_stash, |maybe_prefs| { + maybe_prefs + .as_mut() + .map(|prefs| { + (prefs.commission < min_commission) + .then(|| prefs.commission = min_commission) + }) + .ok_or(Error::::NotStash) + })?; + Ok(()) + } } } diff --git a/frame/staking/src/slashing.rs b/frame/staking/src/slashing.rs index 15ca85b4d046..2f381ad631fe 100644 --- a/frame/staking/src/slashing.rs +++ b/frame/staking/src/slashing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -47,22 +47,23 @@ //! has multiple misbehaviors. However, accounting for such cases is necessary //! to deter a class of "rage-quit" attacks. //! -//! Based on research at +//! Based on research at use crate::{ - BalanceOf, Config, EraIndex, Error, Exposure, NegativeImbalanceOf, Pallet, Perbill, - SessionInterface, Store, UnappliedSlash, + BalanceOf, Config, Error, Exposure, NegativeImbalanceOf, Pallet, Perbill, SessionInterface, + Store, UnappliedSlash, }; use codec::{Decode, Encode}; use frame_support::{ ensure, - traits::{Currency, Imbalance, OnUnbalanced}, + traits::{Currency, Get, Imbalance, OnUnbalanced}, }; use scale_info::TypeInfo; use sp_runtime::{ traits::{Saturating, Zero}, DispatchResult, RuntimeDebug, }; +use sp_staking::{offence::DisableStrategy, EraIndex}; use sp_std::vec::Vec; /// The proportion of the slashing reward to be paid out on the first slashing detection. @@ -190,7 +191,7 @@ pub(crate) struct SpanRecord { impl SpanRecord { /// The value of stash balance slashed in this span. #[cfg(test)] - pub(crate) fn amount_slashed(&self) -> &Balance { + pub(crate) fn amount(&self) -> &Balance { &self.slashed } } @@ -213,6 +214,8 @@ pub(crate) struct SlashParams<'a, T: 'a + Config> { /// The maximum percentage of a slash that ever gets paid out. /// This is f_inf in the paper. pub(crate) reward_proportion: Perbill, + /// When to disable offenders. + pub(crate) disable_strategy: DisableStrategy, } /// Computes a slash of a validator and nominators. It returns an unapplied @@ -224,15 +227,12 @@ pub(crate) struct SlashParams<'a, T: 'a + Config> { pub(crate) fn compute_slash( params: SlashParams, ) -> Option>> { - let SlashParams { stash, slash, exposure, slash_era, window_start, now, reward_proportion } = - params.clone(); - let mut reward_payout = Zero::zero(); let mut val_slashed = Zero::zero(); // is the slash amount here a maximum for the era? - let own_slash = slash * exposure.own; - if slash * exposure.total == Zero::zero() { + let own_slash = params.slash * params.exposure.own; + if params.slash * params.exposure.total == Zero::zero() { // kick out the validator even if they won't be slashed, // as long as the misbehavior is from their most recent slashing span. kick_out_if_recent::(params); @@ -240,13 +240,17 @@ pub(crate) fn compute_slash( } let (prior_slash_p, _era_slash) = - as Store>::ValidatorSlashInEra::get(&slash_era, stash) + as Store>::ValidatorSlashInEra::get(¶ms.slash_era, params.stash) .unwrap_or((Perbill::zero(), Zero::zero())); // compare slash proportions rather than slash values to avoid issues due to rounding // error. - if slash.deconstruct() > prior_slash_p.deconstruct() { - as Store>::ValidatorSlashInEra::insert(&slash_era, stash, &(slash, own_slash)); + if params.slash.deconstruct() > prior_slash_p.deconstruct() { + as Store>::ValidatorSlashInEra::insert( + ¶ms.slash_era, + params.stash, + &(params.slash, own_slash), + ); } else { // we slash based on the max in era - this new event is not the max, // so neither the validator or any nominators will need an update. @@ -261,14 +265,14 @@ pub(crate) fn compute_slash( // apply slash to validator. { let mut spans = fetch_spans::( - stash, - window_start, + params.stash, + params.window_start, &mut reward_payout, &mut val_slashed, - reward_proportion, + params.reward_proportion, ); - let target_span = spans.compare_and_update_span_slash(slash_era, own_slash); + let target_span = spans.compare_and_update_span_slash(params.slash_era, own_slash); if target_span == Some(spans.span_index()) { // misbehavior occurred within the current slashing span - take appropriate @@ -276,22 +280,19 @@ pub(crate) fn compute_slash( // chill the validator - it misbehaved in the current span and should // not continue in the next election. also end the slashing span. - spans.end_span(now); - >::chill_stash(stash); - - // make sure to disable validator till the end of this session - if T::SessionInterface::disable_validator(stash).unwrap_or(false) { - // force a new era, to select a new validator set - >::ensure_new_era() - } + spans.end_span(params.now); + >::chill_stash(params.stash); } } + let disable_when_slashed = params.disable_strategy != DisableStrategy::Never; + add_offending_validator::(params.stash, disable_when_slashed); + let mut nominators_slashed = Vec::new(); - reward_payout += slash_nominators::(params, prior_slash_p, &mut nominators_slashed); + reward_payout += slash_nominators::(params.clone(), prior_slash_p, &mut nominators_slashed); Some(UnappliedSlash { - validator: stash.clone(), + validator: params.stash.clone(), own: val_slashed, others: nominators_slashed, reporters: Vec::new(), @@ -316,13 +317,52 @@ fn kick_out_if_recent(params: SlashParams) { if spans.era_span(params.slash_era).map(|s| s.index) == Some(spans.span_index()) { spans.end_span(params.now); >::chill_stash(params.stash); + } + + let disable_without_slash = params.disable_strategy == DisableStrategy::Always; + add_offending_validator::(params.stash, disable_without_slash); +} + +/// Add the given validator to the offenders list and optionally disable it. +/// If after adding the validator `OffendingValidatorsThreshold` is reached +/// a new era will be forced. +fn add_offending_validator(stash: &T::AccountId, disable: bool) { + as Store>::OffendingValidators::mutate(|offending| { + let validators = T::SessionInterface::validators(); + let validator_index = match validators.iter().position(|i| i == stash) { + Some(index) => index, + None => return, + }; - // make sure to disable validator till the end of this session - if T::SessionInterface::disable_validator(params.stash).unwrap_or(false) { - // force a new era, to select a new validator set - >::ensure_new_era() + let validator_index_u32 = validator_index as u32; + + match offending.binary_search_by_key(&validator_index_u32, |(index, _)| *index) { + // this is a new offending validator + Err(index) => { + offending.insert(index, (validator_index_u32, disable)); + + let offending_threshold = + T::OffendingValidatorsThreshold::get() * validators.len() as u32; + + if offending.len() >= offending_threshold as usize { + // force a new era, to select a new validator set + >::ensure_new_era() + } + + if disable { + T::SessionInterface::disable_validator(validator_index_u32); + } + }, + Ok(index) => { + if disable && !offending[index].1 { + // the validator had previously offended without being disabled, + // let's make sure we disable it now + offending[index].1 = true; + T::SessionInterface::disable_validator(validator_index_u32); + } + }, } - } + }); } /// Slash nominators. Accepts general parameters and the prior slash percentage of the validator. @@ -333,13 +373,10 @@ fn slash_nominators( prior_slash_p: Perbill, nominators_slashed: &mut Vec<(T::AccountId, BalanceOf)>, ) -> BalanceOf { - let SlashParams { stash: _, slash, exposure, slash_era, window_start, now, reward_proportion } = - params; - let mut reward_payout = Zero::zero(); - nominators_slashed.reserve(exposure.others.len()); - for nominator in &exposure.others { + nominators_slashed.reserve(params.exposure.others.len()); + for nominator in ¶ms.exposure.others { let stash = &nominator.who; let mut nom_slashed = Zero::zero(); @@ -347,15 +384,16 @@ fn slash_nominators( // had a new max slash for the era. let era_slash = { let own_slash_prior = prior_slash_p * nominator.value; - let own_slash_by_validator = slash * nominator.value; + let own_slash_by_validator = params.slash * nominator.value; let own_slash_difference = own_slash_by_validator.saturating_sub(own_slash_prior); - let mut era_slash = as Store>::NominatorSlashInEra::get(&slash_era, stash) - .unwrap_or_else(|| Zero::zero()); + let mut era_slash = + as Store>::NominatorSlashInEra::get(¶ms.slash_era, stash) + .unwrap_or_else(|| Zero::zero()); era_slash += own_slash_difference; - as Store>::NominatorSlashInEra::insert(&slash_era, stash, &era_slash); + as Store>::NominatorSlashInEra::insert(¶ms.slash_era, stash, &era_slash); era_slash }; @@ -364,18 +402,18 @@ fn slash_nominators( { let mut spans = fetch_spans::( stash, - window_start, + params.window_start, &mut reward_payout, &mut nom_slashed, - reward_proportion, + params.reward_proportion, ); - let target_span = spans.compare_and_update_span_slash(slash_era, era_slash); + let target_span = spans.compare_and_update_span_slash(params.slash_era, era_slash); if target_span == Some(spans.span_index()) { // End the span, but don't chill the nominator. its nomination // on this validator will be ignored in the future. - spans.end_span(now); + spans.end_span(params.now); } } diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 13762cf5886d..5f9f378b1061 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,13 +36,13 @@ const SEED: u32 = 0; /// This function removes all validators and nominators from storage. pub fn clear_validators_and_nominators() { - Validators::::remove_all(None); - CounterForValidators::::kill(); + Validators::::remove_all(); - // whenever we touch nominators counter we should update `T::SortedListProvider` as well. - Nominators::::remove_all(None); - CounterForNominators::::kill(); - let _ = T::SortedListProvider::clear(None); + // whenever we touch nominators counter we should update `T::VoterList` as well. + Nominators::::remove_all(); + + // NOTE: safe to call outside block production + T::VoterList::unsafe_clear(); } /// Grab a funded user. diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 6f024eb1e6b0..11dfe3e4777f 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,10 +17,10 @@ //! Tests for the module. -use super::{Event, *}; +use super::{ConfigOp, Event, MaxUnlockingChunks, *}; use frame_election_provider_support::{ElectionProvider, SortedListProvider, Support}; use frame_support::{ - assert_noop, assert_ok, + assert_noop, assert_ok, assert_storage_noop, bounded_vec, dispatch::WithPostDispatchInfo, pallet_prelude::*, traits::{Currency, Get, ReservableCurrency}, @@ -34,12 +34,62 @@ use sp_runtime::{ Perbill, Percent, }; use sp_staking::{ - offence::{OffenceDetails, OnOffenceHandler}, + offence::{DisableStrategy, OffenceDetails, OnOffenceHandler}, SessionIndex, }; use sp_std::prelude::*; use substrate_test_utils::assert_eq_uvec; +#[test] +fn set_staking_configs_works() { + ExtBuilder::default().build_and_execute(|| { + // setting works + assert_ok!(Staking::set_staking_configs( + Origin::root(), + ConfigOp::Set(1_500), + ConfigOp::Set(2_000), + ConfigOp::Set(10), + ConfigOp::Set(20), + ConfigOp::Set(Percent::from_percent(75)), + ConfigOp::Set(Zero::zero()) + )); + assert_eq!(MinNominatorBond::::get(), 1_500); + assert_eq!(MinValidatorBond::::get(), 2_000); + assert_eq!(MaxNominatorsCount::::get(), Some(10)); + assert_eq!(MaxValidatorsCount::::get(), Some(20)); + assert_eq!(ChillThreshold::::get(), Some(Percent::from_percent(75))); + assert_eq!(MinCommission::::get(), Perbill::from_percent(0)); + + // noop does nothing + assert_storage_noop!(assert_ok!(Staking::set_staking_configs( + Origin::root(), + ConfigOp::Noop, + ConfigOp::Noop, + ConfigOp::Noop, + ConfigOp::Noop, + ConfigOp::Noop, + ConfigOp::Noop + ))); + + // removing works + assert_ok!(Staking::set_staking_configs( + Origin::root(), + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove + )); + assert_eq!(MinNominatorBond::::get(), 0); + assert_eq!(MinValidatorBond::::get(), 0); + assert_eq!(MaxNominatorsCount::::get(), None); + assert_eq!(MaxValidatorsCount::::get(), None); + assert_eq!(ChillThreshold::::get(), None); + assert_eq!(MinCommission::::get(), Perbill::from_percent(0)); + }); +} + #[test] fn force_unstake_works() { ExtBuilder::default().build_and_execute(|| { @@ -104,7 +154,7 @@ fn basic_setup_works() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![] }) ); @@ -115,7 +165,7 @@ fn basic_setup_works() { stash: 21, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![] }) ); @@ -138,7 +188,7 @@ fn basic_setup_works() { stash: 101, total: 500, active: 500, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![] }) ); @@ -217,16 +267,17 @@ fn rewards_should_work() { Payee::::insert(21, RewardDestination::Controller); Payee::::insert(101, RewardDestination::Controller); - >::reward_by_ids(vec![(11, 50)]); - >::reward_by_ids(vec![(11, 50)]); + Pallet::::reward_by_ids(vec![(11, 50)]); + Pallet::::reward_by_ids(vec![(11, 50)]); // This is the second validator of the current elected set. - >::reward_by_ids(vec![(21, 50)]); + Pallet::::reward_by_ids(vec![(21, 50)]); // Compute total payout now for whole duration of the session. let total_payout_0 = current_total_payout_for_duration(reward_time_per_era()); let maximum_payout = maximum_payout_for_duration(reward_time_per_era()); start_session(1); + assert_eq_uvec!(Session::validators(), vec![11, 21]); assert_eq!(Balances::total_balance(&10), init_balance_10); assert_eq!(Balances::total_balance(&11), init_balance_11); @@ -234,7 +285,6 @@ fn rewards_should_work() { assert_eq!(Balances::total_balance(&21), init_balance_21); assert_eq!(Balances::total_balance(&100), init_balance_100); assert_eq!(Balances::total_balance(&101), init_balance_101); - assert_eq_uvec!(Session::validators(), vec![11, 21]); assert_eq!( Staking::eras_reward_points(active_era()), EraRewardPoints { @@ -283,7 +333,7 @@ fn rewards_should_work() { assert_eq_error_rate!(Balances::total_balance(&101), init_balance_101, 2); assert_eq_uvec!(Session::validators(), vec![11, 21]); - >::reward_by_ids(vec![(11, 1)]); + Pallet::::reward_by_ids(vec![(11, 1)]); // Compute total payout now for whole duration as other parameter won't change let total_payout_1 = current_total_payout_for_duration(reward_time_per_era()); @@ -338,6 +388,7 @@ fn staking_should_work() { // add a new candidate for being a validator. account 3 controlled by 4. assert_ok!(Staking::bond(Origin::signed(3), 4, 1500, RewardDestination::Controller)); assert_ok!(Staking::validate(Origin::signed(4), ValidatorPrefs::default())); + assert_ok!(Session::set_keys(Origin::signed(4), SessionKeys { other: 4.into() }, vec![])); // No effects will be seen so far. assert_eq_uvec!(validator_controllers(), vec![20, 10]); @@ -381,7 +432,7 @@ fn staking_should_work() { stash: 3, total: 1500, active: 1500, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![0], }) ); @@ -519,8 +570,8 @@ fn nominating_and_rewards_should_work() { // the total reward for era 0 let total_payout_0 = current_total_payout_for_duration(reward_time_per_era()); - >::reward_by_ids(vec![(41, 1)]); - >::reward_by_ids(vec![(21, 1)]); + Pallet::::reward_by_ids(vec![(41, 1)]); + Pallet::::reward_by_ids(vec![(21, 1)]); mock::start_active_era(1); @@ -561,8 +612,8 @@ fn nominating_and_rewards_should_work() { // the total reward for era 1 let total_payout_1 = current_total_payout_for_duration(reward_time_per_era()); - >::reward_by_ids(vec![(21, 2)]); - >::reward_by_ids(vec![(11, 1)]); + Pallet::::reward_by_ids(vec![(21, 2)]); + Pallet::::reward_by_ids(vec![(11, 1)]); mock::start_active_era(2); @@ -935,14 +986,14 @@ fn reward_destination_works() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); // Compute total payout now for whole duration as other parameter won't change let total_payout_0 = current_total_payout_for_duration(reward_time_per_era()); - >::reward_by_ids(vec![(11, 1)]); + Pallet::::reward_by_ids(vec![(11, 1)]); mock::start_active_era(1); mock::make_all_reward_payment(0); @@ -958,7 +1009,7 @@ fn reward_destination_works() { stash: 11, total: 1000 + total_payout_0, active: 1000 + total_payout_0, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![0], }) ); @@ -968,7 +1019,7 @@ fn reward_destination_works() { // Compute total payout now for whole duration as other parameter won't change let total_payout_1 = current_total_payout_for_duration(reward_time_per_era()); - >::reward_by_ids(vec![(11, 1)]); + Pallet::::reward_by_ids(vec![(11, 1)]); mock::start_active_era(2); mock::make_all_reward_payment(1); @@ -986,7 +1037,7 @@ fn reward_destination_works() { stash: 11, total: 1000 + total_payout_0, active: 1000 + total_payout_0, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![0, 1], }) ); @@ -999,7 +1050,7 @@ fn reward_destination_works() { // Compute total payout now for whole duration as other parameter won't change let total_payout_2 = current_total_payout_for_duration(reward_time_per_era()); - >::reward_by_ids(vec![(11, 1)]); + Pallet::::reward_by_ids(vec![(11, 1)]); mock::start_active_era(3); mock::make_all_reward_payment(2); @@ -1015,7 +1066,7 @@ fn reward_destination_works() { stash: 11, total: 1000 + total_payout_0, active: 1000 + total_payout_0, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![0, 1, 2], }) ); @@ -1049,7 +1100,7 @@ fn validator_payment_prefs_work() { // Compute total payout now for whole duration as other parameter won't change let total_payout_1 = current_total_payout_for_duration(reward_time_per_era()); let exposure_1 = Staking::eras_stakers(active_era(), 11); - >::reward_by_ids(vec![(11, 1)]); + Pallet::::reward_by_ids(vec![(11, 1)]); mock::start_active_era(2); mock::make_all_reward_payment(1); @@ -1080,7 +1131,7 @@ fn bond_extra_works() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); @@ -1097,7 +1148,7 @@ fn bond_extra_works() { stash: 11, total: 1000 + 100, active: 1000 + 100, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); @@ -1111,7 +1162,7 @@ fn bond_extra_works() { stash: 11, total: 1000000, active: 1000000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); @@ -1149,7 +1200,7 @@ fn bond_extra_and_withdraw_unbonded_works() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); @@ -1167,7 +1218,7 @@ fn bond_extra_and_withdraw_unbonded_works() { stash: 11, total: 1000 + 100, active: 1000 + 100, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); @@ -1188,7 +1239,7 @@ fn bond_extra_and_withdraw_unbonded_works() { stash: 11, total: 1000 + 100, active: 1000 + 100, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); @@ -1206,7 +1257,7 @@ fn bond_extra_and_withdraw_unbonded_works() { stash: 11, total: 1000 + 100, active: 100, - unlocking: vec![UnlockChunk { value: 1000, era: 2 + 3 }], + unlocking: bounded_vec![UnlockChunk { value: 1000, era: 2 + 3 }], claimed_rewards: vec![] }), ); @@ -1219,7 +1270,7 @@ fn bond_extra_and_withdraw_unbonded_works() { stash: 11, total: 1000 + 100, active: 100, - unlocking: vec![UnlockChunk { value: 1000, era: 2 + 3 }], + unlocking: bounded_vec![UnlockChunk { value: 1000, era: 2 + 3 }], claimed_rewards: vec![] }), ); @@ -1235,7 +1286,7 @@ fn bond_extra_and_withdraw_unbonded_works() { stash: 11, total: 1000 + 100, active: 100, - unlocking: vec![UnlockChunk { value: 1000, era: 2 + 3 }], + unlocking: bounded_vec![UnlockChunk { value: 1000, era: 2 + 3 }], claimed_rewards: vec![] }), ); @@ -1251,7 +1302,7 @@ fn bond_extra_and_withdraw_unbonded_works() { stash: 11, total: 100, active: 100, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![] }), ); @@ -1261,23 +1312,35 @@ fn bond_extra_and_withdraw_unbonded_works() { #[test] fn too_many_unbond_calls_should_not_work() { ExtBuilder::default().build_and_execute(|| { - // locked at era 0 until 3 - for _ in 0..MAX_UNLOCKING_CHUNKS - 1 { + let mut current_era = 0; + // locked at era MaxUnlockingChunks - 1 until 3 + for i in 0..MaxUnlockingChunks::get() - 1 { + // There is only 1 chunk per era, so we need to be in a new era to create a chunk. + current_era = i as u32; + mock::start_active_era(current_era); assert_ok!(Staking::unbond(Origin::signed(10), 1)); } - mock::start_active_era(1); + current_era += 1; + mock::start_active_era(current_era); - // locked at era 1 until 4 + // This chunk is locked at `current_era` through `current_era + 2` (because BondingDuration + // == 3). assert_ok!(Staking::unbond(Origin::signed(10), 1)); + assert_eq!( + Staking::ledger(&10).unwrap().unlocking.len(), + MaxUnlockingChunks::get() as usize + ); // can't do more. assert_noop!(Staking::unbond(Origin::signed(10), 1), Error::::NoMoreChunks); - mock::start_active_era(3); + current_era += 2; + mock::start_active_era(current_era); assert_noop!(Staking::unbond(Origin::signed(10), 1), Error::::NoMoreChunks); - // free up. + // free up everything except the most recently added chunk. assert_ok!(Staking::withdraw_unbonded(Origin::signed(10), 0)); + assert_eq!(Staking::ledger(&10).unwrap().unlocking.len(), 1); // Can add again. assert_ok!(Staking::unbond(Origin::signed(10), 1)); @@ -1309,7 +1372,7 @@ fn rebond_works() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); @@ -1328,7 +1391,7 @@ fn rebond_works() { stash: 11, total: 1000, active: 100, - unlocking: vec![UnlockChunk { value: 900, era: 2 + 3 }], + unlocking: bounded_vec![UnlockChunk { value: 900, era: 2 + 3 }], claimed_rewards: vec![], }) ); @@ -1341,7 +1404,7 @@ fn rebond_works() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); @@ -1354,7 +1417,7 @@ fn rebond_works() { stash: 11, total: 1000, active: 100, - unlocking: vec![UnlockChunk { value: 900, era: 5 }], + unlocking: bounded_vec![UnlockChunk { value: 900, era: 5 }], claimed_rewards: vec![], }) ); @@ -1367,7 +1430,7 @@ fn rebond_works() { stash: 11, total: 1000, active: 600, - unlocking: vec![UnlockChunk { value: 400, era: 5 }], + unlocking: bounded_vec![UnlockChunk { value: 400, era: 5 }], claimed_rewards: vec![], }) ); @@ -1380,7 +1443,7 @@ fn rebond_works() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); @@ -1395,11 +1458,7 @@ fn rebond_works() { stash: 11, total: 1000, active: 100, - unlocking: vec![ - UnlockChunk { value: 300, era: 5 }, - UnlockChunk { value: 300, era: 5 }, - UnlockChunk { value: 300, era: 5 }, - ], + unlocking: bounded_vec![UnlockChunk { value: 900, era: 5 }], claimed_rewards: vec![], }) ); @@ -1412,10 +1471,7 @@ fn rebond_works() { stash: 11, total: 1000, active: 600, - unlocking: vec![ - UnlockChunk { value: 300, era: 5 }, - UnlockChunk { value: 100, era: 5 }, - ], + unlocking: bounded_vec![UnlockChunk { value: 400, era: 5 }], claimed_rewards: vec![], }) ); @@ -1442,7 +1498,7 @@ fn rebond_is_fifo() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); @@ -1457,7 +1513,7 @@ fn rebond_is_fifo() { stash: 11, total: 1000, active: 600, - unlocking: vec![UnlockChunk { value: 400, era: 2 + 3 },], + unlocking: bounded_vec![UnlockChunk { value: 400, era: 2 + 3 }], claimed_rewards: vec![], }) ); @@ -1472,7 +1528,7 @@ fn rebond_is_fifo() { stash: 11, total: 1000, active: 300, - unlocking: vec![ + unlocking: bounded_vec![ UnlockChunk { value: 400, era: 2 + 3 }, UnlockChunk { value: 300, era: 3 + 3 }, ], @@ -1490,7 +1546,7 @@ fn rebond_is_fifo() { stash: 11, total: 1000, active: 100, - unlocking: vec![ + unlocking: bounded_vec![ UnlockChunk { value: 400, era: 2 + 3 }, UnlockChunk { value: 300, era: 3 + 3 }, UnlockChunk { value: 200, era: 4 + 3 }, @@ -1507,7 +1563,7 @@ fn rebond_is_fifo() { stash: 11, total: 1000, active: 500, - unlocking: vec![ + unlocking: bounded_vec![ UnlockChunk { value: 400, era: 2 + 3 }, UnlockChunk { value: 100, era: 3 + 3 }, ], @@ -1539,7 +1595,7 @@ fn rebond_emits_right_value_in_event() { stash: 11, total: 1000, active: 100, - unlocking: vec![UnlockChunk { value: 900, era: 1 + 3 }], + unlocking: bounded_vec![UnlockChunk { value: 900, era: 1 + 3 }], claimed_rewards: vec![], }) ); @@ -1552,7 +1608,7 @@ fn rebond_emits_right_value_in_event() { stash: 11, total: 1000, active: 200, - unlocking: vec![UnlockChunk { value: 800, era: 1 + 3 }], + unlocking: bounded_vec![UnlockChunk { value: 800, era: 1 + 3 }], claimed_rewards: vec![], }) ); @@ -1567,7 +1623,7 @@ fn rebond_emits_right_value_in_event() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); @@ -1603,15 +1659,15 @@ fn reward_to_stake_works() { stash: 21, total: 69, active: 69, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }, ); // Compute total payout now for whole duration as other parameter won't change let total_payout_0 = current_total_payout_for_duration(reward_time_per_era()); - >::reward_by_ids(vec![(11, 1)]); - >::reward_by_ids(vec![(21, 1)]); + Pallet::::reward_by_ids(vec![(11, 1)]); + Pallet::::reward_by_ids(vec![(21, 1)]); // New era --> rewards are paid --> stakes are changed mock::start_active_era(1); @@ -1633,115 +1689,49 @@ fn reward_to_stake_works() { } #[test] -fn on_free_balance_zero_stash_removes_validator() { - // Tests that validator storage items are cleaned up when stash is empty - // Tests that storage items are untouched when controller is empty +fn reap_stash_works() { ExtBuilder::default() .existential_deposit(10) .balance_factor(10) .build_and_execute(|| { - // Check the balance of the validator account + // given assert_eq!(Balances::free_balance(10), 10); - // Check the balance of the stash account assert_eq!(Balances::free_balance(11), 10 * 1000); - // Check these two accounts are bonded assert_eq!(Staking::bonded(&11), Some(10)); - // Set payee information - assert_ok!(Staking::set_payee(Origin::signed(10), RewardDestination::Stash)); - - // Check storage items that should be cleaned up assert!(>::contains_key(&10)); assert!(>::contains_key(&11)); assert!(>::contains_key(&11)); assert!(>::contains_key(&11)); - // Reduce free_balance of controller to 0 - let _ = Balances::slash(&10, Balance::max_value()); - - // Check the balance of the stash account has not been touched - assert_eq!(Balances::free_balance(11), 10 * 1000); - // Check these two accounts are still bonded - assert_eq!(Staking::bonded(&11), Some(10)); - - // Check storage items have not changed - assert!(>::contains_key(&10)); - assert!(>::contains_key(&11)); - assert!(>::contains_key(&11)); - assert!(>::contains_key(&11)); - - // Reduce free_balance of stash to 0 - let _ = Balances::slash(&11, Balance::max_value()); - // Check total balance of stash - assert_eq!(Balances::total_balance(&11), 10); - - // Reap the stash - assert_ok!(Staking::reap_stash(Origin::none(), 11, 0)); - - // Check storage items do not exist - assert!(!>::contains_key(&10)); - assert!(!>::contains_key(&11)); - assert!(!>::contains_key(&11)); - assert!(!>::contains_key(&11)); - assert!(!>::contains_key(&11)); - }); -} - -#[test] -fn on_free_balance_zero_stash_removes_nominator() { - // Tests that nominator storage items are cleaned up when stash is empty - // Tests that storage items are untouched when controller is empty - ExtBuilder::default() - .existential_deposit(10) - .balance_factor(10) - .build_and_execute(|| { - // Make 10 a nominator - assert_ok!(Staking::nominate(Origin::signed(10), vec![20])); - // Check that account 10 is a nominator - assert!(>::contains_key(11)); - // Check the balance of the nominator account - assert_eq!(Balances::free_balance(10), 10); - // Check the balance of the stash account - assert_eq!(Balances::free_balance(11), 10_000); - - // Set payee information - assert_ok!(Staking::set_payee(Origin::signed(10), RewardDestination::Stash)); - - // Check storage items that should be cleaned up - assert!(>::contains_key(&10)); - assert!(>::contains_key(&11)); - assert!(>::contains_key(&11)); - assert!(>::contains_key(&11)); - - // Reduce free_balance of controller to 0 - let _ = Balances::slash(&10, Balance::max_value()); - // Check total balance of account 10 - assert_eq!(Balances::total_balance(&10), 0); - - // Check the balance of the stash account has not been touched - assert_eq!(Balances::free_balance(11), 10_000); - // Check these two accounts are still bonded - assert_eq!(Staking::bonded(&11), Some(10)); - - // Check storage items have not changed - assert!(>::contains_key(&10)); - assert!(>::contains_key(&11)); - assert!(>::contains_key(&11)); - assert!(>::contains_key(&11)); + // stash is not reapable + assert_noop!( + Staking::reap_stash(Origin::signed(20), 11, 0), + Error::::FundedTarget + ); + // controller or any other account is not reapable + assert_noop!(Staking::reap_stash(Origin::signed(20), 10, 0), Error::::NotStash); - // Reduce free_balance of stash to 0 - let _ = Balances::slash(&11, Balance::max_value()); - // Check total balance of stash - assert_eq!(Balances::total_balance(&11), 10); + // no easy way to cause an account to go below ED, we tweak their staking ledger + // instead. + Ledger::::insert( + 10, + StakingLedger { + stash: 11, + total: 5, + active: 5, + unlocking: Default::default(), + claimed_rewards: vec![], + }, + ); - // Reap the stash - assert_ok!(Staking::reap_stash(Origin::none(), 11, 0)); + // reap-able + assert_ok!(Staking::reap_stash(Origin::signed(20), 11, 0)); - // Check storage items do not exist + // then assert!(!>::contains_key(&10)); assert!(!>::contains_key(&11)); assert!(!>::contains_key(&11)); - assert!(!>::contains_key(&11)); assert!(!>::contains_key(&11)); }); } @@ -1773,6 +1763,7 @@ fn switching_roles() { // add a new validator candidate assert_ok!(Staking::bond(Origin::signed(5), 6, 1000, RewardDestination::Controller)); assert_ok!(Staking::validate(Origin::signed(6), ValidatorPrefs::default())); + assert_ok!(Session::set_keys(Origin::signed(6), SessionKeys { other: 6.into() }, vec![])); mock::start_active_era(1); @@ -1781,6 +1772,7 @@ fn switching_roles() { // 2 decides to be a validator. Consequences: assert_ok!(Staking::validate(Origin::signed(2), ValidatorPrefs::default())); + assert_ok!(Session::set_keys(Origin::signed(2), SessionKeys { other: 2.into() }, vec![])); // new stakes: // 10: 1000 self vote // 20: 1000 self vote + 250 vote @@ -1847,7 +1839,7 @@ fn bond_with_no_staked_value() { stash: 1, active: 0, total: 5, - unlocking: vec![UnlockChunk { value: 5, era: 3 }], + unlocking: bounded_vec![UnlockChunk { value: 5, era: 3 }], claimed_rewards: vec![], }) ); @@ -1885,6 +1877,11 @@ fn bond_with_little_staked_value_bounded() { // Stingy validator. assert_ok!(Staking::bond(Origin::signed(1), 2, 1, RewardDestination::Controller)); assert_ok!(Staking::validate(Origin::signed(2), ValidatorPrefs::default())); + assert_ok!(Session::set_keys( + Origin::signed(2), + SessionKeys { other: 2.into() }, + vec![] + )); // 1 era worth of reward. BUT, we set the timestamp after on_initialize, so outdated by // one block. @@ -2117,12 +2114,12 @@ fn reward_from_authorship_event_handler_works() { ExtBuilder::default().build_and_execute(|| { use pallet_authorship::EventHandler; - assert_eq!(>::author(), 11); + assert_eq!(>::author(), Some(11)); - >::note_author(11); - >::note_uncle(21, 1); + Pallet::::note_author(11); + Pallet::::note_uncle(21, 1); // Rewarding the same two times works. - >::note_uncle(11, 1); + Pallet::::note_uncle(11, 1); // Not mandatory but must be coherent with rewards assert_eq_uvec!(Session::validators(), vec![11, 21]); @@ -2145,9 +2142,9 @@ fn add_reward_points_fns_works() { // Not mandatory but must be coherent with rewards assert_eq_uvec!(Session::validators(), vec![21, 11]); - >::reward_by_ids(vec![(21, 1), (11, 1), (11, 1)]); + Pallet::::reward_by_ids(vec![(21, 1), (11, 1), (11, 1)]); - >::reward_by_ids(vec![(21, 1), (11, 1), (11, 1)]); + Pallet::::reward_by_ids(vec![(21, 1), (11, 1), (11, 1)]); assert_eq!( ErasRewardPoints::::get(active_era()), @@ -2316,12 +2313,14 @@ fn slash_in_old_span_does_not_deselect() { }], &[Perbill::from_percent(0)], 1, + DisableStrategy::WhenSlashed, ); - // not forcing for zero-slash and previous span. - assert_eq!(Staking::force_era(), Forcing::NotForcing); - assert!(>::contains_key(11)); - assert!(Session::validators().contains(&11)); + // the validator doesn't get chilled again + assert!(::Validators::iter().any(|(stash, _)| stash == 11)); + + // but we are still forcing a new era + assert_eq!(Staking::force_era(), Forcing::ForceNew); on_offence_in_era( &[OffenceDetails { @@ -2331,12 +2330,16 @@ fn slash_in_old_span_does_not_deselect() { // NOTE: A 100% slash here would clean up the account, causing de-registration. &[Perbill::from_percent(95)], 1, + DisableStrategy::WhenSlashed, ); - // or non-zero. - assert_eq!(Staking::force_era(), Forcing::NotForcing); - assert!(>::contains_key(11)); - assert!(Session::validators().contains(&11)); + // the validator doesn't get chilled again + assert!(::Validators::iter().any(|(stash, _)| stash == 11)); + + // but it's disabled + assert!(is_disabled(10)); + // and we are still forcing a new era + assert_eq!(Staking::force_era(), Forcing::ForceNew); }); } @@ -2531,7 +2534,7 @@ fn garbage_collection_after_slashing() { assert_eq!(Balances::free_balance(11), 2000 - 200); assert!(::SlashingSpans::get(&11).is_some()); - assert_eq!(::SpanSlash::get(&(11, 0)).amount_slashed(), &200); + assert_eq!(::SpanSlash::get(&(11, 0)).amount(), &200); on_offence_now( &[OffenceDetails { @@ -2552,13 +2555,13 @@ fn garbage_collection_after_slashing() { // reap_stash respects num_slashing_spans so that weight is accurate assert_noop!( - Staking::reap_stash(Origin::none(), 11, 0), + Staking::reap_stash(Origin::signed(20), 11, 0), Error::::IncorrectSlashingSpans ); - assert_ok!(Staking::reap_stash(Origin::none(), 11, 2)); + assert_ok!(Staking::reap_stash(Origin::signed(20), 11, 2)); assert!(::SlashingSpans::get(&11).is_none()); - assert_eq!(::SpanSlash::get(&(11, 0)).amount_slashed(), &0); + assert_eq!(::SpanSlash::get(&(11, 0)).amount(), &0); }) } @@ -2624,6 +2627,7 @@ fn slashing_nominators_by_span_max() { }], &[Perbill::from_percent(10)], 2, + DisableStrategy::WhenSlashed, ); assert_eq!(Balances::free_balance(11), 900); @@ -2650,6 +2654,7 @@ fn slashing_nominators_by_span_max() { }], &[Perbill::from_percent(30)], 3, + DisableStrategy::WhenSlashed, ); // 11 was not further slashed, but 21 and 101 were. @@ -2671,6 +2676,7 @@ fn slashing_nominators_by_span_max() { }], &[Perbill::from_percent(20)], 2, + DisableStrategy::WhenSlashed, ); // 11 was further slashed, but 21 and 101 were not. @@ -2806,6 +2812,7 @@ fn remove_deferred() { &[OffenceDetails { offender: (11, exposure.clone()), reporters: vec![] }], &[Perbill::from_percent(15)], 1, + DisableStrategy::WhenSlashed, ); // fails if empty @@ -2967,6 +2974,166 @@ fn slash_kicks_validators_not_nominators_and_disables_nominator_for_kicked_valid }); } +#[test] +fn non_slashable_offence_doesnt_disable_validator() { + ExtBuilder::default().build_and_execute(|| { + mock::start_active_era(1); + assert_eq_uvec!(Session::validators(), vec![11, 21]); + + let exposure_11 = Staking::eras_stakers(Staking::active_era().unwrap().index, &11); + let exposure_21 = Staking::eras_stakers(Staking::active_era().unwrap().index, &21); + + // offence with no slash associated + on_offence_now( + &[OffenceDetails { offender: (11, exposure_11.clone()), reporters: vec![] }], + &[Perbill::zero()], + ); + + // offence that slashes 25% of the bond + on_offence_now( + &[OffenceDetails { offender: (21, exposure_21.clone()), reporters: vec![] }], + &[Perbill::from_percent(25)], + ); + + // the offence for validator 10 wasn't slashable so it wasn't disabled + assert!(!is_disabled(10)); + // whereas validator 20 gets disabled + assert!(is_disabled(20)); + }); +} + +#[test] +fn slashing_independent_of_disabling_validator() { + ExtBuilder::default().build_and_execute(|| { + mock::start_active_era(1); + assert_eq_uvec!(Session::validators(), vec![11, 21]); + + let exposure_11 = Staking::eras_stakers(Staking::active_era().unwrap().index, &11); + let exposure_21 = Staking::eras_stakers(Staking::active_era().unwrap().index, &21); + + let now = Staking::active_era().unwrap().index; + + // offence with no slash associated, BUT disabling + on_offence_in_era( + &[OffenceDetails { offender: (11, exposure_11.clone()), reporters: vec![] }], + &[Perbill::zero()], + now, + DisableStrategy::Always, + ); + + // offence that slashes 25% of the bond, BUT not disabling + on_offence_in_era( + &[OffenceDetails { offender: (21, exposure_21.clone()), reporters: vec![] }], + &[Perbill::from_percent(25)], + now, + DisableStrategy::Never, + ); + + // the offence for validator 10 was explicitly disabled + assert!(is_disabled(10)); + // whereas validator 20 is explicitly not disabled + assert!(!is_disabled(20)); + }); +} + +#[test] +fn offence_threshold_triggers_new_era() { + ExtBuilder::default() + .validator_count(4) + .set_status(41, StakerStatus::Validator) + .build_and_execute(|| { + mock::start_active_era(1); + assert_eq_uvec!(Session::validators(), vec![11, 21, 31, 41]); + + assert_eq!( + ::OffendingValidatorsThreshold::get(), + Perbill::from_percent(75), + ); + + // we have 4 validators and an offending validator threshold of 75%, + // once the third validator commits an offence a new era should be forced + + let exposure_11 = Staking::eras_stakers(Staking::active_era().unwrap().index, &11); + let exposure_21 = Staking::eras_stakers(Staking::active_era().unwrap().index, &21); + let exposure_31 = Staking::eras_stakers(Staking::active_era().unwrap().index, &31); + + on_offence_now( + &[OffenceDetails { offender: (11, exposure_11.clone()), reporters: vec![] }], + &[Perbill::zero()], + ); + + assert_eq!(ForceEra::::get(), Forcing::NotForcing); + + on_offence_now( + &[OffenceDetails { offender: (21, exposure_21.clone()), reporters: vec![] }], + &[Perbill::zero()], + ); + + assert_eq!(ForceEra::::get(), Forcing::NotForcing); + + on_offence_now( + &[OffenceDetails { offender: (31, exposure_31.clone()), reporters: vec![] }], + &[Perbill::zero()], + ); + + assert_eq!(ForceEra::::get(), Forcing::ForceNew); + }); +} + +#[test] +fn disabled_validators_are_kept_disabled_for_whole_era() { + ExtBuilder::default() + .validator_count(4) + .set_status(41, StakerStatus::Validator) + .build_and_execute(|| { + mock::start_active_era(1); + assert_eq_uvec!(Session::validators(), vec![11, 21, 31, 41]); + assert_eq!(::SessionsPerEra::get(), 3); + + let exposure_11 = Staking::eras_stakers(Staking::active_era().unwrap().index, &11); + let exposure_21 = Staking::eras_stakers(Staking::active_era().unwrap().index, &21); + + on_offence_now( + &[OffenceDetails { offender: (11, exposure_11.clone()), reporters: vec![] }], + &[Perbill::zero()], + ); + + on_offence_now( + &[OffenceDetails { offender: (21, exposure_21.clone()), reporters: vec![] }], + &[Perbill::from_percent(25)], + ); + + // validator 10 should not be disabled since the offence wasn't slashable + assert!(!is_disabled(10)); + // validator 20 gets disabled since it got slashed + assert!(is_disabled(20)); + + advance_session(); + + // disabled validators should carry-on through all sessions in the era + assert!(!is_disabled(10)); + assert!(is_disabled(20)); + + // validator 10 should now get disabled + on_offence_now( + &[OffenceDetails { offender: (11, exposure_11.clone()), reporters: vec![] }], + &[Perbill::from_percent(25)], + ); + + advance_session(); + + // and both are disabled in the last session of the era + assert!(is_disabled(10)); + assert!(is_disabled(20)); + + mock::start_active_era(2); + + // when a new era starts disabled validators get cleared + assert!(!is_disabled(10)); + assert!(!is_disabled(20)); + }); +} + #[test] fn claim_reward_at_the_last_era_and_no_double_claim_and_invalid_claim() { // should check that: @@ -2987,13 +3154,13 @@ fn claim_reward_at_the_last_era_and_no_double_claim_and_invalid_claim() { Payee::::insert(11, RewardDestination::Controller); Payee::::insert(101, RewardDestination::Controller); - >::reward_by_ids(vec![(11, 1)]); + Pallet::::reward_by_ids(vec![(11, 1)]); // Compute total payout now for whole duration as other parameter won't change let total_payout_0 = current_total_payout_for_duration(reward_time_per_era()); mock::start_active_era(1); - >::reward_by_ids(vec![(11, 1)]); + Pallet::::reward_by_ids(vec![(11, 1)]); // Change total issuance in order to modify total payout let _ = Balances::deposit_creating(&999, 1_000_000_000); // Compute total payout now for whole duration as other parameter won't change @@ -3002,7 +3169,7 @@ fn claim_reward_at_the_last_era_and_no_double_claim_and_invalid_claim() { mock::start_active_era(2); - >::reward_by_ids(vec![(11, 1)]); + Pallet::::reward_by_ids(vec![(11, 1)]); // Change total issuance in order to modify total payout let _ = Balances::deposit_creating(&999, 1_000_000_000); // Compute total payout now for whole duration as other parameter won't change @@ -3162,7 +3329,7 @@ fn test_max_nominator_rewarded_per_validator_and_cant_steal_someone_else_reward( } mock::start_active_era(1); - >::reward_by_ids(vec![(11, 1)]); + Pallet::::reward_by_ids(vec![(11, 1)]); // compute and ensure the reward amount is greater than zero. let _ = current_total_payout_for_duration(reward_time_per_era()); @@ -3242,7 +3409,7 @@ fn test_payout_stakers() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![1] }) ); @@ -3264,7 +3431,7 @@ fn test_payout_stakers() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: (1..=14).collect() }) ); @@ -3285,7 +3452,7 @@ fn test_payout_stakers() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![15, 98] }) ); @@ -3300,7 +3467,7 @@ fn test_payout_stakers() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![15, 23, 42, 69, 98] }) ); @@ -3495,7 +3662,7 @@ fn bond_during_era_correctly_populates_claimed_rewards() { stash: 9, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![], }) ); @@ -3507,7 +3674,7 @@ fn bond_during_era_correctly_populates_claimed_rewards() { stash: 11, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: (0..5).collect(), }) ); @@ -3519,7 +3686,7 @@ fn bond_during_era_correctly_populates_claimed_rewards() { stash: 13, total: 1000, active: 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: (15..99).collect(), }) ); @@ -3531,7 +3698,7 @@ fn offences_weight_calculated_correctly() { ExtBuilder::default().nominate(true).build_and_execute(|| { // On offence with zero offenders: 4 Reads, 1 Write let zero_offence_weight = ::DbWeight::get().reads_writes(4, 1); - assert_eq!(Staking::on_offence(&[], &[Perbill::from_percent(50)], 0), zero_offence_weight); + assert_eq!(Staking::on_offence(&[], &[Perbill::from_percent(50)], 0, DisableStrategy::WhenSlashed), zero_offence_weight); // On Offence with N offenders, Unapplied: 4 Reads, 1 Write + 4 Reads, 5 Writes let n_offence_unapplied_weight = ::DbWeight::get().reads_writes(4, 1) @@ -3544,7 +3711,7 @@ fn offences_weight_calculated_correctly() { reporters: vec![], } ).collect(); - assert_eq!(Staking::on_offence(&offenders, &[Perbill::from_percent(50)], 0), n_offence_unapplied_weight); + assert_eq!(Staking::on_offence(&offenders, &[Perbill::from_percent(50)], 0, DisableStrategy::WhenSlashed), n_offence_unapplied_weight); // On Offence with one offenders, Applied let one_offender = [ @@ -3565,7 +3732,7 @@ fn offences_weight_calculated_correctly() { // `reward_cost` * reporters (1) + ::DbWeight::get().reads_writes(2, 2); - assert_eq!(Staking::on_offence(&one_offender, &[Perbill::from_percent(50)], 0), one_offence_unapplied_weight); + assert_eq!(Staking::on_offence(&one_offender, &[Perbill::from_percent(50)], 0, DisableStrategy::WhenSlashed), one_offence_unapplied_weight); }); } @@ -3738,7 +3905,7 @@ fn cannot_rebond_to_lower_than_ed() { stash: 21, total: 10 * 1000, active: 10 * 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![] } ); @@ -3752,7 +3919,7 @@ fn cannot_rebond_to_lower_than_ed() { stash: 21, total: 10 * 1000, active: 0, - unlocking: vec![UnlockChunk { value: 10 * 1000, era: 3 }], + unlocking: bounded_vec![UnlockChunk { value: 10 * 1000, era: 3 }], claimed_rewards: vec![] } ); @@ -3775,7 +3942,7 @@ fn cannot_bond_extra_to_lower_than_ed() { stash: 21, total: 10 * 1000, active: 10 * 1000, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![] } ); @@ -3789,7 +3956,7 @@ fn cannot_bond_extra_to_lower_than_ed() { stash: 21, total: 10 * 1000, active: 0, - unlocking: vec![UnlockChunk { value: 10 * 1000, era: 3 }], + unlocking: bounded_vec![UnlockChunk { value: 10 * 1000, era: 3 }], claimed_rewards: vec![] } ); @@ -3816,7 +3983,7 @@ fn do_not_die_when_active_is_ed() { stash: 21, total: 1000 * ed, active: 1000 * ed, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![] } ); @@ -3833,7 +4000,7 @@ fn do_not_die_when_active_is_ed() { stash: 21, total: ed, active: ed, - unlocking: vec![], + unlocking: Default::default(), claimed_rewards: vec![] } ); @@ -3888,11 +4055,12 @@ mod election_data_provider { #[test] fn voters_include_self_vote() { ExtBuilder::default().nominate(false).build_and_execute(|| { - assert!(>::iter().map(|(x, _)| x).all(|v| Staking::voters(None) - .unwrap() - .into_iter() - .find(|(w, _, t)| { v == *w && t[0] == *w }) - .is_some())) + assert!(>::iter().map(|(x, _)| x).all(|v| Staking::electing_voters( + None + ) + .unwrap() + .into_iter() + .any(|(w, _, t)| { v == w && t[0] == w }))) }) } @@ -3901,7 +4069,7 @@ mod election_data_provider { ExtBuilder::default().build_and_execute(|| { assert_eq!(Staking::nominators(101).unwrap().targets, vec![11, 21]); assert_eq!( - >::voters(None) + ::electing_voters(None) .unwrap() .iter() .find(|x| x.0 == 101) @@ -3916,7 +4084,7 @@ mod election_data_provider { // 11 is gone. start_active_era(2); assert_eq!( - >::voters(None) + ::electing_voters(None) .unwrap() .iter() .find(|x| x.0 == 101) @@ -3928,7 +4096,7 @@ mod election_data_provider { // resubmit and it is back assert_ok!(Staking::nominate(Origin::signed(100), vec![11, 21])); assert_eq!( - >::voters(None) + ::electing_voters(None) .unwrap() .iter() .find(|x| x.0 == 101) @@ -3945,67 +4113,66 @@ mod election_data_provider { .set_status(41, StakerStatus::Validator) .build_and_execute(|| { // sum of all nominators who'd be voters (1), plus the self-votes (4). - assert_eq!( - ::SortedListProvider::count() + - >::iter().count() as u32, - 5 - ); + assert_eq!(::VoterList::count(), 5); // if limits is less.. - assert_eq!(Staking::voters(Some(1)).unwrap().len(), 1); + assert_eq!(Staking::electing_voters(Some(1)).unwrap().len(), 1); // if limit is equal.. - assert_eq!(Staking::voters(Some(5)).unwrap().len(), 5); + assert_eq!(Staking::electing_voters(Some(5)).unwrap().len(), 5); // if limit is more. - assert_eq!(Staking::voters(Some(55)).unwrap().len(), 5); + assert_eq!(Staking::electing_voters(Some(55)).unwrap().len(), 5); // if target limit is more.. - assert_eq!(Staking::targets(Some(6)).unwrap().len(), 4); - assert_eq!(Staking::targets(Some(4)).unwrap().len(), 4); + assert_eq!(Staking::electable_targets(Some(6)).unwrap().len(), 4); + assert_eq!(Staking::electable_targets(Some(4)).unwrap().len(), 4); // if target limit is less, then we return an error. - assert_eq!(Staking::targets(Some(1)).unwrap_err(), "Target snapshot too big"); + assert_eq!( + Staking::electable_targets(Some(1)).unwrap_err(), + "Target snapshot too big" + ); }); } + // Tests the criteria that in `ElectionDataProvider::voters` function, we try to get at most + // `maybe_max_len` voters, and if some of them end up being skipped, we iterate at most `2 * + // maybe_max_len`. #[test] - fn only_iterates_max_2_times_nominators_quota() { + fn only_iterates_max_2_times_max_allowed_len() { ExtBuilder::default() - .nominate(true) // add nominator 101, who nominates [11, 21] + .nominate(false) // the other nominators only nominate 21 .add_staker(61, 60, 2_000, StakerStatus::::Nominator(vec![21])) .add_staker(71, 70, 2_000, StakerStatus::::Nominator(vec![21])) .add_staker(81, 80, 2_000, StakerStatus::::Nominator(vec![21])) .build_and_execute(|| { - // given our nominators ordered by stake, + // all voters ordered by stake, assert_eq!( - ::SortedListProvider::iter().collect::>(), - vec![61, 71, 81, 101] + ::VoterList::iter().collect::>(), + vec![61, 71, 81, 11, 21, 31] ); - // and total voters - assert_eq!( - ::SortedListProvider::count() + - >::iter().count() as u32, - 7 - ); - - // roll to session 5 run_to_block(25); // slash 21, the only validator nominated by our first 3 nominators add_slash(&21); - // we take 4 voters: 2 validators and 2 nominators (so nominators quota = 2) + // we want 2 voters now, and in maximum we allow 4 iterations. This is what happens: + // 61 is pruned; + // 71 is pruned; + // 81 is pruned; + // 11 is taken; + // we finish since the 2x limit is reached. assert_eq!( - Staking::voters(Some(3)) + Staking::electing_voters(Some(2)) .unwrap() .iter() .map(|(stash, _, _)| stash) .copied() .collect::>(), - vec![31, 11], // 2 validators, but no nominators because we hit the quota + vec![11], ); }); } @@ -4018,57 +4185,43 @@ mod election_data_provider { #[test] fn get_max_len_voters_even_if_some_nominators_are_slashed() { ExtBuilder::default() - .nominate(true) // add nominator 101, who nominates [11, 21] + .nominate(false) .add_staker(61, 60, 20, StakerStatus::::Nominator(vec![21])) - // 61 only nominates validator 21 ^^ .add_staker(71, 70, 10, StakerStatus::::Nominator(vec![11, 21])) + .add_staker(81, 80, 10, StakerStatus::::Nominator(vec![11, 21])) .build_and_execute(|| { - // given our nominators ordered by stake, + // given our voters ordered by stake, assert_eq!( - ::SortedListProvider::iter().collect::>(), - vec![101, 61, 71] + ::VoterList::iter().collect::>(), + vec![11, 21, 31, 61, 71, 81] ); - // and total voters - assert_eq!( - ::SortedListProvider::count() + - >::iter().count() as u32, - 6 - ); - - // we take 5 voters + // we take 4 voters assert_eq!( - Staking::voters(Some(5)) + Staking::electing_voters(Some(4)) .unwrap() .iter() .map(|(stash, _, _)| stash) .copied() .collect::>(), - // then - vec![ - 31, 21, 11, // 3 nominators - 101, 61 // 2 validators, and 71 is excluded - ], + vec![11, 21, 31, 61], ); // roll to session 5 run_to_block(25); - // slash 21, the only validator nominated by 61 + // slash 21, the only validator nominated by 61. add_slash(&21); - // we take 4 voters + // we take 4 voters; 71 and 81 are replacing the ejected ones. assert_eq!( - Staking::voters(Some(4)) + Staking::electing_voters(Some(4)) .unwrap() .iter() .map(|(stash, _, _)| stash) .copied() .collect::>(), - vec![ - 31, 11, // 2 validators (21 was slashed) - 101, 71 // 2 nominators, excluding 61 - ], + vec![11, 31, 71, 81], ); }); } @@ -4137,7 +4290,11 @@ fn count_check_works() { Validators::::insert(987654321, ValidatorPrefs::default()); Nominators::::insert( 987654321, - Nominations { targets: vec![], submitted_in: Default::default(), suppressed: false }, + Nominations { + targets: Default::default(), + submitted_in: Default::default(), + suppressed: false, + }, ); }) } @@ -4196,8 +4353,8 @@ fn chill_other_works() { .min_nominator_bond(1_000) .min_validator_bond(1_500) .build_and_execute(|| { - let initial_validators = CounterForValidators::::get(); - let initial_nominators = CounterForNominators::::get(); + let initial_validators = Validators::::count(); + let initial_nominators = Nominators::::count(); for i in 0..15 { let a = 4 * i; let b = 4 * i + 1; @@ -4246,7 +4403,15 @@ fn chill_other_works() { ); // Change the minimum bond... but no limits. - assert_ok!(Staking::set_staking_limits(Origin::root(), 1_500, 2_000, None, None, None)); + assert_ok!(Staking::set_staking_configs( + Origin::root(), + ConfigOp::Set(1_500), + ConfigOp::Set(2_000), + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove + )); // Still can't chill these users assert_noop!( @@ -4259,13 +4424,14 @@ fn chill_other_works() { ); // Add limits, but no threshold - assert_ok!(Staking::set_staking_limits( + assert_ok!(Staking::set_staking_configs( Origin::root(), - 1_500, - 2_000, - Some(10), - Some(10), - None + ConfigOp::Noop, + ConfigOp::Noop, + ConfigOp::Set(10), + ConfigOp::Set(10), + ConfigOp::Noop, + ConfigOp::Noop )); // Still can't chill these users @@ -4279,13 +4445,14 @@ fn chill_other_works() { ); // Add threshold, but no limits - assert_ok!(Staking::set_staking_limits( + assert_ok!(Staking::set_staking_configs( Origin::root(), - 1_500, - 2_000, - None, - None, - Some(Percent::from_percent(0)) + ConfigOp::Noop, + ConfigOp::Noop, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Noop, + ConfigOp::Noop )); // Still can't chill these users @@ -4299,18 +4466,19 @@ fn chill_other_works() { ); // Add threshold and limits - assert_ok!(Staking::set_staking_limits( + assert_ok!(Staking::set_staking_configs( Origin::root(), - 1_500, - 2_000, - Some(10), - Some(10), - Some(Percent::from_percent(75)) + ConfigOp::Noop, + ConfigOp::Noop, + ConfigOp::Set(10), + ConfigOp::Set(10), + ConfigOp::Set(Percent::from_percent(75)), + ConfigOp::Noop )); // 16 people total because tests start with 2 active one - assert_eq!(CounterForNominators::::get(), 15 + initial_nominators); - assert_eq!(CounterForValidators::::get(), 15 + initial_validators); + assert_eq!(Nominators::::count(), 15 + initial_nominators); + assert_eq!(Validators::::count(), 15 + initial_validators); // Users can now be chilled down to 7 people, so we try to remove 9 of them (starting // with 16) @@ -4322,13 +4490,13 @@ fn chill_other_works() { } // chill a nominator. Limit is not reached, not chill-able - assert_eq!(CounterForNominators::::get(), 7); + assert_eq!(Nominators::::count(), 7); assert_noop!( Staking::chill_other(Origin::signed(1337), 1), Error::::CannotChillOther ); // chill a validator. Limit is reached, chill-able. - assert_eq!(CounterForValidators::::get(), 9); + assert_eq!(Validators::::count(), 9); assert_ok!(Staking::chill_other(Origin::signed(1337), 3)); }) } @@ -4336,20 +4504,21 @@ fn chill_other_works() { #[test] fn capped_stakers_works() { ExtBuilder::default().build_and_execute(|| { - let validator_count = CounterForValidators::::get(); + let validator_count = Validators::::count(); assert_eq!(validator_count, 3); - let nominator_count = CounterForNominators::::get(); + let nominator_count = Nominators::::count(); assert_eq!(nominator_count, 1); // Change the maximums let max = 10; - assert_ok!(Staking::set_staking_limits( + assert_ok!(Staking::set_staking_configs( Origin::root(), - 10, - 10, - Some(max), - Some(max), - Some(Percent::from_percent(0)) + ConfigOp::Set(10), + ConfigOp::Set(10), + ConfigOp::Set(max), + ConfigOp::Set(max), + ConfigOp::Remove, + ConfigOp::Remove, )); // can create `max - validator_count` validators @@ -4412,12 +4581,154 @@ fn capped_stakers_works() { )); // No problem when we set to `None` again - assert_ok!(Staking::set_staking_limits(Origin::root(), 10, 10, None, None, None)); + assert_ok!(Staking::set_staking_configs( + Origin::root(), + ConfigOp::Noop, + ConfigOp::Noop, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Noop, + ConfigOp::Noop, + )); assert_ok!(Staking::nominate(Origin::signed(last_nominator), vec![1])); assert_ok!(Staking::validate(Origin::signed(last_validator), ValidatorPrefs::default())); }) } +#[test] +fn min_commission_works() { + ExtBuilder::default().build_and_execute(|| { + assert_ok!(Staking::validate( + Origin::signed(10), + ValidatorPrefs { commission: Perbill::from_percent(5), blocked: false } + )); + + assert_ok!(Staking::set_staking_configs( + Origin::root(), + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Remove, + ConfigOp::Set(Perbill::from_percent(10)), + )); + + // can't make it less than 10 now + assert_noop!( + Staking::validate( + Origin::signed(10), + ValidatorPrefs { commission: Perbill::from_percent(5), blocked: false } + ), + Error::::CommissionTooLow + ); + + // can only change to higher. + assert_ok!(Staking::validate( + Origin::signed(10), + ValidatorPrefs { commission: Perbill::from_percent(10), blocked: false } + )); + + assert_ok!(Staking::validate( + Origin::signed(10), + ValidatorPrefs { commission: Perbill::from_percent(15), blocked: false } + )); + }) +} + +#[test] +fn change_of_max_nominations() { + use frame_election_provider_support::ElectionDataProvider; + ExtBuilder::default() + .add_staker(60, 61, 10, StakerStatus::Nominator(vec![1])) + .add_staker(70, 71, 10, StakerStatus::Nominator(vec![1, 2, 3])) + .balance_factor(10) + .build_and_execute(|| { + // pre-condition + assert_eq!(MaxNominations::get(), 16); + + assert_eq!( + Nominators::::iter() + .map(|(k, n)| (k, n.targets.len())) + .collect::>(), + vec![(70, 3), (101, 2), (60, 1)] + ); + // 3 validators and 3 nominators + assert_eq!(Staking::electing_voters(None).unwrap().len(), 3 + 3); + + // abrupt change from 16 to 4, everyone should be fine. + MaxNominations::set(4); + + assert_eq!( + Nominators::::iter() + .map(|(k, n)| (k, n.targets.len())) + .collect::>(), + vec![(70, 3), (101, 2), (60, 1)] + ); + assert_eq!(Staking::electing_voters(None).unwrap().len(), 3 + 3); + + // abrupt change from 4 to 3, everyone should be fine. + MaxNominations::set(3); + + assert_eq!( + Nominators::::iter() + .map(|(k, n)| (k, n.targets.len())) + .collect::>(), + vec![(70, 3), (101, 2), (60, 1)] + ); + assert_eq!(Staking::electing_voters(None).unwrap().len(), 3 + 3); + + // abrupt change from 3 to 2, this should cause some nominators to be non-decodable, and + // thus non-existent unless if they update. + MaxNominations::set(2); + + assert_eq!( + Nominators::::iter() + .map(|(k, n)| (k, n.targets.len())) + .collect::>(), + vec![(101, 2), (60, 1)] + ); + // 70 is still in storage.. + assert!(Nominators::::contains_key(70)); + // but its value cannot be decoded and default is returned. + assert!(Nominators::::get(70).is_none()); + + assert_eq!(Staking::electing_voters(None).unwrap().len(), 3 + 2); + assert!(Nominators::::contains_key(101)); + + // abrupt change from 2 to 1, this should cause some nominators to be non-decodable, and + // thus non-existent unless if they update. + MaxNominations::set(1); + + assert_eq!( + Nominators::::iter() + .map(|(k, n)| (k, n.targets.len())) + .collect::>(), + vec![(60, 1)] + ); + assert!(Nominators::::contains_key(70)); + assert!(Nominators::::contains_key(60)); + assert!(Nominators::::get(70).is_none()); + assert!(Nominators::::get(60).is_some()); + assert_eq!(Staking::electing_voters(None).unwrap().len(), 3 + 1); + + // now one of them can revive themselves by re-nominating to a proper value. + assert_ok!(Staking::nominate(Origin::signed(71), vec![1])); + assert_eq!( + Nominators::::iter() + .map(|(k, n)| (k, n.targets.len())) + .collect::>(), + vec![(70, 1), (60, 1)] + ); + + // or they can be chilled by any account. + assert!(Nominators::::contains_key(101)); + assert!(Nominators::::get(101).is_none()); + assert_ok!(Staking::chill_other(Origin::signed(70), 100)); + assert!(!Nominators::::contains_key(101)); + assert!(Nominators::::get(101).is_none()); + }) +} + mod sorted_list_provider { use super::*; use frame_election_provider_support::SortedListProvider; @@ -4426,19 +4737,80 @@ mod sorted_list_provider { fn re_nominate_does_not_change_counters_or_list() { ExtBuilder::default().nominate(true).build_and_execute(|| { // given - let pre_insert_nominator_count = Nominators::::iter().count() as u32; - assert_eq!(::SortedListProvider::count(), pre_insert_nominator_count); - assert!(Nominators::::contains_key(101)); - assert_eq!(::SortedListProvider::iter().collect::>(), vec![101]); + let pre_insert_voter_count = + (Nominators::::count() + Validators::::count()) as u32; + assert_eq!(::VoterList::count(), pre_insert_voter_count); + + assert_eq!( + ::VoterList::iter().collect::>(), + vec![11, 21, 31, 101] + ); // when account 101 renominates assert_ok!(Staking::nominate(Origin::signed(100), vec![41])); // then counts don't change - assert_eq!(::SortedListProvider::count(), pre_insert_nominator_count); - assert_eq!(Nominators::::iter().count() as u32, pre_insert_nominator_count); + assert_eq!(::VoterList::count(), pre_insert_voter_count); + // and the list is the same + assert_eq!( + ::VoterList::iter().collect::>(), + vec![11, 21, 31, 101] + ); + }); + } + + #[test] + fn re_validate_does_not_change_counters_or_list() { + ExtBuilder::default().nominate(false).build_and_execute(|| { + // given + let pre_insert_voter_count = + (Nominators::::count() + Validators::::count()) as u32; + assert_eq!(::VoterList::count(), pre_insert_voter_count); + + assert_eq!(::VoterList::iter().collect::>(), vec![11, 21, 31]); + + // when account 11 re-validates + assert_ok!(Staking::validate(Origin::signed(10), Default::default())); + + // then counts don't change + assert_eq!(::VoterList::count(), pre_insert_voter_count); // and the list is the same - assert_eq!(::SortedListProvider::iter().collect::>(), vec![101]); + assert_eq!(::VoterList::iter().collect::>(), vec![11, 21, 31]); }); } } + +#[test] +fn force_apply_min_commission_works() { + let prefs = |c| ValidatorPrefs { commission: Perbill::from_percent(c), blocked: false }; + let validators = || Validators::::iter().collect::>(); + ExtBuilder::default().build_and_execute(|| { + assert_ok!(Staking::validate(Origin::signed(30), prefs(10))); + assert_ok!(Staking::validate(Origin::signed(20), prefs(5))); + + // Given + assert_eq!(validators(), vec![(31, prefs(10)), (21, prefs(5)), (11, prefs(0))]); + MinCommission::::set(Perbill::from_percent(5)); + + // When applying to a commission greater than min + assert_ok!(Staking::force_apply_min_commission(Origin::signed(1), 31)); + // Then the commission is not changed + assert_eq!(validators(), vec![(31, prefs(10)), (21, prefs(5)), (11, prefs(0))]); + + // When applying to a commission that is equal to min + assert_ok!(Staking::force_apply_min_commission(Origin::signed(1), 21)); + // Then the commission is not changed + assert_eq!(validators(), vec![(31, prefs(10)), (21, prefs(5)), (11, prefs(0))]); + + // When applying to a commission that is less than the min + assert_ok!(Staking::force_apply_min_commission(Origin::signed(1), 11)); + // Then the commission is bumped to the min + assert_eq!(validators(), vec![(31, prefs(10)), (21, prefs(5)), (11, prefs(5))]); + + // When applying commission to a validator that doesn't exist then storage is not altered + assert_noop!( + Staking::force_apply_min_commission(Origin::signed(1), 420), + Error::::NotStash + ); + }); +} diff --git a/frame/staking/src/weights.rs b/frame/staking/src/weights.rs index 32c8dc80da15..9d43cd882260 100644 --- a/frame/staking/src/weights.rs +++ b/frame/staking/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_staking //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-09-04, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-03-02, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -35,7 +35,6 @@ // --output=./frame/staking/src/weights.rs // --template=./.maintain/frame-weight-template.hbs - #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] #![allow(unused_imports)] @@ -71,8 +70,10 @@ pub trait WeightInfo { fn new_era(v: u32, n: u32, ) -> Weight; fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight; fn get_npos_targets(v: u32, ) -> Weight; - fn set_staking_limits() -> Weight; + fn set_staking_configs_all_set() -> Weight; + fn set_staking_configs_all_remove() -> Weight; fn chill_other() -> Weight; + fn force_apply_min_commission() -> Weight; } /// Weights for pallet_staking using the Substrate node and recommended hardware. @@ -85,7 +86,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn bond() -> Weight { - (73_865_000 as Weight) + (37_238_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } @@ -95,7 +96,7 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListNodes (r:3 w:3) // Storage: BagsList ListBags (r:2 w:2) fn bond_extra() -> Weight { - (114_296_000 as Weight) + (64_061_000 as Weight) .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(7 as Weight)) } @@ -109,7 +110,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Bonded (r:1 w:0) // Storage: BagsList ListBags (r:2 w:2) fn unbond() -> Weight { - (121_737_000 as Weight) + (70_069_000 as Weight) .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().writes(8 as Weight)) } @@ -118,9 +119,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn withdraw_unbonded_update(s: u32, ) -> Weight { - (51_631_000 as Weight) + (29_855_000 as Weight) // Standard Error: 0 - .saturating_add((55_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((53_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -137,13 +138,16 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) - fn withdraw_unbonded_kill(_s: u32, ) -> Weight { - (101_870_000 as Weight) + fn withdraw_unbonded_kill(s: u32, ) -> Weight { + (57_313_000 as Weight) + // Standard Error: 0 + .saturating_add((1_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(13 as Weight)) .saturating_add(T::DbWeight::get().writes(11 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking MinValidatorBond (r:1 w:0) + // Storage: Staking MinCommission (r:1 w:0) // Storage: Staking Validators (r:1 w:1) // Storage: Staking MaxValidatorsCount (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) @@ -153,16 +157,16 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Staking CounterForValidators (r:1 w:1) fn validate() -> Weight { - (69_092_000 as Weight) - .saturating_add(T::DbWeight::get().reads(11 as Weight)) + (44_448_000 as Weight) + .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().writes(8 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) fn kick(k: u32, ) -> Weight { - (21_468_000 as Weight) - // Standard Error: 19_000 - .saturating_add((16_415_000 as Weight).saturating_mul(k as Weight)) + (13_902_000 as Weight) + // Standard Error: 12_000 + .saturating_add((7_421_000 as Weight).saturating_mul(k as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) @@ -173,15 +177,15 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking MaxNominatorsCount (r:1 w:0) // Storage: Staking Validators (r:2 w:0) // Storage: Staking CurrentEra (r:1 w:0) - // Storage: Staking CounterForNominators (r:1 w:1) // Storage: Staking Bonded (r:1 w:0) // Storage: BagsList ListNodes (r:2 w:2) // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) fn nominate(n: u32, ) -> Weight { - (82_389_000 as Weight) - // Standard Error: 14_000 - .saturating_add((5_597_000 as Weight).saturating_mul(n as Weight)) + (49_580_000 as Weight) + // Standard Error: 9_000 + .saturating_add((3_362_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(6 as Weight)) @@ -194,49 +198,49 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill() -> Weight { - (69_655_000 as Weight) + (44_180_000 as Weight) .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Payee (r:0 w:1) fn set_payee() -> Weight { - (12_770_000 as Weight) + (7_922_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) // Storage: Staking Ledger (r:2 w:2) fn set_controller() -> Weight { - (27_756_000 as Weight) + (15_436_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } // Storage: Staking ValidatorCount (r:0 w:1) fn set_validator_count() -> Weight { - (2_446_000 as Weight) + (1_091_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_no_eras() -> Weight { - (2_720_000 as Weight) + (1_204_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era() -> Weight { - (2_711_000 as Weight) + (1_208_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era_always() -> Weight { - (2_796_000 as Weight) + (1_220_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Invulnerables (r:0 w:1) fn set_invulnerables(v: u32, ) -> Weight { - (3_141_000 as Weight) + (1_473_000 as Weight) // Standard Error: 0 - .saturating_add((53_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((9_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) @@ -253,18 +257,18 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:2) fn force_unstake(s: u32, ) -> Weight { - (97_394_000 as Weight) - // Standard Error: 2_000 - .saturating_add((2_370_000 as Weight).saturating_mul(s as Weight)) + (55_815_000 as Weight) + // Standard Error: 1_000 + .saturating_add((808_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().writes(12 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking UnappliedSlashes (r:1 w:1) fn cancel_deferred_slash(s: u32, ) -> Weight { - (2_783_746_000 as Weight) - // Standard Error: 182_000 - .saturating_add((16_223_000 as Weight).saturating_mul(s as Weight)) + (903_077_000 as Weight) + // Standard Error: 53_000 + .saturating_add((4_434_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -279,9 +283,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Payee (r:2 w:0) // Storage: System Account (r:2 w:2) fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (109_233_000 as Weight) - // Standard Error: 17_000 - .saturating_add((47_612_000 as Weight).saturating_mul(n as Weight)) + (79_440_000 as Weight) + // Standard Error: 14_000 + .saturating_add((24_005_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) @@ -299,9 +303,9 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:2 w:2) // Storage: Balances Locks (r:2 w:2) fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (177_392_000 as Weight) + (99_118_000 as Weight) // Standard Error: 20_000 - .saturating_add((60_771_000 as Weight).saturating_mul(n as Weight)) + .saturating_add((33_274_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(3 as Weight)) @@ -314,9 +318,9 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Bonded (r:1 w:0) // Storage: BagsList ListBags (r:2 w:2) fn rebond(l: u32, ) -> Weight { - (111_858_000 as Weight) - // Standard Error: 4_000 - .saturating_add((36_000 as Weight).saturating_mul(l as Weight)) + (63_335_000 as Weight) + // Standard Error: 2_000 + .saturating_add((53_000 as Weight).saturating_mul(l as Weight)) .saturating_add(T::DbWeight::get().reads(9 as Weight)) .saturating_add(T::DbWeight::get().writes(8 as Weight)) } @@ -331,14 +335,15 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 68_000 - .saturating_add((33_495_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 58_000 + .saturating_add((20_386_000 as Weight).saturating_mul(e as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) .saturating_add(T::DbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) } // Storage: System Account (r:1 w:1) // Storage: Staking Bonded (r:1 w:1) + // Storage: Staking Ledger (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:1) // Storage: Staking Validators (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) @@ -347,14 +352,13 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Balances Locks (r:1 w:1) - // Storage: Staking Ledger (r:0 w:1) // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:1) fn reap_stash(s: u32, ) -> Weight { - (100_178_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_358_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(11 as Weight)) + (61_486_000 as Weight) + // Standard Error: 0 + .saturating_add((809_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().writes(12 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } @@ -378,10 +382,10 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 860_000 - .saturating_add((298_721_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 43_000 - .saturating_add((49_427_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 878_000 + .saturating_add((212_233_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 44_000 + .saturating_add((30_364_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(208 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -399,12 +403,12 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Nominators (r:1000 w:0) fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 91_000 - .saturating_add((26_605_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 91_000 - .saturating_add((31_481_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_122_000 - .saturating_add((16_672_000 as Weight).saturating_mul(s as Weight)) + // Standard Error: 95_000 + .saturating_add((18_439_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 95_000 + .saturating_add((20_382_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_232_000 + .saturating_add((4_870_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(204 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -413,23 +417,34 @@ impl WeightInfo for SubstrateWeight { // Storage: Staking Validators (r:501 w:0) fn get_npos_targets(v: u32, ) -> Weight { (0 as Weight) - // Standard Error: 34_000 - .saturating_add((10_558_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 29_000 + .saturating_add((7_552_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } + // Storage: Staking MinCommission (r:0 w:1) // Storage: Staking MinValidatorBond (r:0 w:1) // Storage: Staking MaxValidatorsCount (r:0 w:1) // Storage: Staking ChillThreshold (r:0 w:1) // Storage: Staking MaxNominatorsCount (r:0 w:1) // Storage: Staking MinNominatorBond (r:0 w:1) - fn set_staking_limits() -> Weight { - (6_353_000 as Weight) - .saturating_add(T::DbWeight::get().writes(5 as Weight)) + fn set_staking_configs_all_set() -> Weight { + (3_597_000 as Weight) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) + } + // Storage: Staking MinCommission (r:0 w:1) + // Storage: Staking MinValidatorBond (r:0 w:1) + // Storage: Staking MaxValidatorsCount (r:0 w:1) + // Storage: Staking ChillThreshold (r:0 w:1) + // Storage: Staking MaxNominatorsCount (r:0 w:1) + // Storage: Staking MinNominatorBond (r:0 w:1) + fn set_staking_configs_all_remove() -> Weight { + (3_198_000 as Weight) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) - // Storage: Staking ChillThreshold (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking ChillThreshold (r:1 w:0) // Storage: Staking MaxNominatorsCount (r:1 w:0) // Storage: Staking CounterForNominators (r:1 w:1) // Storage: Staking MinNominatorBond (r:1 w:0) @@ -438,10 +453,17 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill_other() -> Weight { - (83_389_000 as Weight) + (55_725_000 as Weight) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) } + // Storage: Staking MinCommission (r:1 w:0) + // Storage: Staking Validators (r:1 w:1) + fn force_apply_min_commission() -> Weight { + (8_946_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } } // For backwards compatibility and tests @@ -453,7 +475,7 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) fn bond() -> Weight { - (73_865_000 as Weight) + (37_238_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } @@ -463,7 +485,7 @@ impl WeightInfo for () { // Storage: BagsList ListNodes (r:3 w:3) // Storage: BagsList ListBags (r:2 w:2) fn bond_extra() -> Weight { - (114_296_000 as Weight) + (64_061_000 as Weight) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(7 as Weight)) } @@ -477,7 +499,7 @@ impl WeightInfo for () { // Storage: Staking Bonded (r:1 w:0) // Storage: BagsList ListBags (r:2 w:2) fn unbond() -> Weight { - (121_737_000 as Weight) + (70_069_000 as Weight) .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } @@ -486,9 +508,9 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn withdraw_unbonded_update(s: u32, ) -> Weight { - (51_631_000 as Weight) + (29_855_000 as Weight) // Standard Error: 0 - .saturating_add((55_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((53_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -505,13 +527,16 @@ impl WeightInfo for () { // Storage: System Account (r:1 w:1) // Storage: Balances Locks (r:1 w:1) // Storage: Staking Payee (r:0 w:1) - fn withdraw_unbonded_kill(_s: u32, ) -> Weight { - (101_870_000 as Weight) + fn withdraw_unbonded_kill(s: u32, ) -> Weight { + (57_313_000 as Weight) + // Standard Error: 0 + .saturating_add((1_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(13 as Weight)) .saturating_add(RocksDbWeight::get().writes(11 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking MinValidatorBond (r:1 w:0) + // Storage: Staking MinCommission (r:1 w:0) // Storage: Staking Validators (r:1 w:1) // Storage: Staking MaxValidatorsCount (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) @@ -521,16 +546,16 @@ impl WeightInfo for () { // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Staking CounterForValidators (r:1 w:1) fn validate() -> Weight { - (69_092_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + (44_448_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) fn kick(k: u32, ) -> Weight { - (21_468_000 as Weight) - // Standard Error: 19_000 - .saturating_add((16_415_000 as Weight).saturating_mul(k as Weight)) + (13_902_000 as Weight) + // Standard Error: 12_000 + .saturating_add((7_421_000 as Weight).saturating_mul(k as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) @@ -541,15 +566,15 @@ impl WeightInfo for () { // Storage: Staking MaxNominatorsCount (r:1 w:0) // Storage: Staking Validators (r:2 w:0) // Storage: Staking CurrentEra (r:1 w:0) - // Storage: Staking CounterForNominators (r:1 w:1) // Storage: Staking Bonded (r:1 w:0) // Storage: BagsList ListNodes (r:2 w:2) // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) + // Storage: Staking CounterForNominators (r:1 w:1) fn nominate(n: u32, ) -> Weight { - (82_389_000 as Weight) - // Standard Error: 14_000 - .saturating_add((5_597_000 as Weight).saturating_mul(n as Weight)) + (49_580_000 as Weight) + // Standard Error: 9_000 + .saturating_add((3_362_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) @@ -562,49 +587,49 @@ impl WeightInfo for () { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill() -> Weight { - (69_655_000 as Weight) + (44_180_000 as Weight) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) // Storage: Staking Payee (r:0 w:1) fn set_payee() -> Weight { - (12_770_000 as Weight) + (7_922_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) // Storage: Staking Ledger (r:2 w:2) fn set_controller() -> Weight { - (27_756_000 as Weight) + (15_436_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } // Storage: Staking ValidatorCount (r:0 w:1) fn set_validator_count() -> Weight { - (2_446_000 as Weight) + (1_091_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_no_eras() -> Weight { - (2_720_000 as Weight) + (1_204_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era() -> Weight { - (2_711_000 as Weight) + (1_208_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking ForceEra (r:0 w:1) fn force_new_era_always() -> Weight { - (2_796_000 as Weight) + (1_220_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Invulnerables (r:0 w:1) fn set_invulnerables(v: u32, ) -> Weight { - (3_141_000 as Weight) + (1_473_000 as Weight) // Standard Error: 0 - .saturating_add((53_000 as Weight).saturating_mul(v as Weight)) + .saturating_add((9_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Staking Bonded (r:1 w:1) @@ -621,18 +646,18 @@ impl WeightInfo for () { // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:2) fn force_unstake(s: u32, ) -> Weight { - (97_394_000 as Weight) - // Standard Error: 2_000 - .saturating_add((2_370_000 as Weight).saturating_mul(s as Weight)) + (55_815_000 as Weight) + // Standard Error: 1_000 + .saturating_add((808_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().writes(12 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } // Storage: Staking UnappliedSlashes (r:1 w:1) fn cancel_deferred_slash(s: u32, ) -> Weight { - (2_783_746_000 as Weight) - // Standard Error: 182_000 - .saturating_add((16_223_000 as Weight).saturating_mul(s as Weight)) + (903_077_000 as Weight) + // Standard Error: 53_000 + .saturating_add((4_434_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -647,9 +672,9 @@ impl WeightInfo for () { // Storage: Staking Payee (r:2 w:0) // Storage: System Account (r:2 w:2) fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (109_233_000 as Weight) - // Standard Error: 17_000 - .saturating_add((47_612_000 as Weight).saturating_mul(n as Weight)) + (79_440_000 as Weight) + // Standard Error: 14_000 + .saturating_add((24_005_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) @@ -667,9 +692,9 @@ impl WeightInfo for () { // Storage: System Account (r:2 w:2) // Storage: Balances Locks (r:2 w:2) fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (177_392_000 as Weight) + (99_118_000 as Weight) // Standard Error: 20_000 - .saturating_add((60_771_000 as Weight).saturating_mul(n as Weight)) + .saturating_add((33_274_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) @@ -682,9 +707,9 @@ impl WeightInfo for () { // Storage: Staking Bonded (r:1 w:0) // Storage: BagsList ListBags (r:2 w:2) fn rebond(l: u32, ) -> Weight { - (111_858_000 as Weight) - // Standard Error: 4_000 - .saturating_add((36_000 as Weight).saturating_mul(l as Weight)) + (63_335_000 as Weight) + // Standard Error: 2_000 + .saturating_add((53_000 as Weight).saturating_mul(l as Weight)) .saturating_add(RocksDbWeight::get().reads(9 as Weight)) .saturating_add(RocksDbWeight::get().writes(8 as Weight)) } @@ -699,14 +724,15 @@ impl WeightInfo for () { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 68_000 - .saturating_add((33_495_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 58_000 + .saturating_add((20_386_000 as Weight).saturating_mul(e as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) .saturating_add(RocksDbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) } // Storage: System Account (r:1 w:1) // Storage: Staking Bonded (r:1 w:1) + // Storage: Staking Ledger (r:1 w:1) // Storage: Staking SlashingSpans (r:1 w:1) // Storage: Staking Validators (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) @@ -715,14 +741,13 @@ impl WeightInfo for () { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) // Storage: Balances Locks (r:1 w:1) - // Storage: Staking Ledger (r:0 w:1) // Storage: Staking Payee (r:0 w:1) // Storage: Staking SpanSlash (r:0 w:1) fn reap_stash(s: u32, ) -> Weight { - (100_178_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_358_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + (61_486_000 as Weight) + // Standard Error: 0 + .saturating_add((809_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().writes(12 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } @@ -746,10 +771,10 @@ impl WeightInfo for () { // Storage: Staking ErasStartSessionIndex (r:0 w:1) fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 860_000 - .saturating_add((298_721_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 43_000 - .saturating_add((49_427_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 878_000 + .saturating_add((212_233_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 44_000 + .saturating_add((30_364_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(208 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -767,12 +792,12 @@ impl WeightInfo for () { // Storage: Staking Nominators (r:1000 w:0) fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 91_000 - .saturating_add((26_605_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 91_000 - .saturating_add((31_481_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_122_000 - .saturating_add((16_672_000 as Weight).saturating_mul(s as Weight)) + // Standard Error: 95_000 + .saturating_add((18_439_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 95_000 + .saturating_add((20_382_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_232_000 + .saturating_add((4_870_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(204 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -781,23 +806,34 @@ impl WeightInfo for () { // Storage: Staking Validators (r:501 w:0) fn get_npos_targets(v: u32, ) -> Weight { (0 as Weight) - // Standard Error: 34_000 - .saturating_add((10_558_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 29_000 + .saturating_add((7_552_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } + // Storage: Staking MinCommission (r:0 w:1) // Storage: Staking MinValidatorBond (r:0 w:1) // Storage: Staking MaxValidatorsCount (r:0 w:1) // Storage: Staking ChillThreshold (r:0 w:1) // Storage: Staking MaxNominatorsCount (r:0 w:1) // Storage: Staking MinNominatorBond (r:0 w:1) - fn set_staking_limits() -> Weight { - (6_353_000 as Weight) - .saturating_add(RocksDbWeight::get().writes(5 as Weight)) + fn set_staking_configs_all_set() -> Weight { + (3_597_000 as Weight) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + } + // Storage: Staking MinCommission (r:0 w:1) + // Storage: Staking MinValidatorBond (r:0 w:1) + // Storage: Staking MaxValidatorsCount (r:0 w:1) + // Storage: Staking ChillThreshold (r:0 w:1) + // Storage: Staking MaxNominatorsCount (r:0 w:1) + // Storage: Staking MinNominatorBond (r:0 w:1) + fn set_staking_configs_all_remove() -> Weight { + (3_198_000 as Weight) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } // Storage: Staking Ledger (r:1 w:0) - // Storage: Staking ChillThreshold (r:1 w:0) // Storage: Staking Nominators (r:1 w:1) + // Storage: Staking ChillThreshold (r:1 w:0) // Storage: Staking MaxNominatorsCount (r:1 w:0) // Storage: Staking CounterForNominators (r:1 w:1) // Storage: Staking MinNominatorBond (r:1 w:0) @@ -806,8 +842,15 @@ impl WeightInfo for () { // Storage: BagsList ListBags (r:1 w:1) // Storage: BagsList CounterForListNodes (r:1 w:1) fn chill_other() -> Weight { - (83_389_000 as Weight) + (55_725_000 as Weight) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } + // Storage: Staking MinCommission (r:1 w:0) + // Storage: Staking Validators (r:1 w:1) + fn force_apply_min_commission() -> Weight { + (8_946_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } } diff --git a/frame/state-trie-migration/Cargo.toml b/frame/state-trie-migration/Cargo.toml new file mode 100644 index 000000000000..762fd85f13b6 --- /dev/null +++ b/frame/state-trie-migration/Cargo.toml @@ -0,0 +1,58 @@ +[package] +name = "pallet-state-trie-migration" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet migration of trie" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +log = { version = "0.4.14", default-features = false } + +sp-std = { default-features = false, path = "../../primitives/std" } +sp-io = { default-features = false, path = "../../primitives/io" } +sp-core = { default-features = false, path = "../../primitives/core" } +sp-runtime = { default-features = false, path = "../../primitives/runtime" } +substrate-state-trie-migration-rpc = { optional = true, path = "../../utils/frame/rpc/state-trie-migration-rpc" } + +frame-support = { default-features = false, path = "../support" } +frame-system = { default-features = false, path = "../system" } +frame-benchmarking = { default-features = false, path = "../benchmarking", optional = true } + +serde = { version = "1.0.133", optional = true } +thousands = { version = "0.2.0", optional = true } +remote-externalities = { path = "../../utils/frame/remote-externalities", optional = true } +zstd = { version = "0.9.0", optional = true } + +[dev-dependencies] +pallet-balances = { path = "../balances" } +parking_lot = "0.12.0" +sp-tracing = { path = "../../primitives/tracing" } +tokio = { version = "1.10", features = ["macros"] } + +[features] +default = ["std"] +std = [ + "log/std", + "scale-info/std", + "codec/std", + "frame-benchmarking/std", + "frame-support/std", + "frame-system/std", + "sp-core/std", + "sp-io/std", + "sp-runtime/std", + "sp-std/std", +] +runtime-benchmarks = ["frame-benchmarking"] +try-runtime = ["frame-support/try-runtime"] + +remote-test = [ "std", "zstd", "serde", "thousands", "remote-externalities", "substrate-state-trie-migration-rpc" ] diff --git a/frame/state-trie-migration/src/lib.rs b/frame/state-trie-migration/src/lib.rs new file mode 100644 index 000000000000..87be4099c89c --- /dev/null +++ b/frame/state-trie-migration/src/lib.rs @@ -0,0 +1,1608 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Pallet State Trie Migration +//! +//! Reads and writes all keys and values in the entire state in a systematic way. This is useful for +//! upgrading a chain to [`sp-core::StateVersion::V1`], where all keys need to be touched. +//! +//! ## Migration Types +//! +//! This pallet provides 2 ways to do this, each of which is suited for a particular use-case, and +//! can be enabled independently. +//! +//! ### Auto migration +//! +//! This system will try and migrate all keys by continuously using `on_initialize`. It is only +//! sensible for a relay chain or a solo chain, where going slightly over weight is not a problem. +//! It can be configured so that the migration takes at most `n` items and tries to not go over `x` +//! bytes, but the latter is not guaranteed. +//! +//! For example, if a chain contains keys of 1 byte size, the `on_initialize` could read up to `x - +//! 1` bytes from `n` different keys, while the next key is suddenly `:code:`, and there is no way +//! to bail out of this. +//! +//! ### Signed migration +//! +//! As a backup, the migration process can be set in motion via signed transactions that basically +//! say in advance how many items and how many bytes they will consume, and pay for it as well. This +//! can be a good safe alternative, if the former system is not desirable. +//! +//! The (minor) caveat of this approach is that we cannot know in advance how many bytes reading a +//! certain number of keys will incur. To overcome this, the runtime needs to configure this pallet +//! with a `SignedDepositPerItem`. This is the per-item deposit that the origin of the signed +//! migration transactions need to have in their account (on top of the normal fee) and if the size +//! witness data that they claim is incorrect, this deposit is slashed. +//! +//! --- +//! +//! Initially, this pallet does not contain any auto migration. They must be manually enabled by the +//! `ControlOrigin`. + +#![cfg_attr(not(feature = "std"), no_std)] + +pub use pallet::*; + +const LOG_TARGET: &'static str = "runtime::state-trie-migration"; + +#[macro_export] +macro_rules! log { + ($level:tt, $patter:expr $(, $values:expr)* $(,)?) => { + log::$level!( + target: crate::LOG_TARGET, + concat!("[{:?}] 🤖 ", $patter), frame_system::Pallet::::block_number() $(, $values)* + ) + }; +} + +#[frame_support::pallet] +pub mod pallet { + use frame_support::{ + dispatch::{DispatchErrorWithPostInfo, PostDispatchInfo}, + ensure, + pallet_prelude::*, + traits::{Currency, Get}, + }; + use frame_system::{self, pallet_prelude::*}; + use sp_core::storage::well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX; + use sp_runtime::{ + self, + traits::{Saturating, Zero}, + }; + use sp_std::prelude::*; + + pub(crate) type BalanceOf = + <::Currency as Currency<::AccountId>>::Balance; + + /// The weight information of this pallet. + pub trait WeightInfo { + fn process_top_key(x: u32) -> Weight; + fn continue_migrate() -> Weight; + fn continue_migrate_wrong_witness() -> Weight; + fn migrate_custom_top_fail() -> Weight; + fn migrate_custom_top_success() -> Weight; + fn migrate_custom_child_fail() -> Weight; + fn migrate_custom_child_success() -> Weight; + } + + impl WeightInfo for () { + fn process_top_key(_: u32) -> Weight { + 1000000 + } + fn continue_migrate() -> Weight { + 1000000 + } + fn continue_migrate_wrong_witness() -> Weight { + 1000000 + } + fn migrate_custom_top_fail() -> Weight { + 1000000 + } + fn migrate_custom_top_success() -> Weight { + 1000000 + } + fn migrate_custom_child_fail() -> Weight { + 1000000 + } + fn migrate_custom_child_success() -> Weight { + 1000000 + } + } + + /// The progress of either the top or child keys. + #[derive(Clone, Encode, Decode, scale_info::TypeInfo, PartialEq, Eq)] + pub enum Progress { + /// Yet to begin. + ToStart, + /// Ongoing, with the last key given. + LastKey(Vec), + /// All done. + Complete, + } + + /// A migration task stored in state. + /// + /// It tracks the last top and child keys read. + #[derive(Clone, Encode, Decode, scale_info::TypeInfo, PartialEq, Eq)] + #[codec(mel_bound(T: Config))] + #[scale_info(skip_type_params(T))] + pub struct MigrationTask { + /// The current top trie migration progress. + pub(crate) progress_top: Progress, + /// The current child trie migration progress. + /// + /// If `ToStart`, no further top keys are processed until the child key migration is + /// `Complete`. + pub(crate) progress_child: Progress, + + /// Dynamic counter for the number of items that we have processed in this execution from + /// the top trie. + /// + /// It is not written to storage. + #[codec(skip)] + pub(crate) dyn_top_items: u32, + /// Dynamic counter for the number of items that we have processed in this execution from + /// any child trie. + /// + /// It is not written to storage. + #[codec(skip)] + pub(crate) dyn_child_items: u32, + + /// Dynamic counter for for the byte size of items that we have processed in this + /// execution. + /// + /// It is not written to storage. + #[codec(skip)] + pub(crate) dyn_size: u32, + + /// The total size of the migration, over all executions. + /// + /// This only kept around for bookkeeping and debugging. + pub(crate) size: u32, + /// The total count of top keys in the migration, over all executions. + /// + /// This only kept around for bookkeeping and debugging. + pub(crate) top_items: u32, + /// The total count of child keys in the migration, over all executions. + /// + /// This only kept around for bookkeeping and debugging. + pub(crate) child_items: u32, + + #[codec(skip)] + pub(crate) _ph: sp_std::marker::PhantomData, + } + + impl sp_std::fmt::Debug for Progress { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { + match self { + Progress::ToStart => f.write_str("To start"), + Progress::LastKey(key) => + write!(f, "Last: {:?}", sp_core::hexdisplay::HexDisplay::from(key)), + Progress::Complete => f.write_str("Complete"), + } + } + } + + impl sp_std::fmt::Debug for MigrationTask { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { + f.debug_struct("MigrationTask") + .field("top", &self.progress_top) + .field("child", &self.progress_child) + .field("dyn_top_items", &self.dyn_top_items) + .field("dyn_child_items", &self.dyn_child_items) + .field("dyn_size", &self.dyn_size) + .field("size", &self.size) + .field("top_items", &self.top_items) + .field("child_items", &self.child_items) + .finish() + } + } + + impl Default for MigrationTask { + fn default() -> Self { + Self { + progress_top: Progress::ToStart, + progress_child: Progress::ToStart, + dyn_child_items: Default::default(), + dyn_top_items: Default::default(), + dyn_size: Default::default(), + _ph: Default::default(), + size: Default::default(), + top_items: Default::default(), + child_items: Default::default(), + } + } + } + + impl MigrationTask { + /// Return true if the task is finished. + pub(crate) fn finished(&self) -> bool { + matches!(self.progress_top, Progress::Complete) + } + + /// Check if there's any work left, or if we have exhausted the limits already. + fn exhausted(&self, limits: MigrationLimits) -> bool { + self.dyn_total_items() >= limits.item || self.dyn_size >= limits.size + } + + /// get the total number of keys affected by the current task. + pub(crate) fn dyn_total_items(&self) -> u32 { + self.dyn_child_items.saturating_add(self.dyn_top_items) + } + + /// Migrate keys until either of the given limits are exhausted, or if no more top keys + /// exist. + /// + /// Note that this can return after the **first** migration tick that causes exhaustion, + /// specifically in the case of the `size` constrain. The reason for this is that before + /// reading a key, we simply cannot know how many bytes it is. In other words, this should + /// not be used in any environment where resources are strictly bounded (e.g. a parachain), + /// but it is acceptable otherwise (relay chain, offchain workers). + pub fn migrate_until_exhaustion(&mut self, limits: MigrationLimits) { + log!(debug, "running migrations on top of {:?} until {:?}", self, limits); + + if limits.item.is_zero() || limits.size.is_zero() { + // handle this minor edge case, else we would call `migrate_tick` at least once. + log!(warn, "limits are zero. stopping"); + return + } + + while !self.exhausted(limits) && !self.finished() { + self.migrate_tick(); + } + + // accumulate dynamic data into the storage items. + self.size = self.size.saturating_add(self.dyn_size); + self.child_items = self.child_items.saturating_add(self.dyn_child_items); + self.top_items = self.top_items.saturating_add(self.dyn_top_items); + log!(debug, "finished with {:?}", self); + } + + /// Migrate AT MOST ONE KEY. This can be either a top or a child key. + /// + /// This function is *the* core of this entire pallet. + fn migrate_tick(&mut self) { + match (&self.progress_top, &self.progress_child) { + (Progress::ToStart, _) => { + self.migrate_top(); + }, + (Progress::LastKey(_), Progress::LastKey(_)) => { + // we're in the middle of doing work on a child tree. + self.migrate_child(); + }, + (Progress::LastKey(top_key), Progress::ToStart) => { + // 3. this is the root of a child key, and we are finishing all child-keys (and + // should call `migrate_top`). + + // NOTE: this block is written intentionally to verbosely for easy of + // verification. + if !top_key.starts_with(DEFAULT_CHILD_STORAGE_KEY_PREFIX) { + // we continue the top key migrations. + // continue the top key migration + self.migrate_top(); + } else { + // this is the root of a child key, and we start processing child keys (and + // should call `migrate_child`). + self.migrate_child(); + } + }, + (Progress::LastKey(_), Progress::Complete) => { + // we're done with migrating a child-root. + self.migrate_top(); + self.progress_child = Progress::ToStart; + }, + (Progress::Complete, _) => { + // nada + }, + } + } + + /// Migrate the current child key, setting it to its new value, if one exists. + /// + /// It updates the dynamic counters. + fn migrate_child(&mut self) { + use sp_io::default_child_storage as child_io; + let (maybe_current_child, child_root) = match (&self.progress_child, &self.progress_top) + { + (Progress::LastKey(last_child), Progress::LastKey(last_top)) => { + let child_root = Pallet::::transform_child_key_or_halt(&last_top); + let maybe_current_child = child_io::next_key(child_root, &last_child); + (maybe_current_child, child_root) + }, + (Progress::ToStart, Progress::LastKey(last_top)) => { + let child_root = Pallet::::transform_child_key_or_halt(&last_top); + // Start with the empty key as first key. + (Some(Vec::new()), child_root) + }, + _ => { + // defensive: there must be an ongoing top migration. + frame_support::defensive!("cannot migrate child key."); + return + }, + }; + + if let Some(current_child) = maybe_current_child.as_ref() { + let added_size = if let Some(data) = child_io::get(child_root, ¤t_child) { + child_io::set(child_root, current_child, &data); + data.len() as u32 + } else { + Zero::zero() + }; + self.dyn_size = self.dyn_size.saturating_add(added_size); + self.dyn_child_items.saturating_inc(); + } + + log!(trace, "migrated a child key, next_child_key: {:?}", maybe_current_child); + self.progress_child = match maybe_current_child { + Some(last_child) => Progress::LastKey(last_child), + None => Progress::Complete, + } + } + + /// Migrate the current top key, setting it to its new value, if one exists. + /// + /// It updates the dynamic counters. + fn migrate_top(&mut self) { + let maybe_current_top = match &self.progress_top { + Progress::LastKey(last_top) => sp_io::storage::next_key(last_top), + // Start with the empty key as first key. + Progress::ToStart => Some(Vec::new()), + Progress::Complete => { + // defensive: there must be an ongoing top migration. + frame_support::defensive!("cannot migrate top key."); + return + }, + }; + + if let Some(current_top) = maybe_current_top.as_ref() { + let added_size = if let Some(data) = sp_io::storage::get(¤t_top) { + sp_io::storage::set(¤t_top, &data); + data.len() as u32 + } else { + Zero::zero() + }; + self.dyn_size = self.dyn_size.saturating_add(added_size); + self.dyn_top_items.saturating_inc(); + } + + log!(trace, "migrated a top key, next_top_key = {:?}", maybe_current_top); + self.progress_top = match maybe_current_top { + Some(last_top) => Progress::LastKey(last_top), + None => Progress::Complete, + } + } + } + + /// The limits of a migration. + #[derive(Clone, Copy, Encode, Decode, scale_info::TypeInfo, Default, Debug, PartialEq, Eq)] + pub struct MigrationLimits { + /// The byte size limit. + pub size: u32, + /// The number of keys limit. + pub item: u32, + } + + /// How a migration was computed. + #[derive(Clone, Copy, Encode, Decode, scale_info::TypeInfo, Debug, PartialEq, Eq)] + pub enum MigrationCompute { + /// A signed origin triggered the migration. + Signed, + /// An automatic task triggered the migration. + Auto, + } + + /// Inner events of this pallet. + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// Given number of `(top, child)` keys were migrated respectively, with the given + /// `compute`. + Migrated { top: u32, child: u32, compute: MigrationCompute }, + /// Some account got slashed by the given amount. + Slashed { who: T::AccountId, amount: BalanceOf }, + /// The auto migration task finished. + AutoMigrationFinished, + /// Migration got halted. + Halted, + } + + /// The outer Pallet struct. + #[pallet::pallet] + #[pallet::generate_store(pub(crate) trait Store)] + #[pallet::without_storage_info] + pub struct Pallet(_); + + /// Configurations of this pallet. + #[pallet::config] + pub trait Config: frame_system::Config { + /// Origin that can control the configurations of this pallet. + type ControlOrigin: frame_support::traits::EnsureOrigin; + + /// Filter on which origin that trigger the manual migrations. + type SignedFilter: EnsureOrigin; + + /// The overarching event type. + type Event: From> + IsType<::Event>; + + /// The currency provider type. + type Currency: Currency; + + /// The amount of deposit collected per item in advance, for signed migrations. + /// + /// This should reflect the average storage value size in the worse case. + type SignedDepositPerItem: Get>; + + /// The base value of [`Config::SignedDepositPerItem`]. + /// + /// Final deposit is `items * SignedDepositPerItem + SignedDepositBase`. + type SignedDepositBase: Get>; + + /// The weight information of this pallet. + type WeightInfo: WeightInfo; + } + + /// Migration progress. + /// + /// This stores the snapshot of the last migrated keys. It can be set into motion and move + /// forward by any of the means provided by this pallet. + #[pallet::storage] + #[pallet::getter(fn migration_process)] + pub type MigrationProcess = StorageValue<_, MigrationTask, ValueQuery>; + + /// The limits that are imposed on automatic migrations. + /// + /// If set to None, then no automatic migration happens. + #[pallet::storage] + #[pallet::getter(fn auto_limits)] + pub type AutoLimits = StorageValue<_, Option, ValueQuery>; + + /// The maximum limits that the signed migration could use. + /// + /// If not set, no signed submission is allowed. + #[pallet::storage] + #[pallet::getter(fn signed_migration_max_limits)] + pub type SignedMigrationMaxLimits = StorageValue<_, MigrationLimits, OptionQuery>; + + #[pallet::error] + pub enum Error { + /// max signed limits not respected. + MaxSignedLimits, + /// submitter does not have enough funds. + NotEnoughFunds, + /// bad witness data provided. + BadWitness, + /// upper bound of size is exceeded, + SizeUpperBoundExceeded, + /// Signed migration is not allowed because the maximum limit is not set yet. + SignedMigrationNotAllowed, + } + + #[pallet::call] + impl Pallet { + /// Control the automatic migration. + /// + /// The dispatch origin of this call must be [`Config::ControlOrigin`]. + #[pallet::weight(T::DbWeight::get().reads_writes(1, 1))] + pub fn control_auto_migration( + origin: OriginFor, + maybe_config: Option, + ) -> DispatchResult { + T::ControlOrigin::ensure_origin(origin)?; + AutoLimits::::put(maybe_config); + Ok(().into()) + } + + /// Continue the migration for the given `limits`. + /// + /// The dispatch origin of this call can be any signed account. + /// + /// This transaction has NO MONETARY INCENTIVES. calling it will not reward anyone. Albeit, + /// Upon successful execution, the transaction fee is returned. + /// + /// The (potentially over-estimated) of the byte length of all the data read must be + /// provided for up-front fee-payment and weighing. In essence, the caller is guaranteeing + /// that executing the current `MigrationTask` with the given `limits` will not exceed + /// `real_size_upper` bytes of read data. + /// + /// The `witness_task` is merely a helper to prevent the caller from being slashed or + /// generally trigger a migration that they do not intend. This parameter is just a message + /// from caller, saying that they believed `witness_task` was the last state of the + /// migration, and they only wish for their transaction to do anything, if this assumption + /// holds. In case `witness_task` does not match, the transaction fails. + /// + /// Based on the documentation of [`MigrationTask::migrate_until_exhaustion`], the + /// recommended way of doing this is to pass a `limit` that only bounds `count`, as the + /// `size` limit can always be overwritten. + #[pallet::weight( + // the migration process + Pallet::::dynamic_weight(limits.item, * real_size_upper) + // rest of the operations, like deposit etc. + + T::WeightInfo::continue_migrate() + )] + pub fn continue_migrate( + origin: OriginFor, + limits: MigrationLimits, + real_size_upper: u32, + witness_task: MigrationTask, + ) -> DispatchResultWithPostInfo { + let who = T::SignedFilter::ensure_origin(origin)?; + + let max_limits = + Self::signed_migration_max_limits().ok_or(Error::::SignedMigrationNotAllowed)?; + ensure!( + limits.size <= max_limits.size && limits.item <= max_limits.item, + Error::::MaxSignedLimits, + ); + + // ensure they can pay more than the fee. + let deposit = T::SignedDepositPerItem::get().saturating_mul(limits.item.into()); + ensure!(T::Currency::can_slash(&who, deposit), Error::::NotEnoughFunds); + + let mut task = Self::migration_process(); + ensure!( + task == witness_task, + DispatchErrorWithPostInfo { + error: Error::::BadWitness.into(), + post_info: PostDispatchInfo { + actual_weight: Some(T::WeightInfo::continue_migrate_wrong_witness()), + pays_fee: Pays::Yes + } + } + ); + task.migrate_until_exhaustion(limits); + + // ensure that the migration witness data was correct. + if real_size_upper < task.dyn_size { + // let the imbalance burn. + let (_imbalance, _remainder) = T::Currency::slash(&who, deposit); + Self::deposit_event(Event::::Slashed { who, amount: deposit }); + debug_assert!(_remainder.is_zero()); + return Err(Error::::SizeUpperBoundExceeded.into()) + } + + Self::deposit_event(Event::::Migrated { + top: task.dyn_top_items, + child: task.dyn_child_items, + compute: MigrationCompute::Signed, + }); + + // refund and correct the weight. + let actual_weight = Some( + Pallet::::dynamic_weight(limits.item, task.dyn_size) + .saturating_add(T::WeightInfo::continue_migrate()), + ); + + MigrationProcess::::put(task); + Ok((actual_weight, Pays::No).into()) + } + + /// Migrate the list of top keys by iterating each of them one by one. + /// + /// This does not affect the global migration process tracker ([`MigrationProcess`]), and + /// should only be used in case any keys are leftover due to a bug. + #[pallet::weight( + T::WeightInfo::migrate_custom_top_success() + .max(T::WeightInfo::migrate_custom_top_fail()) + .saturating_add( + Pallet::::dynamic_weight(keys.len() as u32, *witness_size) + ) + )] + pub fn migrate_custom_top( + origin: OriginFor, + keys: Vec>, + witness_size: u32, + ) -> DispatchResultWithPostInfo { + let who = T::SignedFilter::ensure_origin(origin)?; + + // ensure they can pay more than the fee. + let deposit = T::SignedDepositBase::get().saturating_add( + T::SignedDepositPerItem::get().saturating_mul((keys.len() as u32).into()), + ); + ensure!(T::Currency::can_slash(&who, deposit), "not enough funds"); + + let mut dyn_size = 0u32; + for key in &keys { + if let Some(data) = sp_io::storage::get(&key) { + dyn_size = dyn_size.saturating_add(data.len() as u32); + sp_io::storage::set(key, &data); + } + } + + if dyn_size > witness_size { + let (_imbalance, _remainder) = T::Currency::slash(&who, deposit); + Self::deposit_event(Event::::Slashed { who, amount: deposit }); + debug_assert!(_remainder.is_zero()); + Err("wrong witness data".into()) + } else { + Self::deposit_event(Event::::Migrated { + top: keys.len() as u32, + child: 0, + compute: MigrationCompute::Signed, + }); + Ok(PostDispatchInfo { + actual_weight: Some( + T::WeightInfo::migrate_custom_top_success().saturating_add( + Pallet::::dynamic_weight(keys.len() as u32, dyn_size), + ), + ), + pays_fee: Pays::Yes, + }) + } + } + + /// Migrate the list of child keys by iterating each of them one by one. + /// + /// All of the given child keys must be present under one `child_root`. + /// + /// This does not affect the global migration process tracker ([`MigrationProcess`]), and + /// should only be used in case any keys are leftover due to a bug. + #[pallet::weight( + T::WeightInfo::migrate_custom_child_success() + .max(T::WeightInfo::migrate_custom_child_fail()) + .saturating_add( + Pallet::::dynamic_weight(child_keys.len() as u32, *total_size) + ) + )] + pub fn migrate_custom_child( + origin: OriginFor, + root: Vec, + child_keys: Vec>, + total_size: u32, + ) -> DispatchResultWithPostInfo { + use sp_io::default_child_storage as child_io; + let who = T::SignedFilter::ensure_origin(origin)?; + + // ensure they can pay more than the fee. + let deposit = T::SignedDepositBase::get().saturating_add( + T::SignedDepositPerItem::get().saturating_mul((child_keys.len() as u32).into()), + ); + sp_std::if_std! { + println!("+ {:?} / {:?} / {:?}", who, deposit, T::Currency::free_balance(&who)); + } + ensure!(T::Currency::can_slash(&who, deposit), "not enough funds"); + + let mut dyn_size = 0u32; + let transformed_child_key = Self::transform_child_key(&root).ok_or("bad child key")?; + for child_key in &child_keys { + if let Some(data) = child_io::get(transformed_child_key, &child_key) { + dyn_size = dyn_size.saturating_add(data.len() as u32); + child_io::set(transformed_child_key, &child_key, &data); + } + } + + if dyn_size != total_size { + let (_imbalance, _remainder) = T::Currency::slash(&who, deposit); + debug_assert!(_remainder.is_zero()); + Self::deposit_event(Event::::Slashed { who, amount: deposit }); + Err(DispatchErrorWithPostInfo { + error: "bad witness".into(), + post_info: PostDispatchInfo { + actual_weight: Some(T::WeightInfo::migrate_custom_child_fail()), + pays_fee: Pays::Yes, + }, + }) + } else { + Self::deposit_event(Event::::Migrated { + top: 0, + child: child_keys.len() as u32, + compute: MigrationCompute::Signed, + }); + Ok(PostDispatchInfo { + actual_weight: Some( + T::WeightInfo::migrate_custom_child_success().saturating_add( + Pallet::::dynamic_weight(child_keys.len() as u32, total_size), + ), + ), + pays_fee: Pays::Yes, + }) + } + } + + /// Set the maximum limit of the signed migration. + #[pallet::weight(T::DbWeight::get().reads_writes(1, 1))] + pub fn set_signed_max_limits( + origin: OriginFor, + limits: MigrationLimits, + ) -> DispatchResult { + let _ = T::ControlOrigin::ensure_origin(origin)?; + SignedMigrationMaxLimits::::put(limits); + Ok(()) + } + + /// Forcefully set the progress the running migration. + /// + /// This is only useful in one case: the next key to migrate is too big to be migrated with + /// a signed account, in a parachain context, and we simply want to skip it. A reasonable + /// example of this would be `:code:`, which is both very expensive to migrate, and commonly + /// used, so probably it is already migrated. + /// + /// In case you mess things up, you can also, in principle, use this to reset the migration + /// process. + #[pallet::weight(T::DbWeight::get().reads_writes(1, 1))] + pub fn force_set_progress( + origin: OriginFor, + progress_top: Progress, + progress_child: Progress, + ) -> DispatchResult { + let _ = T::ControlOrigin::ensure_origin(origin)?; + MigrationProcess::::mutate(|task| { + task.progress_top = progress_top; + task.progress_child = progress_child; + }); + Ok(()) + } + } + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_initialize(_: BlockNumberFor) -> Weight { + if let Some(limits) = Self::auto_limits() { + let mut task = Self::migration_process(); + task.migrate_until_exhaustion(limits); + let weight = Self::dynamic_weight(task.dyn_total_items(), task.dyn_size); + + log!( + info, + "migrated {} top keys, {} child keys, and a total of {} bytes.", + task.dyn_top_items, + task.dyn_child_items, + task.dyn_size, + ); + + if task.finished() { + Self::deposit_event(Event::::AutoMigrationFinished); + AutoLimits::::kill(); + } else { + Self::deposit_event(Event::::Migrated { + top: task.dyn_top_items, + child: task.dyn_child_items, + compute: MigrationCompute::Auto, + }); + } + + MigrationProcess::::put(task); + + weight + } else { + T::DbWeight::get().reads(1) + } + } + } + + impl Pallet { + /// The real weight of a migration of the given number of `items` with total `size`. + fn dynamic_weight(items: u32, size: u32) -> frame_support::pallet_prelude::Weight { + let items = items as Weight; + items + .saturating_mul(::DbWeight::get().reads_writes(1, 1)) + // we assume that the read/write per-byte weight is the same for child and top tree. + .saturating_add(T::WeightInfo::process_top_key(size)) + } + + /// Put a stop to all ongoing migrations. + fn halt() { + AutoLimits::::kill(); + Self::deposit_event(Event::::Halted); + } + + /// Convert a child root key, aka. "Child-bearing top key" into the proper format. + fn transform_child_key(root: &Vec) -> Option<&[u8]> { + use sp_core::storage::{ChildType, PrefixedStorageKey}; + match ChildType::from_prefixed_key(PrefixedStorageKey::new_ref(root)) { + Some((ChildType::ParentKeyId, root)) => Some(root), + _ => None, + } + } + + /// Same as [`child_io_key`], and it halts the auto/unsigned migrations if a bad child root + /// is used. + /// + /// This should be used when we are sure that `root` is a correct default child root. + fn transform_child_key_or_halt(root: &Vec) -> &[u8] { + let key = Self::transform_child_key(root); + if key.is_none() { + Self::halt(); + } + key.unwrap_or_default() + } + + /// Convert a child root to be in the default child-tree. + #[cfg(any(test, feature = "runtime-benchmarks"))] + pub(crate) fn childify(root: &'static str) -> Vec { + let mut string = DEFAULT_CHILD_STORAGE_KEY_PREFIX.to_vec(); + string.extend_from_slice(root.as_ref()); + string + } + } +} + +#[cfg(feature = "runtime-benchmarks")] +mod benchmarks { + use super::{pallet::Pallet as StateTrieMigration, *}; + use frame_support::traits::{Currency, Get}; + use sp_runtime::traits::Saturating; + use sp_std::prelude::*; + + // The size of the key seemingly makes no difference in the read/write time, so we make it + // constant. + const KEY: &'static [u8] = b"key"; + + frame_benchmarking::benchmarks! { + continue_migrate { + // note that this benchmark should migrate nothing, as we only want the overhead weight + // of the bookkeeping, and the migration cost itself is noted via the `dynamic_weight` + // function. + let null = MigrationLimits::default(); + let caller = frame_benchmarking::whitelisted_caller(); + }: _(frame_system::RawOrigin::Signed(caller), null, 0, StateTrieMigration::::migration_process()) + verify { + assert_eq!(StateTrieMigration::::migration_process(), Default::default()) + } + + continue_migrate_wrong_witness { + let null = MigrationLimits::default(); + let caller = frame_benchmarking::whitelisted_caller(); + let bad_witness = MigrationTask { progress_top: Progress::LastKey(vec![1u8]), ..Default::default() }; + }: { + assert!( + StateTrieMigration::::continue_migrate( + frame_system::RawOrigin::Signed(caller).into(), + null, + 0, + bad_witness, + ) + .is_err() + ) + } + verify { + assert_eq!(StateTrieMigration::::migration_process(), Default::default()) + } + + migrate_custom_top_success { + let null = MigrationLimits::default(); + let caller = frame_benchmarking::whitelisted_caller(); + let deposit = T::SignedDepositBase::get().saturating_add( + T::SignedDepositPerItem::get().saturating_mul(1u32.into()), + ); + let stash = T::Currency::minimum_balance() * BalanceOf::::from(1000u32) + deposit; + T::Currency::make_free_balance_be(&caller, stash); + }: migrate_custom_top(frame_system::RawOrigin::Signed(caller.clone()), Default::default(), 0) + verify { + assert_eq!(StateTrieMigration::::migration_process(), Default::default()); + assert_eq!(T::Currency::free_balance(&caller), stash) + } + + migrate_custom_top_fail { + let null = MigrationLimits::default(); + let caller = frame_benchmarking::whitelisted_caller(); + let deposit = T::SignedDepositBase::get().saturating_add( + T::SignedDepositPerItem::get().saturating_mul(1u32.into()), + ); + let stash = T::Currency::minimum_balance() * BalanceOf::::from(1000u32) + deposit; + T::Currency::make_free_balance_be(&caller, stash); + // for tests, we need to make sure there is _something_ in storage that is being + // migrated. + sp_io::storage::set(b"foo", vec![1u8;33].as_ref()); + }: { + assert!( + StateTrieMigration::::migrate_custom_top( + frame_system::RawOrigin::Signed(caller.clone()).into(), + vec![b"foo".to_vec()], + 1, + ).is_err() + ) + } + verify { + assert_eq!(StateTrieMigration::::migration_process(), Default::default()); + // must have gotten slashed + assert!(T::Currency::free_balance(&caller) < stash) + } + + migrate_custom_child_success { + let caller = frame_benchmarking::whitelisted_caller(); + let deposit = T::SignedDepositBase::get().saturating_add( + T::SignedDepositPerItem::get().saturating_mul(1u32.into()), + ); + let stash = T::Currency::minimum_balance() * BalanceOf::::from(1000u32) + deposit; + T::Currency::make_free_balance_be(&caller, stash); + }: migrate_custom_child( + frame_system::RawOrigin::Signed(caller.clone()), + StateTrieMigration::::childify(Default::default()), + Default::default(), + 0 + ) + verify { + assert_eq!(StateTrieMigration::::migration_process(), Default::default()); + assert_eq!(T::Currency::free_balance(&caller), stash); + } + + migrate_custom_child_fail { + let caller = frame_benchmarking::whitelisted_caller(); + let deposit = T::SignedDepositBase::get().saturating_add( + T::SignedDepositPerItem::get().saturating_mul(1u32.into()), + ); + let stash = T::Currency::minimum_balance() * BalanceOf::::from(1000u32) + deposit; + T::Currency::make_free_balance_be(&caller, stash); + // for tests, we need to make sure there is _something_ in storage that is being + // migrated. + sp_io::default_child_storage::set(b"top", b"foo", vec![1u8;33].as_ref()); + }: { + assert!( + StateTrieMigration::::migrate_custom_child( + frame_system::RawOrigin::Signed(caller.clone()).into(), + StateTrieMigration::::childify("top"), + vec![b"foo".to_vec()], + 1, + ).is_err() + ) + } + verify { + assert_eq!(StateTrieMigration::::migration_process(), Default::default()); + // must have gotten slashed + assert!(T::Currency::free_balance(&caller) < stash) + } + + process_top_key { + let v in 1 .. (4 * 1024 * 1024); + + let value = sp_std::vec![1u8; v as usize]; + sp_io::storage::set(KEY, &value); + }: { + let data = sp_io::storage::get(KEY).unwrap(); + sp_io::storage::set(KEY, &data); + let _next = sp_io::storage::next_key(KEY); + assert_eq!(data, value); + } + + impl_benchmark_test_suite!( + StateTrieMigration, + crate::mock::new_test_ext(sp_runtime::StateVersion::V0, true, None, None), + crate::mock::Test + ); + } +} + +#[cfg(test)] +mod mock { + use super::*; + use crate as pallet_state_trie_migration; + use frame_support::{parameter_types, traits::Hooks}; + use frame_system::{EnsureRoot, EnsureSigned}; + use sp_core::{ + storage::{ChildInfo, StateVersion}, + H256, + }; + use sp_runtime::{ + traits::{BlakeTwo256, Header as _, IdentityLookup}, + StorageChild, + }; + + type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; + type Block = frame_system::mocking::MockBlock; + + // Configure a mock runtime to test the pallet. + frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + Balances: pallet_balances::{Pallet, Call, Config, Storage, Event}, + StateTrieMigration: pallet_state_trie_migration::{Pallet, Call, Storage, Event}, + } + ); + + parameter_types! { + pub const BlockHashCount: u32 = 250; + pub const SS58Prefix: u8 = 42; + } + + impl frame_system::Config for Test { + type BaseCallFilter = frame_support::traits::Everything; + type BlockWeights = (); + type BlockLength = (); + type Origin = Origin; + type Call = Call; + type Index = u64; + type BlockNumber = u32; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = sp_runtime::generic::Header; + type Event = Event; + type BlockHashCount = BlockHashCount; + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = SS58Prefix; + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; + } + + parameter_types! { + pub const ExistentialDeposit: u64 = 1; + pub const OffchainRepeat: u32 = 1; + pub const SignedDepositPerItem: u64 = 1; + pub const SignedDepositBase: u64 = 5; + } + + impl pallet_balances::Config for Test { + type Balance = u64; + type Event = Event; + type DustRemoval = (); + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; + type MaxLocks = (); + type MaxReserves = (); + type ReserveIdentifier = [u8; 8]; + type WeightInfo = (); + } + + impl pallet_state_trie_migration::Config for Test { + type Event = Event; + type ControlOrigin = EnsureRoot; + type Currency = Balances; + type SignedDepositPerItem = SignedDepositPerItem; + type SignedDepositBase = SignedDepositBase; + type SignedFilter = EnsureSigned; + type WeightInfo = (); + } + + pub fn new_test_ext( + version: StateVersion, + with_pallets: bool, + custom_keys: Option, Vec)>>, + custom_child: Option, Vec, Vec)>>, + ) -> sp_io::TestExternalities { + let minimum_size = sp_core::storage::TRIE_VALUE_NODE_THRESHOLD as usize + 1; + let mut custom_storage = sp_core::storage::Storage { + top: vec![ + (b"key1".to_vec(), vec![1u8; minimum_size + 1]), // 6b657931 + (b"key2".to_vec(), vec![1u8; minimum_size + 2]), // 6b657931 + (b"key3".to_vec(), vec![1u8; minimum_size + 3]), // 6b657931 + (b"key4".to_vec(), vec![1u8; minimum_size + 4]), // 6b657931 + (b"key5".to_vec(), vec![1u8; minimum_size + 5]), // 6b657932 + (b"key6".to_vec(), vec![1u8; minimum_size + 6]), // 6b657934 + (b"key7".to_vec(), vec![1u8; minimum_size + 7]), // 6b657934 + (b"key8".to_vec(), vec![1u8; minimum_size + 8]), // 6b657934 + (b"key9".to_vec(), vec![1u8; minimum_size + 9]), // 6b657934 + (b"CODE".to_vec(), vec![1u8; minimum_size + 100]), // 434f4445 + ] + .into_iter() + .chain(custom_keys.unwrap_or_default()) + .collect(), + children_default: vec![ + ( + b"chk1".to_vec(), // 63686b31 + StorageChild { + data: vec![ + (b"key1".to_vec(), vec![1u8; 55]), + (b"key2".to_vec(), vec![2u8; 66]), + ] + .into_iter() + .collect(), + child_info: ChildInfo::new_default(b"chk1"), + }, + ), + ( + b"chk2".to_vec(), + StorageChild { + data: vec![ + (b"key1".to_vec(), vec![1u8; 54]), + (b"key2".to_vec(), vec![2u8; 64]), + ] + .into_iter() + .collect(), + child_info: ChildInfo::new_default(b"chk2"), + }, + ), + ] + .into_iter() + .chain( + custom_child + .unwrap_or_default() + .into_iter() + .map(|(r, k, v)| { + ( + r.clone(), + StorageChild { + data: vec![(k, v)].into_iter().collect(), + child_info: ChildInfo::new_default(&r), + }, + ) + }) + .collect::>(), + ) + .collect(), + }; + + if with_pallets { + frame_system::GenesisConfig::default() + .assimilate_storage::(&mut custom_storage) + .unwrap(); + pallet_balances::GenesisConfig:: { balances: vec![(1, 1000)] } + .assimilate_storage(&mut custom_storage) + .unwrap(); + } + + sp_tracing::try_init_simple(); + let mut ext: sp_io::TestExternalities = (custom_storage, version).into(); + + // set some genesis values for this pallet as well. + ext.execute_with(|| { + SignedMigrationMaxLimits::::put(MigrationLimits { size: 1024, item: 5 }); + }); + + ext + } + + pub(crate) fn run_to_block(n: u32) -> (H256, u64) { + let mut root = Default::default(); + let mut weight_sum = 0; + log::trace!(target: LOG_TARGET, "running from {:?} to {:?}", System::block_number(), n); + while System::block_number() < n { + System::set_block_number(System::block_number() + 1); + System::on_initialize(System::block_number()); + + weight_sum += StateTrieMigration::on_initialize(System::block_number()); + + root = System::finalize().state_root().clone(); + System::on_finalize(System::block_number()); + } + (root, weight_sum) + } +} + +#[cfg(test)] +mod test { + use super::{mock::*, *}; + use sp_runtime::{traits::Bounded, StateVersion}; + + #[test] + fn fails_if_no_migration() { + let mut ext = new_test_ext(StateVersion::V0, false, None, None); + let root1 = ext.execute_with(|| run_to_block(30).0); + + let mut ext2 = new_test_ext(StateVersion::V1, false, None, None); + let root2 = ext2.execute_with(|| run_to_block(30).0); + + // these two roots should not be the same. + assert_ne!(root1, root2); + } + + #[test] + fn detects_value_in_empty_top_key() { + let limit = MigrationLimits { item: 1, size: 1000 }; + let initial_keys = Some(vec![(vec![], vec![66u8; 77])]); + let mut ext = new_test_ext(StateVersion::V0, false, initial_keys.clone(), None); + + let root_upgraded = ext.execute_with(|| { + AutoLimits::::put(Some(limit)); + let root = run_to_block(30).0; + + // eventually everything is over. + assert!(StateTrieMigration::migration_process().finished()); + root + }); + + let mut ext2 = new_test_ext(StateVersion::V1, false, initial_keys, None); + let root = ext2.execute_with(|| { + AutoLimits::::put(Some(limit)); + run_to_block(30).0 + }); + + assert_eq!(root, root_upgraded); + } + + #[test] + fn detects_value_in_first_child_key() { + let limit = MigrationLimits { item: 1, size: 1000 }; + let initial_child = Some(vec![(b"chk1".to_vec(), vec![], vec![66u8; 77])]); + let mut ext = new_test_ext(StateVersion::V0, false, None, initial_child.clone()); + + let root_upgraded = ext.execute_with(|| { + AutoLimits::::put(Some(limit)); + let root = run_to_block(30).0; + + // eventually everything is over. + assert!(StateTrieMigration::migration_process().finished()); + root + }); + + let mut ext2 = new_test_ext(StateVersion::V1, false, None, initial_child); + let root = ext2.execute_with(|| { + AutoLimits::::put(Some(limit)); + run_to_block(30).0 + }); + + assert_eq!(root, root_upgraded); + } + + #[test] + fn auto_migrate_works() { + let run_with_limits = |limit, from, until| { + let mut ext = new_test_ext(StateVersion::V0, false, None, None); + let root_upgraded = ext.execute_with(|| { + assert_eq!(AutoLimits::::get(), None); + assert_eq!(MigrationProcess::::get(), Default::default()); + + // nothing happens if we don't set the limits. + let _ = run_to_block(from); + assert_eq!(MigrationProcess::::get(), Default::default()); + + // this should allow 1 item per block to be migrated. + AutoLimits::::put(Some(limit)); + + let root = run_to_block(until).0; + + // eventually everything is over. + assert!(matches!( + StateTrieMigration::migration_process(), + MigrationTask { progress_top: Progress::Complete, .. } + )); + root + }); + + let mut ext2 = new_test_ext(StateVersion::V1, false, None, None); + let root = ext2.execute_with(|| { + // update ex2 to contain the new items + let _ = run_to_block(from); + AutoLimits::::put(Some(limit)); + run_to_block(until).0 + }); + assert_eq!(root, root_upgraded); + }; + + // single item + run_with_limits(MigrationLimits { item: 1, size: 1000 }, 10, 100); + // multi-item + run_with_limits(MigrationLimits { item: 5, size: 1000 }, 10, 100); + // multi-item, based on size. Note that largest value is 100 bytes. + run_with_limits(MigrationLimits { item: 1000, size: 128 }, 10, 100); + // unbounded + run_with_limits( + MigrationLimits { item: Bounded::max_value(), size: Bounded::max_value() }, + 10, + 100, + ); + } + + #[test] + fn signed_migrate_works() { + new_test_ext(StateVersion::V0, true, None, None).execute_with(|| { + assert_eq!(MigrationProcess::::get(), Default::default()); + + // can't submit if limit is too high. + frame_support::assert_err!( + StateTrieMigration::continue_migrate( + Origin::signed(1), + MigrationLimits { item: 5, size: sp_runtime::traits::Bounded::max_value() }, + Bounded::max_value(), + MigrationProcess::::get() + ), + Error::::MaxSignedLimits, + ); + + // can't submit if poor. + frame_support::assert_err!( + StateTrieMigration::continue_migrate( + Origin::signed(2), + MigrationLimits { item: 5, size: 100 }, + 100, + MigrationProcess::::get() + ), + Error::::NotEnoughFunds, + ); + + // can't submit with bad witness. + frame_support::assert_err_ignore_postinfo!( + StateTrieMigration::continue_migrate( + Origin::signed(1), + MigrationLimits { item: 5, size: 100 }, + 100, + MigrationTask { + progress_top: Progress::LastKey(vec![1u8]), + ..Default::default() + } + ), + Error::::BadWitness + ); + + // migrate all keys in a series of submissions + while !MigrationProcess::::get().finished() { + // first we compute the task to get the accurate consumption. + let mut task = StateTrieMigration::migration_process(); + task.migrate_until_exhaustion( + StateTrieMigration::signed_migration_max_limits().unwrap(), + ); + + frame_support::assert_ok!(StateTrieMigration::continue_migrate( + Origin::signed(1), + StateTrieMigration::signed_migration_max_limits().unwrap(), + task.dyn_size, + MigrationProcess::::get() + )); + + // no funds should remain reserved. + assert_eq!(Balances::reserved_balance(&1), 0); + + // and the task should be updated + assert!(matches!( + StateTrieMigration::migration_process(), + MigrationTask { size: x, .. } if x > 0, + )); + } + }); + } + + #[test] + fn custom_migrate_top_works() { + let correct_witness = 3 + sp_core::storage::TRIE_VALUE_NODE_THRESHOLD * 3 + 1 + 2 + 3; + new_test_ext(StateVersion::V0, true, None, None).execute_with(|| { + frame_support::assert_ok!(StateTrieMigration::migrate_custom_top( + Origin::signed(1), + vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], + correct_witness, + )); + + // no funds should remain reserved. + assert_eq!(Balances::reserved_balance(&1), 0); + assert_eq!(Balances::free_balance(&1), 1000); + }); + + new_test_ext(StateVersion::V0, true, None, None).execute_with(|| { + // works if the witness is an overestimate + frame_support::assert_ok!(StateTrieMigration::migrate_custom_top( + Origin::signed(1), + vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], + correct_witness + 99, + )); + + // no funds should remain reserved. + assert_eq!(Balances::reserved_balance(&1), 0); + assert_eq!(Balances::free_balance(&1), 1000); + }); + + new_test_ext(StateVersion::V0, true, None, None).execute_with(|| { + assert_eq!(Balances::free_balance(&1), 1000); + + // note that we don't expect this to be a noop -- we do slash. + frame_support::assert_err!( + StateTrieMigration::migrate_custom_top( + Origin::signed(1), + vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], + correct_witness - 1, + ), + "wrong witness data" + ); + + // no funds should remain reserved. + assert_eq!(Balances::reserved_balance(&1), 0); + assert_eq!( + Balances::free_balance(&1), + 1000 - (3 * SignedDepositPerItem::get() + SignedDepositBase::get()) + ); + }); + } + + #[test] + fn custom_migrate_child_works() { + new_test_ext(StateVersion::V0, true, None, None).execute_with(|| { + frame_support::assert_ok!(StateTrieMigration::migrate_custom_child( + Origin::signed(1), + StateTrieMigration::childify("chk1"), + vec![b"key1".to_vec(), b"key2".to_vec()], + 55 + 66, + )); + + // no funds should remain reserved. + assert_eq!(Balances::reserved_balance(&1), 0); + assert_eq!(Balances::free_balance(&1), 1000); + }); + + new_test_ext(StateVersion::V0, true, None, None).execute_with(|| { + assert_eq!(Balances::free_balance(&1), 1000); + + // note that we don't expect this to be a noop -- we do slash. + assert!(StateTrieMigration::migrate_custom_child( + Origin::signed(1), + StateTrieMigration::childify("chk1"), + vec![b"key1".to_vec(), b"key2".to_vec()], + 999999, // wrong witness + ) + .is_err()); + + // no funds should remain reserved. + assert_eq!(Balances::reserved_balance(&1), 0); + assert_eq!( + Balances::free_balance(&1), + 1000 - (2 * SignedDepositPerItem::get() + SignedDepositBase::get()) + ); + }); + } +} + +/// Exported set of tests to be called against different runtimes. +#[cfg(feature = "remote-test")] +pub(crate) mod remote_tests { + use crate::{AutoLimits, MigrationLimits, Pallet as StateTrieMigration, LOG_TARGET}; + use codec::Encode; + use frame_benchmarking::Zero; + use frame_support::traits::{Get, Hooks}; + use frame_system::Pallet as System; + use remote_externalities::Mode; + use sp_core::H256; + use sp_runtime::traits::{Block as BlockT, HashFor, Header as _, One}; + use thousands::Separable; + + fn run_to_block>( + n: ::BlockNumber, + ) -> (H256, u64) { + let mut root = Default::default(); + let mut weight_sum = 0; + while System::::block_number() < n { + System::::set_block_number(System::::block_number() + One::one()); + System::::on_initialize(System::::block_number()); + + weight_sum += + StateTrieMigration::::on_initialize(System::::block_number()); + + root = System::::finalize().state_root().clone(); + System::::on_finalize(System::::block_number()); + } + (root, weight_sum) + } + + /// Run the entire migration, against the given `Runtime`, until completion. + /// + /// This will print some very useful statistics, make sure [`crate::LOG_TARGET`] is enabled. + pub(crate) async fn run_with_limits< + Runtime: crate::Config, + Block: BlockT + serde::de::DeserializeOwned, + >( + limits: MigrationLimits, + mode: Mode, + ) { + let mut ext = remote_externalities::Builder::::new() + .mode(mode) + .state_version(sp_core::storage::StateVersion::V0) + .build() + .await + .unwrap(); + + let mut now = ext.execute_with(|| { + AutoLimits::::put(Some(limits)); + // requires the block number type in our tests to be same as with mainnet, u32. + frame_system::Pallet::::block_number() + }); + + let mut duration: ::BlockNumber = Zero::zero(); + // set the version to 1, as if the upgrade happened. + ext.state_version = sp_core::storage::StateVersion::V1; + + let (top_left, child_left) = + substrate_state_trie_migration_rpc::migration_status(&ext.as_backend()).unwrap(); + assert!( + top_left > 0, + "no node needs migrating, this probably means that state was initialized with `StateVersion::V1`", + ); + + log::info!( + target: LOG_TARGET, + "initial check: top_left: {}, child_left: {}", + top_left.separate_with_commas(), + child_left.separate_with_commas(), + ); + + loop { + let last_state_root = ext.backend.root().clone(); + let ((finished, weight), proof) = ext.execute_and_prove(|| { + let weight = run_to_block::(now + One::one()).1; + if StateTrieMigration::::migration_process().finished() { + return (true, weight) + } + duration += One::one(); + now += One::one(); + (false, weight) + }); + + let compact_proof = + proof.clone().into_compact_proof::>(last_state_root).unwrap(); + log::info!( + target: LOG_TARGET, + "proceeded to #{}, weight: [{} / {}], proof: [{} / {} / {}]", + now, + weight.separate_with_commas(), + ::BlockWeights::get() + .max_block + .separate_with_commas(), + proof.encoded_size().separate_with_commas(), + compact_proof.encoded_size().separate_with_commas(), + zstd::stream::encode_all(&compact_proof.encode()[..], 0) + .unwrap() + .len() + .separate_with_commas(), + ); + ext.commit_all().unwrap(); + + if finished { + break + } + } + + ext.execute_with(|| { + log::info!( + target: LOG_TARGET, + "finished on_initialize migration in {} block, final state of the task: {:?}", + duration, + StateTrieMigration::::migration_process(), + ) + }); + + let (top_left, child_left) = + substrate_state_trie_migration_rpc::migration_status(&ext.as_backend()).unwrap(); + assert_eq!(top_left, 0); + assert_eq!(child_left, 0); + } +} + +#[cfg(all(test, feature = "remote-test"))] +mod remote_tests_local { + use super::{ + mock::{Call as MockCall, *}, + remote_tests::run_with_limits, + *, + }; + use remote_externalities::{Mode, OfflineConfig, OnlineConfig}; + use sp_runtime::traits::Bounded; + + // we only use the hash type from this, so using the mock should be fine. + type Extrinsic = sp_runtime::testing::TestXt; + type Block = sp_runtime::testing::Block; + + #[tokio::test] + async fn on_initialize_migration() { + sp_tracing::try_init_simple(); + let mode = Mode::OfflineOrElseOnline( + OfflineConfig { state_snapshot: env!("SNAP").to_owned().into() }, + OnlineConfig { + transport: std::env!("WS_API").to_owned().into(), + state_snapshot: Some(env!("SNAP").to_owned().into()), + ..Default::default() + }, + ); + + // item being the bottleneck + run_with_limits::( + MigrationLimits { item: 8 * 1024, size: 128 * 1024 * 1024 }, + mode.clone(), + ) + .await; + // size being the bottleneck + run_with_limits::( + MigrationLimits { item: Bounded::max_value(), size: 64 * 1024 }, + mode, + ) + .await; + } +} diff --git a/frame/state-trie-migration/src/weights.rs b/frame/state-trie-migration/src/weights.rs new file mode 100644 index 000000000000..f08b115378f2 --- /dev/null +++ b/frame/state-trie-migration/src/weights.rs @@ -0,0 +1,137 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for pallet_state_trie_migration +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2022-03-04, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 + +// Executed Command: +// target/production/substrate +// benchmark +// --chain=dev +// --steps=50 +// --repeat=20 +// --pallet=pallet_state_trie_migration +// --extrinsic=* +// --execution=wasm +// --wasm-execution=compiled +// --heap-pages=4096 +// --output=./frame/state-trie-migration/src/weights.rs +// --template=./.maintain/frame-weight-template.hbs + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use sp_std::marker::PhantomData; + +/// Weight functions needed for pallet_state_trie_migration. +pub trait WeightInfo { + fn continue_migrate() -> Weight; + fn continue_migrate_wrong_witness() -> Weight; + fn migrate_custom_top_success() -> Weight; + fn migrate_custom_top_fail() -> Weight; + fn migrate_custom_child_success() -> Weight; + fn migrate_custom_child_fail() -> Weight; + fn process_top_key(v: u32, ) -> Weight; +} + +/// Weights for pallet_state_trie_migration using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + // Storage: StateTrieMigration MigrationProcess (r:1 w:1) + fn continue_migrate() -> Weight { + (13_385_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: StateTrieMigration MigrationProcess (r:1 w:0) + fn continue_migrate_wrong_witness() -> Weight { + (1_757_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + } + fn migrate_custom_top_success() -> Weight { + (12_813_000 as Weight) + } + // Storage: unknown [0x666f6f] (r:1 w:1) + fn migrate_custom_top_fail() -> Weight { + (24_961_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + fn migrate_custom_child_success() -> Weight { + (13_132_000 as Weight) + } + // Storage: unknown [0x666f6f] (r:1 w:1) + fn migrate_custom_child_fail() -> Weight { + (29_215_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } + // Storage: unknown [0x6b6579] (r:1 w:1) + fn process_top_key(v: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(v as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } +} + +// For backwards compatibility and tests +impl WeightInfo for () { + // Storage: StateTrieMigration MigrationProcess (r:1 w:1) + fn continue_migrate() -> Weight { + (13_385_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: StateTrieMigration MigrationProcess (r:1 w:0) + fn continue_migrate_wrong_witness() -> Weight { + (1_757_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + } + fn migrate_custom_top_success() -> Weight { + (12_813_000 as Weight) + } + // Storage: unknown [0x666f6f] (r:1 w:1) + fn migrate_custom_top_fail() -> Weight { + (24_961_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + fn migrate_custom_child_success() -> Weight { + (13_132_000 as Weight) + } + // Storage: unknown [0x666f6f] (r:1 w:1) + fn migrate_custom_child_fail() -> Weight { + (29_215_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } + // Storage: unknown [0x6b6579] (r:1 w:1) + fn process_top_key(v: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(v as Weight)) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } +} diff --git a/frame/sudo/Cargo.toml b/frame/sudo/Cargo.toml index baacb66d5c75..b209351ddaf0 100644 --- a/frame/sudo/Cargo.toml +++ b/frame/sudo/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-sudo" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for sudo" readme = "README.md" @@ -13,16 +13,16 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } [features] default = ["std"] diff --git a/frame/sudo/README.md b/frame/sudo/README.md index ac7de01615f3..e8f688091e32 100644 --- a/frame/sudo/README.md +++ b/frame/sudo/README.md @@ -35,15 +35,22 @@ Learn more about privileged functions and `Root` origin in the [`Origin`] type d This is an example of a module that exposes a privileged function: ```rust -use frame_support::{decl_module, dispatch}; -use frame_system::ensure_root; - -pub trait Config: frame_system::Config {} - -decl_module! { - pub struct Module for enum Call where origin: T::Origin { - #[weight = 0] - pub fn privileged_function(origin) -> dispatch::DispatchResult { +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::call] + impl Pallet { + #[pallet::weight(0)] + pub fn privileged_function(origin: OriginFor) -> DispatchResult { ensure_root(origin)?; // do something... @@ -65,6 +72,6 @@ You need to set an initial superuser account as the sudo `key`. [`Call`]: ./enum.Call.html [`Config`]: ./trait.Config.html -[`Origin`]: https://docs.substrate.dev/docs/substrate-types +[`Origin`]: https://docs.substrate.io/v3/runtime/origins License: Apache-2.0 diff --git a/frame/sudo/src/lib.rs b/frame/sudo/src/lib.rs index bab93ffcee16..d9e72b37f297 100644 --- a/frame/sudo/src/lib.rs +++ b/frame/sudo/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -52,28 +52,27 @@ //! This is an example of a pallet that exposes a privileged function: //! //! ``` -//! //! #[frame_support::pallet] -//! pub mod logger { +//! pub mod pallet { +//! use super::*; //! use frame_support::pallet_prelude::*; //! use frame_system::pallet_prelude::*; -//! use super::*; +//! +//! #[pallet::pallet] +//! pub struct Pallet(_); //! //! #[pallet::config] //! pub trait Config: frame_system::Config {} //! -//! #[pallet::pallet] -//! pub struct Pallet(PhantomData); -//! //! #[pallet::call] //! impl Pallet { //! #[pallet::weight(0)] -//! pub fn privileged_function(origin: OriginFor) -> DispatchResultWithPostInfo { +//! pub fn privileged_function(origin: OriginFor) -> DispatchResult { //! ensure_root(origin)?; //! //! // do something... //! -//! Ok(().into()) +//! Ok(()) //! } //! } //! } @@ -89,7 +88,7 @@ //! //! * [Democracy](../pallet_democracy/index.html) //! -//! [`Origin`]: https://docs.substrate.dev/docs/substrate-types +//! [`Origin`]: https://docs.substrate.io/v3/runtime/origins #![cfg_attr(not(feature = "std"), no_std)] @@ -122,7 +121,6 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] - #[pallet::generate_storage_info] pub struct Pallet(PhantomData); #[pallet::call] @@ -147,10 +145,10 @@ pub mod pallet { ) -> DispatchResultWithPostInfo { // This is a public call, so we ensure that the origin is some signed account. let sender = ensure_signed(origin)?; - ensure!(sender == Self::key(), Error::::RequireSudo); + ensure!(Self::key().map_or(false, |k| sender == k), Error::::RequireSudo); let res = call.dispatch_bypass_filter(frame_system::RawOrigin::Root.into()); - Self::deposit_event(Event::Sudid(res.map(|_| ()).map_err(|e| e.error))); + Self::deposit_event(Event::Sudid { sudo_result: res.map(|_| ()).map_err(|e| e.error) }); // Sudo user does not pay a fee. Ok(Pays::No.into()) } @@ -173,10 +171,10 @@ pub mod pallet { ) -> DispatchResultWithPostInfo { // This is a public call, so we ensure that the origin is some signed account. let sender = ensure_signed(origin)?; - ensure!(sender == Self::key(), Error::::RequireSudo); + ensure!(Self::key().map_or(false, |k| sender == k), Error::::RequireSudo); let res = call.dispatch_bypass_filter(frame_system::RawOrigin::Root.into()); - Self::deposit_event(Event::Sudid(res.map(|_| ()).map_err(|e| e.error))); + Self::deposit_event(Event::Sudid { sudo_result: res.map(|_| ()).map_err(|e| e.error) }); // Sudo user does not pay a fee. Ok(Pays::No.into()) } @@ -198,11 +196,11 @@ pub mod pallet { ) -> DispatchResultWithPostInfo { // This is a public call, so we ensure that the origin is some signed account. let sender = ensure_signed(origin)?; - ensure!(sender == Self::key(), Error::::RequireSudo); + ensure!(Self::key().map_or(false, |k| sender == k), Error::::RequireSudo); let new = T::Lookup::lookup(new)?; - Self::deposit_event(Event::KeyChanged(Self::key())); - >::put(new); + Self::deposit_event(Event::KeyChanged { old_sudoer: Key::::get() }); + Key::::put(&new); // Sudo user does not pay a fee. Ok(Pays::No.into()) } @@ -235,13 +233,15 @@ pub mod pallet { ) -> DispatchResultWithPostInfo { // This is a public call, so we ensure that the origin is some signed account. let sender = ensure_signed(origin)?; - ensure!(sender == Self::key(), Error::::RequireSudo); + ensure!(Self::key().map_or(false, |k| sender == k), Error::::RequireSudo); let who = T::Lookup::lookup(who)?; let res = call.dispatch_bypass_filter(frame_system::RawOrigin::Signed(who).into()); - Self::deposit_event(Event::SudoAsDone(res.map(|_| ()).map_err(|e| e.error))); + Self::deposit_event(Event::SudoAsDone { + sudo_result: res.map(|_| ()).map_err(|e| e.error), + }); // Sudo user does not pay a fee. Ok(Pays::No.into()) } @@ -251,11 +251,11 @@ pub mod pallet { #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { /// A sudo just took place. \[result\] - Sudid(DispatchResult), - /// The \[sudoer\] just switched identity; the old key is supplied. - KeyChanged(T::AccountId), + Sudid { sudo_result: DispatchResult }, + /// The \[sudoer\] just switched identity; the old key is supplied if one existed. + KeyChanged { old_sudoer: Option }, /// A sudo just took place. \[result\] - SudoAsDone(DispatchResult), + SudoAsDone { sudo_result: DispatchResult }, } #[pallet::error] @@ -268,25 +268,27 @@ pub mod pallet { /// The `AccountId` of the sudo key. #[pallet::storage] #[pallet::getter(fn key)] - pub(super) type Key = StorageValue<_, T::AccountId, ValueQuery>; + pub(super) type Key = StorageValue<_, T::AccountId, OptionQuery>; #[pallet::genesis_config] pub struct GenesisConfig { /// The `AccountId` of the sudo key. - pub key: T::AccountId, + pub key: Option, } #[cfg(feature = "std")] impl Default for GenesisConfig { fn default() -> Self { - Self { key: Default::default() } + Self { key: None } } } #[pallet::genesis_build] impl GenesisBuild for GenesisConfig { fn build(&self) { - >::put(&self.key); + if let Some(ref key) = self.key { + Key::::put(key); + } } } } diff --git a/frame/sudo/src/mock.rs b/frame/sudo/src/mock.rs index dad17384d560..2e2a4abafcd9 100644 --- a/frame/sudo/src/mock.rs +++ b/frame/sudo/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use super::*; use crate as sudo; use frame_support::{ parameter_types, - traits::{Contains, GenesisBuild}, + traits::{ConstU32, ConstU64, Contains, GenesisBuild}, }; use frame_system::limits; use sp_core::H256; @@ -34,7 +34,6 @@ use sp_runtime::{ // Logger module to track execution. #[frame_support::pallet] pub mod logger { - use super::*; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; @@ -45,6 +44,7 @@ pub mod logger { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(PhantomData); #[pallet::call] @@ -58,7 +58,7 @@ pub mod logger { // Ensure that the `origin` is `Root`. ensure_root(origin)?; >::append(i); - Self::deposit_event(Event::AppendI32(i, weight)); + Self::deposit_event(Event::AppendI32 { value: i, weight }); Ok(().into()) } @@ -72,7 +72,7 @@ pub mod logger { let sender = ensure_signed(origin)?; >::append(i); >::append(sender.clone()); - Self::deposit_event(Event::AppendI32AndAccount(sender, i, weight)); + Self::deposit_event(Event::AppendI32AndAccount { sender, value: i, weight }); Ok(().into()) } } @@ -80,8 +80,8 @@ pub mod logger { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - AppendI32(i32, Weight), - AppendI32AndAccount(T::AccountId, i32, Weight), + AppendI32 { value: i32, weight: Weight }, + AppendI32AndAccount { sender: T::AccountId, value: i32, weight: Weight }, } #[pallet::storage] @@ -109,7 +109,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: limits::BlockWeights = limits::BlockWeights::simple_max(1024); } @@ -135,7 +134,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -144,6 +143,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } // Implement the logger module's `Config` on the Test runtime. @@ -164,7 +164,7 @@ pub type LoggerCall = logger::Call; // Build test environment by setting the root `key` for the Genesis. pub fn new_test_ext(root_key: u64) -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); - sudo::GenesisConfig:: { key: root_key } + sudo::GenesisConfig:: { key: Some(root_key) } .assimilate_storage(&mut t) .unwrap(); t.into() diff --git a/frame/sudo/src/tests.rs b/frame/sudo/src/tests.rs index 2eb558e9471c..84c8e0c5c254 100644 --- a/frame/sudo/src/tests.rs +++ b/frame/sudo/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,7 +28,7 @@ use mock::{ fn test_setup_works() { // Environment setup, logger storage, and sudo `key` retrieval should work as expected. new_test_ext(1).execute_with(|| { - assert_eq!(Sudo::key(), 1u64); + assert_eq!(Sudo::key(), Some(1u64)); assert!(Logger::i32_log().is_empty()); assert!(Logger::account_log().is_empty()); }); @@ -58,7 +58,7 @@ fn sudo_emits_events_correctly() { // Should emit event to indicate success when called with the root `key` and `call` is `Ok`. let call = Box::new(Call::Logger(LoggerCall::privileged_i32_log { i: 42, weight: 1 })); assert_ok!(Sudo::sudo(Origin::signed(1), call)); - System::assert_has_event(TestEvent::Sudo(Event::Sudid(Ok(())))); + System::assert_has_event(TestEvent::Sudo(Event::Sudid { sudo_result: Ok(()) })); }) } @@ -96,7 +96,7 @@ fn sudo_unchecked_weight_emits_events_correctly() { // Should emit event to indicate success when called with the root `key` and `call` is `Ok`. let call = Box::new(Call::Logger(LoggerCall::privileged_i32_log { i: 42, weight: 1 })); assert_ok!(Sudo::sudo_unchecked_weight(Origin::signed(1), call, 1_000)); - System::assert_has_event(TestEvent::Sudo(Event::Sudid(Ok(())))); + System::assert_has_event(TestEvent::Sudo(Event::Sudid { sudo_result: Ok(()) })); }) } @@ -105,7 +105,7 @@ fn set_key_basics() { new_test_ext(1).execute_with(|| { // A root `key` can change the root `key` assert_ok!(Sudo::set_key(Origin::signed(1), 2)); - assert_eq!(Sudo::key(), 2u64); + assert_eq!(Sudo::key(), Some(2u64)); }); new_test_ext(1).execute_with(|| { @@ -123,10 +123,10 @@ fn set_key_emits_events_correctly() { // A root `key` can change the root `key`. assert_ok!(Sudo::set_key(Origin::signed(1), 2)); - System::assert_has_event(TestEvent::Sudo(Event::KeyChanged(1))); + System::assert_has_event(TestEvent::Sudo(Event::KeyChanged { old_sudoer: Some(1) })); // Double check. assert_ok!(Sudo::set_key(Origin::signed(2), 4)); - System::assert_has_event(TestEvent::Sudo(Event::KeyChanged(2))); + System::assert_has_event(TestEvent::Sudo(Event::KeyChanged { old_sudoer: Some(2) })); }); } @@ -161,6 +161,6 @@ fn sudo_as_emits_events_correctly() { // A non-privileged function will work when passed to `sudo_as` with the root `key`. let call = Box::new(Call::Logger(LoggerCall::non_privileged_log { i: 42, weight: 1 })); assert_ok!(Sudo::sudo_as(Origin::signed(1), 2, call)); - System::assert_has_event(TestEvent::Sudo(Event::SudoAsDone(Ok(())))); + System::assert_has_event(TestEvent::Sudo(Event::SudoAsDone { sudo_result: Ok(()) })); }); } diff --git a/frame/support/Cargo.toml b/frame/support/Cargo.toml index f4af38db54e2..ec596494c601 100644 --- a/frame/support/Cargo.toml +++ b/frame/support/Cargo.toml @@ -2,9 +2,9 @@ name = "frame-support" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Support code for the runtime." readme = "README.md" @@ -13,32 +13,34 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -serde = { version = "1.0.126", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "2.2.0", default-features = false, features = ["derive", "max-encoded-len"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -frame-metadata = { version = "14.0.0", default-features = false, features = ["v14"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-tracing = { version = "4.0.0-dev", default-features = false, path = "../../primitives/tracing" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-arithmetic = { version = "4.0.0-dev", default-features = false, path = "../../primitives/arithmetic" } +serde = { version = "1.0.136", optional = true, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +frame-metadata = { version = "15.0.0", default-features = false, features = ["v14"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-tracing = { version = "5.0.0", default-features = false, path = "../../primitives/tracing" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-arithmetic = { version = "5.0.0", default-features = false, path = "../../primitives/arithmetic" } sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../primitives/inherents" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } +tt-call = "1.0.8" frame-support-procedural = { version = "4.0.0-dev", default-features = false, path = "./procedural" } paste = "1.0" once_cell = { version = "1", default-features = false, optional = true } -sp-state-machine = { version = "0.10.0-dev", optional = true, path = "../../primitives/state-machine" } +sp-state-machine = { version = "0.12.0", optional = true, path = "../../primitives/state-machine" } bitflags = "1.3" -impl-trait-for-tuples = "0.2.1" -smallvec = "1.7.0" +impl-trait-for-tuples = "0.2.2" +smallvec = "1.8.0" log = { version = "0.4.14", default-features = false } +sp-core-hashing-proc-macro = { version = "5.0.0", path = "../../primitives/core/hashing/proc-macro" } [dev-dependencies] assert_matches = "1.3.0" -pretty_assertions = "0.6.1" +pretty_assertions = "1.0.0" frame-system = { version = "4.0.0-dev", path = "../system" } -parity-util-mem = { version = "0.10.0", default-features = false, features = ["primitive-types"] } +parity-util-mem = { version = "0.11.0", default-features = false, features = ["primitive-types"] } [features] default = ["std"] @@ -61,3 +63,9 @@ std = [ ] runtime-benchmarks = [] try-runtime = [] +# By default some types have documentation, `no-metadata-docs` allows to reduce the documentation +# in the metadata. +no-metadata-docs = ["frame-support-procedural/no-metadata-docs"] +# By default some types have documentation, `full-metadata-docs` allows to add documentation to +# more types in the metadata. +full-metadata-docs = ["scale-info/docs"] diff --git a/frame/support/procedural/Cargo.toml b/frame/support/procedural/Cargo.toml index e1ff6dcf39b7..4e9618b5bc16 100644 --- a/frame/support/procedural/Cargo.toml +++ b/frame/support/procedural/Cargo.toml @@ -2,9 +2,9 @@ name = "frame-support-procedural" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Proc macro of Support code for the runtime." @@ -16,11 +16,12 @@ proc-macro = true [dependencies] frame-support-procedural-tools = { version = "4.0.0-dev", path = "./tools" } -proc-macro2 = "1.0.29" -quote = "1.0.3" +proc-macro2 = "1.0.36" +quote = "1.0.10" Inflector = "0.11.4" -syn = { version = "1.0.58", features = ["full"] } +syn = { version = "1.0.82", features = ["full"] } [features] default = ["std"] std = [] +no-metadata-docs = [] diff --git a/frame/support/procedural/src/clone_no_bound.rs b/frame/support/procedural/src/clone_no_bound.rs index 747900fd023f..bd2741c0d47a 100644 --- a/frame/support/procedural/src/clone_no_bound.rs +++ b/frame/support/procedural/src/clone_no_bound.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/construct_runtime/expand/call.rs b/frame/support/procedural/src/construct_runtime/expand/call.rs index 2532a680e21b..b09ef126ac6d 100644 --- a/frame/support/procedural/src/construct_runtime/expand/call.rs +++ b/frame/support/procedural/src/construct_runtime/expand/call.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,6 +22,7 @@ use syn::Ident; pub fn expand_outer_dispatch( runtime: &Ident, + system_pallet: &Pallet, pallet_decls: &[Pallet], scrate: &TokenStream, ) -> TokenStream { @@ -29,6 +30,7 @@ pub fn expand_outer_dispatch( let mut variant_patterns = Vec::new(); let mut query_call_part_macros = Vec::new(); let mut pallet_names = Vec::new(); + let system_path = &system_pallet.path; let pallets_with_call = pallet_decls.iter().filter(|decl| decl.exists_part("Call")); @@ -106,7 +108,9 @@ pub fn expand_outer_dispatch( type PostInfo = #scrate::weights::PostDispatchInfo; fn dispatch(self, origin: Origin) -> #scrate::dispatch::DispatchResultWithPostInfo { if !::filter_call(&origin, &self) { - return #scrate::sp_std::result::Result::Err(#scrate::dispatch::DispatchError::BadOrigin.into()); + return #scrate::sp_std::result::Result::Err( + #system_path::Error::<#runtime>::CallFiltered.into() + ); } #scrate::traits::UnfilteredDispatchable::dispatch_bypass_filter(self, origin) diff --git a/frame/support/procedural/src/construct_runtime/expand/config.rs b/frame/support/procedural/src/construct_runtime/expand/config.rs index 5e1b9d94700e..79176fa7385e 100644 --- a/frame/support/procedural/src/construct_runtime/expand/config.rs +++ b/frame/support/procedural/src/construct_runtime/expand/config.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/construct_runtime/expand/event.rs b/frame/support/procedural/src/construct_runtime/expand/event.rs index 798646bf2733..b242f9641562 100644 --- a/frame/support/procedural/src/construct_runtime/expand/event.rs +++ b/frame/support/procedural/src/construct_runtime/expand/event.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -127,7 +127,7 @@ fn expand_event_conversion( Event::#variant_name(x) } } - impl #scrate::sp_std::convert::TryInto<#pallet_event> for Event { + impl TryInto<#pallet_event> for Event { type Error = (); fn try_into(self) -> #scrate::sp_std::result::Result<#pallet_event, Self::Error> { diff --git a/frame/support/procedural/src/construct_runtime/expand/inherent.rs b/frame/support/procedural/src/construct_runtime/expand/inherent.rs index fd3041678268..0f0d53864324 100644 --- a/frame/support/procedural/src/construct_runtime/expand/inherent.rs +++ b/frame/support/procedural/src/construct_runtime/expand/inherent.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/construct_runtime/expand/metadata.rs b/frame/support/procedural/src/construct_runtime/expand/metadata.rs index c8445e0bbc25..6e2dd5fc002c 100644 --- a/frame/support/procedural/src/construct_runtime/expand/metadata.rs +++ b/frame/support/procedural/src/construct_runtime/expand/metadata.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/construct_runtime/expand/mod.rs b/frame/support/procedural/src/construct_runtime/expand/mod.rs index cf8b5eef8d10..6c92d2c3444e 100644 --- a/frame/support/procedural/src/construct_runtime/expand/mod.rs +++ b/frame/support/procedural/src/construct_runtime/expand/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/construct_runtime/expand/origin.rs b/frame/support/procedural/src/construct_runtime/expand/origin.rs index a65ad78527ff..342a002b52b9 100644 --- a/frame/support/procedural/src/construct_runtime/expand/origin.rs +++ b/frame/support/procedural/src/construct_runtime/expand/origin.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,23 +18,14 @@ use crate::construct_runtime::{Pallet, SYSTEM_PALLET_NAME}; use proc_macro2::TokenStream; use quote::quote; -use syn::{token, Generics, Ident}; +use syn::{Generics, Ident}; pub fn expand_outer_origin( runtime: &Ident, + system_pallet: &Pallet, pallets: &[Pallet], - pallets_token: token::Brace, scrate: &TokenStream, ) -> syn::Result { - let system_pallet = - pallets.iter().find(|decl| decl.name == SYSTEM_PALLET_NAME).ok_or_else(|| { - syn::Error::new( - pallets_token.span, - "`System` pallet declaration is missing. \ - Please add this line: `System: frame_system::{Pallet, Call, Storage, Config, Event},`", - ) - })?; - let mut caller_variants = TokenStream::new(); let mut pallet_conversions = TokenStream::new(); let mut query_origin_part_macros = Vec::new(); @@ -77,13 +68,39 @@ pub fn expand_outer_origin( } let system_path = &system_pallet.path; + let system_index = system_pallet.index; + let system_path_name = system_path.module_name(); + + let doc_string = get_intra_doc_string( + "Origin is always created with the base filter configured in", + &system_path_name, + ); + + let doc_string_none_origin = + get_intra_doc_string("Create with system none origin and", &system_path_name); + + let doc_string_root_origin = + get_intra_doc_string("Create with system root origin and", &system_path_name); + + let doc_string_signed_origin = + get_intra_doc_string("Create with system signed origin and", &system_path_name); + + let doc_string_runtime_origin = + get_intra_doc_string("Convert to runtime origin, using as filter:", &system_path_name); + + let doc_string_runtime_origin_with_caller = get_intra_doc_string( + "Convert to runtime origin with caller being system signed or none and use filter", + &system_path_name, + ); + Ok(quote! { #( #query_origin_part_macros )* - // WARNING: All instance must hold the filter `frame_system::Config::BaseCallFilter`, except - // when caller is system Root. One can use `OriginTrait::reset_filter` to do so. + /// The runtime origin type representing the origin of a call. + /// + #[doc = #doc_string] #[derive(Clone)] pub struct Origin { caller: OriginCaller, @@ -140,7 +157,11 @@ pub fn expand_outer_origin( } fn filter_call(&self, call: &Self::Call) -> bool { - (self.filter)(call) + match self.caller { + // Root bypasses all filters + OriginCaller::system(#system_path::Origin::<#runtime>::Root) => true, + _ => (self.filter)(call), + } } fn caller(&self) -> &Self::PalletsOrigin { @@ -157,15 +178,14 @@ pub fn expand_outer_origin( } } - /// Create with system none origin and `frame-system::Config::BaseCallFilter`. fn none() -> Self { #system_path::RawOrigin::None.into() } - /// Create with system root origin and no filter. + fn root() -> Self { #system_path::RawOrigin::Root.into() } - /// Create with system signed origin and `frame-system::Config::BaseCallFilter`. + fn signed(by: <#runtime as #system_path::Config>::AccountId) -> Self { #system_path::RawOrigin::Signed(by).into() } @@ -187,15 +207,18 @@ pub fn expand_outer_origin( // For backwards compatibility and ease of accessing these functions. #[allow(dead_code)] impl Origin { - /// Create with system none origin and `frame-system::Config::BaseCallFilter`. + + #[doc = #doc_string_none_origin] pub fn none() -> Self { ::none() } - /// Create with system root origin and no filter. + + #[doc = #doc_string_root_origin] pub fn root() -> Self { ::root() } - /// Create with system signed origin and `frame-system::Config::BaseCallFilter`. + + #[doc = #doc_string_signed_origin] pub fn signed(by: <#runtime as #system_path::Config>::AccountId) -> Self { ::signed(by) } @@ -207,7 +230,7 @@ pub fn expand_outer_origin( } } - impl #scrate::sp_std::convert::TryFrom for #system_path::Origin<#runtime> { + impl TryFrom for #system_path::Origin<#runtime> { type Error = OriginCaller; fn try_from(x: OriginCaller) -> #scrate::sp_std::result::Result<#system_path::Origin<#runtime>, OriginCaller> @@ -221,9 +244,8 @@ pub fn expand_outer_origin( } impl From<#system_path::Origin<#runtime>> for Origin { - /// Convert to runtime origin: - /// * root origin is built with no filter - /// * others use `frame-system::Config::BaseCallFilter` + + #[doc = #doc_string_runtime_origin] fn from(x: #system_path::Origin<#runtime>) -> Self { let o: OriginCaller = x.into(); o.into() @@ -237,10 +259,7 @@ pub fn expand_outer_origin( filter: #scrate::sp_std::rc::Rc::new(Box::new(|_| true)), }; - // Root has no filter - if !matches!(o.caller, OriginCaller::system(#system_path::Origin::<#runtime>::Root)) { - #scrate::traits::OriginTrait::reset_filter(&mut o); - } + #scrate::traits::OriginTrait::reset_filter(&mut o); o } @@ -257,8 +276,7 @@ pub fn expand_outer_origin( } } impl From::AccountId>> for Origin { - /// Convert to runtime origin with caller being system signed or none and use filter - /// `frame-system::Config::BaseCallFilter`. + #[doc = #doc_string_runtime_origin_with_caller] fn from(x: Option<<#runtime as #system_path::Config>::AccountId>) -> Self { <#system_path::Origin<#runtime>>::from(x).into() } @@ -313,6 +331,8 @@ fn expand_origin_pallet_conversions( None => quote!(#path::Origin), }; + let doc_string = get_intra_doc_string(" Convert to runtime origin using", &path.module_name()); + quote! { impl From<#pallet_origin> for OriginCaller { fn from(x: #pallet_origin) -> Self { @@ -321,7 +341,7 @@ fn expand_origin_pallet_conversions( } impl From<#pallet_origin> for Origin { - /// Convert to runtime origin using `frame-system::Config::BaseCallFilter`. + #[doc = #doc_string] fn from(x: #pallet_origin) -> Self { let x: OriginCaller = x.into(); x.into() @@ -339,7 +359,7 @@ fn expand_origin_pallet_conversions( } } - impl #scrate::sp_std::convert::TryFrom for #pallet_origin { + impl TryFrom for #pallet_origin { type Error = OriginCaller; fn try_from( x: OriginCaller, @@ -353,3 +373,8 @@ fn expand_origin_pallet_conversions( } } } + +// Get the actual documentation using the doc information and system path name +fn get_intra_doc_string(doc_info: &str, system_path_name: &String) -> String { + format!(" {} [`{}::Config::BaseCallFilter`].", doc_info, system_path_name) +} diff --git a/frame/support/procedural/src/construct_runtime/expand/unsigned.rs b/frame/support/procedural/src/construct_runtime/expand/unsigned.rs index d51792dd4a8d..c03067680209 100644 --- a/frame/support/procedural/src/construct_runtime/expand/unsigned.rs +++ b/frame/support/procedural/src/construct_runtime/expand/unsigned.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/construct_runtime/mod.rs b/frame/support/procedural/src/construct_runtime/mod.rs index 04bb2ead645d..2a86869382c9 100644 --- a/frame/support/procedural/src/construct_runtime/mod.rs +++ b/frame/support/procedural/src/construct_runtime/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,114 +15,214 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Implementation of `construct_runtime`. +//! +//! `construct_runtime` implementation is recursive and can generate code which will call itself in +//! order to get all the pallet parts for each pallet. +//! +//! Pallets define their parts (`Call`, `Storage`, ..) either explicitly with the syntax +//! `::{Call, ...}` or implicitly. +//! +//! In case a pallet defines its parts implicitly, then the pallet must provide the +//! `tt_default_parts` macro. `construct_rutime` will generate some code which utilizes `tt_call` +//! to call the `tt_default_parts` macro of the pallet. `tt_default_parts` will then return the +//! default pallet parts as input tokens to the `match_and_replace` macro, which ultimately +//! generates a call to `construct_runtime` again, this time with all the pallet parts explicitly +//! defined. +//! +//! E.g. +//! ```ignore +//! construct_runtime!( +//! //... +//! { +//! System: frame_system = 0, // Implicit definition of parts +//! Balances: pallet_balances = 1, // Implicit definition of parts +//! } +//! ); +//! ``` +//! This call has some implicit pallet parts, thus it will expand to: +//! ```ignore +//! frame_support::tt_call! { +//! macro = [{ pallet_balances::tt_default_parts }] +//! ~~> frame_support::match_and_insert! { +//! target = [{ +//! frame_support::tt_call! { +//! macro = [{ frame_system::tt_default_parts }] +//! ~~> frame_support::match_and_insert! { +//! target = [{ +//! construct_runtime!( +//! //... +//! { +//! System: frame_system = 0, +//! Balances: pallet_balances = 1, +//! } +//! ); +//! }] +//! pattern = [{ System: frame_system }] +//! } +//! } +//! }] +//! pattern = [{ Balances: pallet_balances }] +//! } +//! } +//! ``` +//! `tt_default_parts` must be defined. It returns the pallet parts inside some tokens, and +//! then `tt_call` will pipe the returned pallet parts into the input of `match_and_insert`. +//! Thus `match_and_insert` will initially receive the following inputs: +//! ```ignore +//! frame_support::match_and_insert! { +//! target = [{ +//! frame_support::match_and_insert! { +//! target = [{ +//! construct_runtime!( +//! //... +//! { +//! System: frame_system = 0, +//! Balances: pallet_balances = 1, +//! } +//! ) +//! }] +//! pattern = [{ System: frame_system }] +//! tokens = [{ ::{Pallet, Call} }] +//! } +//! }] +//! pattern = [{ Balances: pallet_balances }] +//! tokens = [{ ::{Pallet, Call} }] +//! } +//! ``` +//! After dealing with `pallet_balances`, the inner `match_and_insert` will expand to: +//! ```ignore +//! frame_support::match_and_insert! { +//! target = [{ +//! construct_runtime!( +//! //... +//! { +//! System: frame_system = 0, // Implicit definition of parts +//! Balances: pallet_balances::{Pallet, Call} = 1, // Explicit definition of parts +//! } +//! ) +//! }] +//! pattern = [{ System: frame_system }] +//! tokens = [{ ::{Pallet, Call} }] +//! } +//! ``` +//! Which will then finally expand to the following: +//! ```ignore +//! construct_runtime!( +//! //... +//! { +//! System: frame_system::{Pallet, Call}, +//! Balances: pallet_balances::{Pallet, Call}, +//! } +//! ) +//! ``` +//! This call has no implicit pallet parts, thus it will expand to the runtime construction: +//! ```ignore +//! pub struct Runtime { ... } +//! pub struct Call { ... } +//! impl Call ... +//! pub enum Origin { ... } +//! ... +//! ``` +//! +//! Visualizing the entire flow of `construct_runtime!`, it would look like the following: +//! +//! ```ignore +//! +--------------------+ +---------------------+ +-------------------+ +//! | | | (defined in pallet) | | | +//! | construct_runtime! | --> | tt_default_parts! | --> | match_and_insert! | +//! | w/ no pallet parts | | | | | +//! +--------------------+ +---------------------+ +-------------------+ +//! +//! +--------------------+ +//! | | +//! --> | construct_runtime! | +//! | w/ pallet parts | +//! +--------------------+ +//! ``` + mod expand; mod parse; use frame_support_procedural_tools::{ - generate_crate_access, generate_hidden_includes, syn_ext as ext, + generate_crate_access, generate_crate_access_2018, generate_hidden_includes, +}; +use parse::{ + ExplicitRuntimeDeclaration, ImplicitRuntimeDeclaration, Pallet, RuntimeDeclaration, + WhereSection, }; -use parse::{PalletDeclaration, PalletPart, PalletPath, RuntimeDefinition, WhereSection}; use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use quote::quote; -use std::collections::HashMap; use syn::{Ident, Result}; /// The fixed name of the system pallet. const SYSTEM_PALLET_NAME: &str = "System"; -/// The complete definition of a pallet with the resulting fixed index. -#[derive(Debug, Clone)] -pub struct Pallet { - pub name: Ident, - pub index: u8, - pub path: PalletPath, - pub instance: Option, - pub pallet_parts: Vec, -} - -impl Pallet { - /// Get resolved pallet parts - fn pallet_parts(&self) -> &[PalletPart] { - &self.pallet_parts - } +/// Implementation of `construct_runtime` macro. Either expand to some code which will call +/// `construct_runtime` again, or expand to the final runtime definition. +pub fn construct_runtime(input: TokenStream) -> TokenStream { + let input_copy = input.clone(); + let definition = syn::parse_macro_input!(input as RuntimeDeclaration); - /// Find matching parts - fn find_part(&self, name: &str) -> Option<&PalletPart> { - self.pallet_parts.iter().find(|part| part.name() == name) - } + let res = match definition { + RuntimeDeclaration::Implicit(implicit_def) => + construct_runtime_intermediary_expansion(input_copy.into(), implicit_def), + RuntimeDeclaration::Explicit(explicit_decl) => + construct_runtime_final_expansion(explicit_decl), + }; - /// Return whether pallet contains part - fn exists_part(&self, name: &str) -> bool { - self.find_part(name).is_some() - } + res.unwrap_or_else(|e| e.to_compile_error()).into() } -/// Convert from the parsed pallet to their final information. -/// Assign index to each pallet using same rules as rust for fieldless enum. -/// I.e. implicit are assigned number incrementedly from last explicit or 0. -fn complete_pallets(decl: impl Iterator) -> syn::Result> { - let mut indices = HashMap::new(); - let mut last_index: Option = None; - let mut names = HashMap::new(); - - decl.map(|pallet| { - let final_index = match pallet.index { - Some(i) => i, - None => last_index.map_or(Some(0), |i| i.checked_add(1)).ok_or_else(|| { - let msg = "Pallet index doesn't fit into u8, index is 256"; - syn::Error::new(pallet.name.span(), msg) - })?, - }; - - last_index = Some(final_index); - - if let Some(used_pallet) = indices.insert(final_index, pallet.name.clone()) { - let msg = format!( - "Pallet indices are conflicting: Both pallets {} and {} are at index {}", - used_pallet, pallet.name, final_index, - ); - let mut err = syn::Error::new(used_pallet.span(), &msg); - err.combine(syn::Error::new(pallet.name.span(), msg)); - return Err(err) - } - - if let Some(used_pallet) = names.insert(pallet.name.clone(), pallet.name.span()) { - let msg = "Two pallets with the same name!"; - - let mut err = syn::Error::new(used_pallet, &msg); - err.combine(syn::Error::new(pallet.name.span(), &msg)); - return Err(err) - } - - Ok(Pallet { - name: pallet.name, - index: final_index, - path: pallet.path, - instance: pallet.instance, - pallet_parts: pallet.pallet_parts, - }) - }) - .collect() -} +/// When some pallet have implicit parts definition then the macro will expand into a macro call to +/// `construct_runtime_args` of each pallets, see root documentation. +fn construct_runtime_intermediary_expansion( + input: TokenStream2, + definition: ImplicitRuntimeDeclaration, +) -> Result { + let frame_support = generate_crate_access_2018("frame-support")?; + let mut expansion = quote::quote!( + #frame_support::construct_runtime! { #input } + ); + for pallet in definition.pallets.iter().filter(|pallet| pallet.pallet_parts.is_none()) { + let pallet_path = &pallet.path; + let pallet_name = &pallet.name; + let pallet_instance = pallet.instance.as_ref().map(|instance| quote::quote!(::<#instance>)); + expansion = quote::quote!( + #frame_support::tt_call! { + macro = [{ #pallet_path::tt_default_parts }] + frame_support = [{ #frame_support }] + ~~> #frame_support::match_and_insert! { + target = [{ #expansion }] + pattern = [{ #pallet_name: #pallet_path #pallet_instance }] + } + } + ); + } -pub fn construct_runtime(input: TokenStream) -> TokenStream { - let definition = syn::parse_macro_input!(input as RuntimeDefinition); - construct_runtime_parsed(definition) - .unwrap_or_else(|e| e.to_compile_error()) - .into() + Ok(expansion.into()) } -fn construct_runtime_parsed(definition: RuntimeDefinition) -> Result { - let RuntimeDefinition { +/// All pallets have explicit definition of parts, this will expand to the runtime declaration. +fn construct_runtime_final_expansion( + definition: ExplicitRuntimeDeclaration, +) -> Result { + let ExplicitRuntimeDeclaration { name, - where_section: WhereSection { block, node_block, unchecked_extrinsic, .. }, - pallets: - ext::Braces { content: ext::Punctuated { inner: pallets, .. }, token: pallets_token }, - .. + where_section: WhereSection { block, node_block, unchecked_extrinsic }, + pallets, + pallets_token, } = definition; - let pallets = complete_pallets(pallets.into_iter())?; + let system_pallet = + pallets.iter().find(|decl| decl.name == SYSTEM_PALLET_NAME).ok_or_else(|| { + syn::Error::new( + pallets_token.span, + "`System` pallet declaration is missing. \ + Please add this line: `System: frame_system::{Pallet, Call, Storage, Config, Event},`", + ) + })?; let hidden_crate_name = "construct_runtime"; let scrate = generate_crate_access(&hidden_crate_name, "frame-support"); @@ -130,17 +230,18 @@ fn construct_runtime_parsed(definition: RuntimeDefinition) -> Result Result( types.extend(type_decl); names.push(&pallet_declaration.name); } - // Make nested tuple structure like (((Babe, Consensus), Grandpa), ...) + + // Make nested tuple structure like: + // `((FirstPallet, (SecondPallet, ( ... , LastPallet) ... ))))` // But ignore the system pallet. - let all_pallets = names + let all_pallets_without_system = names .iter() .filter(|n| **n != SYSTEM_PALLET_NAME) + .rev() .fold(TokenStream2::default(), |combined, name| quote!((#name, #combined))); + // Make nested tuple structure like: + // `((FirstPallet, (SecondPallet, ( ... , LastPallet) ... ))))` let all_pallets_with_system = names .iter() + .rev() + .fold(TokenStream2::default(), |combined, name| quote!((#name, #combined))); + + // Make nested tuple structure like: + // `((LastPallet, (SecondLastPallet, ( ... , FirstPallet) ... ))))` + // But ignore the system pallet. + let all_pallets_without_system_reversed = names + .iter() + .filter(|n| **n != SYSTEM_PALLET_NAME) .fold(TokenStream2::default(), |combined, name| quote!((#name, #combined))); + // Make nested tuple structure like: + // `((LastPallet, (SecondLastPallet, ( ... , FirstPallet) ... ))))` + let all_pallets_with_system_reversed = names + .iter() + .fold(TokenStream2::default(), |combined, name| quote!((#name, #combined))); + + let system_pallet = match names.iter().find(|n| **n == SYSTEM_PALLET_NAME) { + Some(name) => name, + None => + return syn::Error::new( + proc_macro2::Span::call_site(), + "`System` pallet declaration is missing. \ + Please add this line: `System: frame_system::{Pallet, Call, Storage, Config, Event},`", + ) + .into_compile_error(), + }; + quote!( #types + /// All pallets included in the runtime as a nested tuple of types. - /// Excludes the System pallet. - pub type AllPallets = ( #all_pallets ); + #[deprecated(note = "The type definition has changed from representing all pallets \ + excluding system, in reversed order to become the representation of all pallets \ + including system pallet in regular order. For this reason it is encouraged to use \ + explicitly one of `AllPalletsWithSystem`, `AllPalletsWithoutSystem`, \ + `AllPalletsWithSystemReversed`, `AllPalletsWithoutSystemReversed`. \ + Note that the type `frame_executive::Executive` expects one of `AllPalletsWithSystem` \ + , `AllPalletsWithSystemReversed`, `AllPalletsReversedWithSystemFirst`. More details in \ + https://github.com/paritytech/substrate/pull/10043")] + pub type AllPallets = AllPalletsWithSystem; + /// All pallets included in the runtime as a nested tuple of types. pub type AllPalletsWithSystem = ( #all_pallets_with_system ); - /// All modules included in the runtime as a nested tuple of types. + /// All pallets included in the runtime as a nested tuple of types. /// Excludes the System pallet. - #[deprecated(note = "use `AllPallets` instead")] - #[allow(dead_code)] - pub type AllModules = ( #all_pallets ); - /// All modules included in the runtime as a nested tuple of types. - #[deprecated(note = "use `AllPalletsWithSystem` instead")] - #[allow(dead_code)] - pub type AllModulesWithSystem = ( #all_pallets_with_system ); + pub type AllPalletsWithoutSystem = ( #all_pallets_without_system ); + + /// All pallets included in the runtime as a nested tuple of types in reversed order. + /// Excludes the System pallet. + pub type AllPalletsWithoutSystemReversed = ( #all_pallets_without_system_reversed ); + + /// All pallets included in the runtime as a nested tuple of types in reversed order. + pub type AllPalletsWithSystemReversed = ( #all_pallets_with_system_reversed ); + + /// All pallets included in the runtime as a nested tuple of types in reversed order. + /// With the system pallet first. + pub type AllPalletsReversedWithSystemFirst = ( + #system_pallet, + AllPalletsWithoutSystemReversed + ); ) } @@ -323,3 +474,34 @@ fn decl_integrity_test(scrate: &TokenStream2) -> TokenStream2 { } ) } + +fn decl_static_assertions( + runtime: &Ident, + pallet_decls: &[Pallet], + scrate: &TokenStream2, +) -> TokenStream2 { + let error_encoded_size_check = pallet_decls.iter().map(|decl| { + let path = &decl.path; + let assert_message = format!( + "The maximum encoded size of the error type in the `{}` pallet exceeds \ + `MAX_MODULE_ERROR_ENCODED_SIZE`", + decl.name, + ); + + quote! { + #scrate::tt_call! { + macro = [{ #path::tt_error_token }] + frame_support = [{ #scrate }] + ~~> #scrate::assert_error_encoded_size! { + path = [{ #path }] + runtime = [{ #runtime }] + assert_message = [{ #assert_message }] + } + } + } + }); + + quote! { + #(#error_encoded_size_check)* + } +} diff --git a/frame/support/procedural/src/construct_runtime/parse.rs b/frame/support/procedural/src/construct_runtime/parse.rs index a0ec6dfa5803..a2cda6a0777b 100644 --- a/frame/support/procedural/src/construct_runtime/parse.rs +++ b/frame/support/procedural/src/construct_runtime/parse.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,13 +17,13 @@ use frame_support_procedural_tools::syn_ext as ext; use proc_macro2::{Span, TokenStream}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use syn::{ ext::IdentExt, parse::{Parse, ParseStream}, punctuated::Punctuated, spanned::Spanned, - token, Error, Ident, Path, PathArguments, PathSegment, Result, Token, + token, Error, Ident, Path, Result, Token, }; mod keyword { @@ -38,26 +38,63 @@ mod keyword { syn::custom_keyword!(Origin); syn::custom_keyword!(Inherent); syn::custom_keyword!(ValidateUnsigned); + syn::custom_keyword!(exclude_parts); + syn::custom_keyword!(use_parts); } +/// Declaration of a runtime. +/// +/// Pallet declare their part either explicitly or implicitly (using no part declaration) +/// If all pallet have explicit parts then the runtime declaration is explicit, otherwise it is +/// implicit. +#[derive(Debug)] +pub enum RuntimeDeclaration { + Implicit(ImplicitRuntimeDeclaration), + Explicit(ExplicitRuntimeDeclaration), +} + +/// Declaration of a runtime with some pallet with implicit declaration of parts. +#[derive(Debug)] +pub struct ImplicitRuntimeDeclaration { + pub name: Ident, + pub where_section: WhereSection, + pub pallets: Vec, +} + +/// Declaration of a runtime with all pallet having explicit declaration of parts. #[derive(Debug)] -pub struct RuntimeDefinition { - pub visibility_token: Token![pub], - pub enum_token: Token![enum], +pub struct ExplicitRuntimeDeclaration { pub name: Ident, pub where_section: WhereSection, - pub pallets: ext::Braces>, + pub pallets: Vec, + pub pallets_token: token::Brace, } -impl Parse for RuntimeDefinition { +impl Parse for RuntimeDeclaration { fn parse(input: ParseStream) -> Result { - Ok(Self { - visibility_token: input.parse()?, - enum_token: input.parse()?, - name: input.parse()?, - where_section: input.parse()?, - pallets: input.parse()?, - }) + input.parse::()?; + input.parse::()?; + let name = input.parse::()?; + let where_section = input.parse()?; + let pallets = + input.parse::>>()?; + let pallets_token = pallets.token; + + match convert_pallets(pallets.content.inner.into_iter().collect())? { + PalletsConversion::Implicit(pallets) => + Ok(RuntimeDeclaration::Implicit(ImplicitRuntimeDeclaration { + name, + where_section, + pallets, + })), + PalletsConversion::Explicit(pallets) => + Ok(RuntimeDeclaration::Explicit(ExplicitRuntimeDeclaration { + name, + where_section, + pallets, + pallets_token, + })), + } } } @@ -136,14 +173,34 @@ impl Parse for WhereDefinition { } } +/// The declaration of a pallet. #[derive(Debug, Clone)] pub struct PalletDeclaration { + /// The name of the pallet, e.g.`System` in `System: frame_system`. pub name: Ident, - /// Optional fixed index (e.g. `MyPallet ... = 3,`) + /// Optional fixed index, e.g. `MyPallet ... = 3,`. pub index: Option, + /// The path of the pallet, e.g. `frame_system` in `System: frame_system`. pub path: PalletPath, + /// The instance of the pallet, e.g. `Instance1` in `Council: pallet_collective::`. pub instance: Option, - pub pallet_parts: Vec, + /// The declared pallet parts, + /// e.g. `Some([Pallet, Call])` for `System: system::{Pallet, Call}` + /// or `None` for `System: system`. + pub pallet_parts: Option>, + /// The specified parts, either use_parts or exclude_parts. + pub specified_parts: SpecifiedParts, +} + +/// The possible declaration of pallet parts to use. +#[derive(Debug, Clone)] +pub enum SpecifiedParts { + /// Use all the pallet parts except those specified. + Exclude(Vec), + /// Use only the specified pallet parts. + Use(Vec), + /// Use the all the pallet parts. + All, } impl Parse for PalletDeclaration { @@ -151,38 +208,78 @@ impl Parse for PalletDeclaration { let name = input.parse()?; let _: Token![:] = input.parse()?; let path = input.parse()?; - let instance = if input.peek(Token![<]) { + + // Parse for instance. + let instance = if input.peek(Token![::]) && input.peek3(Token![<]) { + let _: Token![::] = input.parse()?; let _: Token![<] = input.parse()?; let res = Some(input.parse()?); let _: Token![>] = input.parse()?; - let _: Token![::] = input.parse()?; res + } else if !(input.peek(Token![::]) && input.peek3(token::Brace)) && + !input.peek(keyword::exclude_parts) && + !input.peek(keyword::use_parts) && + !input.peek(Token![=]) && + !input.peek(Token![,]) && + !input.is_empty() + { + return Err(input.error( + "Unexpected tokens, expected one of `::$ident` `::{`, `exclude_parts`, `use_parts`, `=`, `,`", + )); } else { None }; - let pallet_parts = parse_pallet_parts(input)?; + // Parse for explicit parts + let pallet_parts = if input.peek(Token![::]) && input.peek3(token::Brace) { + let _: Token![::] = input.parse()?; + Some(parse_pallet_parts(input)?) + } else if !input.peek(keyword::exclude_parts) && + !input.peek(keyword::use_parts) && + !input.peek(Token![=]) && + !input.peek(Token![,]) && + !input.is_empty() + { + return Err(input.error( + "Unexpected tokens, expected one of `::{`, `exclude_parts`, `use_parts`, `=`, `,`", + )) + } else { + None + }; + + // Parse for specified parts + let specified_parts = if input.peek(keyword::exclude_parts) { + let _: keyword::exclude_parts = input.parse()?; + SpecifiedParts::Exclude(parse_pallet_parts_no_generic(input)?) + } else if input.peek(keyword::use_parts) { + let _: keyword::use_parts = input.parse()?; + SpecifiedParts::Use(parse_pallet_parts_no_generic(input)?) + } else if !input.peek(Token![=]) && !input.peek(Token![,]) && !input.is_empty() { + return Err(input.error("Unexpected tokens, expected one of `exclude_parts`, `=`, `,`")) + } else { + SpecifiedParts::All + }; + // Parse for pallet index let index = if input.peek(Token![=]) { input.parse::()?; let index = input.parse::()?; let index = index.base10_parse::()?; Some(index) + } else if !input.peek(Token![,]) && !input.is_empty() { + return Err(input.error("Unexpected tokens, expected one of `=`, `,`")) } else { None }; - let parsed = Self { name, path, instance, pallet_parts, index }; - - Ok(parsed) + Ok(Self { name, path, instance, pallet_parts, specified_parts, index }) } } /// A struct representing a path to a pallet. `PalletPath` is almost identical to the standard /// Rust path with a few restrictions: /// - No leading colons allowed -/// - Path segments can only consist of identifers; angle-bracketed or parenthesized segments will -/// result in a parsing error (except when specifying instances) +/// - Path segments can only consist of identifers separated by colons #[derive(Debug, Clone)] pub struct PalletPath { pub inner: Path, @@ -202,34 +299,27 @@ impl PalletPath { impl Parse for PalletPath { fn parse(input: ParseStream) -> Result { - let mut lookahead = input.lookahead1(); - let mut segments = Punctuated::new(); + let mut res = + PalletPath { inner: Path { leading_colon: None, segments: Punctuated::new() } }; + let lookahead = input.lookahead1(); if lookahead.peek(Token![crate]) || lookahead.peek(Token![self]) || lookahead.peek(Token![super]) || lookahead.peek(Ident) { let ident = input.call(Ident::parse_any)?; - segments.push(PathSegment { ident, arguments: PathArguments::None }); - let _: Token![::] = input.parse()?; - lookahead = input.lookahead1(); + res.inner.segments.push(ident.into()); } else { return Err(lookahead.error()) } - while lookahead.peek(Ident) { - let ident = input.parse()?; - segments.push(PathSegment { ident, arguments: PathArguments::None }); - let _: Token![::] = input.parse()?; - lookahead = input.lookahead1(); - } - - if !lookahead.peek(token::Brace) && !lookahead.peek(Token![<]) { - return Err(lookahead.error()) + while input.peek(Token![::]) && input.peek3(Ident) { + input.parse::()?; + let ident = input.parse::()?; + res.inner.segments.push(ident.into()); } - - Ok(Self { inner: Path { leading_colon: None, segments } }) + Ok(res) } } @@ -391,3 +481,174 @@ fn remove_kind( Err(input.error(msg)) } } + +/// The declaration of a part without its generics +#[derive(Debug, Clone)] +pub struct PalletPartNoGeneric { + keyword: PalletPartKeyword, +} + +impl Parse for PalletPartNoGeneric { + fn parse(input: ParseStream) -> Result { + Ok(Self { keyword: input.parse()? }) + } +} + +/// Parse [`PalletPartNoGeneric`]'s from a braces enclosed list that is split by commas, e.g. +/// +/// `{ Call, Event }` +fn parse_pallet_parts_no_generic(input: ParseStream) -> Result> { + let pallet_parts: ext::Braces> = + input.parse()?; + + let mut resolved = HashSet::new(); + for part in pallet_parts.content.inner.iter() { + if !resolved.insert(part.keyword.name()) { + let msg = format!( + "`{}` was already declared before. Please remove the duplicate declaration", + part.keyword.name(), + ); + return Err(Error::new(part.keyword.span(), msg)) + } + } + + Ok(pallet_parts.content.inner.into_iter().collect()) +} + +/// The final definition of a pallet with the resulting fixed index and explicit parts. +#[derive(Debug, Clone)] +pub struct Pallet { + /// The name of the pallet, e.g.`System` in `System: frame_system`. + pub name: Ident, + /// Either automatically infered, or defined (e.g. `MyPallet ... = 3,`). + pub index: u8, + /// The path of the pallet, e.g. `frame_system` in `System: frame_system`. + pub path: PalletPath, + /// The instance of the pallet, e.g. `Instance1` in `Council: pallet_collective::`. + pub instance: Option, + /// The pallet parts to use for the pallet. + pub pallet_parts: Vec, +} + +impl Pallet { + /// Get resolved pallet parts + pub fn pallet_parts(&self) -> &[PalletPart] { + &self.pallet_parts + } + + /// Find matching parts + pub fn find_part(&self, name: &str) -> Option<&PalletPart> { + self.pallet_parts.iter().find(|part| part.name() == name) + } + + /// Return whether pallet contains part + pub fn exists_part(&self, name: &str) -> bool { + self.find_part(name).is_some() + } +} + +/// Result of a conversion of a declaration of pallets. +enum PalletsConversion { + Implicit(Vec), + Explicit(Vec), +} + +/// Convert from the parsed pallet declaration to their final information. +/// +/// Check if all pallet have explicit declaration of their parts, if so then assign index to each +/// pallet using same rules as rust for fieldless enum. I.e. implicit are assigned number +/// incrementedly from last explicit or 0. +fn convert_pallets(pallets: Vec) -> syn::Result { + if pallets.iter().any(|pallet| pallet.pallet_parts.is_none()) { + return Ok(PalletsConversion::Implicit(pallets)) + } + + let mut indices = HashMap::new(); + let mut last_index: Option = None; + let mut names = HashMap::new(); + + let pallets = pallets + .into_iter() + .map(|pallet| { + let final_index = match pallet.index { + Some(i) => i, + None => last_index.map_or(Some(0), |i| i.checked_add(1)).ok_or_else(|| { + let msg = "Pallet index doesn't fit into u8, index is 256"; + syn::Error::new(pallet.name.span(), msg) + })?, + }; + + last_index = Some(final_index); + + if let Some(used_pallet) = indices.insert(final_index, pallet.name.clone()) { + let msg = format!( + "Pallet indices are conflicting: Both pallets {} and {} are at index {}", + used_pallet, pallet.name, final_index, + ); + let mut err = syn::Error::new(used_pallet.span(), &msg); + err.combine(syn::Error::new(pallet.name.span(), msg)); + return Err(err) + } + + if let Some(used_pallet) = names.insert(pallet.name.clone(), pallet.name.span()) { + let msg = "Two pallets with the same name!"; + + let mut err = syn::Error::new(used_pallet, &msg); + err.combine(syn::Error::new(pallet.name.span(), &msg)); + return Err(err) + } + + let mut pallet_parts = pallet.pallet_parts.expect("Checked above"); + + let available_parts = + pallet_parts.iter().map(|part| part.keyword.name()).collect::>(); + + // Check parts are correctly specified + match &pallet.specified_parts { + SpecifiedParts::Exclude(parts) | SpecifiedParts::Use(parts) => + for part in parts { + if !available_parts.contains(part.keyword.name()) { + let msg = format!( + "Invalid pallet part specified, the pallet `{}` doesn't have the \ + `{}` part. Available parts are: {}.", + pallet.name, + part.keyword.name(), + pallet_parts.iter().fold(String::new(), |fold, part| { + if fold.is_empty() { + format!("`{}`", part.keyword.name()) + } else { + format!("{}, `{}`", fold, part.keyword.name()) + } + }) + ); + return Err(syn::Error::new(part.keyword.span(), msg)) + } + }, + SpecifiedParts::All => (), + } + + // Set only specified parts. + match pallet.specified_parts { + SpecifiedParts::Exclude(excluded_parts) => pallet_parts.retain(|part| { + !excluded_parts + .iter() + .any(|excluded_part| excluded_part.keyword.name() == part.keyword.name()) + }), + SpecifiedParts::Use(used_parts) => pallet_parts.retain(|part| { + used_parts.iter().any(|use_part| use_part.keyword.name() == part.keyword.name()) + }), + SpecifiedParts::All => (), + } + + Ok(Pallet { + name: pallet.name, + index: final_index, + path: pallet.path, + instance: pallet.instance, + pallet_parts, + }) + }) + .collect::>>()?; + + Ok(PalletsConversion::Explicit(pallets)) +} diff --git a/frame/support/procedural/src/crate_version.rs b/frame/support/procedural/src/crate_version.rs index cfa35c6190e1..e2be6dd889db 100644 --- a/frame/support/procedural/src/crate_version.rs +++ b/frame/support/procedural/src/crate_version.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/debug_no_bound.rs b/frame/support/procedural/src/debug_no_bound.rs index acfd8d0cabc8..56168edb87e8 100644 --- a/frame/support/procedural/src/debug_no_bound.rs +++ b/frame/support/procedural/src/debug_no_bound.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/default_no_bound.rs b/frame/support/procedural/src/default_no_bound.rs index 38d6e19b1732..192be0786d96 100644 --- a/frame/support/procedural/src/default_no_bound.rs +++ b/frame/support/procedural/src/default_no_bound.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/key_prefix.rs b/frame/support/procedural/src/key_prefix.rs index 3f424e8b8b8d..05582f1297ee 100644 --- a/frame/support/procedural/src/key_prefix.rs +++ b/frame/support/procedural/src/key_prefix.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 6987fc49b9a8..92564e94493c 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,10 +26,13 @@ mod debug_no_bound; mod default_no_bound; mod dummy_part_checker; mod key_prefix; +mod match_and_insert; mod pallet; +mod pallet_error; mod partial_eq_no_bound; mod storage; mod transactional; +mod tt_macro; use proc_macro::TokenStream; use std::{cell::RefCell, str::FromStr}; @@ -40,9 +43,9 @@ thread_local! { static COUNTER: RefCell = RefCell::new(Counter(0)); } -/// Counter to generate a relatively unique identifier for macros querying for the existence of -/// pallet parts. This is necessary because declarative macros gets hoisted to the crate root, -/// which shares the namespace with other pallets containing the very same query macros. +/// Counter to generate a relatively unique identifier for macros. This is necessary because +/// declarative macros gets hoisted to the crate root, which shares the namespace with other pallets +/// containing the very same macros. struct Counter(u64); impl Counter { @@ -297,52 +300,91 @@ pub fn decl_storage(input: TokenStream) -> TokenStream { /// /// # Example: /// -/// ```nocompile +/// ```ignore /// construct_runtime!( /// pub enum Runtime where /// Block = Block, /// NodeBlock = node::Block, /// UncheckedExtrinsic = UncheckedExtrinsic /// { -/// System: system::{Pallet, Call, Event, Config} = 0, -/// Test: test::{Pallet, Call} = 1, -/// Test2: test_with_long_module::{Pallet, Event}, +/// System: frame_system::{Pallet, Call, Event, Config} = 0, +/// Test: path::to::test::{Pallet, Call} = 1, /// /// // Pallets with instances -/// Test3_Instance1: test3::::{Pallet, Call, Storage, Event, Config, Origin}, -/// Test3_DefaultInstance: test3::{Pallet, Call, Storage, Event, Config, Origin} = 4, +/// Test2_Instance1: test2::::{Pallet, Call, Storage, Event, Config, Origin}, +/// Test2_DefaultInstance: test2::{Pallet, Call, Storage, Event, Config, Origin} = 4, +/// +/// // Pallets declared with `pallet` attribute macro: no need to define the parts +/// Test3_Instance1: test3::, +/// Test3_DefaultInstance: test3, +/// +/// // with `exclude_parts` keyword some part can be excluded. +/// Test4_Instance1: test4:: exclude_parts { Call, Origin }, +/// Test4_DefaultInstance: test4 exclude_parts { Storage }, +/// +/// // with `use_parts` keyword, a subset of the pallet parts can be specified. +/// Test4_Instance1: test4:: use_parts { Pallet, Call}, +/// Test4_DefaultInstance: test4 use_parts { Pallet }, /// } /// ) /// ``` /// -/// The identifier `System` is the name of the pallet and the lower case identifier `system` is the -/// name of the Rust module/crate for this Substrate pallet. The identifiers between the braces are -/// the pallet parts provided by the pallet. It is important to list these parts here to export -/// them correctly in the metadata or to make the pallet usable in the runtime. +/// Each pallet is declared as such: +/// * `Identifier`: name given to the pallet that uniquely identifies it. /// -/// We provide support for the following module parts in a pallet: +/// * `:`: colon separator /// -/// - `Pallet` - Required for all pallets -/// - `Call` - If the pallet has callable functions -/// - `Storage` - If the pallet uses storage -/// - `Event` or `Event` (if the event is generic) - If the pallet emits events -/// - `Origin` or `Origin` (if the origin is generic) - If the pallet has instanciable origins -/// - `Config` or `Config` (if the config is generic) - If the pallet builds the genesis storage -/// with `GenesisConfig` -/// - `Inherent` - If the pallet provides/can check inherents. -/// - `ValidateUnsigned` - If the pallet validates unsigned extrinsics. +/// * `path::to::pallet`: identifiers separated by colons which declare the path to a pallet +/// definition. /// -/// `= $n` is an optional part allowing to define at which index the pallet variants in -/// `OriginCaller`, `Call` and `Event` are encoded, and to define the ModuleToIndex value. +/// * `::` optional: specify the instance of the pallet to use. If not specified it will +/// use the default instance (or the only instance in case of non-instantiable pallets). /// -/// if `= $n` is not given, then index is resolved same as fieldless enum in Rust -/// (i.e. incrementedly from previous index): -/// ```nocompile -/// pallet1 .. = 2, -/// pallet2 .., // Here pallet2 is given index 3 -/// pallet3 .. = 0, -/// pallet4 .., // Here pallet4 is given index 1 -/// ``` +/// * `::{ Part1, Part2, .. }` optional if pallet declared with `frame_support::pallet`: Comma +/// separated parts declared with their generic. If a pallet is declared with +/// `frame_support::pallet` macro then the parts can be automatically derived if not explicitly +/// provided. We provide support for the following module parts in a pallet: +/// +/// - `Pallet` - Required for all pallets +/// - `Call` - If the pallet has callable functions +/// - `Storage` - If the pallet uses storage +/// - `Event` or `Event` (if the event is generic) - If the pallet emits events +/// - `Origin` or `Origin` (if the origin is generic) - If the pallet has instanciable origins +/// - `Config` or `Config` (if the config is generic) - If the pallet builds the genesis +/// storage with `GenesisConfig` +/// - `Inherent` - If the pallet provides/can check inherents. +/// - `ValidateUnsigned` - If the pallet validates unsigned extrinsics. +/// +/// It is important to list these parts here to export them correctly in the metadata or to make +/// the pallet usable in the runtime. +/// +/// * `exclude_parts { Part1, Part2 }` optional: comma separated parts without generics. I.e. one of +/// `Pallet`, `Call`, `Storage`, `Event`, `Origin`, `Config`, `Inherent`, `ValidateUnsigned`. It +/// is incompatible with `use_parts`. This specifies the part to exclude. In order to select +/// subset of the pallet parts. +/// +/// For example excluding the part `Call` can be useful if the runtime doesn't want to make the +/// pallet calls available. +/// +/// * `use_parts { Part1, Part2 }` optional: comma separated parts without generics. I.e. one of +/// `Pallet`, `Call`, `Storage`, `Event`, `Origin`, `Config`, `Inherent`, `ValidateUnsigned`. It +/// is incompatible with `exclude_parts`. This specifies the part to use. In order to select a +/// subset of the pallet parts. +/// +/// For example not using the part `Call` can be useful if the runtime doesn't want to make the +/// pallet calls available. +/// +/// * `= $n` optional: number to define at which index the pallet variants in `OriginCaller`, `Call` +/// and `Event` are encoded, and to define the ModuleToIndex value. +/// +/// if `= $n` is not given, then index is resolved in the same way as fieldless enum in Rust +/// (i.e. incrementedly from previous index): +/// ```nocompile +/// pallet1 .. = 2, +/// pallet2 .., // Here pallet2 is given index 3 +/// pallet3 .. = 0, +/// pallet4 .., // Here pallet4 is given index 1 +/// ``` /// /// # Note /// @@ -352,8 +394,8 @@ pub fn decl_storage(input: TokenStream) -> TokenStream { /// /// # Type definitions /// -/// * The macro generates a type alias for each pallet to their `Module` (or `Pallet`). E.g. `type -/// System = frame_system::Pallet` +/// * The macro generates a type alias for each pallet to their `Pallet`. E.g. `type System = +/// frame_system::Pallet` #[proc_macro] pub fn construct_runtime(input: TokenStream) -> TokenStream { construct_runtime::construct_runtime(input) @@ -498,3 +540,38 @@ pub fn impl_key_prefix_for_tuples(input: TokenStream) -> TokenStream { pub fn __generate_dummy_part_checker(input: TokenStream) -> TokenStream { dummy_part_checker::generate_dummy_part_checker(input) } + +/// Macro that inserts some tokens after the first match of some pattern. +/// +/// # Example: +/// +/// ```nocompile +/// match_and_insert!( +/// target = [{ Some content with { at some point match pattern } other match pattern are ignored }] +/// pattern = [{ match pattern }] // the match pattern cannot contain any group: `[]`, `()`, `{}` +/// // can relax this constraint, but will require modifying the match logic in code +/// tokens = [{ expansion tokens }] // content inside braces can be anything including groups +/// ); +/// ``` +/// +/// will generate: +/// +/// ```nocompile +/// Some content with { at some point match pattern expansion tokens } other match patterns are +/// ignored +/// ``` +#[proc_macro] +pub fn match_and_insert(input: TokenStream) -> TokenStream { + match_and_insert::match_and_insert(input) +} + +#[proc_macro_derive(PalletError, attributes(codec))] +pub fn derive_pallet_error(input: TokenStream) -> TokenStream { + pallet_error::derive_pallet_error(input) +} + +/// Internal macro used by `frame_support` to create tt-call-compliant macros +#[proc_macro] +pub fn __create_tt_macro(input: TokenStream) -> TokenStream { + tt_macro::create_tt_return_macro(input) +} diff --git a/frame/support/procedural/src/match_and_insert.rs b/frame/support/procedural/src/match_and_insert.rs new file mode 100644 index 000000000000..79d1da7549c1 --- /dev/null +++ b/frame/support/procedural/src/match_and_insert.rs @@ -0,0 +1,159 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implementation of the `match_and_insert` macro. + +use proc_macro2::{Group, Span, TokenStream, TokenTree}; +use std::iter::once; +use syn::spanned::Spanned; + +mod keyword { + syn::custom_keyword!(target); + syn::custom_keyword!(pattern); + syn::custom_keyword!(tokens); +} + +pub fn match_and_insert(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let MatchAndInsertDef { pattern, tokens, target } = + syn::parse_macro_input!(input as MatchAndInsertDef); + + match expand_in_stream(&pattern, &mut Some(tokens), target) { + Ok(stream) => stream.into(), + Err(err) => err.to_compile_error().into(), + } +} + +struct MatchAndInsertDef { + // Token stream to search and insert tokens into. + target: TokenStream, + // Pattern to match against, this is ensured to have no TokenTree::Group nor TokenTree::Literal + // (i.e. contains only Punct or Ident), and not being empty. + pattern: Vec, + // Token stream to insert after the match pattern. + tokens: TokenStream, +} + +impl syn::parse::Parse for MatchAndInsertDef { + fn parse(input: syn::parse::ParseStream) -> syn::Result { + let mut target; + let _ = input.parse::()?; + let _ = input.parse::()?; + let _replace_with_bracket: syn::token::Bracket = syn::bracketed!(target in input); + let _replace_with_brace: syn::token::Brace = syn::braced!(target in target); + let target = target.parse()?; + + let mut pattern; + let _ = input.parse::()?; + let _ = input.parse::()?; + let _replace_with_bracket: syn::token::Bracket = syn::bracketed!(pattern in input); + let _replace_with_brace: syn::token::Brace = syn::braced!(pattern in pattern); + let pattern = pattern.parse::()?.into_iter().collect::>(); + + if let Some(t) = pattern.iter().find(|t| matches!(t, TokenTree::Group(_))) { + return Err(syn::Error::new(t.span(), "Unexpected group token tree")) + } + if let Some(t) = pattern.iter().find(|t| matches!(t, TokenTree::Literal(_))) { + return Err(syn::Error::new(t.span(), "Unexpected literal token tree")) + } + + if pattern.is_empty() { + return Err(syn::Error::new(Span::call_site(), "empty match pattern is invalid")) + } + + let mut tokens; + let _ = input.parse::()?; + let _ = input.parse::()?; + let _replace_with_bracket: syn::token::Bracket = syn::bracketed!(tokens in input); + let _replace_with_brace: syn::token::Brace = syn::braced!(tokens in tokens); + let tokens = tokens.parse()?; + + Ok(Self { tokens, pattern, target }) + } +} + +// Insert `tokens` after the first matching `pattern`. +// `tokens` must be some (Option is used for internal simplification). +// `pattern` must not be empty and should only contain Ident or Punct. +fn expand_in_stream( + pattern: &[TokenTree], + tokens: &mut Option, + stream: TokenStream, +) -> syn::Result { + assert!( + tokens.is_some(), + "`tokens` must be some, Option is used because `tokens` is used only once" + ); + assert!( + !pattern.is_empty(), + "`pattern` must not be empty, otherwise there is nothing to match against" + ); + + let stream_span = stream.span(); + let mut stream = stream.into_iter(); + let mut extended = TokenStream::new(); + let mut match_cursor = 0; + + while let Some(token) = stream.next() { + match token { + TokenTree::Group(group) => { + match_cursor = 0; + let group_stream = group.stream(); + match expand_in_stream(pattern, tokens, group_stream) { + Ok(s) => { + extended.extend(once(TokenTree::Group(Group::new(group.delimiter(), s)))); + extended.extend(stream); + return Ok(extended) + }, + Err(_) => { + extended.extend(once(TokenTree::Group(group))); + }, + } + }, + other => { + advance_match_cursor(&other, pattern, &mut match_cursor); + + extended.extend(once(other)); + + if match_cursor == pattern.len() { + extended + .extend(once(tokens.take().expect("tokens is used to replace only once"))); + extended.extend(stream); + return Ok(extended) + } + }, + } + } + // if we reach this point, it means the stream is empty and we haven't found a matching pattern + let msg = format!("Cannot find pattern `{:?}` in given token stream", pattern); + Err(syn::Error::new(stream_span, msg)) +} + +fn advance_match_cursor(other: &TokenTree, pattern: &[TokenTree], match_cursor: &mut usize) { + use TokenTree::{Ident, Punct}; + + let does_match_other_pattern = match (other, &pattern[*match_cursor]) { + (Ident(i1), Ident(i2)) => i1 == i2, + (Punct(p1), Punct(p2)) => p1.as_char() == p2.as_char(), + _ => false, + }; + + if does_match_other_pattern { + *match_cursor += 1; + } else { + *match_cursor = 0; + } +} diff --git a/frame/support/procedural/src/pallet/expand/call.rs b/frame/support/procedural/src/pallet/expand/call.rs index 8f7bcdccaf22..2d7c550a4486 100644 --- a/frame/support/procedural/src/pallet/expand/call.rs +++ b/frame/support/procedural/src/pallet/expand/call.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -137,6 +137,8 @@ pub fn expand_call(def: &mut Def) -> proc_macro2::TokenStream { let count = COUNTER.with(|counter| counter.borrow_mut().inc()); let macro_ident = syn::Ident::new(&format!("__is_call_part_defined_{}", count), span); + let capture_docs = if cfg!(feature = "no-metadata-docs") { "never" } else { "always" }; + quote::quote_spanned!(span => #[doc(hidden)] pub mod __substrate_call_check { @@ -164,7 +166,7 @@ pub fn expand_call(def: &mut Def) -> proc_macro2::TokenStream { )] #[codec(encode_bound())] #[codec(decode_bound())] - #[scale_info(skip_type_params(#type_use_gen), capture_docs = "always")] + #[scale_info(skip_type_params(#type_use_gen), capture_docs = #capture_docs)] #[allow(non_camel_case_types)] pub enum #call_ident<#type_decl_bounded_gen> #where_clause { #[doc(hidden)] @@ -176,7 +178,10 @@ pub fn expand_call(def: &mut Def) -> proc_macro2::TokenStream { #( #( #[doc = #fn_doc] )* #fn_name { - #( #args_compact_attr #args_name_stripped: #args_type ),* + #( + #[allow(missing_docs)] + #args_compact_attr #args_name_stripped: #args_type + ),* }, )* } diff --git a/frame/support/procedural/src/pallet/expand/config.rs b/frame/support/procedural/src/pallet/expand/config.rs index dad26ccad6dc..8ad34361d684 100644 --- a/frame/support/procedural/src/pallet/expand/config.rs +++ b/frame/support/procedural/src/pallet/expand/config.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/expand/constants.rs b/frame/support/procedural/src/pallet/expand/constants.rs index 7cc245e8089d..1f9526f9cebe 100644 --- a/frame/support/procedural/src/pallet/expand/constants.rs +++ b/frame/support/procedural/src/pallet/expand/constants.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,6 +26,8 @@ struct ConstDef { pub doc: Vec, /// default_byte implementation pub default_byte_impl: proc_macro2::TokenStream, + /// Constant name for Metadata (optional) + pub metadata_name: Option, } /// @@ -35,6 +37,7 @@ pub fn expand_constants(def: &mut Def) -> proc_macro2::TokenStream { let type_impl_gen = &def.type_impl_generics(proc_macro2::Span::call_site()); let type_use_gen = &def.type_use_generics(proc_macro2::Span::call_site()); let pallet_ident = &def.pallet_struct.pallet; + let trait_use_gen = &def.trait_use_generics(proc_macro2::Span::call_site()); let mut where_clauses = vec![&def.config.where_clause]; where_clauses.extend(def.extra_constants.iter().map(|d| &d.where_clause)); @@ -49,9 +52,11 @@ pub fn expand_constants(def: &mut Def) -> proc_macro2::TokenStream { type_: const_.type_.clone(), doc: const_.doc.clone(), default_byte_impl: quote::quote!( - let value = >::get(); + let value = <::#ident as + #frame_support::traits::Get<#const_type>>::get(); #frame_support::codec::Encode::encode(&value) ), + metadata_name: None, } }); @@ -66,14 +71,17 @@ pub fn expand_constants(def: &mut Def) -> proc_macro2::TokenStream { let value = >::#ident(); #frame_support::codec::Encode::encode(&value) ), + metadata_name: const_.metadata_name.clone(), } }); let consts = config_consts.chain(extra_consts).map(|const_| { let const_type = &const_.type_; - let ident = &const_.ident; - let ident_str = format!("{}", ident); - let doc = const_.doc.clone().into_iter(); + let ident_str = format!("{}", const_.metadata_name.unwrap_or(const_.ident)); + + let no_docs = vec![]; + let doc = if cfg!(feature = "no-metadata-docs") { &no_docs } else { &const_.doc }; + let default_byte_impl = &const_.default_byte_impl; quote::quote!({ diff --git a/frame/support/procedural/src/pallet/expand/error.rs b/frame/support/procedural/src/pallet/expand/error.rs index 7a058bb32c92..124e8b312ce3 100644 --- a/frame/support/procedural/src/pallet/expand/error.rs +++ b/frame/support/procedural/src/pallet/expand/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,20 +15,48 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::pallet::Def; +use crate::{ + pallet::{parse::error::VariantField, Def}, + COUNTER, +}; use frame_support_procedural_tools::get_doc_literals; +use syn::spanned::Spanned; /// /// * impl various trait on Error pub fn expand_error(def: &mut Def) -> proc_macro2::TokenStream { - let error = if let Some(error) = &def.error { error } else { return Default::default() }; + let count = COUNTER.with(|counter| counter.borrow_mut().inc()); + let error_token_unique_id = + syn::Ident::new(&format!("__tt_error_token_{}", count), def.item.span()); - let error_ident = &error.error; let frame_support = &def.frame_support; let frame_system = &def.frame_system; + let config_where_clause = &def.config.where_clause; + + let error = if let Some(error) = &def.error { + error + } else { + return quote::quote! { + #[macro_export] + #[doc(hidden)] + macro_rules! #error_token_unique_id { + { + $caller:tt + frame_support = [{ $($frame_support:ident)::* }] + } => { + $($frame_support::)*tt_return! { + $caller + } + }; + } + + pub use #error_token_unique_id as tt_error_token; + } + }; + + let error_ident = &error.error; let type_impl_gen = &def.type_impl_generics(error.attr_span); let type_use_gen = &def.type_use_generics(error.attr_span); - let config_where_clause = &def.config.where_clause; let phantom_variant: syn::Variant = syn::parse_quote!( #[doc(hidden)] @@ -39,13 +67,19 @@ pub fn expand_error(def: &mut Def) -> proc_macro2::TokenStream { ) ); - let as_u8_matches = error.variants.iter().enumerate().map( - |(i, (variant, _))| quote::quote_spanned!(error.attr_span => Self::#variant => #i as u8,), - ); - - let as_str_matches = error.variants.iter().map(|(variant, _)| { - let variant_str = format!("{}", variant); - quote::quote_spanned!(error.attr_span => Self::#variant => #variant_str,) + let as_str_matches = error.variants.iter().map(|(variant, field_ty, _)| { + let variant_str = variant.to_string(); + match field_ty { + Some(VariantField { is_named: true }) => { + quote::quote_spanned!(error.attr_span => Self::#variant { .. } => #variant_str,) + }, + Some(VariantField { is_named: false }) => { + quote::quote_spanned!(error.attr_span => Self::#variant(..) => #variant_str,) + }, + None => { + quote::quote_spanned!(error.attr_span => Self::#variant => #variant_str,) + }, + } }); let error_item = { @@ -58,18 +92,26 @@ pub fn expand_error(def: &mut Def) -> proc_macro2::TokenStream { }; error_item.variants.insert(0, phantom_variant); + + let capture_docs = if cfg!(feature = "no-metadata-docs") { "never" } else { "always" }; + // derive TypeInfo for error metadata - error_item - .attrs - .push(syn::parse_quote!( #[derive(#frame_support::scale_info::TypeInfo)] )); + error_item.attrs.push(syn::parse_quote! { + #[derive( + #frame_support::codec::Encode, + #frame_support::codec::Decode, + #frame_support::scale_info::TypeInfo, + #frame_support::PalletError, + )] + }); error_item.attrs.push(syn::parse_quote!( - #[scale_info(skip_type_params(#type_use_gen), capture_docs = "always")] + #[scale_info(skip_type_params(#type_use_gen), capture_docs = #capture_docs)] )); if get_doc_literals(&error_item.attrs).is_empty() { error_item.attrs.push(syn::parse_quote!( #[doc = r" - Custom [dispatch errors](https://substrate.dev/docs/en/knowledgebase/runtime/errors) + Custom [dispatch errors](https://docs.substrate.io/v3/runtime/events-and-errors) of this pallet. "] )); @@ -87,13 +129,7 @@ pub fn expand_error(def: &mut Def) -> proc_macro2::TokenStream { } impl<#type_impl_gen> #error_ident<#type_use_gen> #config_where_clause { - pub fn as_u8(&self) -> u8 { - match &self { - Self::__Ignore(_, _) => unreachable!("`__Ignore` can never be constructed"), - #( #as_u8_matches )* - } - } - + #[doc(hidden)] pub fn as_str(&self) -> &'static str { match &self { Self::__Ignore(_, _) => unreachable!("`__Ignore` can never be constructed"), @@ -115,18 +151,37 @@ pub fn expand_error(def: &mut Def) -> proc_macro2::TokenStream { #config_where_clause { fn from(err: #error_ident<#type_use_gen>) -> Self { + use #frame_support::codec::Encode; let index = < ::PalletInfo as #frame_support::traits::PalletInfo >::index::>() .expect("Every active module has an index in the runtime; qed") as u8; + let mut encoded = err.encode(); + encoded.resize(#frame_support::MAX_MODULE_ERROR_ENCODED_SIZE, 0); - #frame_support::sp_runtime::DispatchError::Module { + #frame_support::sp_runtime::DispatchError::Module(#frame_support::sp_runtime::ModuleError { index, - error: err.as_u8(), + error: TryInto::try_into(encoded).expect("encoded error is resized to be equal to the maximum encoded error size; qed"), message: Some(err.as_str()), - } + }) } } + + #[macro_export] + #[doc(hidden)] + macro_rules! #error_token_unique_id { + { + $caller:tt + frame_support = [{ $($frame_support:ident)::* }] + } => { + $($frame_support::)*tt_return! { + $caller + error = [{ #error_ident }] + } + }; + } + + pub use #error_token_unique_id as tt_error_token; ) } diff --git a/frame/support/procedural/src/pallet/expand/event.rs b/frame/support/procedural/src/pallet/expand/event.rs index ebd2d7aeabaf..acd60ab959c6 100644 --- a/frame/support/procedural/src/pallet/expand/event.rs +++ b/frame/support/procedural/src/pallet/expand/event.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -98,7 +98,7 @@ pub fn expand_event(def: &mut Def) -> proc_macro2::TokenStream { if get_doc_literals(&event_item.attrs).is_empty() { event_item.attrs.push(syn::parse_quote!( #[doc = r" - The [event](https://substrate.dev/docs/en/knowledgebase/runtime/events) emitted + The [event](https://docs.substrate.io/v3/runtime/events-and-errors) emitted by this pallet. "] )); @@ -117,9 +117,11 @@ pub fn expand_event(def: &mut Def) -> proc_macro2::TokenStream { )] )); - // skip requirement for type params to implement `TypeInfo`, and require docs capture + let capture_docs = if cfg!(feature = "no-metadata-docs") { "never" } else { "always" }; + + // skip requirement for type params to implement `TypeInfo`, and set docs capture event_item.attrs.push(syn::parse_quote!( - #[scale_info(skip_type_params(#event_use_gen), capture_docs = "always")] + #[scale_info(skip_type_params(#event_use_gen), capture_docs = #capture_docs)] )); let deposit_event = if let Some(deposit_event) = &event.deposit_event { @@ -134,12 +136,12 @@ pub fn expand_event(def: &mut Def) -> proc_macro2::TokenStream { impl<#type_impl_gen> Pallet<#type_use_gen> #completed_where_clause { #fn_vis fn deposit_event(event: Event<#event_use_gen>) { let event = < - ::Event as + ::Event as From> >::from(event); let event = < - ::Event as + ::Event as Into<::Event> >::into(event); diff --git a/frame/support/procedural/src/pallet/expand/genesis_build.rs b/frame/support/procedural/src/pallet/expand/genesis_build.rs index 06acaf324254..53d0695e5f97 100644 --- a/frame/support/procedural/src/pallet/expand/genesis_build.rs +++ b/frame/support/procedural/src/pallet/expand/genesis_build.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/expand/genesis_config.rs b/frame/support/procedural/src/pallet/expand/genesis_config.rs index 4bbba2c05908..18fa87a26253 100644 --- a/frame/support/procedural/src/pallet/expand/genesis_config.rs +++ b/frame/support/procedural/src/pallet/expand/genesis_config.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -89,7 +89,7 @@ pub fn expand_genesis_config(def: &mut Def) -> proc_macro2::TokenStream { attrs.push(syn::parse_quote!( #[doc = r" Can be used to configure the - [genesis state](https://substrate.dev/docs/en/knowledgebase/integrate/chain-spec#the-genesis-state) + [genesis state](https://docs.substrate.io/v3/runtime/chain-specs#the-genesis-state) of this pallet. "] )); diff --git a/frame/support/procedural/src/pallet/expand/hooks.rs b/frame/support/procedural/src/pallet/expand/hooks.rs index e0b7e3669da4..7a1a94cf46d3 100644 --- a/frame/support/procedural/src/pallet/expand/hooks.rs +++ b/frame/support/procedural/src/pallet/expand/hooks.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/expand/inherent.rs b/frame/support/procedural/src/pallet/expand/inherent.rs index 185211ecd4df..71d95f958fcd 100644 --- a/frame/support/procedural/src/pallet/expand/inherent.rs +++ b/frame/support/procedural/src/pallet/expand/inherent.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/expand/instances.rs b/frame/support/procedural/src/pallet/expand/instances.rs index 2ecb5ec481ac..52dc88763550 100644 --- a/frame/support/procedural/src/pallet/expand/instances.rs +++ b/frame/support/procedural/src/pallet/expand/instances.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/expand/mod.rs b/frame/support/procedural/src/pallet/expand/mod.rs index 1c8883977c76..83bef7a97af1 100644 --- a/frame/support/procedural/src/pallet/expand/mod.rs +++ b/frame/support/procedural/src/pallet/expand/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,6 +29,7 @@ mod origin; mod pallet_struct; mod storage; mod store_trait; +mod tt_default_parts; mod type_value; mod validate_unsigned; @@ -67,14 +68,15 @@ pub fn expand(mut def: Def) -> proc_macro2::TokenStream { let type_values = type_value::expand_type_values(&mut def); let origins = origin::expand_origins(&mut def); let validate_unsigned = validate_unsigned::expand_validate_unsigned(&mut def); + let tt_default_parts = tt_default_parts::expand_tt_default_parts(&mut def); if get_doc_literals(&def.item.attrs).is_empty() { def.item.attrs.push(syn::parse_quote!( #[doc = r" The module that hosts all the - [FRAME](https://substrate.dev/docs/en/knowledgebase/runtime/frame) + [FRAME](https://docs.substrate.io/v3/runtime/frame) types needed to add this pallet to a - [runtime](https://substrate.dev/docs/en/knowledgebase/runtime/). + runtime. "] )); } @@ -96,6 +98,7 @@ pub fn expand(mut def: Def) -> proc_macro2::TokenStream { #type_values #origins #validate_unsigned + #tt_default_parts ); def.item diff --git a/frame/support/procedural/src/pallet/expand/origin.rs b/frame/support/procedural/src/pallet/expand/origin.rs index 987512f69a02..721fc781a907 100644 --- a/frame/support/procedural/src/pallet/expand/origin.rs +++ b/frame/support/procedural/src/pallet/expand/origin.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/expand/pallet_struct.rs b/frame/support/procedural/src/pallet/expand/pallet_struct.rs index ea601f138ea0..52586a70a521 100644 --- a/frame/support/procedural/src/pallet/expand/pallet_struct.rs +++ b/frame/support/procedural/src/pallet/expand/pallet_struct.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -62,7 +62,7 @@ pub fn expand_pallet_struct(def: &mut Def) -> proc_macro2::TokenStream { if get_doc_literals(&pallet_item.attrs).is_empty() { pallet_item.attrs.push(syn::parse_quote!( #[doc = r" - The [pallet](https://substrate.dev/docs/en/knowledgebase/runtime/pallets) implementing + The [pallet](https://docs.substrate.io/v3/runtime/frame#pallets) implementing the on-chain logic. "] )); @@ -81,6 +81,7 @@ pub fn expand_pallet_struct(def: &mut Def) -> proc_macro2::TokenStream { let error_ident = &error_def.error; quote::quote_spanned!(def.pallet_struct.attr_span => impl<#type_impl_gen> #pallet_ident<#type_use_gen> #config_where_clause { + #[doc(hidden)] pub fn error_metadata() -> Option<#frame_support::metadata::PalletErrorMetadata> { Some(#frame_support::metadata::PalletErrorMetadata { ty: #frame_support::scale_info::meta_type::<#error_ident<#type_use_gen>>() @@ -91,6 +92,7 @@ pub fn expand_pallet_struct(def: &mut Def) -> proc_macro2::TokenStream { } else { quote::quote_spanned!(def.pallet_struct.attr_span => impl<#type_impl_gen> #pallet_ident<#type_use_gen> #config_where_clause { + #[doc(hidden)] pub fn error_metadata() -> Option<#frame_support::metadata::PalletErrorMetadata> { None } @@ -99,19 +101,19 @@ pub fn expand_pallet_struct(def: &mut Def) -> proc_macro2::TokenStream { }; let storage_info_span = - def.pallet_struct.generate_storage_info.unwrap_or(def.pallet_struct.attr_span); + def.pallet_struct.without_storage_info.unwrap_or(def.pallet_struct.attr_span); let storage_names = &def.storages.iter().map(|storage| &storage.ident).collect::>(); let storage_cfg_attrs = &def.storages.iter().map(|storage| &storage.cfg_attrs).collect::>(); - // Depending on the flag `generate_storage_info` and the storage attribute `unbounded`, we use + // Depending on the flag `without_storage_info` and the storage attribute `unbounded`, we use // partial or full storage info from storage. let storage_info_traits = &def .storages .iter() .map(|storage| { - if storage.unbounded || def.pallet_struct.generate_storage_info.is_none() { + if storage.unbounded || def.pallet_struct.without_storage_info.is_some() { quote::quote_spanned!(storage_info_span => PartialStorageInfoTrait) } else { quote::quote_spanned!(storage_info_span => StorageInfoTrait) @@ -123,7 +125,7 @@ pub fn expand_pallet_struct(def: &mut Def) -> proc_macro2::TokenStream { .storages .iter() .map(|storage| { - if storage.unbounded || def.pallet_struct.generate_storage_info.is_none() { + if storage.unbounded || def.pallet_struct.without_storage_info.is_some() { quote::quote_spanned!(storage_info_span => partial_storage_info) } else { quote::quote_spanned!(storage_info_span => storage_info) @@ -233,6 +235,25 @@ pub fn expand_pallet_struct(def: &mut Def) -> proc_macro2::TokenStream { } } + impl<#type_impl_gen> #frame_support::traits::PalletsInfoAccess + for #pallet_ident<#type_use_gen> + #config_where_clause + { + fn count() -> usize { 1 } + fn accumulate( + acc: &mut #frame_support::sp_std::vec::Vec<#frame_support::traits::PalletInfoData> + ) { + use #frame_support::traits::PalletInfoAccess; + let item = #frame_support::traits::PalletInfoData { + index: Self::index(), + name: Self::name(), + module_name: Self::module_name(), + crate_version: Self::crate_version(), + }; + acc.push(item); + } + } + #storage_info ) } diff --git a/frame/support/procedural/src/pallet/expand/storage.rs b/frame/support/procedural/src/pallet/expand/storage.rs index a4f030722f1c..d59dd4605139 100644 --- a/frame/support/procedural/src/pallet/expand/storage.rs +++ b/frame/support/procedural/src/pallet/expand/storage.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -234,7 +234,8 @@ pub fn expand_storages(def: &mut Def) -> proc_macro2::TokenStream { let pallet_ident = &def.pallet_struct.pallet; let entries_builder = def.storages.iter().map(|storage| { - let docs = &storage.docs; + let no_docs = vec![]; + let docs = if cfg!(feature = "no-metadata-docs") { &no_docs } else { &storage.docs }; let ident = &storage.ident; let gen = &def.type_use_generics(storage.attr_span); @@ -405,6 +406,7 @@ pub fn expand_storages(def: &mut Def) -> proc_macro2::TokenStream { quote::quote_spanned!(storage_def.attr_span => #(#cfg_attrs)* + #[doc(hidden)] #prefix_struct_vis struct #counter_prefix_struct_ident<#type_use_gen>( core::marker::PhantomData<(#type_use_gen,)> ); @@ -438,6 +440,7 @@ pub fn expand_storages(def: &mut Def) -> proc_macro2::TokenStream { #maybe_counter #(#cfg_attrs)* + #[doc(hidden)] #prefix_struct_vis struct #prefix_struct_ident<#type_use_gen>( core::marker::PhantomData<(#type_use_gen,)> ); diff --git a/frame/support/procedural/src/pallet/expand/store_trait.rs b/frame/support/procedural/src/pallet/expand/store_trait.rs index 36cc08b732fe..4fb4f46143a1 100644 --- a/frame/support/procedural/src/pallet/expand/store_trait.rs +++ b/frame/support/procedural/src/pallet/expand/store_trait.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/expand/tt_default_parts.rs b/frame/support/procedural/src/pallet/expand/tt_default_parts.rs new file mode 100644 index 000000000000..173dcc0e2ac3 --- /dev/null +++ b/frame/support/procedural/src/pallet/expand/tt_default_parts.rs @@ -0,0 +1,82 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{pallet::Def, COUNTER}; +use syn::spanned::Spanned; + +/// Generate the `tt_default_parts` macro. +pub fn expand_tt_default_parts(def: &mut Def) -> proc_macro2::TokenStream { + let count = COUNTER.with(|counter| counter.borrow_mut().inc()); + let default_parts_unique_id = + syn::Ident::new(&format!("__tt_default_parts_{}", count), def.item.span()); + + let call_part = def.call.as_ref().map(|_| quote::quote!(Call,)); + + let storage_part = (!def.storages.is_empty()).then(|| quote::quote!(Storage,)); + + let event_part = def.event.as_ref().map(|event| { + let gen = event.gen_kind.is_generic().then(|| quote::quote!( )); + quote::quote!( Event #gen , ) + }); + + let origin_part = def.origin.as_ref().map(|origin| { + let gen = origin.is_generic.then(|| quote::quote!( )); + quote::quote!( Origin #gen , ) + }); + + let config_part = def.genesis_config.as_ref().map(|genesis_config| { + let gen = genesis_config.gen_kind.is_generic().then(|| quote::quote!( )); + quote::quote!( Config #gen , ) + }); + + let inherent_part = def.inherent.as_ref().map(|_| quote::quote!(Inherent,)); + + let validate_unsigned_part = + def.validate_unsigned.as_ref().map(|_| quote::quote!(ValidateUnsigned,)); + + quote::quote!( + // This macro follows the conventions as laid out by the `tt-call` crate. It does not + // accept any arguments and simply returns the pallet parts, separated by commas, then + // wrapped inside of braces and finally prepended with double colons, to the caller inside + // of a key named `tokens`. + // + // We need to accept a frame_support argument here, because this macro gets expanded on the + // crate that called the `construct_runtime!` macro, and said crate may have renamed + // frame-support, and so we need to pass in the frame-support path that said crate + // recognizes. + #[macro_export] + #[doc(hidden)] + macro_rules! #default_parts_unique_id { + { + $caller:tt + frame_support = [{ $($frame_support:ident)::* }] + } => { + $($frame_support)*::tt_return! { + $caller + tokens = [{ + ::{ + Pallet, #call_part #storage_part #event_part #origin_part #config_part + #inherent_part #validate_unsigned_part + } + }] + } + }; + } + + pub use #default_parts_unique_id as tt_default_parts; + ) +} diff --git a/frame/support/procedural/src/pallet/expand/type_value.rs b/frame/support/procedural/src/pallet/expand/type_value.rs index 535a18777380..abe52459a791 100644 --- a/frame/support/procedural/src/pallet/expand/type_value.rs +++ b/frame/support/procedural/src/pallet/expand/type_value.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -59,7 +59,10 @@ pub fn expand_type_values(def: &mut Def) -> proc_macro2::TokenStream { (Default::default(), Default::default()) }; + let docs = &type_value.docs; + expand.extend(quote::quote_spanned!(type_value.attr_span => + #( #[doc = #docs] )* #vis struct #ident<#struct_use_gen>(core::marker::PhantomData<((), #struct_use_gen)>); impl<#struct_impl_gen> #frame_support::traits::Get<#type_> for #ident<#struct_use_gen> #where_clause diff --git a/frame/support/procedural/src/pallet/expand/validate_unsigned.rs b/frame/support/procedural/src/pallet/expand/validate_unsigned.rs index 5f30d712e9a5..b49ba2b5b02e 100644 --- a/frame/support/procedural/src/pallet/expand/validate_unsigned.rs +++ b/frame/support/procedural/src/pallet/expand/validate_unsigned.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/mod.rs b/frame/support/procedural/src/pallet/mod.rs index 93797906d04d..ff9f12286774 100644 --- a/frame/support/procedural/src/pallet/mod.rs +++ b/frame/support/procedural/src/pallet/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/parse/call.rs b/frame/support/procedural/src/pallet/parse/call.rs index 0563568f3331..5468b1352197 100644 --- a/frame/support/procedural/src/pallet/parse/call.rs +++ b/frame/support/procedural/src/pallet/parse/call.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/parse/config.rs b/frame/support/procedural/src/pallet/parse/config.rs index 712c20ffc7b4..526c7eda2fd5 100644 --- a/frame/support/procedural/src/pallet/parse/config.rs +++ b/frame/support/procedural/src/pallet/parse/config.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,6 @@ // limitations under the License. use super::helper; -use core::convert::TryFrom; use frame_support_procedural_tools::get_doc_literals; use quote::ToTokens; use syn::spanned::Spanned; diff --git a/frame/support/procedural/src/pallet/parse/error.rs b/frame/support/procedural/src/pallet/parse/error.rs index 9c9a95105c53..0ec49aa0adb4 100644 --- a/frame/support/procedural/src/pallet/parse/error.rs +++ b/frame/support/procedural/src/pallet/parse/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,20 +18,26 @@ use super::helper; use frame_support_procedural_tools::get_doc_literals; use quote::ToTokens; -use syn::spanned::Spanned; +use syn::{spanned::Spanned, Fields}; /// List of additional token to be used for parsing. mod keyword { syn::custom_keyword!(Error); } +/// Records information about the error enum variants. +pub struct VariantField { + /// Whether or not the field is named, i.e. whether it is a tuple variant or struct variant. + pub is_named: bool, +} + /// This checks error declaration as a enum declaration with only variants without fields nor /// discriminant. pub struct ErrorDef { /// The index of error item in pallet module. pub index: usize, - /// Variants ident and doc literals (ordered as declaration order) - pub variants: Vec<(syn::Ident, Vec)>, + /// Variants ident, optional field and doc literals (ordered as declaration order) + pub variants: Vec<(syn::Ident, Option, Vec)>, /// A set of usage of instance, must be check for consistency with trait. pub instances: Vec, /// The keyword error used (contains span). @@ -70,18 +76,19 @@ impl ErrorDef { .variants .iter() .map(|variant| { - if !matches!(variant.fields, syn::Fields::Unit) { - let msg = "Invalid pallet::error, unexpected fields, must be `Unit`"; - return Err(syn::Error::new(variant.fields.span(), msg)) - } + let field_ty = match &variant.fields { + Fields::Unit => None, + Fields::Named(_) => Some(VariantField { is_named: true }), + Fields::Unnamed(_) => Some(VariantField { is_named: false }), + }; if variant.discriminant.is_some() { - let msg = "Invalid pallet::error, unexpected discriminant, discriminant \ + let msg = "Invalid pallet::error, unexpected discriminant, discriminants \ are not supported"; let span = variant.discriminant.as_ref().unwrap().0.span(); return Err(syn::Error::new(span, msg)) } - Ok((variant.ident.clone(), get_doc_literals(&variant.attrs))) + Ok((variant.ident.clone(), field_ty, get_doc_literals(&variant.attrs))) }) .collect::>()?; diff --git a/frame/support/procedural/src/pallet/parse/event.rs b/frame/support/procedural/src/pallet/parse/event.rs index 33de4aca8b59..e046cacac88e 100644 --- a/frame/support/procedural/src/pallet/parse/event.rs +++ b/frame/support/procedural/src/pallet/parse/event.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/parse/extra_constants.rs b/frame/support/procedural/src/pallet/parse/extra_constants.rs index c1324df6c22f..7163b4b63208 100644 --- a/frame/support/procedural/src/pallet/parse/extra_constants.rs +++ b/frame/support/procedural/src/pallet/parse/extra_constants.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,6 +28,7 @@ mod keyword { syn::custom_keyword!(compact); syn::custom_keyword!(T); syn::custom_keyword!(pallet); + syn::custom_keyword!(constant_name); } /// Definition of extra constants typically `impl Pallet { ... }` @@ -50,6 +51,29 @@ pub struct ExtraConstantDef { pub type_: syn::Type, /// The doc associated pub doc: Vec, + /// Optional MetaData Name + pub metadata_name: Option, +} + +/// Attributes for functions in extra_constants impl block. +/// Parse for `#[pallet::constant_name(ConstantName)]` +pub struct ExtraConstAttr { + metadata_name: syn::Ident, +} + +impl syn::parse::Parse for ExtraConstAttr { + fn parse(input: syn::parse::ParseStream) -> syn::Result { + input.parse::()?; + let content; + syn::bracketed!(content in input); + content.parse::()?; + content.parse::()?; + content.parse::()?; + + let metadata_name; + syn::parenthesized!(metadata_name in content); + Ok(ExtraConstAttr { metadata_name: metadata_name.parse::()? }) + } } impl ExtraConstantsDef { @@ -57,7 +81,10 @@ impl ExtraConstantsDef { let item = if let syn::Item::Impl(item) = item { item } else { - return Err(syn::Error::new(item.span(), "Invalid pallet::call, expected item impl")) + return Err(syn::Error::new( + item.span(), + "Invalid pallet::extra_constants, expected item impl", + )) }; let mut instances = vec![]; @@ -102,10 +129,23 @@ impl ExtraConstantsDef { syn::ReturnType::Type(_, type_) => *type_.clone(), }; + // parse metadata_name + let mut extra_constant_attrs: Vec = + helper::take_item_pallet_attrs(method)?; + + if extra_constant_attrs.len() > 1 { + let msg = + "Invalid attribute in pallet::constant_name, only one attribute is expected"; + return Err(syn::Error::new(extra_constant_attrs[1].metadata_name.span(), msg)) + } + + let metadata_name = extra_constant_attrs.pop().map(|attr| attr.metadata_name); + extra_constants.push(ExtraConstantDef { ident: method.sig.ident.clone(), type_, doc: get_doc_literals(&method.attrs), + metadata_name, }); } diff --git a/frame/support/procedural/src/pallet/parse/genesis_build.rs b/frame/support/procedural/src/pallet/parse/genesis_build.rs index 82e297b4e26e..79ee08306982 100644 --- a/frame/support/procedural/src/pallet/parse/genesis_build.rs +++ b/frame/support/procedural/src/pallet/parse/genesis_build.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/parse/genesis_config.rs b/frame/support/procedural/src/pallet/parse/genesis_config.rs index a0cf7de1a846..875d15bdc061 100644 --- a/frame/support/procedural/src/pallet/parse/genesis_config.rs +++ b/frame/support/procedural/src/pallet/parse/genesis_config.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/parse/helper.rs b/frame/support/procedural/src/pallet/parse/helper.rs index 2590e86b58b0..824407917358 100644 --- a/frame/support/procedural/src/pallet/parse/helper.rs +++ b/frame/support/procedural/src/pallet/parse/helper.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -139,6 +139,12 @@ impl MutItemAttrs for syn::ItemMod { } } +impl MutItemAttrs for syn::ImplItemMethod { + fn mut_item_attrs(&mut self) -> Option<&mut Vec> { + Some(&mut self.attrs) + } +} + /// Parse for `()` struct Unit; impl syn::parse::Parse for Unit { diff --git a/frame/support/procedural/src/pallet/parse/hooks.rs b/frame/support/procedural/src/pallet/parse/hooks.rs index 1dd86498f22d..cacc149e4866 100644 --- a/frame/support/procedural/src/pallet/parse/hooks.rs +++ b/frame/support/procedural/src/pallet/parse/hooks.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/parse/inherent.rs b/frame/support/procedural/src/pallet/parse/inherent.rs index de5ad8f795db..2833b3ef5c72 100644 --- a/frame/support/procedural/src/pallet/parse/inherent.rs +++ b/frame/support/procedural/src/pallet/parse/inherent.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/parse/mod.rs b/frame/support/procedural/src/pallet/parse/mod.rs index 96d4776e805b..a436f7e09c1d 100644 --- a/frame/support/procedural/src/pallet/parse/mod.rs +++ b/frame/support/procedural/src/pallet/parse/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/parse/origin.rs b/frame/support/procedural/src/pallet/parse/origin.rs index c4e1197ac511..2d729376f5de 100644 --- a/frame/support/procedural/src/pallet/parse/origin.rs +++ b/frame/support/procedural/src/pallet/parse/origin.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet/parse/pallet_struct.rs b/frame/support/procedural/src/pallet/parse/pallet_struct.rs index 278f46e13818..d98862be8f78 100644 --- a/frame/support/procedural/src/pallet/parse/pallet_struct.rs +++ b/frame/support/procedural/src/pallet/parse/pallet_struct.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,7 +24,7 @@ mod keyword { syn::custom_keyword!(pallet); syn::custom_keyword!(Pallet); syn::custom_keyword!(generate_store); - syn::custom_keyword!(generate_storage_info); + syn::custom_keyword!(without_storage_info); syn::custom_keyword!(storage_version); syn::custom_keyword!(Store); } @@ -43,18 +43,18 @@ pub struct PalletStructDef { pub attr_span: proc_macro2::Span, /// Whether to specify the storages max encoded len when implementing `StorageInfoTrait`. /// Contains the span of the attribute. - pub generate_storage_info: Option, + pub without_storage_info: Option, /// The current storage version of the pallet. pub storage_version: Option, } /// Parse for one variant of: /// * `#[pallet::generate_store($vis trait Store)]` -/// * `#[pallet::generate_storage_info]` +/// * `#[pallet::without_storage_info]` /// * `#[pallet::storage_version(STORAGE_VERSION)]` pub enum PalletStructAttr { GenerateStore { span: proc_macro2::Span, vis: syn::Visibility, keyword: keyword::Store }, - GenerateStorageInfoTrait(proc_macro2::Span), + WithoutStorageInfoTrait(proc_macro2::Span), StorageVersion { storage_version: syn::Path, span: proc_macro2::Span }, } @@ -62,7 +62,7 @@ impl PalletStructAttr { fn span(&self) -> proc_macro2::Span { match self { Self::GenerateStore { span, .. } => *span, - Self::GenerateStorageInfoTrait(span) => *span, + Self::WithoutStorageInfoTrait(span) => *span, Self::StorageVersion { span, .. } => *span, } } @@ -86,9 +86,9 @@ impl syn::parse::Parse for PalletStructAttr { generate_content.parse::()?; let keyword = generate_content.parse::()?; Ok(Self::GenerateStore { vis, keyword, span }) - } else if lookahead.peek(keyword::generate_storage_info) { - let span = content.parse::()?.span(); - Ok(Self::GenerateStorageInfoTrait(span)) + } else if lookahead.peek(keyword::without_storage_info) { + let span = content.parse::()?.span(); + Ok(Self::WithoutStorageInfoTrait(span)) } else if lookahead.peek(keyword::storage_version) { let span = content.parse::()?.span(); @@ -117,7 +117,7 @@ impl PalletStructDef { }; let mut store = None; - let mut generate_storage_info = None; + let mut without_storage_info = None; let mut storage_version_found = None; let struct_attrs: Vec = helper::take_item_pallet_attrs(&mut item.attrs)?; @@ -126,16 +126,16 @@ impl PalletStructDef { PalletStructAttr::GenerateStore { vis, keyword, .. } if store.is_none() => { store = Some((vis, keyword)); }, - PalletStructAttr::GenerateStorageInfoTrait(span) - if generate_storage_info.is_none() => + PalletStructAttr::WithoutStorageInfoTrait(span) + if without_storage_info.is_none() => { - generate_storage_info = Some(span); - } + without_storage_info = Some(span); + }, PalletStructAttr::StorageVersion { storage_version, .. } if storage_version_found.is_none() => { storage_version_found = Some(storage_version); - } + }, attr => { let msg = "Unexpected duplicated attribute"; return Err(syn::Error::new(attr.span(), msg)) @@ -164,7 +164,7 @@ impl PalletStructDef { pallet, store, attr_span, - generate_storage_info, + without_storage_info, storage_version: storage_version_found, }) } diff --git a/frame/support/procedural/src/pallet/parse/storage.rs b/frame/support/procedural/src/pallet/parse/storage.rs index cd29baf93d84..effe0ce6c55d 100644 --- a/frame/support/procedural/src/pallet/parse/storage.rs +++ b/frame/support/procedural/src/pallet/parse/storage.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -704,11 +704,11 @@ impl StorageDef { }) .unwrap_or(Some(QueryKind::OptionQuery)); // This value must match the default generic. - if query_kind.is_none() && getter.is_some() { + if let (None, Some(getter)) = (query_kind.as_ref(), getter.as_ref()) { let msg = "Invalid pallet::storage, cannot generate getter because QueryKind is not \ identifiable. QueryKind must be `OptionQuery`, `ValueQuery`, or default one to be \ identifiable."; - return Err(syn::Error::new(getter.unwrap().span(), msg)) + return Err(syn::Error::new(getter.span(), msg)) } Ok(StorageDef { diff --git a/frame/support/procedural/src/pallet/parse/type_value.rs b/frame/support/procedural/src/pallet/parse/type_value.rs index 7b9d57472db4..a3d004cd8a53 100644 --- a/frame/support/procedural/src/pallet/parse/type_value.rs +++ b/frame/support/procedural/src/pallet/parse/type_value.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -38,6 +38,8 @@ pub struct TypeValueDef { pub where_clause: Option, /// The span of the pallet::type_value attribute. pub attr_span: proc_macro2::Span, + /// Docs on the item. + pub docs: Vec, } impl TypeValueDef { @@ -53,9 +55,18 @@ impl TypeValueDef { return Err(syn::Error::new(item.span(), msg)) }; - if !item.attrs.is_empty() { - let msg = "Invalid pallet::type_value, unexpected attribute"; - return Err(syn::Error::new(item.attrs[0].span(), msg)) + let mut docs = vec![]; + for attr in &item.attrs { + if let Ok(syn::Meta::NameValue(meta)) = attr.parse_meta() { + if meta.path.get_ident().map_or(false, |ident| ident == "doc") { + docs.push(meta.lit); + continue + } + } + + let msg = "Invalid pallet::type_value, unexpected attribute, only doc attribute are \ + allowed"; + return Err(syn::Error::new(attr.span(), msg)) } if let Some(span) = item @@ -106,6 +117,7 @@ impl TypeValueDef { type_, instances, where_clause, + docs, }) } } diff --git a/frame/support/procedural/src/pallet/parse/validate_unsigned.rs b/frame/support/procedural/src/pallet/parse/validate_unsigned.rs index 87e2a326f186..a58671d9762d 100644 --- a/frame/support/procedural/src/pallet/parse/validate_unsigned.rs +++ b/frame/support/procedural/src/pallet/parse/validate_unsigned.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/pallet_error.rs b/frame/support/procedural/src/pallet_error.rs new file mode 100644 index 000000000000..216168131e43 --- /dev/null +++ b/frame/support/procedural/src/pallet_error.rs @@ -0,0 +1,197 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use frame_support_procedural_tools::generate_crate_access_2018; +use quote::ToTokens; +use std::str::FromStr; + +// Derive `PalletError` +pub fn derive_pallet_error(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let syn::DeriveInput { ident: name, generics, data, .. } = match syn::parse(input) { + Ok(input) => input, + Err(e) => return e.to_compile_error().into(), + }; + + let frame_support = match generate_crate_access_2018("frame-support") { + Ok(c) => c, + Err(e) => return e.into_compile_error().into(), + }; + let frame_support = &frame_support; + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + let max_encoded_size = match data { + syn::Data::Struct(syn::DataStruct { fields, .. }) => match fields { + syn::Fields::Named(syn::FieldsNamed { named: fields, .. }) | + syn::Fields::Unnamed(syn::FieldsUnnamed { unnamed: fields, .. }) => { + let maybe_field_tys = fields + .iter() + .map(|f| generate_field_types(f, &frame_support)) + .collect::>>(); + let field_tys = match maybe_field_tys { + Ok(tys) => tys.into_iter().flatten(), + Err(e) => return e.into_compile_error().into(), + }; + quote::quote! { + 0_usize + #( + .saturating_add(< + #field_tys as #frame_support::traits::PalletError + >::MAX_ENCODED_SIZE) + )* + } + }, + syn::Fields::Unit => quote::quote!(0), + }, + syn::Data::Enum(syn::DataEnum { variants, .. }) => { + let field_tys = variants + .iter() + .map(|variant| generate_variant_field_types(variant, &frame_support)) + .collect::>>, syn::Error>>(); + + let field_tys = match field_tys { + Ok(tys) => tys.into_iter().flatten().collect::>(), + Err(e) => return e.to_compile_error().into(), + }; + + // We start with `1`, because the discriminant of an enum is stored as u8 + if field_tys.is_empty() { + quote::quote!(1) + } else { + let variant_sizes = field_tys.into_iter().map(|variant_field_tys| { + quote::quote! { + 1_usize + #(.saturating_add(< + #variant_field_tys as #frame_support::traits::PalletError + >::MAX_ENCODED_SIZE))* + } + }); + + quote::quote! {{ + let mut size = 1_usize; + let mut tmp = 0_usize; + #( + tmp = #variant_sizes; + size = if tmp > size { tmp } else { size }; + tmp = 0_usize; + )* + size + }} + } + }, + syn::Data::Union(syn::DataUnion { union_token, .. }) => { + let msg = "Cannot derive `PalletError` for union; please implement it directly"; + return syn::Error::new(union_token.span, msg).into_compile_error().into() + }, + }; + + quote::quote!( + const _: () = { + impl #impl_generics #frame_support::traits::PalletError + for #name #ty_generics #where_clause + { + const MAX_ENCODED_SIZE: usize = #max_encoded_size; + } + }; + ) + .into() +} + +fn generate_field_types( + field: &syn::Field, + scrate: &syn::Ident, +) -> syn::Result> { + let attrs = &field.attrs; + + for attr in attrs { + if attr.path.is_ident("codec") { + match attr.parse_meta()? { + syn::Meta::List(ref meta_list) if meta_list.nested.len() == 1 => { + match meta_list + .nested + .first() + .expect("Just checked that there is one item; qed") + { + syn::NestedMeta::Meta(syn::Meta::Path(path)) + if path.get_ident().map_or(false, |i| i == "skip") => + return Ok(None), + + syn::NestedMeta::Meta(syn::Meta::Path(path)) + if path.get_ident().map_or(false, |i| i == "compact") => + { + let field_ty = &field.ty; + return Ok(Some(quote::quote!(#scrate::codec::Compact<#field_ty>))) + }, + + syn::NestedMeta::Meta(syn::Meta::NameValue(syn::MetaNameValue { + path, + lit: syn::Lit::Str(lit_str), + .. + })) if path.get_ident().map_or(false, |i| i == "encoded_as") => { + let ty = proc_macro2::TokenStream::from_str(&lit_str.value())?; + return Ok(Some(ty)) + }, + + _ => (), + } + }, + _ => (), + } + } + } + + Ok(Some(field.ty.to_token_stream())) +} + +fn generate_variant_field_types( + variant: &syn::Variant, + scrate: &syn::Ident, +) -> syn::Result>> { + let attrs = &variant.attrs; + + for attr in attrs { + if attr.path.is_ident("codec") { + match attr.parse_meta()? { + syn::Meta::List(ref meta_list) if meta_list.nested.len() == 1 => { + match meta_list + .nested + .first() + .expect("Just checked that there is one item; qed") + { + syn::NestedMeta::Meta(syn::Meta::Path(path)) + if path.get_ident().map_or(false, |i| i == "skip") => + return Ok(None), + + _ => (), + } + }, + _ => (), + } + } + } + + match &variant.fields { + syn::Fields::Named(syn::FieldsNamed { named: fields, .. }) | + syn::Fields::Unnamed(syn::FieldsUnnamed { unnamed: fields, .. }) => { + let field_tys = fields + .iter() + .map(|field| generate_field_types(field, scrate)) + .collect::>>()?; + Ok(Some(field_tys.into_iter().flatten().collect())) + }, + syn::Fields::Unit => Ok(None), + } +} diff --git a/frame/support/procedural/src/partial_eq_no_bound.rs b/frame/support/procedural/src/partial_eq_no_bound.rs index 3dbabf3f5d39..31930c0c3dae 100644 --- a/frame/support/procedural/src/partial_eq_no_bound.rs +++ b/frame/support/procedural/src/partial_eq_no_bound.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/storage/genesis_config/builder_def.rs b/frame/support/procedural/src/storage/genesis_config/builder_def.rs index 001cea0f2b78..975791881da2 100644 --- a/frame/support/procedural/src/storage/genesis_config/builder_def.rs +++ b/frame/support/procedural/src/storage/genesis_config/builder_def.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/storage/genesis_config/genesis_config_def.rs b/frame/support/procedural/src/storage/genesis_config/genesis_config_def.rs index fbdaab06b489..d24e50096f25 100644 --- a/frame/support/procedural/src/storage/genesis_config/genesis_config_def.rs +++ b/frame/support/procedural/src/storage/genesis_config/genesis_config_def.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/storage/genesis_config/mod.rs b/frame/support/procedural/src/storage/genesis_config/mod.rs index d2d1afb01773..d4348ee19171 100644 --- a/frame/support/procedural/src/storage/genesis_config/mod.rs +++ b/frame/support/procedural/src/storage/genesis_config/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -75,12 +75,12 @@ fn decl_genesis_config_and_impl_default( #[serde(deny_unknown_fields)] #[serde(crate = #serde_crate)] #serde_bug_bound - pub struct GenesisConfig#genesis_struct_decl #genesis_where_clause { + pub struct GenesisConfig #genesis_struct_decl #genesis_where_clause { #( #config_fields )* } #[cfg(feature = "std")] - impl#genesis_impl Default for GenesisConfig#genesis_struct #genesis_where_clause { + impl #genesis_impl Default for GenesisConfig #genesis_struct #genesis_where_clause { fn default() -> Self { GenesisConfig { #( #config_field_defaults )* @@ -137,7 +137,7 @@ fn impl_build_storage( quote! { #[cfg(feature = "std")] - impl#genesis_impl GenesisConfig#genesis_struct #genesis_where_clause { + impl #genesis_impl GenesisConfig #genesis_struct #genesis_where_clause { /// Build the storage for this module. pub fn build_storage #fn_generic (&self) -> std::result::Result< #scrate::sp_runtime::Storage, @@ -161,7 +161,7 @@ fn impl_build_storage( } #[cfg(feature = "std")] - impl#build_storage_impl #build_storage_impl_trait for GenesisConfig#genesis_struct + impl #build_storage_impl #build_storage_impl_trait for GenesisConfig #genesis_struct #where_clause { fn build_module_genesis_storage( diff --git a/frame/support/procedural/src/storage/getters.rs b/frame/support/procedural/src/storage/getters.rs index 988e6fa09624..9fe3734e6889 100644 --- a/frame/support/procedural/src/storage/getters.rs +++ b/frame/support/procedural/src/storage/getters.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -92,7 +92,7 @@ pub fn impl_getters(def: &DeclStorageDefExt) -> TokenStream { let where_clause = &def.where_clause; quote!( - impl#module_impl #module_struct #where_clause { + impl #module_impl #module_struct #where_clause { #getters } ) diff --git a/frame/support/procedural/src/storage/instance_trait.rs b/frame/support/procedural/src/storage/instance_trait.rs index 00a73d6fbd6e..14e968112029 100644 --- a/frame/support/procedural/src/storage/instance_trait.rs +++ b/frame/support/procedural/src/storage/instance_trait.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/storage/metadata.rs b/frame/support/procedural/src/storage/metadata.rs index a90e5051c5b2..d1879d1509f8 100644 --- a/frame/support/procedural/src/storage/metadata.rs +++ b/frame/support/procedural/src/storage/metadata.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -207,7 +207,7 @@ pub fn impl_metadata(def: &DeclStorageDefExt) -> TokenStream { quote!( #default_byte_getter_struct_defs - impl#module_impl #module_struct #where_clause { + impl #module_impl #module_struct #where_clause { #[doc(hidden)] pub fn storage_metadata() -> #scrate::metadata::PalletStorageMetadata { #store_metadata diff --git a/frame/support/procedural/src/storage/mod.rs b/frame/support/procedural/src/storage/mod.rs index 27964d7012a2..b89e75633498 100644 --- a/frame/support/procedural/src/storage/mod.rs +++ b/frame/support/procedural/src/storage/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/storage/parse.rs b/frame/support/procedural/src/storage/parse.rs index 3a11846181a8..54026b7d78b1 100644 --- a/frame/support/procedural/src/storage/parse.rs +++ b/frame/support/procedural/src/storage/parse.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/storage/storage_info.rs b/frame/support/procedural/src/storage/storage_info.rs index 844896409f85..77515fa739b2 100644 --- a/frame/support/procedural/src/storage/storage_info.rs +++ b/frame/support/procedural/src/storage/storage_info.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -48,7 +48,7 @@ pub fn impl_storage_info(def: &DeclStorageDefExt) -> TokenStream { let where_clause = &def.where_clause; quote!( - impl#module_impl #scrate::traits::StorageInfoTrait for #module_struct #where_clause { + impl #module_impl #scrate::traits::StorageInfoTrait for #module_struct #where_clause { fn storage_info() -> #scrate::sp_std::vec::Vec<#scrate::traits::StorageInfo> { let mut res = #scrate::sp_std::vec![]; #res_append_storage diff --git a/frame/support/procedural/src/storage/storage_struct.rs b/frame/support/procedural/src/storage/storage_struct.rs index b318225681c1..649a41bab5ec 100644 --- a/frame/support/procedural/src/storage/storage_struct.rs +++ b/frame/support/procedural/src/storage/storage_struct.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/src/storage/store_trait.rs b/frame/support/procedural/src/storage/store_trait.rs index 7dde92cf9a75..daf933b3b770 100644 --- a/frame/support/procedural/src/storage/store_trait.rs +++ b/frame/support/procedural/src/storage/store_trait.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -48,7 +48,7 @@ pub fn decl_and_impl(def: &DeclStorageDefExt) -> TokenStream { #visibility trait #store_trait { #decl_store_items } - impl#module_impl #store_trait for #module_struct #where_clause { + impl #module_impl #store_trait for #module_struct #where_clause { #impl_store_items } ) diff --git a/frame/support/procedural/src/transactional.rs b/frame/support/procedural/src/transactional.rs index 403f1cd02bac..eb77a320a509 100644 --- a/frame/support/procedural/src/transactional.rs +++ b/frame/support/procedural/src/transactional.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -49,7 +49,9 @@ pub fn require_transactional(_attr: TokenStream, input: TokenStream) -> Result, +} + +impl syn::parse::Parse for CreateTtReturnMacroDef { + fn parse(input: syn::parse::ParseStream) -> syn::Result { + let name = input.parse()?; + let _ = input.parse::()?; + + let mut args = Vec::new(); + while !input.is_empty() { + let mut value; + let key: Ident = input.parse()?; + let _ = input.parse::()?; + let _: syn::token::Bracket = syn::bracketed!(value in input); + let _: syn::token::Brace = syn::braced!(value in value); + let value: TokenStream = value.parse()?; + + args.push((key, value)) + } + + Ok(Self { name, args }) + } +} + +/// A proc macro that accepts a name and any number of key-value pairs, to be used to create a +/// declarative macro that follows tt-call conventions and simply calls [`tt_call::tt_return`], +/// accepting an optional `frame-support` argument and returning the key-value pairs that were +/// supplied to the proc macro. +/// +/// # Example +/// ```ignore +/// __create_tt_macro! { +/// my_tt_macro, +/// foo = [{ bar }] +/// } +/// +/// // Creates the following declarative macro: +/// +/// macro_rules! my_tt_macro { +/// { +/// $caller:tt +/// $(frame_support = [{ $($frame_support:ident)::* }])? +/// } => { +/// frame_support::tt_return! { +/// $caller +/// foo = [{ bar }] +/// } +/// } +/// } +/// ``` +pub fn create_tt_return_macro(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let CreateTtReturnMacroDef { name, args } = + syn::parse_macro_input!(input as CreateTtReturnMacroDef); + + let frame_support = match generate_crate_access_2018("frame-support") { + Ok(i) => i, + Err(e) => return e.into_compile_error().into(), + }; + let (keys, values): (Vec<_>, Vec<_>) = args.into_iter().unzip(); + let count = COUNTER.with(|counter| counter.borrow_mut().inc()); + let unique_name = format_ident!("{}_{}", name, count); + + let decl_macro = quote::quote! { + #[macro_export] + #[doc(hidden)] + macro_rules! #unique_name { + { + $caller:tt + $(frame_support = [{ $($frame_support:ident)::* }])? + } => { + #frame_support::tt_return! { + $caller + #( + #keys = [{ #values }] + )* + } + } + } + + pub use #unique_name as #name; + }; + + decl_macro.into() +} diff --git a/frame/support/procedural/tools/Cargo.toml b/frame/support/procedural/tools/Cargo.toml index ee59f53287ef..b38071dd3158 100644 --- a/frame/support/procedural/tools/Cargo.toml +++ b/frame/support/procedural/tools/Cargo.toml @@ -2,9 +2,9 @@ name = "frame-support-procedural-tools" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Proc macro helpers for procedural macros" @@ -13,7 +13,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] frame-support-procedural-tools-derive = { version = "3.0.0", path = "./derive" } -proc-macro2 = "1.0.29" -quote = "1.0.3" -syn = { version = "1.0.58", features = ["full", "visit", "extra-traits"] } -proc-macro-crate = "1.0.0" +proc-macro2 = "1.0.36" +quote = "1.0.10" +syn = { version = "1.0.82", features = ["full", "visit", "extra-traits"] } +proc-macro-crate = "1.1.3" diff --git a/frame/support/procedural/tools/derive/Cargo.toml b/frame/support/procedural/tools/derive/Cargo.toml index 12ec6a69f396..5ed1b506dfb9 100644 --- a/frame/support/procedural/tools/derive/Cargo.toml +++ b/frame/support/procedural/tools/derive/Cargo.toml @@ -2,9 +2,9 @@ name = "frame-support-procedural-tools-derive" version = "3.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Use to derive parsing for parsing struct." @@ -15,6 +15,6 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -proc-macro2 = "1.0.29" -quote = { version = "1.0.3", features = ["proc-macro"] } -syn = { version = "1.0.58", features = ["proc-macro" ,"full", "extra-traits", "parsing"] } +proc-macro2 = "1.0.36" +quote = { version = "1.0.10", features = ["proc-macro"] } +syn = { version = "1.0.82", features = ["proc-macro" ,"full", "extra-traits", "parsing"] } diff --git a/frame/support/procedural/tools/derive/src/lib.rs b/frame/support/procedural/tools/derive/src/lib.rs index 792210589560..392a43834201 100644 --- a/frame/support/procedural/tools/derive/src/lib.rs +++ b/frame/support/procedural/tools/derive/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -131,7 +131,7 @@ fn derive_totokens_enum(input: syn::ItemEnum) -> TokenStream { }; let field = fields_idents(v.fields.iter().map(Clone::clone)); quote! { - #ident::#v_ident#fields_build => { + #ident::#v_ident #fields_build => { #( #field.to_tokens(tokens); )* diff --git a/frame/support/procedural/tools/src/lib.rs b/frame/support/procedural/tools/src/lib.rs index d7aba4c7cbf1..3abde5957411 100644 --- a/frame/support/procedural/tools/src/lib.rs +++ b/frame/support/procedural/tools/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/procedural/tools/src/syn_ext.rs b/frame/support/procedural/tools/src/syn_ext.rs index a9e9ef573985..25c98faaf388 100644 --- a/frame/support/procedural/tools/src/syn_ext.rs +++ b/frame/support/procedural/tools/src/syn_ext.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/dispatch.rs b/frame/support/src/dispatch.rs index b4e9071e361a..7ccf79616729 100644 --- a/frame/support/src/dispatch.rs +++ b/frame/support/src/dispatch.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,6 +20,7 @@ pub use crate::{ codec::{Codec, Decode, Encode, EncodeAsRef, EncodeLike, HasCompact, Input, Output}, + scale_info::TypeInfo, sp_std::{ fmt, marker, prelude::{Clone, Eq, PartialEq, Vec}, @@ -33,7 +34,7 @@ pub use crate::{ TransactionPriority, WeighData, Weight, WithPostDispatchInfo, }, }; -pub use sp_runtime::{traits::Dispatchable, DispatchError}; +pub use sp_runtime::{traits::Dispatchable, DispatchError, RuntimeDebug}; /// The return type of a `Dispatchable` in frame. When returned explicitly from /// a dispatchable function it allows overriding the default `PostDispatchInfo` @@ -60,6 +61,28 @@ pub trait Callable { // https://github.com/rust-lang/rust/issues/51331 pub type CallableCallFor = >::Call; +/// Origin for the System pallet. +#[derive(PartialEq, Eq, Clone, RuntimeDebug, Encode, Decode, TypeInfo)] +pub enum RawOrigin { + /// The system itself ordained this dispatch to happen: this is the highest privilege level. + Root, + /// It is signed by some public key and we provide the `AccountId`. + Signed(AccountId), + /// It is signed by nobody, can be either: + /// * included and agreed upon by the validators anyway, + /// * or unsigned transaction validated by a pallet. + None, +} + +impl From> for RawOrigin { + fn from(s: Option) -> RawOrigin { + match s { + Some(who) => RawOrigin::Signed(who), + None => RawOrigin::None, + } + } +} + /// A type that can be used as a parameter in a dispatchable function. /// /// When using `decl_module` all arguments for call functions must implement this trait. @@ -280,7 +303,7 @@ impl Parameter for T where T: Codec + EncodeLike + Clone + Eq + fmt::Debug + /// /// The following are reserved function signatures: /// -/// * `deposit_event`: Helper function for depositing an [event](https://docs.substrate.dev/docs/event-enum). +/// * `deposit_event`: Helper function for depositing an [event](https://docs.substrate.io/v3/runtime/events-and-errors). /// The default behavior is to call `deposit_event` from the [System /// module](../frame_system/index.html). However, you can write your own implementation for events /// in your runtime. To use the default behavior, add `fn deposit_event() = default;` to your @@ -1546,7 +1569,7 @@ macro_rules! decl_module { ::current_storage_version(), ); - (|| { $( $impl )* })() + { $( $impl )* } } #[cfg(feature = "try-runtime")] @@ -1980,6 +2003,10 @@ macro_rules! decl_module { pub type Pallet<$trait_instance $(, $instance $( = $module_default_instance)?)?> = $mod_type<$trait_instance $(, $instance)?>; + $crate::__create_tt_macro! { + tt_error_token, + } + $crate::decl_module! { @impl_on_initialize { $system } @@ -2165,6 +2192,22 @@ macro_rules! decl_module { } } + impl<$trait_instance: $trait_name $(, $instance: $instantiable)?> $crate::traits::PalletsInfoAccess + for $mod_type<$trait_instance $(, $instance)?> where $( $other_where_bounds )* + { + fn count() -> usize { 1 } + fn accumulate(acc: &mut $crate::sp_std::vec::Vec<$crate::traits::PalletInfoData>) { + use $crate::traits::PalletInfoAccess; + let item = $crate::traits::PalletInfoData { + index: Self::index(), + name: Self::name(), + module_name: Self::module_name(), + crate_version: Self::crate_version(), + }; + acc.push(item); + } + } + // Implement GetCallName for the Call. impl<$trait_instance: $trait_name $(, $instance: $instantiable)?> $crate::dispatch::GetCallName for $call_type<$trait_instance $(, $instance)?> where $( $other_where_bounds )* @@ -2566,21 +2609,7 @@ mod tests { type DbWeight: Get; } - #[derive(Clone, PartialEq, Eq, Debug, Encode, Decode, scale_info::TypeInfo)] - pub enum RawOrigin { - Root, - Signed(AccountId), - None, - } - - impl From> for RawOrigin { - fn from(s: Option) -> RawOrigin { - match s { - Some(who) => RawOrigin::Signed(who), - None => RawOrigin::None, - } - } - } + pub use super::super::RawOrigin; pub type Origin = RawOrigin<::AccountId>; } @@ -2622,7 +2651,7 @@ mod tests { } } - #[derive(scale_info::TypeInfo)] + #[derive(Eq, PartialEq, Clone, crate::RuntimeDebug, scale_info::TypeInfo)] pub struct TraitImpl {} impl Config for TraitImpl {} @@ -2663,8 +2692,15 @@ mod tests { } } + #[derive(TypeInfo, crate::RuntimeDebug, Eq, PartialEq, Clone, Encode, Decode)] pub struct OuterOrigin; + impl From::AccountId>> for OuterOrigin { + fn from(_: RawOrigin<::AccountId>) -> Self { + unimplemented!("Not required in tests!") + } + } + impl crate::traits::OriginTrait for OuterOrigin { type Call = ::Call; type PalletsOrigin = OuterOrigin; diff --git a/frame/support/src/error.rs b/frame/support/src/error.rs index 836428c6bc7d..0ffe4334e2e3 100644 --- a/frame/support/src/error.rs +++ b/frame/support/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -85,7 +85,12 @@ macro_rules! decl_error { } ) => { $(#[$attr])* - #[derive($crate::scale_info::TypeInfo)] + #[derive( + $crate::codec::Encode, + $crate::codec::Decode, + $crate::scale_info::TypeInfo, + $crate::PalletError, + )] #[scale_info(skip_type_params($generic $(, $inst_generic)?), capture_docs = "always")] pub enum $error<$generic: $trait $(, $inst_generic: $instance)?> $( where $( $where_ty: $where_bound ),* )? @@ -114,17 +119,6 @@ macro_rules! decl_error { impl<$generic: $trait $(, $inst_generic: $instance)?> $error<$generic $(, $inst_generic)?> $( where $( $where_ty: $where_bound ),* )? { - fn as_u8(&self) -> u8 { - $crate::decl_error! { - @GENERATE_AS_U8 - self - $error - {} - 0, - $( $name ),* - } - } - fn as_str(&self) -> &'static str { match self { Self::__Ignore(_, _) => unreachable!("`__Ignore` can never be constructed"), @@ -149,47 +143,19 @@ macro_rules! decl_error { $( where $( $where_ty: $where_bound ),* )? { fn from(err: $error<$generic $(, $inst_generic)?>) -> Self { + use $crate::codec::Encode; let index = <$generic::PalletInfo as $crate::traits::PalletInfo> ::index::<$module<$generic $(, $inst_generic)?>>() .expect("Every active module has an index in the runtime; qed") as u8; + let mut error = err.encode(); + error.resize($crate::MAX_MODULE_ERROR_ENCODED_SIZE, 0); - $crate::sp_runtime::DispatchError::Module { + $crate::sp_runtime::DispatchError::Module($crate::sp_runtime::ModuleError { index, - error: err.as_u8(), + error: TryInto::try_into(error).expect("encoded error is resized to be equal to the maximum encoded error size; qed"), message: Some(err.as_str()), - } - } - } - }; - (@GENERATE_AS_U8 - $self:ident - $error:ident - { $( $generated:tt )* } - $index:expr, - $name:ident - $( , $rest:ident )* - ) => { - $crate::decl_error! { - @GENERATE_AS_U8 - $self - $error - { - $( $generated )* - $error::$name => $index, + }) } - $index + 1, - $( $rest ),* } }; - (@GENERATE_AS_U8 - $self:ident - $error:ident - { $( $generated:tt )* } - $index:expr, - ) => { - match $self { - $error::__Ignore(_, _) => unreachable!("`__Ignore` can never be constructed"), - $( $generated )* - } - } } diff --git a/frame/support/src/event.rs b/frame/support/src/event.rs index 3d042a3122db..dc45f2bf69d9 100644 --- a/frame/support/src/event.rs +++ b/frame/support/src/event.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/hash.rs b/frame/support/src/hash.rs index f943bcf32309..9ce0968351a4 100644 --- a/frame/support/src/hash.rs +++ b/frame/support/src/hash.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/inherent.rs b/frame/support/src/inherent.rs index 2125f3e7f50a..0aa6b9d3f75a 100644 --- a/frame/support/src/inherent.rs +++ b/frame/support/src/inherent.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,9 +24,12 @@ pub use sp_inherents::{ CheckInherentsResult, InherentData, InherentIdentifier, IsFatalError, MakeFatalError, }; -/// A pallet that provides or verifies an inherent extrinsic. +/// A pallet that provides or verifies an inherent extrinsic will implement this trait. /// -/// The pallet may provide the inherent, verify an inherent, or both provide and verify. +/// The pallet may provide an inherent, verify an inherent, or both provide and verify. +/// +/// Briefly, inherent extrinsics ("inherents") are extrinsics that are added to a block by the block +/// producer. See [`sp_inherents`] for more documentation on inherents. pub trait ProvideInherent { /// The call type of the pallet. type Call; @@ -36,6 +39,12 @@ pub trait ProvideInherent { const INHERENT_IDENTIFIER: self::InherentIdentifier; /// Create an inherent out of the given `InherentData`. + /// + /// NOTE: All checks necessary to ensure that the inherent is correct and that can be done in + /// the runtime should happen in the returned `Call`. + /// E.g. if this provides the timestamp, the call will check that the given timestamp is + /// increasing the old timestamp by more than a minimum and it will also check that the + /// timestamp hasn't already been set in the current block. fn create_inherent(data: &InherentData) -> Option; /// Determines whether this inherent is required in this block. @@ -44,15 +53,17 @@ pub trait ProvideInherent { /// implementation returns this. /// /// - `Ok(Some(e))` indicates that this inherent is required in this block. `construct_runtime!` - /// will call this function from in its implementation of `fn check_extrinsics`. + /// will call this function in its implementation of `fn check_extrinsics`. /// If the inherent is not present, it will return `e`. /// /// - `Err(_)` indicates that this function failed and further operations should be aborted. /// - /// NOTE: If inherent is required then the runtime asserts that the block contains at least + /// NOTE: If the inherent is required then the runtime asserts that the block contains at least /// one inherent for which: /// * type is [`Self::Call`], /// * [`Self::is_inherent`] returns true. + /// + /// NOTE: This is currently only checked by block producers, not all full nodes. fn is_inherent_required(_: &InherentData) -> Result, Self::Error> { Ok(None) } @@ -64,21 +75,27 @@ pub trait ProvideInherent { /// included in the block by its author. Whereas the second parameter represents the inherent /// data that the verifying node calculates. /// - /// NOTE: A block can contains multiple inherent. + /// This is intended to allow for checks that cannot be done within the runtime such as, e.g., + /// the timestamp. + /// + /// # Warning + /// + /// This check is not guaranteed to be run by all full nodes and cannot be relied upon for + /// ensuring that the block is correct. fn check_inherent(_: &Self::Call, _: &InherentData) -> Result<(), Self::Error> { Ok(()) } /// Return whether the call is an inherent call. /// - /// NOTE: Signed extrinsics are not inherent, but signed extrinsic with the given call variant - /// can be dispatched. + /// NOTE: Signed extrinsics are not inherents, but a signed extrinsic with the given call + /// variant can be dispatched. /// /// # Warning /// - /// In FRAME, inherent are enforced to be before other extrinsics, for this reason, + /// In FRAME, inherents are enforced to be executed before other extrinsics. For this reason, /// pallets with unsigned transactions **must ensure** that no unsigned transaction call /// is an inherent call, when implementing `ValidateUnsigned::validate_unsigned`. - /// Otherwise block producer can produce invalid blocks by including them after non inherent. + /// Otherwise block producers can produce invalid blocks by including them after non inherents. fn is_inherent(call: &Self::Call) -> bool; } diff --git a/frame/support/src/instances.rs b/frame/support/src/instances.rs index 9908d16076a0..81139f867181 100644 --- a/frame/support/src/instances.rs +++ b/frame/support/src/instances.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index f3b00c764bb3..714449eec784 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -42,14 +42,18 @@ pub use scale_info; pub use serde; pub use sp_core::Void; #[doc(hidden)] +pub use sp_core_hashing_proc_macro; +#[doc(hidden)] pub use sp_io::{self, storage::root as storage_root}; #[doc(hidden)] -pub use sp_runtime::RuntimeDebug; +pub use sp_runtime::{RuntimeDebug, StateVersion}; #[cfg(feature = "std")] #[doc(hidden)] pub use sp_state_machine::BasicExternalities; #[doc(hidden)] pub use sp_std; +#[doc(hidden)] +pub use tt_call::*; #[macro_use] pub mod dispatch; @@ -89,7 +93,9 @@ pub use self::{ StorageMap, StorageNMap, StoragePrefixedMap, StorageValue, }, }; -pub use sp_runtime::{self, print, traits::Printable, ConsensusEngineId}; +pub use sp_runtime::{ + self, print, traits::Printable, ConsensusEngineId, MAX_MODULE_ERROR_ENCODED_SIZE, +}; use codec::{Decode, Encode}; use scale_info::TypeInfo; @@ -99,7 +105,7 @@ use sp_runtime::TypeId; pub const LOG_TARGET: &'static str = "runtime::frame-support"; /// A type that cannot be instantiated. -#[derive(Debug, PartialEq, Eq, Clone, TypeInfo)] +#[derive(Encode, Decode, Debug, PartialEq, Eq, Clone, TypeInfo)] pub enum Never {} /// A pallet identifier. These are per pallet and should be stored in a registry somewhere. @@ -110,6 +116,27 @@ impl TypeId for PalletId { const TYPE_ID: [u8; 4] = *b"modl"; } +/// Build a bounded vec from the given literals. +/// +/// The type of the outcome must be known. +/// +/// Will not handle any errors and just panic if the given literals cannot fit in the corresponding +/// bounded vec type. Thus, this is only suitable for testing and non-consensus code. +#[macro_export] +#[cfg(feature = "std")] +macro_rules! bounded_vec { + ($ ($values:expr),* $(,)?) => { + { + $crate::sp_std::vec![$($values),*].try_into().unwrap() + } + }; + ( $value:expr ; $repetition:expr ) => { + { + $crate::sp_std::vec![$value ; $repetition].try_into().unwrap() + } + } +} + /// Generate a new type alias for [`storage::types::StorageValue`], /// [`storage::types::StorageMap`], [`storage::types::StorageDoubleMap`] /// and [`storage::types::StorageNMap`]. @@ -127,8 +154,8 @@ impl TypeId for PalletId { /// // to `Vec` /// generate_storage_alias!( /// OtherPrefix, OtherStorageName => DoubleMap< -/// (u32, Twox64Concat), -/// (u32, Twox64Concat), +/// (Twox64Concat, u32), +/// (Twox64Concat, u32), /// Vec /// > /// ); @@ -138,8 +165,8 @@ impl TypeId for PalletId { /// generate_storage_alias!(Prefix, ValueName => Value); /// generate_storage_alias!( /// Prefix, SomeStorageName => DoubleMap< -/// (u32, Twox64Concat), -/// (u32, Twox64Concat), +/// (Twox64Concat, u32), +/// (Twox64Concat, u32), /// Vec, /// ValueQuery /// > @@ -148,14 +175,14 @@ impl TypeId for PalletId { /// // generate a map from `Config::AccountId` (with hasher `Twox64Concat`) to `Vec` /// trait Config { type AccountId: codec::FullCodec; } /// generate_storage_alias!( -/// Prefix, GenericStorage => Map<(T::AccountId, Twox64Concat), Vec> +/// Prefix, GenericStorage => Map<(Twox64Concat, T::AccountId), Vec> /// ); /// # fn main() {} /// ``` #[macro_export] macro_rules! generate_storage_alias { // without generic for $name. - ($pallet:ident, $name:ident => Map<($key:ty, $hasher:ty), $value:ty $(, $querytype:ty)?>) => { + ($pallet:ident, $name:ident => Map<($hasher:ty, $key:ty), $value:ty $(, $querytype:ty)?>) => { $crate::paste::paste! { $crate::generate_storage_alias!(@GENERATE_INSTANCE_STRUCT $pallet, $name); type $name = $crate::storage::types::StorageMap< @@ -170,7 +197,7 @@ macro_rules! generate_storage_alias { ( $pallet:ident, $name:ident - => DoubleMap<($key1:ty, $hasher1:ty), ($key2:ty, $hasher2:ty), $value:ty $(, $querytype:ty)?> + => DoubleMap<($hasher1:ty, $key1:ty), ($hasher2:ty, $key2:ty), $value:ty $(, $querytype:ty)?> ) => { $crate::paste::paste! { $crate::generate_storage_alias!(@GENERATE_INSTANCE_STRUCT $pallet, $name); @@ -188,7 +215,7 @@ macro_rules! generate_storage_alias { ( $pallet:ident, $name:ident - => NMap, $value:ty $(, $querytype:ty)?> + => NMap, $value:ty $(, $querytype:ty)?> ) => { $crate::paste::paste! { $crate::generate_storage_alias!(@GENERATE_INSTANCE_STRUCT $pallet, $name); @@ -216,15 +243,15 @@ macro_rules! generate_storage_alias { ( $pallet:ident, $name:ident<$t:ident : $bounds:tt> - => Map<($key:ty, $hasher:ty), $value:ty $(, $querytype:ty)?> + => Map<($hasher:ty, $key:ty), $value:ty $(, $querytype:ty)?> ) => { $crate::paste::paste! { $crate::generate_storage_alias!(@GENERATE_INSTANCE_STRUCT $pallet, $name); #[allow(type_alias_bounds)] type $name<$t : $bounds> = $crate::storage::types::StorageMap< [<$name Instance>], - $key, $hasher, + $key, $value, $( $querytype )? >; @@ -233,17 +260,17 @@ macro_rules! generate_storage_alias { ( $pallet:ident, $name:ident<$t:ident : $bounds:tt> - => DoubleMap<($key1:ty, $hasher1:ty), ($key2:ty, $hasher2:ty), $value:ty $(, $querytype:ty)?> + => DoubleMap<($hasher1:ty, $key1:ty), ($hasher2:ty, $key2:ty), $value:ty $(, $querytype:ty)?> ) => { $crate::paste::paste! { $crate::generate_storage_alias!(@GENERATE_INSTANCE_STRUCT $pallet, $name); #[allow(type_alias_bounds)] type $name<$t : $bounds> = $crate::storage::types::StorageDoubleMap< [<$name Instance>], - $key1, $hasher1, - $key2, + $key1, $hasher2, + $key2, $value, $( $querytype )? >; @@ -252,7 +279,7 @@ macro_rules! generate_storage_alias { ( $pallet:ident, $name:ident<$t:ident : $bounds:tt> - => NMap<$(($key:ty, $hasher:ty),)+ $value:ty $(, $querytype:ty)?> + => NMap<$(($hasher:ty, $key:ty),)+ $value:ty $(, $querytype:ty)?> ) => { $crate::paste::paste! { $crate::generate_storage_alias!(@GENERATE_INSTANCE_STRUCT $pallet, $name); @@ -425,9 +452,7 @@ macro_rules! parameter_types { /// Returns the key for this parameter type. #[allow(unused)] pub fn key() -> [u8; 16] { - $crate::sp_io::hashing::twox_128( - concat!(":", stringify!($name), ":").as_bytes() - ) + $crate::sp_core_hashing_proc_macro::twox_128!(b":", $name, b":") } /// Set the value of this parameter type in the storage. @@ -573,11 +598,12 @@ pub fn debug(data: &impl sp_std::fmt::Debug) { #[doc(inline)] pub use frame_support_procedural::{ - construct_runtime, decl_storage, transactional, RuntimeDebugNoBound, + construct_runtime, decl_storage, match_and_insert, transactional, PalletError, + RuntimeDebugNoBound, }; #[doc(hidden)] -pub use frame_support_procedural::__generate_dummy_part_checker; +pub use frame_support_procedural::{__create_tt_macro, __generate_dummy_part_checker}; /// Derive [`Clone`] but do not bound any generic. /// @@ -749,9 +775,9 @@ macro_rules! assert_noop { $x:expr, $y:expr $(,)? ) => { - let h = $crate::storage_root(); + let h = $crate::storage_root($crate::StateVersion::V1); $crate::assert_err!($x, $y); - assert_eq!(h, $crate::storage_root()); + assert_eq!(h, $crate::storage_root($crate::StateVersion::V1)); }; } @@ -764,9 +790,9 @@ macro_rules! assert_storage_noop { ( $x:expr ) => { - let h = $crate::storage_root(); + let h = $crate::storage_root($crate::StateVersion::V1); $x; - assert_eq!(h, $crate::storage_root()); + assert_eq!(h, $crate::storage_root($crate::StateVersion::V1)); }; } @@ -822,6 +848,32 @@ macro_rules! assert_ok { }; } +/// Assert that the maximum encoding size does not exceed the value defined in +/// [`MAX_MODULE_ERROR_ENCODED_SIZE`] during compilation. +/// +/// This macro is intended to be used in conjunction with `tt_call!`. +#[macro_export] +macro_rules! assert_error_encoded_size { + { + path = [{ $($path:ident)::+ }] + runtime = [{ $runtime:ident }] + assert_message = [{ $assert_message:literal }] + error = [{ $error:ident }] + } => { + const _: () = assert!( + < + $($path::)+$error<$runtime> as $crate::traits::PalletError + >::MAX_ENCODED_SIZE <= $crate::MAX_MODULE_ERROR_ENCODED_SIZE, + $assert_message + ); + }; + { + path = [{ $($path:ident)::+ }] + runtime = [{ $runtime:ident }] + assert_message = [{ $assert_message:literal }] + } => {}; +} + #[cfg(feature = "std")] #[doc(hidden)] pub use serde::{Deserialize, Serialize}; @@ -920,6 +972,20 @@ pub mod tests { } } + #[test] + fn generate_storage_alias_works() { + new_test_ext().execute_with(|| { + generate_storage_alias!( + Test, + GenericData2 => Map<(Blake2_128Concat, T::BlockNumber), T::BlockNumber> + ); + + assert_eq!(Module::::generic_data2(5), None); + GenericData2::::insert(5, 5); + assert_eq!(Module::::generic_data2(5), Some(5)); + }); + } + #[test] fn map_issue_3318() { new_test_ext().execute_with(|| { @@ -1328,6 +1394,7 @@ pub mod pallet_prelude { PartialEqNoBound, RuntimeDebug, RuntimeDebugNoBound, Twox128, Twox256, Twox64Concat, }; pub use codec::{Decode, Encode, MaxEncodedLen}; + pub use scale_info::TypeInfo; pub use sp_runtime::{ traits::{MaybeSerializeDeserialize, Member, ValidateUnsigned}, transaction_validity::{ @@ -1335,6 +1402,7 @@ pub mod pallet_prelude { TransactionTag, TransactionValidity, TransactionValidityError, UnknownTransaction, ValidTransaction, }, + MAX_MODULE_ERROR_ENCODED_SIZE, }; pub use sp_std::marker::PhantomData; } @@ -1476,11 +1544,11 @@ pub mod pallet_prelude { /// * [`traits::OnGenesis`]: contains some logic to write pallet version into storage. /// * `PalletErrorTypeInfo`: provides the type information for the pallet error, if defined. /// -/// It declare `type Module` type alias for `Pallet`, used by [`construct_runtime`]. +/// It declares `type Module` type alias for `Pallet`, used by [`construct_runtime`]. /// /// It implements [`traits::PalletInfoAccess`] on `Pallet` to ease access to pallet /// informations given by [`frame_support::traits::PalletInfo`]. -/// (The implementation use the associated type `frame_system::Config::PalletInfo`). +/// (The implementation uses the associated type `frame_system::Config::PalletInfo`). /// /// It implements [`traits::StorageInfoTrait`] on `Pallet` which give information about all /// storages. @@ -1612,10 +1680,25 @@ pub mod pallet_prelude { /// pub enum Error { /// /// $some_optional_doc /// $SomeFieldLessVariant, +/// /// $some_more_optional_doc +/// $SomeVariantWithOneField(FieldType), /// ... /// } /// ``` -/// I.e. a regular rust enum named `Error`, with generic `T` and fieldless variants. +/// I.e. a regular rust enum named `Error`, with generic `T` and fieldless or multiple-field +/// variants. +/// +/// Any field type in the enum variants must implement [`scale_info::TypeInfo`] in order to be +/// properly used in the metadata, and its encoded size should be as small as possible, +/// preferably 1 byte in size in order to reduce storage size. The error enum itself has an +/// absolute maximum encoded size specified by [`MAX_MODULE_ERROR_ENCODED_SIZE`]. +/// +/// Field types in enum variants must also implement [`PalletError`](traits::PalletError), +/// otherwise the pallet will fail to compile. Rust primitive types have already implemented +/// the [`PalletError`](traits::PalletError) trait along with some commonly used stdlib types +/// such as `Option` and `PhantomData`, and hence in most use cases, a manual implementation is +/// not necessary and is discouraged. +/// /// The generic `T` mustn't bound anything and where clause is not allowed. But bounds and /// where clause shouldn't be needed for any usecase. /// @@ -1746,7 +1829,7 @@ pub mod pallet_prelude { /// ``` /// /// The optional attribute `#[pallet::unbounded]` allows to declare the storage as unbounded. -/// When implementating the storage info (when #[pallet::generate_storage_info]` is specified +/// When implementating the storage info (when `#[pallet::generate_storage_info]` is specified /// on the pallet struct placeholder), the size of the storage will be declared as unbounded. /// This can be useful for storage which can never go into PoV (Proof of Validity). /// @@ -1964,7 +2047,7 @@ pub mod pallet_prelude { /// pub trait Config: frame_system::Config { /// #[pallet::constant] // put the constant in metadata /// type MyGetParam: Get; -/// type Balance: Parameter + From; +/// type Balance: Parameter + MaxEncodedLen + From; /// type Event: From> + IsType<::Event>; /// } /// @@ -2153,7 +2236,7 @@ pub mod pallet_prelude { /// pub trait Config: frame_system::Config { /// #[pallet::constant] /// type MyGetParam: Get; -/// type Balance: Parameter + From; +/// type Balance: Parameter + MaxEncodedLen + From; /// type Event: From> + IsType<::Event>; /// } /// diff --git a/frame/support/src/migrations.rs b/frame/support/src/migrations.rs index dc3402440fdd..05833e0515c0 100644 --- a/frame/support/src/migrations.rs +++ b/frame/support/src/migrations.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -58,9 +58,9 @@ impl PalletVersionToStorageVersionHelper for T { /// /// This will remove all `PalletVersion's` from the state and insert the current storage version. pub fn migrate_from_pallet_version_to_storage_version< - AllPallets: PalletVersionToStorageVersionHelper, + Pallets: PalletVersionToStorageVersionHelper, >( db_weight: &RuntimeDbWeight, ) -> Weight { - AllPallets::migrate(db_weight) + Pallets::migrate(db_weight) } diff --git a/frame/support/src/storage/bounded_btree_map.rs b/frame/support/src/storage/bounded_btree_map.rs index 404814cb8169..eca4b17821c7 100644 --- a/frame/support/src/storage/bounded_btree_map.rs +++ b/frame/support/src/storage/bounded_btree_map.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,12 +17,12 @@ //! Traits, types and structs to support a bounded BTreeMap. -use crate::{storage::StorageDecodeLength, traits::Get}; -use codec::{Decode, Encode, MaxEncodedLen}; -use sp_std::{ - borrow::Borrow, collections::btree_map::BTreeMap, convert::TryFrom, marker::PhantomData, - ops::Deref, +use crate::{ + storage::StorageDecodeLength, + traits::{Get, TryCollect}, }; +use codec::{Decode, Encode, MaxEncodedLen}; +use sp_std::{borrow::Borrow, collections::btree_map::BTreeMap, marker::PhantomData, ops::Deref}; /// A bounded map based on a B-Tree. /// @@ -69,6 +69,11 @@ where K: Ord, S: Get, { + /// Create `Self` from `t` without any checks. + fn unchecked_from(t: BTreeMap) -> Self { + Self(t, Default::default()) + } + /// Create a new `BoundedBTreeMap`. /// /// Does not allocate. @@ -183,16 +188,23 @@ where } } -impl PartialEq for BoundedBTreeMap +impl PartialEq> for BoundedBTreeMap where BTreeMap: PartialEq, + S1: Get, + S2: Get, { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 + fn eq(&self, other: &BoundedBTreeMap) -> bool { + S1::get() == S2::get() && self.0 == other.0 } } -impl Eq for BoundedBTreeMap where BTreeMap: Eq {} +impl Eq for BoundedBTreeMap +where + BTreeMap: Eq, + S: Get, +{ +} impl PartialEq> for BoundedBTreeMap where @@ -206,6 +218,7 @@ where impl PartialOrd for BoundedBTreeMap where BTreeMap: PartialOrd, + S: Get, { fn partial_cmp(&self, other: &Self) -> Option { self.0.partial_cmp(&other.0) @@ -215,6 +228,7 @@ where impl Ord for BoundedBTreeMap where BTreeMap: Ord, + S: Get, { fn cmp(&self, other: &Self) -> sp_std::cmp::Ordering { self.0.cmp(&other.0) @@ -302,23 +316,35 @@ impl codec::EncodeLike> for BoundedBTreeMap whe { } +impl TryCollect> for I +where + K: Ord, + I: ExactSizeIterator + Iterator, + Bound: Get, +{ + type Error = &'static str; + + fn try_collect(self) -> Result, Self::Error> { + if self.len() > Bound::get() as usize { + Err("iterator length too big") + } else { + Ok(BoundedBTreeMap::::unchecked_from(self.collect::>())) + } + } +} + #[cfg(test)] pub mod test { use super::*; use crate::Twox128; + use frame_support::traits::ConstU32; use sp_io::TestExternalities; - use sp_std::convert::TryInto; - - crate::parameter_types! { - pub const Seven: u32 = 7; - pub const Four: u32 = 4; - } - crate::generate_storage_alias! { Prefix, Foo => Value> } - crate::generate_storage_alias! { Prefix, FooMap => Map<(u32, Twox128), BoundedBTreeMap> } + crate::generate_storage_alias! { Prefix, Foo => Value>> } + crate::generate_storage_alias! { Prefix, FooMap => Map<(Twox128, u32), BoundedBTreeMap>> } crate::generate_storage_alias! { Prefix, - FooDoubleMap => DoubleMap<(u32, Twox128), (u32, Twox128), BoundedBTreeMap> + FooDoubleMap => DoubleMap<(Twox128, u32), (Twox128, u32), BoundedBTreeMap>> } fn map_from_keys(keys: &[K]) -> BTreeMap @@ -339,13 +365,13 @@ pub mod test { #[test] fn decode_len_works() { TestExternalities::default().execute_with(|| { - let bounded = boundedmap_from_keys::(&[1, 2, 3]); + let bounded = boundedmap_from_keys::>(&[1, 2, 3]); Foo::put(bounded); assert_eq!(Foo::decode_len().unwrap(), 3); }); TestExternalities::default().execute_with(|| { - let bounded = boundedmap_from_keys::(&[1, 2, 3]); + let bounded = boundedmap_from_keys::>(&[1, 2, 3]); FooMap::insert(1, bounded); assert_eq!(FooMap::decode_len(1).unwrap(), 3); assert!(FooMap::decode_len(0).is_none()); @@ -353,7 +379,7 @@ pub mod test { }); TestExternalities::default().execute_with(|| { - let bounded = boundedmap_from_keys::(&[1, 2, 3]); + let bounded = boundedmap_from_keys::>(&[1, 2, 3]); FooDoubleMap::insert(1, 1, bounded); assert_eq!(FooDoubleMap::decode_len(1, 1).unwrap(), 3); assert!(FooDoubleMap::decode_len(2, 1).is_none()); @@ -364,7 +390,7 @@ pub mod test { #[test] fn try_insert_works() { - let mut bounded = boundedmap_from_keys::(&[1, 2, 3]); + let mut bounded = boundedmap_from_keys::>(&[1, 2, 3]); bounded.try_insert(0, ()).unwrap(); assert_eq!(*bounded, map_from_keys(&[1, 0, 2, 3])); @@ -374,7 +400,7 @@ pub mod test { #[test] fn deref_coercion_works() { - let bounded = boundedmap_from_keys::(&[1, 2, 3]); + let bounded = boundedmap_from_keys::>(&[1, 2, 3]); // these methods come from deref-ed vec. assert_eq!(bounded.len(), 3); assert!(bounded.iter().next().is_some()); @@ -383,7 +409,7 @@ pub mod test { #[test] fn try_mutate_works() { - let bounded = boundedmap_from_keys::(&[1, 2, 3, 4, 5, 6]); + let bounded = boundedmap_from_keys::>(&[1, 2, 3, 4, 5, 6]); let bounded = bounded .try_mutate(|v| { v.insert(7, ()); @@ -399,7 +425,7 @@ pub mod test { #[test] fn btree_map_eq_works() { - let bounded = boundedmap_from_keys::(&[1, 2, 3, 4, 5, 6]); + let bounded = boundedmap_from_keys::>(&[1, 2, 3, 4, 5, 6]); assert_eq!(bounded, map_from_keys(&[1, 2, 3, 4, 5, 6])); } @@ -407,7 +433,7 @@ pub mod test { fn too_big_fail_to_decode() { let v: Vec<(u32, u32)> = vec![(1, 1), (2, 2), (3, 3), (4, 4), (5, 5)]; assert_eq!( - BoundedBTreeMap::::decode(&mut &v.encode()[..]), + BoundedBTreeMap::>::decode(&mut &v.encode()[..]), Err("BoundedBTreeMap exceeds its limit".into()), ); } @@ -437,7 +463,7 @@ pub mod test { } } - let mut map = BoundedBTreeMap::::new(); + let mut map = BoundedBTreeMap::>::new(); // when the set is full @@ -457,4 +483,53 @@ pub mod test { assert_eq!(zero_key.1, false); assert_eq!(*zero_value, 6); } + + #[test] + fn can_be_collected() { + let b1 = boundedmap_from_keys::>(&[1, 2, 3, 4]); + let b2: BoundedBTreeMap> = + b1.iter().map(|(k, v)| (k + 1, *v)).try_collect().unwrap(); + assert_eq!(b2.into_iter().map(|(k, _)| k).collect::>(), vec![2, 3, 4, 5]); + + // can also be collected into a collection of length 4. + let b2: BoundedBTreeMap> = + b1.iter().map(|(k, v)| (k + 1, *v)).try_collect().unwrap(); + assert_eq!(b2.into_iter().map(|(k, _)| k).collect::>(), vec![2, 3, 4, 5]); + + // can be mutated further into iterators that are `ExactSizedIterator`. + let b2: BoundedBTreeMap> = + b1.iter().map(|(k, v)| (k + 1, *v)).rev().skip(2).try_collect().unwrap(); + // note that the binary tree will re-sort this, so rev() is not really seen + assert_eq!(b2.into_iter().map(|(k, _)| k).collect::>(), vec![2, 3]); + + let b2: BoundedBTreeMap> = + b1.iter().map(|(k, v)| (k + 1, *v)).take(2).try_collect().unwrap(); + assert_eq!(b2.into_iter().map(|(k, _)| k).collect::>(), vec![2, 3]); + + // but these worn't work + let b2: Result>, _> = + b1.iter().map(|(k, v)| (k + 1, *v)).try_collect(); + assert!(b2.is_err()); + + let b2: Result>, _> = + b1.iter().map(|(k, v)| (k + 1, *v)).skip(2).try_collect(); + assert!(b2.is_err()); + } + + #[test] + fn eq_works() { + // of same type + let b1 = boundedmap_from_keys::>(&[1, 2]); + let b2 = boundedmap_from_keys::>(&[1, 2]); + assert_eq!(b1, b2); + + // of different type, but same value and bound. + crate::parameter_types! { + B1: u32 = 7; + B2: u32 = 7; + } + let b1 = boundedmap_from_keys::(&[1, 2]); + let b2 = boundedmap_from_keys::(&[1, 2]); + assert_eq!(b1, b2); + } } diff --git a/frame/support/src/storage/bounded_btree_set.rs b/frame/support/src/storage/bounded_btree_set.rs index f74ff12854a5..f38952bf545d 100644 --- a/frame/support/src/storage/bounded_btree_set.rs +++ b/frame/support/src/storage/bounded_btree_set.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,12 +17,12 @@ //! Traits, types and structs to support a bounded `BTreeSet`. -use crate::{storage::StorageDecodeLength, traits::Get}; -use codec::{Decode, Encode, MaxEncodedLen}; -use sp_std::{ - borrow::Borrow, collections::btree_set::BTreeSet, convert::TryFrom, marker::PhantomData, - ops::Deref, +use crate::{ + storage::StorageDecodeLength, + traits::{Get, TryCollect}, }; +use codec::{Decode, Encode, MaxEncodedLen}; +use sp_std::{borrow::Borrow, collections::btree_set::BTreeSet, marker::PhantomData, ops::Deref}; /// A bounded set based on a B-Tree. /// @@ -31,7 +31,8 @@ use sp_std::{ /// /// Unlike a standard `BTreeSet`, there is an enforced upper limit to the number of items in the /// set. All internal operations ensure this bound is respected. -#[derive(Encode)] +#[derive(Encode, scale_info::TypeInfo)] +#[scale_info(skip_type_params(S))] pub struct BoundedBTreeSet(BTreeSet, PhantomData); impl Decode for BoundedBTreeSet @@ -67,6 +68,11 @@ where T: Ord, S: Get, { + /// Create `Self` from `t` without any checks. + fn unchecked_from(t: BTreeSet) -> Self { + Self(t, Default::default()) + } + /// Create a new `BoundedBTreeSet`. /// /// Does not allocate. @@ -167,20 +173,28 @@ where } } -impl PartialEq for BoundedBTreeSet +impl PartialEq> for BoundedBTreeSet where BTreeSet: PartialEq, + S1: Get, + S2: Get, { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 + fn eq(&self, other: &BoundedBTreeSet) -> bool { + S1::get() == S2::get() && self.0 == other.0 } } -impl Eq for BoundedBTreeSet where BTreeSet: Eq {} +impl Eq for BoundedBTreeSet +where + BTreeSet: Eq, + S: Get, +{ +} impl PartialEq> for BoundedBTreeSet where BTreeSet: PartialEq, + S: Get, { fn eq(&self, other: &BTreeSet) -> bool { self.0 == *other @@ -190,6 +204,7 @@ where impl PartialOrd for BoundedBTreeSet where BTreeSet: PartialOrd, + S: Get, { fn partial_cmp(&self, other: &Self) -> Option { self.0.partial_cmp(&other.0) @@ -199,6 +214,7 @@ where impl Ord for BoundedBTreeSet where BTreeSet: Ord, + S: Get, { fn cmp(&self, other: &Self) -> sp_std::cmp::Ordering { self.0.cmp(&other.0) @@ -282,50 +298,62 @@ impl StorageDecodeLength for BoundedBTreeSet {} impl codec::EncodeLike> for BoundedBTreeSet where BTreeSet: Encode {} +impl TryCollect> for I +where + T: Ord, + I: ExactSizeIterator + Iterator, + Bound: Get, +{ + type Error = &'static str; + + fn try_collect(self) -> Result, Self::Error> { + if self.len() > Bound::get() as usize { + Err("iterator length too big") + } else { + Ok(BoundedBTreeSet::::unchecked_from(self.collect::>())) + } + } +} + #[cfg(test)] pub mod test { use super::*; use crate::Twox128; + use frame_support::traits::ConstU32; use sp_io::TestExternalities; - use sp_std::convert::TryInto; - - crate::parameter_types! { - pub const Seven: u32 = 7; - pub const Four: u32 = 4; - } - crate::generate_storage_alias! { Prefix, Foo => Value> } - crate::generate_storage_alias! { Prefix, FooMap => Map<(u32, Twox128), BoundedBTreeSet> } + crate::generate_storage_alias! { Prefix, Foo => Value>> } + crate::generate_storage_alias! { Prefix, FooMap => Map<(Twox128, u32), BoundedBTreeSet>> } crate::generate_storage_alias! { Prefix, - FooDoubleMap => DoubleMap<(u32, Twox128), (u32, Twox128), BoundedBTreeSet> + FooDoubleMap => DoubleMap<(Twox128, u32), (Twox128, u32), BoundedBTreeSet>> } - fn map_from_keys(keys: &[T]) -> BTreeSet + fn set_from_keys(keys: &[T]) -> BTreeSet where T: Ord + Copy, { keys.iter().copied().collect() } - fn boundedmap_from_keys(keys: &[T]) -> BoundedBTreeSet + fn boundedset_from_keys(keys: &[T]) -> BoundedBTreeSet where T: Ord + Copy, S: Get, { - map_from_keys(keys).try_into().unwrap() + set_from_keys(keys).try_into().unwrap() } #[test] fn decode_len_works() { TestExternalities::default().execute_with(|| { - let bounded = boundedmap_from_keys::(&[1, 2, 3]); + let bounded = boundedset_from_keys::>(&[1, 2, 3]); Foo::put(bounded); assert_eq!(Foo::decode_len().unwrap(), 3); }); TestExternalities::default().execute_with(|| { - let bounded = boundedmap_from_keys::(&[1, 2, 3]); + let bounded = boundedset_from_keys::>(&[1, 2, 3]); FooMap::insert(1, bounded); assert_eq!(FooMap::decode_len(1).unwrap(), 3); assert!(FooMap::decode_len(0).is_none()); @@ -333,7 +361,7 @@ pub mod test { }); TestExternalities::default().execute_with(|| { - let bounded = boundedmap_from_keys::(&[1, 2, 3]); + let bounded = boundedset_from_keys::>(&[1, 2, 3]); FooDoubleMap::insert(1, 1, bounded); assert_eq!(FooDoubleMap::decode_len(1, 1).unwrap(), 3); assert!(FooDoubleMap::decode_len(2, 1).is_none()); @@ -344,17 +372,17 @@ pub mod test { #[test] fn try_insert_works() { - let mut bounded = boundedmap_from_keys::(&[1, 2, 3]); + let mut bounded = boundedset_from_keys::>(&[1, 2, 3]); bounded.try_insert(0).unwrap(); - assert_eq!(*bounded, map_from_keys(&[1, 0, 2, 3])); + assert_eq!(*bounded, set_from_keys(&[1, 0, 2, 3])); assert!(bounded.try_insert(9).is_err()); - assert_eq!(*bounded, map_from_keys(&[1, 0, 2, 3])); + assert_eq!(*bounded, set_from_keys(&[1, 0, 2, 3])); } #[test] fn deref_coercion_works() { - let bounded = boundedmap_from_keys::(&[1, 2, 3]); + let bounded = boundedset_from_keys::>(&[1, 2, 3]); // these methods come from deref-ed vec. assert_eq!(bounded.len(), 3); assert!(bounded.iter().next().is_some()); @@ -363,7 +391,7 @@ pub mod test { #[test] fn try_mutate_works() { - let bounded = boundedmap_from_keys::(&[1, 2, 3, 4, 5, 6]); + let bounded = boundedset_from_keys::>(&[1, 2, 3, 4, 5, 6]); let bounded = bounded .try_mutate(|v| { v.insert(7); @@ -379,15 +407,15 @@ pub mod test { #[test] fn btree_map_eq_works() { - let bounded = boundedmap_from_keys::(&[1, 2, 3, 4, 5, 6]); - assert_eq!(bounded, map_from_keys(&[1, 2, 3, 4, 5, 6])); + let bounded = boundedset_from_keys::>(&[1, 2, 3, 4, 5, 6]); + assert_eq!(bounded, set_from_keys(&[1, 2, 3, 4, 5, 6])); } #[test] fn too_big_fail_to_decode() { let v: Vec = vec![1, 2, 3, 4, 5]; assert_eq!( - BoundedBTreeSet::::decode(&mut &v.encode()[..]), + BoundedBTreeSet::>::decode(&mut &v.encode()[..]), Err("BoundedBTreeSet exceeds its limit".into()), ); } @@ -417,7 +445,7 @@ pub mod test { } } - let mut set = BoundedBTreeSet::::new(); + let mut set = BoundedBTreeSet::>::new(); // when the set is full @@ -436,4 +464,51 @@ pub mod test { assert_eq!(zero_item.0, 0); assert_eq!(zero_item.1, false); } + + #[test] + fn can_be_collected() { + let b1 = boundedset_from_keys::>(&[1, 2, 3, 4]); + let b2: BoundedBTreeSet> = b1.iter().map(|k| k + 1).try_collect().unwrap(); + assert_eq!(b2.into_iter().collect::>(), vec![2, 3, 4, 5]); + + // can also be collected into a collection of length 4. + let b2: BoundedBTreeSet> = b1.iter().map(|k| k + 1).try_collect().unwrap(); + assert_eq!(b2.into_iter().collect::>(), vec![2, 3, 4, 5]); + + // can be mutated further into iterators that are `ExactSizedIterator`. + let b2: BoundedBTreeSet> = + b1.iter().map(|k| k + 1).rev().skip(2).try_collect().unwrap(); + // note that the binary tree will re-sort this, so rev() is not really seen + assert_eq!(b2.into_iter().collect::>(), vec![2, 3]); + + let b2: BoundedBTreeSet> = + b1.iter().map(|k| k + 1).take(2).try_collect().unwrap(); + assert_eq!(b2.into_iter().collect::>(), vec![2, 3]); + + // but these worn't work + let b2: Result>, _> = + b1.iter().map(|k| k + 1).try_collect(); + assert!(b2.is_err()); + + let b2: Result>, _> = + b1.iter().map(|k| k + 1).skip(2).try_collect(); + assert!(b2.is_err()); + } + + #[test] + fn eq_works() { + // of same type + let b1 = boundedset_from_keys::>(&[1, 2]); + let b2 = boundedset_from_keys::>(&[1, 2]); + assert_eq!(b1, b2); + + // of different type, but same value and bound. + crate::parameter_types! { + B1: u32 = 7; + B2: u32 = 7; + } + let b1 = boundedset_from_keys::(&[1, 2]); + let b2 = boundedset_from_keys::(&[1, 2]); + assert_eq!(b1, b2); + } } diff --git a/frame/support/src/storage/bounded_vec.rs b/frame/support/src/storage/bounded_vec.rs index 44eaab905423..137015098cfa 100644 --- a/frame/support/src/storage/bounded_vec.rs +++ b/frame/support/src/storage/bounded_vec.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,15 +20,15 @@ use crate::{ storage::{StorageDecodeLength, StorageTryAppend}, - traits::Get, + traits::{Get, TryCollect}, WeakBoundedVec, }; use codec::{Decode, Encode, EncodeLike, MaxEncodedLen}; use core::{ - ops::{Deref, Index, IndexMut}, + ops::{Deref, Index, IndexMut, RangeBounds}, slice::SliceIndex, }; -use sp_std::{convert::TryFrom, marker::PhantomData, prelude::*}; +use sp_std::{marker::PhantomData, prelude::*}; /// A bounded vector. /// @@ -44,7 +44,8 @@ pub struct BoundedVec(Vec, PhantomData); /// A bounded slice. /// /// Similar to a `BoundedVec`, but not owned and cannot be decoded. -#[derive(Encode)] +#[derive(Encode, scale_info::TypeInfo)] +#[scale_info(skip_type_params(S))] pub struct BoundedSlice<'a, T, S>(&'a [T], PhantomData); // `BoundedSlice`s encode to something which will always decode into a `BoundedVec`, @@ -56,6 +57,18 @@ impl<'a, T: Encode + Decode, S: Get> EncodeLike> } impl<'a, T: Encode + Decode, S: Get> EncodeLike> for BoundedSlice<'a, T, S> {} +impl> PartialOrd for BoundedVec { + fn partial_cmp(&self, other: &Self) -> Option { + self.0.partial_cmp(&other.0) + } +} + +impl> Ord for BoundedVec { + fn cmp(&self, other: &Self) -> sp_std::cmp::Ordering { + self.0.cmp(&other.0) + } +} + impl<'a, T, S: Get> TryFrom<&'a [T]> for BoundedSlice<'a, T, S> { type Error = (); fn try_from(t: &'a [T]) -> Result { @@ -106,7 +119,17 @@ impl BoundedVec { self.0 } - /// Exactly the same semantics as [`Vec::remove`]. + /// Exactly the same semantics as [`slice::sort_by`]. + /// + /// This is safe since sorting cannot change the number of elements in the vector. + pub fn sort_by(&mut self, compare: F) + where + F: FnMut(&T, &T) -> sp_std::cmp::Ordering, + { + self.0.sort_by(compare) + } + + /// Exactly the same semantics as `Vec::remove`. /// /// # Panics /// @@ -115,7 +138,7 @@ impl BoundedVec { self.0.remove(index) } - /// Exactly the same semantics as [`Vec::swap_remove`]. + /// Exactly the same semantics as `slice::swap_remove`. /// /// # Panics /// @@ -124,18 +147,50 @@ impl BoundedVec { self.0.swap_remove(index) } - /// Exactly the same semantics as [`Vec::retain`]. + /// Exactly the same semantics as `Vec::retain`. pub fn retain bool>(&mut self, f: F) { self.0.retain(f) } - /// Exactly the same semantics as [`Vec::get_mut`]. + /// Exactly the same semantics as `slice::get_mut`. pub fn get_mut>( &mut self, index: I, ) -> Option<&mut >::Output> { self.0.get_mut(index) } + + /// Exactly the same semantics as `Vec::truncate`. + /// + /// This is safe because `truncate` can never increase the length of the internal vector. + pub fn truncate(&mut self, s: usize) { + self.0.truncate(s); + } + + /// Exactly the same semantics as `Vec::pop`. + /// + /// This is safe since popping can only shrink the inner vector. + pub fn pop(&mut self) -> Option { + self.0.pop() + } + + /// Exactly the same semantics as [`slice::iter_mut`]. + pub fn iter_mut(&mut self) -> core::slice::IterMut<'_, T> { + self.0.iter_mut() + } + + /// Exactly the same semantics as [`slice::last_mut`]. + pub fn last_mut(&mut self) -> Option<&mut T> { + self.0.last_mut() + } + + /// Exact same semantics as [`Vec::drain`]. + pub fn drain(&mut self, range: R) -> sp_std::vec::Drain<'_, T> + where + R: RangeBounds, + { + self.0.drain(range) + } } impl> From> for Vec { @@ -145,11 +200,180 @@ impl> From> for Vec { } impl> BoundedVec { + /// Pre-allocate `capacity` items in self. + /// + /// If `capacity` is greater than [`Self::bound`], then the minimum of the two is used. + pub fn with_bounded_capacity(capacity: usize) -> Self { + let capacity = capacity.min(Self::bound()); + Self(Vec::with_capacity(capacity), Default::default()) + } + + /// Allocate self with the maximum possible capacity. + pub fn with_max_capacity() -> Self { + Self::with_bounded_capacity(Self::bound()) + } + /// Get the bound of the type in `usize`. pub fn bound() -> usize { S::get() as usize } + /// Returns true of this collection is full. + pub fn is_full(&self) -> bool { + self.len() >= Self::bound() + } + + /// Forces the insertion of `element` into `self` retaining all items with index at least + /// `index`. + /// + /// If `index == 0` and `self.len() == Self::bound()`, then this is a no-op. + /// + /// If `Self::bound() < index` or `self.len() < index`, then this is also a no-op. + /// + /// Returns `Ok(maybe_removed)` if the item was inserted, where `maybe_removed` is + /// `Some(removed)` if an item was removed to make room for the new one. Returns `Err(())` if + /// `element` cannot be inserted. + pub fn force_insert_keep_right( + &mut self, + index: usize, + mut element: T, + ) -> Result, ()> { + // Check against panics. + if Self::bound() < index || self.len() < index { + Err(()) + } else if self.len() < Self::bound() { + // Cannot panic since self.len() >= index; + self.0.insert(index, element); + Ok(None) + } else { + if index == 0 { + return Err(()) + } + sp_std::mem::swap(&mut self[0], &mut element); + // `[0..index] cannot panic since self.len() >= index. + // `rotate_left(1)` cannot panic because there is at least 1 element. + self[0..index].rotate_left(1); + Ok(Some(element)) + } + } + + /// Forces the insertion of `element` into `self` retaining all items with index at most + /// `index`. + /// + /// If `index == Self::bound()` and `self.len() == Self::bound()`, then this is a no-op. + /// + /// If `Self::bound() < index` or `self.len() < index`, then this is also a no-op. + /// + /// Returns `Ok(maybe_removed)` if the item was inserted, where `maybe_removed` is + /// `Some(removed)` if an item was removed to make room for the new one. Returns `Err(())` if + /// `element` cannot be inserted. + pub fn force_insert_keep_left(&mut self, index: usize, element: T) -> Result, ()> { + // Check against panics. + if Self::bound() < index || self.len() < index || Self::bound() == 0 { + return Err(()) + } + // Noop condition. + if Self::bound() == index && self.len() <= Self::bound() { + return Err(()) + } + let maybe_removed = if self.is_full() { + // defensive-only: since we are at capacity, this is a noop. + self.0.truncate(Self::bound()); + // if we truncate anything, it will be the last one. + self.0.pop() + } else { + None + }; + + // Cannot panic since `self.len() >= index`; + self.0.insert(index, element); + Ok(maybe_removed) + } + + /// Move the position of an item from one location to another in the slice. + /// + /// Except for the item being moved, the order of the slice remains the same. + /// + /// - `index` is the location of the item to be moved. + /// - `insert_position` is the index of the item in the slice which should *immediately follow* + /// the item which is being moved. + /// + /// Returns `true` of the operation was successful, otherwise `false` if a noop. + pub fn slide(&mut self, index: usize, insert_position: usize) -> bool { + // Check against panics. + if self.len() <= index || self.len() < insert_position || index == usize::MAX { + return false + } + // Noop conditions. + if index == insert_position || index + 1 == insert_position { + return false + } + if insert_position < index && index < self.len() { + // --- --- --- === === === === @@@ --- --- --- + // ^-- N ^O^ + // ... + // /-----<<<-----\ + // --- --- --- === === === === @@@ --- --- --- + // >>> >>> >>> >>> + // ... + // --- --- --- @@@ === === === === --- --- --- + // ^N^ + self[insert_position..index + 1].rotate_right(1); + return true + } else if insert_position > 0 && index + 1 < insert_position { + // Note that the apparent asymmetry of these two branches is due to the + // fact that the "new" position is the position to be inserted *before*. + // --- --- --- @@@ === === === === --- --- --- + // ^O^ ^-- N + // ... + // /----->>>-----\ + // --- --- --- @@@ === === === === --- --- --- + // <<< <<< <<< <<< + // ... + // --- --- --- === === === === @@@ --- --- --- + // ^N^ + self[index..insert_position].rotate_left(1); + return true + } + + debug_assert!(false, "all noop conditions should have been covered above"); + false + } + + /// Forces the insertion of `s` into `self` truncating first if necessary. + /// + /// Infallible, but if the bound is zero, then it's a no-op. + pub fn force_push(&mut self, element: T) { + if Self::bound() > 0 { + self.0.truncate(Self::bound() as usize - 1); + self.0.push(element); + } + } + + /// Same as `Vec::resize`, but if `size` is more than [`Self::bound`], then [`Self::bound`] is + /// used. + pub fn bounded_resize(&mut self, size: usize, value: T) + where + T: Clone, + { + let size = size.min(Self::bound()); + self.0.resize(size, value); + } + + /// Exactly the same semantics as [`Vec::extend`], but returns an error and does nothing if the + /// length of the outcome is larger than the bound. + pub fn try_extend( + &mut self, + with: impl IntoIterator + ExactSizeIterator, + ) -> Result<(), ()> { + if with.len().saturating_add(self.len()) <= Self::bound() { + self.0.extend(with); + Ok(()) + } else { + Err(()) + } + } + /// Consumes self and mutates self via the given `mutate` function. /// /// If the outcome of mutation is within bounds, `Some(Self)` is returned. Else, `None` is @@ -200,13 +424,12 @@ impl Default for BoundedVec { } } -#[cfg(feature = "std")] -impl std::fmt::Debug for BoundedVec +impl sp_std::fmt::Debug for BoundedVec where - T: std::fmt::Debug, + T: sp_std::fmt::Debug, S: Get, { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { f.debug_tuple("BoundedVec").field(&self.0).field(&Self::bound()).finish() } } @@ -300,15 +523,14 @@ impl codec::DecodeLength for BoundedVec { } } -// NOTE: we could also implement this as: -// impl, S2: Get> PartialEq> for BoundedVec -// to allow comparison of bounded vectors with different bounds. -impl PartialEq for BoundedVec +impl PartialEq> for BoundedVec where T: PartialEq, + BoundSelf: Get, + BoundRhs: Get, { - fn eq(&self, rhs: &Self) -> bool { - self.0 == rhs.0 + fn eq(&self, rhs: &BoundedVec) -> bool { + BoundSelf::get() == BoundRhs::get() && self.0 == rhs.0 } } @@ -318,7 +540,7 @@ impl> PartialEq> for BoundedVec { } } -impl Eq for BoundedVec where T: Eq {} +impl> Eq for BoundedVec where T: Eq {} impl StorageDecodeLength for BoundedVec {} @@ -337,47 +559,154 @@ where fn max_encoded_len() -> usize { // BoundedVec encodes like Vec which encodes like [T], which is a compact u32 // plus each item in the slice: - // https://substrate.dev/rustdocs/v3.0.0/src/parity_scale_codec/codec.rs.html#798-808 + // https://docs.substrate.io/v3/advanced/scale-codec codec::Compact(S::get()) .encoded_size() .saturating_add(Self::bound().saturating_mul(T::max_encoded_len())) } } +impl TryCollect> for I +where + I: ExactSizeIterator + Iterator, + Bound: Get, +{ + type Error = &'static str; + + fn try_collect(self) -> Result, Self::Error> { + if self.len() > Bound::get() as usize { + Err("iterator length too big") + } else { + Ok(BoundedVec::::unchecked_from(self.collect::>())) + } + } +} + #[cfg(test)] pub mod test { use super::*; - use crate::Twox128; + use crate::{bounded_vec, traits::ConstU32, Twox128}; use sp_io::TestExternalities; - use sp_std::convert::TryInto; - - crate::parameter_types! { - pub const Seven: u32 = 7; - pub const Four: u32 = 4; - } - crate::generate_storage_alias! { Prefix, Foo => Value> } - crate::generate_storage_alias! { Prefix, FooMap => Map<(u32, Twox128), BoundedVec> } + crate::generate_storage_alias! { Prefix, Foo => Value>> } + crate::generate_storage_alias! { Prefix, FooMap => Map<(Twox128, u32), BoundedVec>> } crate::generate_storage_alias! { Prefix, - FooDoubleMap => DoubleMap<(u32, Twox128), (u32, Twox128), BoundedVec> + FooDoubleMap => DoubleMap<(Twox128, u32), (Twox128, u32), BoundedVec>> } #[test] - fn try_append_is_correct() { - assert_eq!(BoundedVec::::bound(), 7); + fn slide_works() { + let mut b: BoundedVec> = bounded_vec![0, 1, 2, 3, 4, 5]; + assert!(b.slide(1, 5)); + assert_eq!(*b, vec![0, 2, 3, 4, 1, 5]); + assert!(b.slide(4, 0)); + assert_eq!(*b, vec![1, 0, 2, 3, 4, 5]); + assert!(b.slide(0, 2)); + assert_eq!(*b, vec![0, 1, 2, 3, 4, 5]); + assert!(b.slide(1, 6)); + assert_eq!(*b, vec![0, 2, 3, 4, 5, 1]); + assert!(b.slide(0, 6)); + assert_eq!(*b, vec![2, 3, 4, 5, 1, 0]); + assert!(b.slide(5, 0)); + assert_eq!(*b, vec![0, 2, 3, 4, 5, 1]); + assert!(!b.slide(6, 0)); + assert!(!b.slide(7, 0)); + assert_eq!(*b, vec![0, 2, 3, 4, 5, 1]); + + let mut c: BoundedVec> = bounded_vec![0, 1, 2]; + assert!(!c.slide(1, 5)); + assert_eq!(*c, vec![0, 1, 2]); + assert!(!c.slide(4, 0)); + assert_eq!(*c, vec![0, 1, 2]); + assert!(!c.slide(3, 0)); + assert_eq!(*c, vec![0, 1, 2]); + assert!(c.slide(2, 0)); + assert_eq!(*c, vec![2, 0, 1]); + } + + #[test] + fn slide_noops_work() { + let mut b: BoundedVec> = bounded_vec![0, 1, 2, 3, 4, 5]; + assert!(!b.slide(3, 3)); + assert_eq!(*b, vec![0, 1, 2, 3, 4, 5]); + assert!(!b.slide(3, 4)); + assert_eq!(*b, vec![0, 1, 2, 3, 4, 5]); + } + + #[test] + fn force_insert_keep_left_works() { + let mut b: BoundedVec> = bounded_vec![]; + assert_eq!(b.force_insert_keep_left(1, 10), Err(())); + assert!(b.is_empty()); + + assert_eq!(b.force_insert_keep_left(0, 30), Ok(None)); + assert_eq!(b.force_insert_keep_left(0, 10), Ok(None)); + assert_eq!(b.force_insert_keep_left(1, 20), Ok(None)); + assert_eq!(b.force_insert_keep_left(3, 40), Ok(None)); + assert_eq!(*b, vec![10, 20, 30, 40]); + // at capacity. + assert_eq!(b.force_insert_keep_left(4, 41), Err(())); + assert_eq!(*b, vec![10, 20, 30, 40]); + assert_eq!(b.force_insert_keep_left(3, 31), Ok(Some(40))); + assert_eq!(*b, vec![10, 20, 30, 31]); + assert_eq!(b.force_insert_keep_left(1, 11), Ok(Some(31))); + assert_eq!(*b, vec![10, 11, 20, 30]); + assert_eq!(b.force_insert_keep_left(0, 1), Ok(Some(30))); + assert_eq!(*b, vec![1, 10, 11, 20]); + + let mut z: BoundedVec> = bounded_vec![]; + assert!(z.is_empty()); + assert_eq!(z.force_insert_keep_left(0, 10), Err(())); + assert!(z.is_empty()); + } + + #[test] + fn force_insert_keep_right_works() { + let mut b: BoundedVec> = bounded_vec![]; + assert_eq!(b.force_insert_keep_right(1, 10), Err(())); + assert!(b.is_empty()); + + assert_eq!(b.force_insert_keep_right(0, 30), Ok(None)); + assert_eq!(b.force_insert_keep_right(0, 10), Ok(None)); + assert_eq!(b.force_insert_keep_right(1, 20), Ok(None)); + assert_eq!(b.force_insert_keep_right(3, 40), Ok(None)); + assert_eq!(*b, vec![10, 20, 30, 40]); + + // at capacity. + assert_eq!(b.force_insert_keep_right(0, 0), Err(())); + assert_eq!(*b, vec![10, 20, 30, 40]); + assert_eq!(b.force_insert_keep_right(1, 11), Ok(Some(10))); + assert_eq!(*b, vec![11, 20, 30, 40]); + assert_eq!(b.force_insert_keep_right(3, 31), Ok(Some(11))); + assert_eq!(*b, vec![20, 30, 31, 40]); + assert_eq!(b.force_insert_keep_right(4, 41), Ok(Some(20))); + assert_eq!(*b, vec![30, 31, 40, 41]); + + assert_eq!(b.force_insert_keep_right(5, 69), Err(())); + assert_eq!(*b, vec![30, 31, 40, 41]); + + let mut z: BoundedVec> = bounded_vec![]; + assert!(z.is_empty()); + assert_eq!(z.force_insert_keep_right(0, 10), Err(())); + assert!(z.is_empty()); + } + + #[test] + fn bound_returns_correct_value() { + assert_eq!(BoundedVec::>::bound(), 7); } #[test] fn decode_len_works() { TestExternalities::default().execute_with(|| { - let bounded: BoundedVec = vec![1, 2, 3].try_into().unwrap(); + let bounded: BoundedVec> = bounded_vec![1, 2, 3]; Foo::put(bounded); assert_eq!(Foo::decode_len().unwrap(), 3); }); TestExternalities::default().execute_with(|| { - let bounded: BoundedVec = vec![1, 2, 3].try_into().unwrap(); + let bounded: BoundedVec> = bounded_vec![1, 2, 3]; FooMap::insert(1, bounded); assert_eq!(FooMap::decode_len(1).unwrap(), 3); assert!(FooMap::decode_len(0).is_none()); @@ -385,7 +714,7 @@ pub mod test { }); TestExternalities::default().execute_with(|| { - let bounded: BoundedVec = vec![1, 2, 3].try_into().unwrap(); + let bounded: BoundedVec> = bounded_vec![1, 2, 3]; FooDoubleMap::insert(1, 1, bounded); assert_eq!(FooDoubleMap::decode_len(1, 1).unwrap(), 3); assert!(FooDoubleMap::decode_len(2, 1).is_none()); @@ -396,7 +725,7 @@ pub mod test { #[test] fn try_insert_works() { - let mut bounded: BoundedVec = vec![1, 2, 3].try_into().unwrap(); + let mut bounded: BoundedVec> = bounded_vec![1, 2, 3]; bounded.try_insert(1, 0).unwrap(); assert_eq!(*bounded, vec![1, 0, 2, 3]); @@ -404,16 +733,29 @@ pub mod test { assert_eq!(*bounded, vec![1, 0, 2, 3]); } + #[test] + fn constructor_macro_works() { + use frame_support::bounded_vec; + + // With values. Use some brackets to make sure the macro doesn't expand. + let bv: BoundedVec<(u32, u32), ConstU32<3>> = bounded_vec![(1, 2), (1, 2), (1, 2)]; + assert_eq!(bv, vec![(1, 2), (1, 2), (1, 2)]); + + // With repetition. + let bv: BoundedVec<(u32, u32), ConstU32<3>> = bounded_vec![(1, 2); 3]; + assert_eq!(bv, vec![(1, 2), (1, 2), (1, 2)]); + } + #[test] #[should_panic(expected = "insertion index (is 9) should be <= len (is 3)")] fn try_inert_panics_if_oob() { - let mut bounded: BoundedVec = vec![1, 2, 3].try_into().unwrap(); + let mut bounded: BoundedVec> = bounded_vec![1, 2, 3]; bounded.try_insert(9, 0).unwrap(); } #[test] fn try_push_works() { - let mut bounded: BoundedVec = vec![1, 2, 3].try_into().unwrap(); + let mut bounded: BoundedVec> = bounded_vec![1, 2, 3]; bounded.try_push(0).unwrap(); assert_eq!(*bounded, vec![1, 2, 3, 0]); @@ -422,7 +764,7 @@ pub mod test { #[test] fn deref_coercion_works() { - let bounded: BoundedVec = vec![1, 2, 3].try_into().unwrap(); + let bounded: BoundedVec> = bounded_vec![1, 2, 3]; // these methods come from deref-ed vec. assert_eq!(bounded.len(), 3); assert!(bounded.iter().next().is_some()); @@ -431,7 +773,7 @@ pub mod test { #[test] fn try_mutate_works() { - let bounded: BoundedVec = vec![1, 2, 3, 4, 5, 6].try_into().unwrap(); + let bounded: BoundedVec> = bounded_vec![1, 2, 3, 4, 5, 6]; let bounded = bounded.try_mutate(|v| v.push(7)).unwrap(); assert_eq!(bounded.len(), 7); assert!(bounded.try_mutate(|v| v.push(8)).is_none()); @@ -439,13 +781,13 @@ pub mod test { #[test] fn slice_indexing_works() { - let bounded: BoundedVec = vec![1, 2, 3, 4, 5, 6].try_into().unwrap(); + let bounded: BoundedVec> = bounded_vec![1, 2, 3, 4, 5, 6]; assert_eq!(&bounded[0..=2], &[1, 2, 3]); } #[test] fn vec_eq_works() { - let bounded: BoundedVec = vec![1, 2, 3, 4, 5, 6].try_into().unwrap(); + let bounded: BoundedVec> = bounded_vec![1, 2, 3, 4, 5, 6]; assert_eq!(bounded, vec![1, 2, 3, 4, 5, 6]); } @@ -453,8 +795,96 @@ pub mod test { fn too_big_vec_fail_to_decode() { let v: Vec = vec![1, 2, 3, 4, 5]; assert_eq!( - BoundedVec::::decode(&mut &v.encode()[..]), + BoundedVec::>::decode(&mut &v.encode()[..]), Err("BoundedVec exceeds its limit".into()), ); } + + #[test] + fn can_be_collected() { + let b1: BoundedVec> = bounded_vec![1, 2, 3, 4]; + let b2: BoundedVec> = b1.iter().map(|x| x + 1).try_collect().unwrap(); + assert_eq!(b2, vec![2, 3, 4, 5]); + + // can also be collected into a collection of length 4. + let b2: BoundedVec> = b1.iter().map(|x| x + 1).try_collect().unwrap(); + assert_eq!(b2, vec![2, 3, 4, 5]); + + // can be mutated further into iterators that are `ExactSizedIterator`. + let b2: BoundedVec> = + b1.iter().map(|x| x + 1).rev().try_collect().unwrap(); + assert_eq!(b2, vec![5, 4, 3, 2]); + + let b2: BoundedVec> = + b1.iter().map(|x| x + 1).rev().skip(2).try_collect().unwrap(); + assert_eq!(b2, vec![3, 2]); + let b2: BoundedVec> = + b1.iter().map(|x| x + 1).rev().skip(2).try_collect().unwrap(); + assert_eq!(b2, vec![3, 2]); + + let b2: BoundedVec> = + b1.iter().map(|x| x + 1).rev().take(2).try_collect().unwrap(); + assert_eq!(b2, vec![5, 4]); + let b2: BoundedVec> = + b1.iter().map(|x| x + 1).rev().take(2).try_collect().unwrap(); + assert_eq!(b2, vec![5, 4]); + + // but these worn't work + let b2: Result>, _> = b1.iter().map(|x| x + 1).try_collect(); + assert!(b2.is_err()); + + let b2: Result>, _> = + b1.iter().map(|x| x + 1).rev().take(2).try_collect(); + assert!(b2.is_err()); + } + + #[test] + fn eq_works() { + // of same type + let b1: BoundedVec> = bounded_vec![1, 2, 3]; + let b2: BoundedVec> = bounded_vec![1, 2, 3]; + assert_eq!(b1, b2); + + // of different type, but same value and bound. + crate::parameter_types! { + B1: u32 = 7; + B2: u32 = 7; + } + let b1: BoundedVec = bounded_vec![1, 2, 3]; + let b2: BoundedVec = bounded_vec![1, 2, 3]; + assert_eq!(b1, b2); + } + + #[test] + fn ord_works() { + use std::cmp::Ordering; + let b1: BoundedVec> = bounded_vec![1, 2, 3]; + let b2: BoundedVec> = bounded_vec![1, 3, 2]; + + // ordering for vec is lexicographic. + assert_eq!(b1.cmp(&b2), Ordering::Less); + assert_eq!(b1.cmp(&b2), b1.into_inner().cmp(&b2.into_inner())); + } + + #[test] + fn try_extend_works() { + let mut b: BoundedVec> = bounded_vec![1, 2, 3]; + + assert!(b.try_extend(vec![4].into_iter()).is_ok()); + assert_eq!(*b, vec![1, 2, 3, 4]); + + assert!(b.try_extend(vec![5].into_iter()).is_ok()); + assert_eq!(*b, vec![1, 2, 3, 4, 5]); + + assert!(b.try_extend(vec![6].into_iter()).is_err()); + assert_eq!(*b, vec![1, 2, 3, 4, 5]); + + let mut b: BoundedVec> = bounded_vec![1, 2, 3]; + assert!(b.try_extend(vec![4, 5].into_iter()).is_ok()); + assert_eq!(*b, vec![1, 2, 3, 4, 5]); + + let mut b: BoundedVec> = bounded_vec![1, 2, 3]; + assert!(b.try_extend(vec![4, 5, 6].into_iter()).is_err()); + assert_eq!(*b, vec![1, 2, 3]); + } } diff --git a/frame/support/src/storage/child.rs b/frame/support/src/storage/child.rs index 4b237aaa561f..949df84e7e76 100644 --- a/frame/support/src/storage/child.rs +++ b/frame/support/src/storage/child.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,7 +24,7 @@ pub use crate::sp_io::KillStorageResult; use crate::sp_std::prelude::*; use codec::{Codec, Decode, Encode}; -pub use sp_core::storage::{ChildInfo, ChildType}; +pub use sp_core::storage::{ChildInfo, ChildType, StateVersion}; /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. pub fn get(child_info: &ChildInfo, key: &[u8]) -> Option { @@ -34,8 +34,9 @@ pub fn get(child_info: &ChildInfo, key: &[u8]) -> Option { sp_io::default_child_storage::get(storage_key, key).and_then(|v| { Decode::decode(&mut &v[..]).map(Some).unwrap_or_else(|_| { // TODO #3700: error should be handleable. - crate::runtime_print!( - "ERROR: Corrupted state in child trie at {:?}/{:?}", + log::error!( + target: "runtime::storage", + "Corrupted state in child trie at {:?}/{:?}", storage_key, key, ); @@ -112,8 +113,7 @@ pub fn take_or_else T>( pub fn exists(child_info: &ChildInfo, key: &[u8]) -> bool { match child_info.child_type() { ChildType::ParentKeyId => - sp_io::default_child_storage::read(child_info.storage_key(), key, &mut [0; 0][..], 0) - .is_some(), + sp_io::default_child_storage::exists(child_info.storage_key(), key), } } @@ -168,9 +168,10 @@ pub fn put_raw(child_info: &ChildInfo, key: &[u8], value: &[u8]) { } /// Calculate current child root value. -pub fn root(child_info: &ChildInfo) -> Vec { +pub fn root(child_info: &ChildInfo, version: StateVersion) -> Vec { match child_info.child_type() { - ChildType::ParentKeyId => sp_io::default_child_storage::root(child_info.storage_key()), + ChildType::ParentKeyId => + sp_io::default_child_storage::root(child_info.storage_key(), version), } } diff --git a/frame/support/src/storage/generator/double_map.rs b/frame/support/src/storage/generator/double_map.rs index 636a10feb1ab..12e1764bfb65 100644 --- a/frame/support/src/storage/generator/double_map.rs +++ b/frame/support/src/storage/generator/double_map.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -527,7 +527,7 @@ mod test_iterators { use crate::hash::Identity; crate::generate_storage_alias!( MyModule, - MyDoubleMap => DoubleMap<(u64, Identity), (u64, Identity), u64> + MyDoubleMap => DoubleMap<(Identity, u64), (Identity, u64), u64> ); MyDoubleMap::insert(1, 10, 100); diff --git a/frame/support/src/storage/generator/map.rs b/frame/support/src/storage/generator/map.rs index 1a4225173c4a..da48952bcba8 100644 --- a/frame/support/src/storage/generator/map.rs +++ b/frame/support/src/storage/generator/map.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -384,7 +384,7 @@ mod test_iterators { fn map_iter_from() { sp_io::TestExternalities::default().execute_with(|| { use crate::hash::Identity; - crate::generate_storage_alias!(MyModule, MyMap => Map<(u64, Identity), u64>); + crate::generate_storage_alias!(MyModule, MyMap => Map<(Identity, u64), u64>); MyMap::insert(1, 10); MyMap::insert(2, 20); diff --git a/frame/support/src/storage/generator/mod.rs b/frame/support/src/storage/generator/mod.rs index 576bada2e262..ca893f44b3cb 100644 --- a/frame/support/src/storage/generator/mod.rs +++ b/frame/support/src/storage/generator/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/storage/generator/nmap.rs b/frame/support/src/storage/generator/nmap.rs index 4845673d3d8c..be085ca2d9db 100755 --- a/frame/support/src/storage/generator/nmap.rs +++ b/frame/support/src/storage/generator/nmap.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -477,7 +477,7 @@ mod test_iterators { use crate::{hash::Identity, storage::Key as NMapKey}; crate::generate_storage_alias!( MyModule, - MyNMap => NMap, u64> + MyNMap => NMap, u64> ); MyNMap::insert((1, 1, 1), 11); @@ -519,8 +519,8 @@ mod test_iterators { { crate::generate_storage_alias!(Test, NMap => DoubleMap< - (u16, crate::Blake2_128Concat), - (u32, crate::Twox64Concat), + (crate::Blake2_128Concat, u16), + (crate::Twox64Concat, u32), u64 >); diff --git a/frame/support/src/storage/generator/value.rs b/frame/support/src/storage/generator/value.rs index 3486eaa005c0..55b3487b1324 100644 --- a/frame/support/src/storage/generator/value.rs +++ b/frame/support/src/storage/generator/value.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/storage/hashed.rs b/frame/support/src/storage/hashed.rs index 241caff809b3..a07db73c947d 100644 --- a/frame/support/src/storage/hashed.rs +++ b/frame/support/src/storage/hashed.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/storage/migration.rs b/frame/support/src/storage/migration.rs index 59422a282aab..713c2b0f3fe0 100644 --- a/frame/support/src/storage/migration.rs +++ b/frame/support/src/storage/migration.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -181,8 +181,8 @@ pub fn storage_iter_with_suffix( prefix.extend_from_slice(&storage_prefix); prefix.extend_from_slice(suffix); let previous_key = prefix.clone(); - let closure = |raw_key_without_prefix: &[u8], raw_value: &[u8]| { - let value = T::decode(&mut &raw_value[..])?; + let closure = |raw_key_without_prefix: &[u8], mut raw_value: &[u8]| { + let value = T::decode(&mut raw_value)?; Ok((raw_key_without_prefix.to_vec(), value)) }; @@ -213,10 +213,10 @@ pub fn storage_key_iter_with_suffix< prefix.extend_from_slice(&storage_prefix); prefix.extend_from_slice(suffix); let previous_key = prefix.clone(); - let closure = |raw_key_without_prefix: &[u8], raw_value: &[u8]| { + let closure = |raw_key_without_prefix: &[u8], mut raw_value: &[u8]| { let mut key_material = H::reverse(raw_key_without_prefix); let key = K::decode(&mut key_material)?; - let value = T::decode(&mut &raw_value[..])?; + let value = T::decode(&mut raw_value)?; Ok((key, value)) }; PrefixIterator { prefix, previous_key, drain: false, closure, phantom: Default::default() } diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index 35552e08fef1..066422ad456a 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +17,6 @@ //! Stuff to do with the runtime's storage. -pub use self::types::StorageEntryMetadataBuilder; use crate::{ hash::{ReversibleStorageHasher, StorageHasher}, storage::types::{ @@ -28,8 +27,13 @@ use crate::{ use codec::{Decode, Encode, EncodeLike, FullCodec, FullEncode}; use sp_core::storage::ChildInfo; use sp_runtime::generic::{Digest, DigestItem}; -pub use sp_runtime::TransactionOutcome; use sp_std::prelude::*; + +pub use self::{ + transactional::{with_transaction, with_transaction_unchecked}, + types::StorageEntryMetadataBuilder, +}; +pub use sp_runtime::TransactionOutcome; pub use types::Key; pub mod bounded_btree_map; @@ -40,88 +44,11 @@ pub mod child; pub mod generator; pub mod hashed; pub mod migration; +pub mod transactional; pub mod types; pub mod unhashed; pub mod weak_bounded_vec; -#[cfg(all(feature = "std", any(test, debug_assertions)))] -mod debug_helper { - use std::cell::RefCell; - - thread_local! { - static TRANSACTION_LEVEL: RefCell = RefCell::new(0); - } - - pub fn require_transaction() { - let level = TRANSACTION_LEVEL.with(|v| *v.borrow()); - if level == 0 { - panic!("Require transaction not called within with_transaction"); - } - } - - pub struct TransactionLevelGuard; - - impl Drop for TransactionLevelGuard { - fn drop(&mut self) { - TRANSACTION_LEVEL.with(|v| *v.borrow_mut() -= 1); - } - } - - /// Increments the transaction level. - /// - /// Returns a guard that when dropped decrements the transaction level automatically. - pub fn inc_transaction_level() -> TransactionLevelGuard { - TRANSACTION_LEVEL.with(|v| { - let mut val = v.borrow_mut(); - *val += 1; - if *val > 10 { - log::warn!( - "Detected with_transaction with nest level {}. Nested usage of with_transaction is not recommended.", - *val - ); - } - }); - - TransactionLevelGuard - } -} - -/// Assert this method is called within a storage transaction. -/// This will **panic** if is not called within a storage transaction. -/// -/// This assertion is enabled for native execution and when `debug_assertions` are enabled. -pub fn require_transaction() { - #[cfg(all(feature = "std", any(test, debug_assertions)))] - debug_helper::require_transaction(); -} - -/// Execute the supplied function in a new storage transaction. -/// -/// All changes to storage performed by the supplied function are discarded if the returned -/// outcome is `TransactionOutcome::Rollback`. -/// -/// Transactions can be nested to any depth. Commits happen to the parent transaction. -pub fn with_transaction(f: impl FnOnce() -> TransactionOutcome) -> R { - use sp_io::storage::{commit_transaction, rollback_transaction, start_transaction}; - use TransactionOutcome::*; - - start_transaction(); - - #[cfg(all(feature = "std", any(test, debug_assertions)))] - let _guard = debug_helper::inc_transaction_level(); - - match f() { - Commit(res) => { - commit_transaction(); - res - }, - Rollback(res) => { - rollback_transaction(); - res - }, - } -} - /// A trait for working with macro-generated storage values under the substrate storage API. /// /// Details on implementation can be found at [`generator::StorageValue`]. @@ -266,6 +193,8 @@ pub trait StorageMap { ) -> R; /// Mutate the item, only if an `Ok` value is returned. Deletes the item if mutated to a `None`. + /// `f` will always be called with an option representing if the storage item exists (`Some`) + /// or if the storage item does not exist (`None`), independent of the `QueryType`. fn try_mutate_exists, R, E, F: FnOnce(&mut Option) -> Result>( key: KeyArg, f: F, @@ -515,9 +444,6 @@ pub trait IterableStorageNMap: StorageN /// An implementation of a map with a two keys. /// -/// It provides an important ability to efficiently remove all entries -/// that have a common first key. -/// /// Details on implementation can be found at [`generator::StorageDoubleMap`]. pub trait StorageDoubleMap { /// The type that get/take returns. @@ -576,7 +502,18 @@ pub trait StorageDoubleMap { KArg1: EncodeLike, KArg2: EncodeLike; - /// Remove all values under the first key. + /// Remove all values under the first key `k1` in the overlay and up to `limit` in the + /// backend. + /// + /// All values in the client overlay will be deleted, if there is some `limit` then up to + /// `limit` values are deleted from the client backend, if `limit` is none then all values in + /// the client backend are deleted. + /// + /// # Note + /// + /// Calling this multiple times per block with a `limit` set leads always to the same keys being + /// removed and the same result being returned. This happens because the keys to delete in the + /// overlay are not taken into account when deleting keys in the backend. fn remove_prefix(k1: KArg1, limit: Option) -> sp_io::KillStorageResult where KArg1: ?Sized + EncodeLike; @@ -608,6 +545,8 @@ pub trait StorageDoubleMap { F: FnOnce(&mut Option) -> R; /// Mutate the item, only if an `Ok` value is returned. Deletes the item if mutated to a `None`. + /// `f` will always be called with an option representing if the storage item exists (`Some`) + /// or if the storage item does not exist (`None`), independent of the `QueryType`. fn try_mutate_exists(k1: KArg1, k2: KArg2, f: F) -> Result where KArg1: EncodeLike, @@ -704,7 +643,18 @@ pub trait StorageNMap { /// Remove the value under a key. fn remove + TupleToEncodedIter>(key: KArg); - /// Remove all values under the partial prefix key. + /// Remove all values starting with `partial_key` in the overlay and up to `limit` in the + /// backend. + /// + /// All values in the client overlay will be deleted, if there is some `limit` then up to + /// `limit` values are deleted from the client backend, if `limit` is none then all values in + /// the client backend are deleted. + /// + /// # Note + /// + /// Calling this multiple times per block with a `limit` set leads always to the same keys being + /// removed and the same result being returned. This happens because the keys to delete in the + /// overlay are not taken into account when deleting keys in the backend. fn remove_prefix(partial_key: KP, limit: Option) -> sp_io::KillStorageResult where K: HasKeyPrefix; @@ -735,6 +685,8 @@ pub trait StorageNMap { F: FnOnce(&mut Option) -> R; /// Mutate the item, only if an `Ok` value is returned. Deletes the item if mutated to a `None`. + /// `f` will always be called with an option representing if the storage item exists (`Some`) + /// or if the storage item does not exist (`None`), independent of the `QueryType`. fn try_mutate_exists(key: KArg, f: F) -> Result where KArg: EncodeLikeTuple + TupleToEncodedIter, @@ -802,6 +754,19 @@ pub struct PrefixIterator { phantom: core::marker::PhantomData, } +impl PrefixIterator { + /// Converts to the same iterator but with the different 'OnRemoval' type + pub fn convert_on_removal(self) -> PrefixIterator { + PrefixIterator:: { + prefix: self.prefix, + previous_key: self.previous_key, + drain: self.drain, + closure: self.closure, + phantom: Default::default(), + } + } +} + /// Trait for specialising on removal logic of [`PrefixIterator`]. pub trait PrefixIteratorOnRemoval { /// This function is called whenever a key/value is removed. @@ -1018,8 +983,8 @@ impl ChildTriePrefixIterator<(Vec, T)> { pub fn with_prefix(child_info: &ChildInfo, prefix: &[u8]) -> Self { let prefix = prefix.to_vec(); let previous_key = prefix.clone(); - let closure = |raw_key_without_prefix: &[u8], raw_value: &[u8]| { - let value = T::decode(&mut &raw_value[..])?; + let closure = |raw_key_without_prefix: &[u8], mut raw_value: &[u8]| { + let value = T::decode(&mut raw_value)?; Ok((raw_key_without_prefix.to_vec(), value)) }; @@ -1045,10 +1010,10 @@ impl ChildTriePrefixIterator<(K, T)> { ) -> Self { let prefix = prefix.to_vec(); let previous_key = prefix.clone(); - let closure = |raw_key_without_prefix: &[u8], raw_value: &[u8]| { + let closure = |raw_key_without_prefix: &[u8], mut raw_value: &[u8]| { let mut key_material = H::reverse(raw_key_without_prefix); let key = K::decode(&mut key_material)?; - let value = T::decode(&mut &raw_value[..])?; + let value = T::decode(&mut raw_value)?; Ok((key, value)) }; @@ -1133,7 +1098,17 @@ pub trait StoragePrefixedMap { crate::storage::storage_prefix(Self::module_prefix(), Self::storage_prefix()) } - /// Remove all value of the storage. + /// Remove all values in the overlay and up to `limit` in the backend. + /// + /// All values in the client overlay will be deleted, if there is some `limit` then up to + /// `limit` values are deleted from the client backend, if `limit` is none then all values in + /// the client backend are deleted. + /// + /// # Note + /// + /// Calling this multiple times per block with a `limit` set leads always to the same keys being + /// removed and the same result being returned. This happens because the keys to delete in the + /// overlay are not taken into account when deleting keys in the backend. fn remove_all(limit: Option) -> sp_io::KillStorageResult { sp_io::storage::clear_prefix(&Self::final_prefix(), limit) } @@ -1223,7 +1198,7 @@ mod private { pub trait Sealed {} impl Sealed for Vec {} - impl Sealed for Digest {} + impl Sealed for Digest {} impl Sealed for BoundedVec {} impl Sealed for WeakBoundedVec {} impl Sealed for bounded_btree_map::BoundedBTreeMap {} @@ -1263,7 +1238,7 @@ impl StorageDecodeLength for Vec {} /// We abuse the fact that SCALE does not put any marker into the encoding, i.e. we only encode the /// internal vec and we can append to this vec. We have a test that ensures that if the `Digest` /// format ever changes, we need to remove this here. -impl StorageAppend> for Digest {} +impl StorageAppend for Digest {} /// Marker trait that is implemented for types that support the `storage::append` api with a limit /// on the number of element. @@ -1400,7 +1375,7 @@ mod test { use super::*; use crate::{assert_ok, hash::Identity, Twox128}; use bounded_vec::BoundedVec; - use core::convert::{TryFrom, TryInto}; + use frame_support::traits::ConstU32; use generator::StorageValue as _; use sp_core::hashing::twox_128; use sp_io::TestExternalities; @@ -1485,8 +1460,8 @@ mod test { fn digest_storage_append_works_as_expected() { TestExternalities::default().execute_with(|| { struct Storage; - impl generator::StorageValue> for Storage { - type Query = Digest; + impl generator::StorageValue for Storage { + type Query = Digest; fn module_prefix() -> &'static [u8] { b"MyModule" @@ -1496,50 +1471,24 @@ mod test { b"Storage" } - fn from_optional_value_to_query(v: Option>) -> Self::Query { + fn from_optional_value_to_query(v: Option) -> Self::Query { v.unwrap() } - fn from_query_to_optional_value(v: Self::Query) -> Option> { + fn from_query_to_optional_value(v: Self::Query) -> Option { Some(v) } } - Storage::append(DigestItem::ChangesTrieRoot(1)); Storage::append(DigestItem::Other(Vec::new())); let value = unhashed::get_raw(&Storage::storage_value_final_key()).unwrap(); - let expected = Digest { - logs: vec![DigestItem::ChangesTrieRoot(1), DigestItem::Other(Vec::new())], - }; + let expected = Digest { logs: vec![DigestItem::Other(Vec::new())] }; assert_eq!(Digest::decode(&mut &value[..]).unwrap(), expected); }); } - #[test] - #[should_panic(expected = "Require transaction not called within with_transaction")] - fn require_transaction_should_panic() { - TestExternalities::default().execute_with(|| { - require_transaction(); - }); - } - - #[test] - fn require_transaction_should_not_panic_in_with_transaction() { - TestExternalities::default().execute_with(|| { - with_transaction(|| { - require_transaction(); - TransactionOutcome::Commit(()) - }); - - with_transaction(|| { - require_transaction(); - TransactionOutcome::Rollback(()) - }); - }); - } - #[test] fn key_prefix_iterator_works() { TestExternalities::default().execute_with(|| { @@ -1598,7 +1547,7 @@ mod test { use crate::{hash::Identity, storage::generator::map::StorageMap}; crate::generate_storage_alias! { MyModule, - MyStorageMap => Map<(u64, Identity), u64> + MyStorageMap => Map<(Identity, u64), u64> } MyStorageMap::insert(1, 10); @@ -1714,22 +1663,17 @@ mod test { }); } - crate::parameter_types! { - pub const Seven: u32 = 7; - pub const Four: u32 = 4; - } - - crate::generate_storage_alias! { Prefix, Foo => Value> } - crate::generate_storage_alias! { Prefix, FooMap => Map<(u32, Twox128), BoundedVec> } + crate::generate_storage_alias! { Prefix, Foo => Value>> } + crate::generate_storage_alias! { Prefix, FooMap => Map<(Twox128, u32), BoundedVec>> } crate::generate_storage_alias! { Prefix, - FooDoubleMap => DoubleMap<(u32, Twox128), (u32, Twox128), BoundedVec> + FooDoubleMap => DoubleMap<(Twox128, u32), (Twox128, u32), BoundedVec>> } #[test] fn try_append_works() { TestExternalities::default().execute_with(|| { - let bounded: WeakBoundedVec = vec![1, 2, 3].try_into().unwrap(); + let bounded: WeakBoundedVec> = vec![1, 2, 3].try_into().unwrap(); Foo::put(bounded); assert_ok!(Foo::try_append(4)); assert_ok!(Foo::try_append(5)); @@ -1740,7 +1684,7 @@ mod test { }); TestExternalities::default().execute_with(|| { - let bounded: BoundedVec = vec![1, 2, 3].try_into().unwrap(); + let bounded: BoundedVec> = vec![1, 2, 3].try_into().unwrap(); FooMap::insert(1, bounded); assert_ok!(FooMap::try_append(1, 4)); @@ -1755,17 +1699,17 @@ mod test { assert_ok!(FooMap::try_append(2, 4)); assert_eq!( FooMap::get(2).unwrap(), - BoundedVec::::try_from(vec![4]).unwrap(), + BoundedVec::>::try_from(vec![4]).unwrap(), ); assert_ok!(FooMap::try_append(2, 5)); assert_eq!( FooMap::get(2).unwrap(), - BoundedVec::::try_from(vec![4, 5]).unwrap(), + BoundedVec::>::try_from(vec![4, 5]).unwrap(), ); }); TestExternalities::default().execute_with(|| { - let bounded: BoundedVec = vec![1, 2, 3].try_into().unwrap(); + let bounded: BoundedVec> = vec![1, 2, 3].try_into().unwrap(); FooDoubleMap::insert(1, 1, bounded); assert_ok!(FooDoubleMap::try_append(1, 1, 4)); @@ -1780,12 +1724,12 @@ mod test { assert_ok!(FooDoubleMap::try_append(2, 1, 4)); assert_eq!( FooDoubleMap::get(2, 1).unwrap(), - BoundedVec::::try_from(vec![4]).unwrap(), + BoundedVec::>::try_from(vec![4]).unwrap(), ); assert_ok!(FooDoubleMap::try_append(2, 1, 5)); assert_eq!( FooDoubleMap::get(2, 1).unwrap(), - BoundedVec::::try_from(vec![4, 5]).unwrap(), + BoundedVec::>::try_from(vec![4, 5]).unwrap(), ); }); } diff --git a/frame/support/src/storage/transactional.rs b/frame/support/src/storage/transactional.rs new file mode 100644 index 000000000000..d1c59d44e258 --- /dev/null +++ b/frame/support/src/storage/transactional.rs @@ -0,0 +1,232 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Provides functionality around the transaction storage. +//! +//! Transactional storage provides functionality to run an entire code block +//! in a storage transaction. This means that either the entire changes to the +//! storage are committed or everything is thrown away. This simplifies the +//! writing of functionality that may bail at any point of operation. Otherwise +//! you would need to first verify all storage accesses and then do the storage +//! modifications. +//! +//! [`with_transaction`] provides a way to run a given closure in a transactional context. + +use sp_io::storage::{commit_transaction, rollback_transaction, start_transaction}; +use sp_runtime::{DispatchError, TransactionOutcome, TransactionalError}; + +/// The type that is being used to store the current number of active layers. +type Layer = u32; +/// The key that is holds the current number of active layers. +const TRANSACTION_LEVEL_KEY: &[u8] = b":transaction_level:"; +/// The maximum number of nested layers. +const TRANSACTIONAL_LIMIT: Layer = 255; + +/// Returns the current number of nested transactional layers. +fn get_transaction_level() -> Layer { + crate::storage::unhashed::get_or_default::(TRANSACTION_LEVEL_KEY) +} + +/// Set the current number of nested transactional layers. +fn set_transaction_level(level: Layer) { + crate::storage::unhashed::put::(TRANSACTION_LEVEL_KEY, &level); +} + +/// Kill the transactional layers storage. +fn kill_transaction_level() { + crate::storage::unhashed::kill(TRANSACTION_LEVEL_KEY); +} + +/// Increments the transaction level. Returns an error if levels go past the limit. +/// +/// Returns a guard that when dropped decrements the transaction level automatically. +fn inc_transaction_level() -> Result { + let existing_levels = get_transaction_level(); + if existing_levels >= TRANSACTIONAL_LIMIT { + return Err(()) + } + // Cannot overflow because of check above. + set_transaction_level(existing_levels + 1); + Ok(StorageLayerGuard) +} + +fn dec_transaction_level() { + let existing_levels = get_transaction_level(); + if existing_levels == 0 { + log::warn!( + "We are underflowing with calculating transactional levels. Not great, but let's not panic...", + ); + } else if existing_levels == 1 { + // Don't leave any trace of this storage item. + kill_transaction_level(); + } else { + // Cannot underflow because of checks above. + set_transaction_level(existing_levels - 1); + } +} + +struct StorageLayerGuard; + +impl Drop for StorageLayerGuard { + fn drop(&mut self) { + dec_transaction_level() + } +} + +/// Check if the current call is within a transactional layer. +pub fn is_transactional() -> bool { + get_transaction_level() > 0 +} + +/// Execute the supplied function in a new storage transaction. +/// +/// All changes to storage performed by the supplied function are discarded if the returned +/// outcome is `TransactionOutcome::Rollback`. +/// +/// Transactions can be nested up to `TRANSACTIONAL_LIMIT` times; more than that will result in an +/// error. +/// +/// Commits happen to the parent transaction. +pub fn with_transaction(f: impl FnOnce() -> TransactionOutcome>) -> Result +where + E: From, +{ + // This needs to happen before `start_transaction` below. + // Otherwise we may rollback the increase, then decrease as the guard goes out of scope + // and then end in some bad state. + let _guard = inc_transaction_level().map_err(|()| TransactionalError::LimitReached.into())?; + + start_transaction(); + + match f() { + TransactionOutcome::Commit(res) => { + commit_transaction(); + res + }, + TransactionOutcome::Rollback(res) => { + rollback_transaction(); + res + }, + } +} + +/// Same as [`with_transaction`] but without a limit check on nested transactional layers. +/// +/// This is mostly for backwards compatibility before there was a transactional layer limit. +/// It is recommended to only use [`with_transaction`] to avoid users from generating too many +/// transactional layers. +pub fn with_transaction_unchecked(f: impl FnOnce() -> TransactionOutcome) -> R { + // This needs to happen before `start_transaction` below. + // Otherwise we may rollback the increase, then decrease as the guard goes out of scope + // and then end in some bad state. + let maybe_guard = inc_transaction_level(); + + if maybe_guard.is_err() { + log::warn!( + "The transactional layer limit has been reached, and new transactional layers are being + spawned with `with_transaction_unchecked`. This could be caused by someone trying to + attack your chain, and you should investigate usage of `with_transaction_unchecked` and + potentially migrate to `with_transaction`, which enforces a transactional limit.", + ); + } + + start_transaction(); + + match f() { + TransactionOutcome::Commit(res) => { + commit_transaction(); + res + }, + TransactionOutcome::Rollback(res) => { + rollback_transaction(); + res + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{assert_noop, assert_ok}; + use sp_io::TestExternalities; + use sp_runtime::DispatchResult; + + #[test] + fn is_transactional_should_return_false() { + TestExternalities::default().execute_with(|| { + assert!(!is_transactional()); + }); + } + + #[test] + fn is_transactional_should_not_error_in_with_transaction() { + TestExternalities::default().execute_with(|| { + assert_ok!(with_transaction(|| -> TransactionOutcome { + assert!(is_transactional()); + TransactionOutcome::Commit(Ok(())) + })); + + assert_noop!( + with_transaction(|| -> TransactionOutcome { + assert!(is_transactional()); + TransactionOutcome::Rollback(Err("revert".into())) + }), + "revert" + ); + }); + } + + fn recursive_transactional(num: u32) -> DispatchResult { + if num == 0 { + return Ok(()) + } + + with_transaction(|| -> TransactionOutcome { + let res = recursive_transactional(num - 1); + TransactionOutcome::Commit(res) + }) + } + + #[test] + fn transaction_limit_should_work() { + TestExternalities::default().execute_with(|| { + assert_eq!(get_transaction_level(), 0); + + assert_ok!(with_transaction(|| -> TransactionOutcome { + assert_eq!(get_transaction_level(), 1); + TransactionOutcome::Commit(Ok(())) + })); + + assert_ok!(with_transaction(|| -> TransactionOutcome { + assert_eq!(get_transaction_level(), 1); + let res = with_transaction(|| -> TransactionOutcome { + assert_eq!(get_transaction_level(), 2); + TransactionOutcome::Commit(Ok(())) + }); + TransactionOutcome::Commit(res) + })); + + assert_ok!(recursive_transactional(255)); + assert_noop!( + recursive_transactional(256), + sp_runtime::TransactionalError::LimitReached + ); + + assert_eq!(get_transaction_level(), 0); + }); + } +} diff --git a/frame/support/src/storage/types/counted_map.rs b/frame/support/src/storage/types/counted_map.rs index 0860a4ed541c..9c48267af86e 100644 --- a/frame/support/src/storage/types/counted_map.rs +++ b/frame/support/src/storage/types/counted_map.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -98,6 +98,17 @@ where OnEmpty: Get + 'static, MaxValues: Get>, { + /// The key used to store the counter of the map. + pub fn counter_storage_final_key() -> [u8; 32] { + CounterFor::::hashed_key() + } + + /// The prefix used to generate the key of the map. + pub fn map_storage_final_prefix() -> Vec { + use crate::storage::generator::StorageMap; + ::Map::prefix_hash() + } + /// Get the storage key used to fetch a value corresponding to a specific key. pub fn hashed_key_for>(key: KeyArg) -> Vec { ::Map::hashed_key_for(key) @@ -179,6 +190,8 @@ where } /// Mutate the item, only if an `Ok` value is returned. Deletes the item if mutated to a `None`. + /// `f` will always be called with an option representing if the storage item exists (`Some`) + /// or if the storage item does not exist (`None`), independent of the `QueryType`. pub fn try_mutate_exists(key: KeyArg, f: F) -> Result where KeyArg: EncodeLike + Clone, @@ -263,6 +276,9 @@ where /// Remove all value of the storage. pub fn remove_all() { + // NOTE: it is not possible to remove up to some limit because + // `sp_io::storage::clear_prefix` and `StorageMap::remove_all` don't give the number of + // value removed from the overlay. CounterFor::::set(0u32); ::Map::remove_all(None); } @@ -271,14 +287,7 @@ where /// /// NOTE: If a value failed to decode because storage is corrupted then it is skipped. pub fn iter_values() -> crate::storage::PrefixIterator> { - let map_iterator = ::Map::iter_values(); - crate::storage::PrefixIterator { - prefix: map_iterator.prefix, - previous_key: map_iterator.previous_key, - drain: map_iterator.drain, - closure: map_iterator.closure, - phantom: Default::default(), - } + ::Map::iter_values().convert_on_removal() } /// Translate the values of all elements by a function `f`, in the map in no particular order. @@ -360,28 +369,14 @@ where /// /// If you alter the map while doing this, you'll get undefined results. pub fn iter() -> crate::storage::PrefixIterator<(Key, Value), OnRemovalCounterUpdate> { - let map_iterator = ::Map::iter(); - crate::storage::PrefixIterator { - prefix: map_iterator.prefix, - previous_key: map_iterator.previous_key, - drain: map_iterator.drain, - closure: map_iterator.closure, - phantom: Default::default(), - } + ::Map::iter().convert_on_removal() } /// Remove all elements from the map and iterate through them in no particular order. /// /// If you add elements to the map while doing this, you'll get undefined results. pub fn drain() -> crate::storage::PrefixIterator<(Key, Value), OnRemovalCounterUpdate> { - let map_iterator = ::Map::drain(); - crate::storage::PrefixIterator { - prefix: map_iterator.prefix, - previous_key: map_iterator.previous_key, - drain: map_iterator.drain, - closure: map_iterator.closure, - phantom: Default::default(), - } + ::Map::drain().convert_on_removal() } /// Translate the values of all elements by a function `f`, in the map in no particular order. @@ -398,6 +393,23 @@ where res }) } + + /// Enumerate all elements in the counted map after a specified `starting_raw_key` in no + /// particular order. + /// + /// If you alter the map while doing this, you'll get undefined results. + pub fn iter_from( + starting_raw_key: Vec, + ) -> crate::storage::PrefixIterator<(Key, Value), OnRemovalCounterUpdate> { + ::Map::iter_from(starting_raw_key).convert_on_removal() + } + + /// Enumerate all keys in the counted map. + /// + /// If you alter the map while doing this, you'll get undefined results. + pub fn iter_keys() -> crate::storage::KeyPrefixIterator { + ::Map::iter_keys() + } } impl StorageEntryMetadataBuilder @@ -414,7 +426,11 @@ where fn build_metadata(docs: Vec<&'static str>, entries: &mut Vec) { ::Map::build_metadata(docs, entries); CounterFor::::build_metadata( - vec![&"Counter for the related counted storage map"], + if cfg!(feature = "no-metadata-docs") { + vec![] + } else { + vec![&"Counter for the related counted storage map"] + }, entries, ); } @@ -1008,6 +1024,25 @@ mod test { }) } + #[test] + fn test_iter_from() { + type A = CountedStorageMap; + TestExternalities::default().execute_with(|| { + A::insert(1, 1); + A::insert(2, 2); + A::insert(3, 3); + A::insert(4, 4); + + // no prefix is same as normal iter. + assert_eq!(A::iter_from(vec![]).collect::>(), A::iter().collect::>()); + + let iter_all = A::iter().collect::>(); + let (before, after) = iter_all.split_at(2); + let last_key = before.last().map(|(k, _)| k).unwrap(); + assert_eq!(A::iter_from(A::hashed_key_for(last_key)).collect::>(), after); + }) + } + #[test] fn test_metadata() { type A = CountedStorageMap; @@ -1032,7 +1067,11 @@ mod test { modifier: StorageEntryModifier::Default, ty: StorageEntryType::Plain(scale_info::meta_type::()), default: vec![0, 0, 0, 0], - docs: vec!["Counter for the related counted storage map"], + docs: if cfg!(feature = "no-metadata-docs") { + vec![] + } else { + vec!["Counter for the related counted storage map"] + }, }, ] ); diff --git a/frame/support/src/storage/types/double_map.rs b/frame/support/src/storage/types/double_map.rs index b9af4a621b92..42b903128934 100644 --- a/frame/support/src/storage/types/double_map.rs +++ b/frame/support/src/storage/types/double_map.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -217,7 +217,18 @@ where >::remove(k1, k2) } - /// Remove all values under the first key. + /// Remove all values under `k1` in the overlay and up to `limit` in the + /// backend. + /// + /// All values in the client overlay will be deleted, if there is some `limit` then up to + /// `limit` values are deleted from the client backend, if `limit` is none then all values in + /// the client backend are deleted. + /// + /// # Note + /// + /// Calling this multiple times per block with a `limit` set leads always to the same keys being + /// removed and the same result being returned. This happens because the keys to delete in the + /// overlay are not taken into account when deleting keys in the backend. pub fn remove_prefix(k1: KArg1, limit: Option) -> sp_io::KillStorageResult where KArg1: ?Sized + EncodeLike, @@ -264,6 +275,8 @@ where } /// Mutate the item, only if an `Ok` value is returned. Deletes the item if mutated to a `None`. + /// `f` will always be called with an option representing if the storage item exists (`Some`) + /// or if the storage item does not exist (`None`), independent of the `QueryType`. pub fn try_mutate_exists(k1: KArg1, k2: KArg2, f: F) -> Result where KArg1: EncodeLike, @@ -335,7 +348,17 @@ where >(key1, key2) } - /// Remove all value of the storage. + /// Remove all values in the overlay and up to `limit` in the backend. + /// + /// All values in the client overlay will be deleted, if there is some `limit` then up to + /// `limit` values are deleted from the client backend, if `limit` is none then all values in + /// the client backend are deleted. + /// + /// # Note + /// + /// Calling this multiple times per block with a `limit` set leads always to the same keys being + /// removed and the same result being returned. This happens because the keys to delete in the + /// overlay are not taken into account when deleting keys in the backend. pub fn remove_all(limit: Option) -> sp_io::KillStorageResult { >::remove_all(limit) } @@ -526,6 +549,8 @@ where MaxValues: Get>, { fn build_metadata(docs: Vec<&'static str>, entries: &mut Vec) { + let docs = if cfg!(feature = "no-metadata-docs") { vec![] } else { docs }; + let entry = StorageEntryMetadata { name: Prefix::STORAGE_PREFIX, modifier: QueryKind::METADATA, diff --git a/frame/support/src/storage/types/key.rs b/frame/support/src/storage/types/key.rs index da265fd6e6c8..182ddbedd9b8 100755 --- a/frame/support/src/storage/types/key.rs +++ b/frame/support/src/storage/types/key.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/storage/types/map.rs b/frame/support/src/storage/types/map.rs index 45340f9015ea..bc32c58da8db 100644 --- a/frame/support/src/storage/types/map.rs +++ b/frame/support/src/storage/types/map.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -174,6 +174,8 @@ where } /// Mutate the item, only if an `Ok` value is returned. Deletes the item if mutated to a `None`. + /// `f` will always be called with an option representing if the storage item exists (`Some`) + /// or if the storage item does not exist (`None`), independent of the `QueryType`. pub fn try_mutate_exists(key: KeyArg, f: F) -> Result where KeyArg: EncodeLike, @@ -234,7 +236,17 @@ where >::migrate_key::(key) } - /// Remove all value of the storage. + /// Remove all values of the storage in the overlay and up to `limit` in the backend. + /// + /// All values in the client overlay will be deleted, if there is some `limit` then up to + /// `limit` values are deleted from the client backend, if `limit` is none then all values in + /// the client backend are deleted. + /// + /// # Note + /// + /// Calling this multiple times per block with a `limit` set leads always to the same keys being + /// removed and the same result being returned. This happens because the keys to delete in the + /// overlay are not taken into account when deleting keys in the backend. pub fn remove_all(limit: Option) -> sp_io::KillStorageResult { >::remove_all(limit) } @@ -348,6 +360,8 @@ where MaxValues: Get>, { fn build_metadata(docs: Vec<&'static str>, entries: &mut Vec) { + let docs = if cfg!(feature = "no-metadata-docs") { vec![] } else { docs }; + let entry = StorageEntryMetadata { name: Prefix::STORAGE_PREFIX, modifier: QueryKind::METADATA, diff --git a/frame/support/src/storage/types/mod.rs b/frame/support/src/storage/types/mod.rs index bcab996f6832..0706e9fb377e 100644 --- a/frame/support/src/storage/types/mod.rs +++ b/frame/support/src/storage/types/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/storage/types/nmap.rs b/frame/support/src/storage/types/nmap.rs index 96d6f383ae11..aff32f211000 100755 --- a/frame/support/src/storage/types/nmap.rs +++ b/frame/support/src/storage/types/nmap.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -173,7 +173,18 @@ where >::remove(key) } - /// Remove all values under the first key. + /// Remove all values starting with `partial_key` in the overlay and up to `limit` in the + /// backend. + /// + /// All values in the client overlay will be deleted, if there is some `limit` then up to + /// `limit` values are deleted from the client backend, if `limit` is none then all values in + /// the client backend are deleted. + /// + /// # Note + /// + /// Calling this multiple times per block with a `limit` set leads always to the same keys being + /// removed and the same result being returned. This happens because the keys to delete in the + /// overlay are not taken into account when deleting keys in the backend. pub fn remove_prefix(partial_key: KP, limit: Option) -> sp_io::KillStorageResult where Key: HasKeyPrefix, @@ -217,6 +228,8 @@ where } /// Mutate the item, only if an `Ok` value is returned. Deletes the item if mutated to a `None`. + /// `f` will always be called with an option representing if the storage item exists (`Some`) + /// or if the storage item does not exist (`None`), independent of the `QueryType`. pub fn try_mutate_exists(key: KArg, f: F) -> Result where KArg: EncodeLikeTuple + TupleToEncodedIter, @@ -275,7 +288,17 @@ where >::migrate_keys::<_>(key, hash_fns) } - /// Remove all value of the storage. + /// Remove all values in the overlay and up to `limit` in the backend. + /// + /// All values in the client overlay will be deleted, if there is some `limit` then up to + /// `limit` values are deleted from the client backend, if `limit` is none then all values in + /// the client backend are deleted. + /// + /// # Note + /// + /// Calling this multiple times per block with a `limit` set leads always to the same keys being + /// removed and the same result being returned. This happens because the keys to delete in the + /// overlay are not taken into account when deleting keys in the backend. pub fn remove_all(limit: Option) -> sp_io::KillStorageResult { >::remove_all(limit) } @@ -451,6 +474,8 @@ where MaxValues: Get>, { fn build_metadata(docs: Vec<&'static str>, entries: &mut Vec) { + let docs = if cfg!(feature = "no-metadata-docs") { vec![] } else { docs }; + let entry = StorageEntryMetadata { name: Prefix::STORAGE_PREFIX, modifier: QueryKind::METADATA, @@ -565,7 +590,7 @@ mod test { { crate::generate_storage_alias!(test, Foo => NMap< - Key<(u16, Blake2_128Concat)>, + Key<(Blake2_128Concat, u16)>, u32 >); diff --git a/frame/support/src/storage/types/value.rs b/frame/support/src/storage/types/value.rs index c5e7173bd0af..f145e9fb3041 100644 --- a/frame/support/src/storage/types/value.rs +++ b/frame/support/src/storage/types/value.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -210,6 +210,8 @@ where OnEmpty: crate::traits::Get + 'static, { fn build_metadata(docs: Vec<&'static str>, entries: &mut Vec) { + let docs = if cfg!(feature = "no-metadata-docs") { vec![] } else { docs }; + let entry = StorageEntryMetadata { name: Prefix::STORAGE_PREFIX, modifier: QueryKind::METADATA, diff --git a/frame/support/src/storage/unhashed.rs b/frame/support/src/storage/unhashed.rs index f700771b2d5c..96bccc6ae0fe 100644 --- a/frame/support/src/storage/unhashed.rs +++ b/frame/support/src/storage/unhashed.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,7 +25,11 @@ pub fn get(key: &[u8]) -> Option { sp_io::storage::get(key).and_then(|val| { Decode::decode(&mut &val[..]).map(Some).unwrap_or_else(|_| { // TODO #3700: error should be handleable. - crate::runtime_print!("ERROR: Corrupted state at {:?}", key); + log::error!( + target: "runtime::storage", + "Corrupted state at {:?}", + key, + ); None }) }) diff --git a/frame/support/src/storage/weak_bounded_vec.rs b/frame/support/src/storage/weak_bounded_vec.rs index 4655c809e014..21cc487b4908 100644 --- a/frame/support/src/storage/weak_bounded_vec.rs +++ b/frame/support/src/storage/weak_bounded_vec.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,7 +27,7 @@ use core::{ ops::{Deref, Index, IndexMut}, slice::SliceIndex, }; -use sp_std::{convert::TryFrom, marker::PhantomData, prelude::*}; +use sp_std::{marker::PhantomData, prelude::*}; /// A weakly bounded vector. /// @@ -90,7 +90,7 @@ impl WeakBoundedVec { self.0.retain(f) } - /// Exactly the same semantics as [`Vec::get_mut`]. + /// Exactly the same semantics as [`slice::get_mut`]. pub fn get_mut>( &mut self, index: I, @@ -307,7 +307,7 @@ where fn max_encoded_len() -> usize { // WeakBoundedVec encodes like Vec which encodes like [T], which is a compact u32 // plus each item in the slice: - // https://substrate.dev/rustdocs/v3.0.0/src/parity_scale_codec/codec.rs.html#798-808 + // https://docs.substrate.io/v3/advanced/scale-codec codec::Compact(S::get()) .encoded_size() .saturating_add(Self::bound().saturating_mul(T::max_encoded_len())) @@ -318,36 +318,31 @@ where pub mod test { use super::*; use crate::Twox128; + use frame_support::traits::ConstU32; use sp_io::TestExternalities; - use sp_std::convert::TryInto; - crate::parameter_types! { - pub const Seven: u32 = 7; - pub const Four: u32 = 4; - } - - crate::generate_storage_alias! { Prefix, Foo => Value> } - crate::generate_storage_alias! { Prefix, FooMap => Map<(u32, Twox128), WeakBoundedVec> } + crate::generate_storage_alias! { Prefix, Foo => Value>> } + crate::generate_storage_alias! { Prefix, FooMap => Map<(Twox128, u32), WeakBoundedVec>> } crate::generate_storage_alias! { Prefix, - FooDoubleMap => DoubleMap<(u32, Twox128), (u32, Twox128), WeakBoundedVec> + FooDoubleMap => DoubleMap<(Twox128, u32), (Twox128, u32), WeakBoundedVec>> } #[test] - fn try_append_is_correct() { - assert_eq!(WeakBoundedVec::::bound(), 7); + fn bound_returns_correct_value() { + assert_eq!(WeakBoundedVec::>::bound(), 7); } #[test] fn decode_len_works() { TestExternalities::default().execute_with(|| { - let bounded: WeakBoundedVec = vec![1, 2, 3].try_into().unwrap(); + let bounded: WeakBoundedVec> = vec![1, 2, 3].try_into().unwrap(); Foo::put(bounded); assert_eq!(Foo::decode_len().unwrap(), 3); }); TestExternalities::default().execute_with(|| { - let bounded: WeakBoundedVec = vec![1, 2, 3].try_into().unwrap(); + let bounded: WeakBoundedVec> = vec![1, 2, 3].try_into().unwrap(); FooMap::insert(1, bounded); assert_eq!(FooMap::decode_len(1).unwrap(), 3); assert!(FooMap::decode_len(0).is_none()); @@ -355,7 +350,7 @@ pub mod test { }); TestExternalities::default().execute_with(|| { - let bounded: WeakBoundedVec = vec![1, 2, 3].try_into().unwrap(); + let bounded: WeakBoundedVec> = vec![1, 2, 3].try_into().unwrap(); FooDoubleMap::insert(1, 1, bounded); assert_eq!(FooDoubleMap::decode_len(1, 1).unwrap(), 3); assert!(FooDoubleMap::decode_len(2, 1).is_none()); @@ -366,7 +361,7 @@ pub mod test { #[test] fn try_insert_works() { - let mut bounded: WeakBoundedVec = vec![1, 2, 3].try_into().unwrap(); + let mut bounded: WeakBoundedVec> = vec![1, 2, 3].try_into().unwrap(); bounded.try_insert(1, 0).unwrap(); assert_eq!(*bounded, vec![1, 0, 2, 3]); @@ -377,13 +372,13 @@ pub mod test { #[test] #[should_panic(expected = "insertion index (is 9) should be <= len (is 3)")] fn try_inert_panics_if_oob() { - let mut bounded: WeakBoundedVec = vec![1, 2, 3].try_into().unwrap(); + let mut bounded: WeakBoundedVec> = vec![1, 2, 3].try_into().unwrap(); bounded.try_insert(9, 0).unwrap(); } #[test] fn try_push_works() { - let mut bounded: WeakBoundedVec = vec![1, 2, 3].try_into().unwrap(); + let mut bounded: WeakBoundedVec> = vec![1, 2, 3].try_into().unwrap(); bounded.try_push(0).unwrap(); assert_eq!(*bounded, vec![1, 2, 3, 0]); @@ -392,7 +387,7 @@ pub mod test { #[test] fn deref_coercion_works() { - let bounded: WeakBoundedVec = vec![1, 2, 3].try_into().unwrap(); + let bounded: WeakBoundedVec> = vec![1, 2, 3].try_into().unwrap(); // these methods come from deref-ed vec. assert_eq!(bounded.len(), 3); assert!(bounded.iter().next().is_some()); @@ -401,7 +396,7 @@ pub mod test { #[test] fn try_mutate_works() { - let bounded: WeakBoundedVec = vec![1, 2, 3, 4, 5, 6].try_into().unwrap(); + let bounded: WeakBoundedVec> = vec![1, 2, 3, 4, 5, 6].try_into().unwrap(); let bounded = bounded.try_mutate(|v| v.push(7)).unwrap(); assert_eq!(bounded.len(), 7); assert!(bounded.try_mutate(|v| v.push(8)).is_none()); @@ -409,20 +404,20 @@ pub mod test { #[test] fn slice_indexing_works() { - let bounded: WeakBoundedVec = vec![1, 2, 3, 4, 5, 6].try_into().unwrap(); + let bounded: WeakBoundedVec> = vec![1, 2, 3, 4, 5, 6].try_into().unwrap(); assert_eq!(&bounded[0..=2], &[1, 2, 3]); } #[test] fn vec_eq_works() { - let bounded: WeakBoundedVec = vec![1, 2, 3, 4, 5, 6].try_into().unwrap(); + let bounded: WeakBoundedVec> = vec![1, 2, 3, 4, 5, 6].try_into().unwrap(); assert_eq!(bounded, vec![1, 2, 3, 4, 5, 6]); } #[test] fn too_big_succeed_to_decode() { let v: Vec = vec![1, 2, 3, 4, 5]; - let w = WeakBoundedVec::::decode(&mut &v.encode()[..]).unwrap(); + let w = WeakBoundedVec::>::decode(&mut &v.encode()[..]).unwrap(); assert_eq!(v, *w); } } diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index 5ac0208dc203..40afc0d337f4 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,7 +23,7 @@ pub mod tokens; pub use tokens::{ currency::{ Currency, LockIdentifier, LockableCurrency, NamedReservableCurrency, ReservableCurrency, - VestingSchedule, + TotalIssuanceOf, VestingSchedule, }, fungible, fungibles, imbalance::{Imbalance, OnUnbalanced, SignedImbalance}, @@ -34,8 +34,9 @@ mod members; #[allow(deprecated)] pub use members::{AllowAll, DenyAll, Filter}; pub use members::{ - AsContains, ChangeMembers, Contains, ContainsLengthBound, Everything, InitializeMembers, - IsInVec, Nothing, SortedMembers, + AsContains, ChangeMembers, Contains, ContainsLengthBound, ContainsPair, Everything, + EverythingBut, FromContainsPair, InitializeMembers, InsideBoth, IsInVec, Nothing, + SortedMembers, TheseExcept, }; mod validation; @@ -45,15 +46,24 @@ pub use validation::{ ValidatorSetWithIdentification, VerifySeal, }; +mod error; +pub use error::PalletError; + mod filter; pub use filter::{ClearFilterGuard, FilterStack, FilterStackGuard, InstanceFilter, IntegrityTest}; mod misc; pub use misc::{ - Backing, ConstU32, EnsureInherentsAreFirst, EstimateCallFee, ExecuteBlock, ExtrinsicCall, Get, - GetBacking, GetDefault, HandleLifetime, IsSubType, IsType, Len, OffchainWorker, - OnKilledAccount, OnNewAccount, SameOrOther, Time, TryDrop, UnixTime, WrapperOpaque, + defensive_prelude::{self, *}, + Backing, ConstBool, ConstI128, ConstI16, ConstI32, ConstI64, ConstI8, ConstU128, ConstU16, + ConstU32, ConstU64, ConstU8, DefensiveSaturating, EnsureInherentsAreFirst, EqualPrivilegeOnly, + EstimateCallFee, ExecuteBlock, ExtrinsicCall, Get, GetBacking, GetDefault, HandleLifetime, + IsSubType, IsType, Len, OffchainWorker, OnKilledAccount, OnNewAccount, PreimageProvider, + PreimageRecipient, PrivilegeCmp, SameOrOther, Time, TryCollect, TryDrop, UnixTime, + WrapperKeepOpaque, WrapperOpaque, }; +#[doc(hidden)] +pub use misc::{DEFENSIVE_OP_INTERNAL_ERROR, DEFENSIVE_OP_PUBLIC_ERROR}; mod stored_map; pub use stored_map::{StorageMapShim, StoredMap}; @@ -63,7 +73,8 @@ pub use randomness::Randomness; mod metadata; pub use metadata::{ CallMetadata, CrateVersion, GetCallMetadata, GetCallName, GetStorageVersion, PalletInfo, - PalletInfoAccess, StorageVersion, STORAGE_VERSION_STORAGE_KEY_POSTFIX, + PalletInfoAccess, PalletInfoData, PalletsInfoAccess, StorageVersion, + STORAGE_VERSION_STORAGE_KEY_POSTFIX, }; mod hooks; @@ -82,7 +93,12 @@ pub use storage::{ }; mod dispatch; -pub use dispatch::{EnsureOrigin, OriginTrait, UnfilteredDispatchable}; +pub use dispatch::{ + AsEnsureOriginWithArg, EnsureOneOf, EnsureOrigin, EnsureOriginWithArg, OriginTrait, + UnfilteredDispatchable, +}; mod voting; -pub use voting::{CurrencyToVote, SaturatingCurrencyToVote, U128CurrencyToVote}; +pub use voting::{ + CurrencyToVote, PollStatus, Polling, SaturatingCurrencyToVote, U128CurrencyToVote, VoteTally, +}; diff --git a/frame/support/src/traits/dispatch.rs b/frame/support/src/traits/dispatch.rs index f82628ede18c..250a31ebfb17 100644 --- a/frame/support/src/traits/dispatch.rs +++ b/frame/support/src/traits/dispatch.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,17 +17,22 @@ //! Traits for dealing with dispatching calls and the origin from which they are dispatched. -use crate::dispatch::DispatchResultWithPostInfo; -use sp_runtime::traits::BadOrigin; +use crate::dispatch::{DispatchResultWithPostInfo, Parameter, RawOrigin}; +use sp_runtime::{ + traits::{BadOrigin, Member}, + Either, +}; /// Some sort of check on the origin is performed by this object. pub trait EnsureOrigin { /// A return type. type Success; + /// Perform the origin check. fn ensure_origin(o: OuterOrigin) -> Result { Self::try_origin(o).map_err(|_| BadOrigin) } + /// Perform the origin check. fn try_origin(o: OuterOrigin) -> Result; @@ -38,6 +43,52 @@ pub trait EnsureOrigin { fn successful_origin() -> OuterOrigin; } +/// Some sort of check on the origin is performed by this object. +pub trait EnsureOriginWithArg { + /// A return type. + type Success; + + /// Perform the origin check. + fn ensure_origin(o: OuterOrigin, a: &Argument) -> Result { + Self::try_origin(o, a).map_err(|_| BadOrigin) + } + + /// Perform the origin check, returning the origin value if unsuccessful. This allows chaining. + fn try_origin(o: OuterOrigin, a: &Argument) -> Result; + + /// Returns an outer origin capable of passing `try_origin` check. + /// + /// ** Should be used for benchmarking only!!! ** + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin(a: &Argument) -> OuterOrigin; +} + +pub struct AsEnsureOriginWithArg(sp_std::marker::PhantomData); +impl> + EnsureOriginWithArg for AsEnsureOriginWithArg +{ + /// A return type. + type Success = EO::Success; + + /// Perform the origin check. + fn ensure_origin(o: OuterOrigin, _: &Argument) -> Result { + EO::ensure_origin(o) + } + + /// Perform the origin check, returning the origin value if unsuccessful. This allows chaining. + fn try_origin(o: OuterOrigin, _: &Argument) -> Result { + EO::try_origin(o) + } + + /// Returns an outer origin capable of passing `try_origin` check. + /// + /// ** Should be used for benchmarking only!!! ** + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin(_: &Argument) -> OuterOrigin { + EO::successful_origin() + } +} + /// Type that can be dispatched with an origin but without checking the origin filter. /// /// Implemented for pallet dispatchable type by `decl_module` and for runtime dispatchable by @@ -56,7 +107,7 @@ pub trait OriginTrait: Sized { type Call; /// The caller origin, overarching type of all pallets origins. - type PalletsOrigin; + type PalletsOrigin: Parameter + Member + Into + From>; /// The AccountId used across the system. type AccountId; @@ -70,7 +121,10 @@ pub trait OriginTrait: Sized { /// Replace the caller with caller from the other origin fn set_caller_from(&mut self, other: impl Into); - /// Filter the call, if false then call is filtered out. + /// Filter the call if caller is not root, if false is returned then the call must be filtered + /// out. + /// + /// For root origin caller, the filters are bypassed and true is returned. fn filter_call(&self, call: &Self::Call) -> bool; /// Get the caller. @@ -82,12 +136,70 @@ pub trait OriginTrait: Sized { f: impl FnOnce(Self::PalletsOrigin) -> Result, ) -> Result; - /// Create with system none origin and `frame-system::Config::BaseCallFilter`. + /// Create with system none origin and `frame_system::Config::BaseCallFilter`. fn none() -> Self; - /// Create with system root origin and no filter. + /// Create with system root origin and `frame_system::Config::BaseCallFilter`. fn root() -> Self; - /// Create with system signed origin and `frame-system::Config::BaseCallFilter`. + /// Create with system signed origin and `frame_system::Config::BaseCallFilter`. fn signed(by: Self::AccountId) -> Self; } + +/// The "OR gate" implementation of `EnsureOrigin`. +/// +/// Origin check will pass if `L` or `R` origin check passes. `L` is tested first. +pub struct EnsureOneOf(sp_std::marker::PhantomData<(L, R)>); + +impl, R: EnsureOrigin> + EnsureOrigin for EnsureOneOf +{ + type Success = Either; + fn try_origin(o: OuterOrigin) -> Result { + L::try_origin(o) + .map_or_else(|o| R::try_origin(o).map(|o| Either::Right(o)), |o| Ok(Either::Left(o))) + } + + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> OuterOrigin { + L::successful_origin() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct EnsureSuccess; + struct EnsureFail; + + impl EnsureOrigin<()> for EnsureSuccess { + type Success = (); + fn try_origin(_: ()) -> Result { + Ok(()) + } + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> () { + () + } + } + + impl EnsureOrigin<()> for EnsureFail { + type Success = (); + fn try_origin(_: ()) -> Result { + Err(()) + } + #[cfg(feature = "runtime-benchmarks")] + fn successful_origin() -> () { + () + } + } + + #[test] + fn ensure_one_of_test() { + assert!(>::try_origin(()).is_ok()); + assert!(>::try_origin(()).is_ok()); + assert!(>::try_origin(()).is_ok()); + assert!(>::try_origin(()).is_err()); + } +} diff --git a/frame/support/src/traits/error.rs b/frame/support/src/traits/error.rs new file mode 100644 index 000000000000..8e26891669e6 --- /dev/null +++ b/frame/support/src/traits/error.rs @@ -0,0 +1,95 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Traits for describing and constraining pallet error types. +use codec::{Compact, Decode, Encode}; +use sp_std::marker::PhantomData; + +/// Trait indicating that the implementing type is going to be included as a field in a variant of +/// the `#[pallet::error]` enum type. +/// +/// ## Notes +/// +/// The pallet error enum has a maximum encoded size as defined by +/// [`frame_support::MAX_MODULE_ERROR_ENCODED_SIZE`]. If the pallet error type exceeds this size +/// limit, a static assertion during compilation will fail. The compilation error will be in the +/// format of `error[E0080]: evaluation of constant value failed` due to the usage of +/// const assertions. +pub trait PalletError: Encode + Decode { + /// The maximum encoded size for the implementing type. + /// + /// This will be used to check whether the pallet error type is less than or equal to + /// [`frame_support::MAX_MODULE_ERROR_ENCODED_SIZE`], and if it is, a compilation error will be + /// thrown. + const MAX_ENCODED_SIZE: usize; +} + +macro_rules! impl_for_types { + (size: $size:expr, $($typ:ty),+) => { + $( + impl PalletError for $typ { + const MAX_ENCODED_SIZE: usize = $size; + } + )+ + }; +} + +impl_for_types!(size: 0, (), crate::Never); +impl_for_types!(size: 1, u8, i8, bool); +impl_for_types!(size: 2, u16, i16, Compact); +impl_for_types!(size: 4, u32, i32, Compact); +impl_for_types!(size: 5, Compact); +impl_for_types!(size: 8, u64, i64); +impl_for_types!(size: 9, Compact); +// Contains a u64 for secs and u32 for nanos, hence 12 bytes +impl_for_types!(size: 12, core::time::Duration); +impl_for_types!(size: 16, u128, i128); +impl_for_types!(size: 17, Compact); + +impl PalletError for PhantomData { + const MAX_ENCODED_SIZE: usize = 0; +} + +impl PalletError for core::ops::Range { + const MAX_ENCODED_SIZE: usize = T::MAX_ENCODED_SIZE.saturating_mul(2); +} + +impl PalletError for [T; N] { + const MAX_ENCODED_SIZE: usize = T::MAX_ENCODED_SIZE.saturating_mul(N); +} + +impl PalletError for Option { + const MAX_ENCODED_SIZE: usize = T::MAX_ENCODED_SIZE.saturating_add(1); +} + +impl PalletError for Result { + const MAX_ENCODED_SIZE: usize = if T::MAX_ENCODED_SIZE > E::MAX_ENCODED_SIZE { + T::MAX_ENCODED_SIZE + } else { + E::MAX_ENCODED_SIZE + } + .saturating_add(1); +} + +#[impl_trait_for_tuples::impl_for_tuples(1, 18)] +impl PalletError for Tuple { + const MAX_ENCODED_SIZE: usize = { + let mut size = 0_usize; + for_tuples!( #(size = size.saturating_add(Tuple::MAX_ENCODED_SIZE);)* ); + size + }; +} diff --git a/frame/support/src/traits/filter.rs b/frame/support/src/traits/filter.rs index c67ffc3c3a11..95e5954184b4 100644 --- a/frame/support/src/traits/filter.rs +++ b/frame/support/src/traits/filter.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/traits/hooks.rs b/frame/support/src/traits/hooks.rs index 2a8b0a156247..385db4e4d1ad 100644 --- a/frame/support/src/traits/hooks.rs +++ b/frame/support/src/traits/hooks.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -244,10 +244,15 @@ pub trait Hooks { /// # Warning /// /// This function will be called before we initialized any runtime state, aka `on_initialize` - /// wasn't called yet. So, information like the block number and any other - /// block local data are not accessible. + /// wasn't called yet. So, information like the block number and any other block local data are + /// not accessible. /// /// Return the non-negotiable weight consumed for runtime upgrade. + /// + /// While this function can be freely implemented, using `on_runtime_upgrade` from inside the + /// pallet is discouraged and might get deprecated in the future. Alternatively, export the same + /// logic as a free-function from your pallet, and pass it to `type Executive` from the + /// top-level runtime. fn on_runtime_upgrade() -> crate::weights::Weight { 0 } diff --git a/frame/support/src/traits/members.rs b/frame/support/src/traits/members.rs index a59869c2fc9a..f3c586b64af0 100644 --- a/frame/support/src/traits/members.rs +++ b/frame/support/src/traits/members.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,6 +25,40 @@ pub trait Contains { fn contains(t: &T) -> bool; } +#[impl_trait_for_tuples::impl_for_tuples(1, 30)] +impl Contains for Tuple { + fn contains(t: &T) -> bool { + for_tuples!( #( + if Tuple::contains(t) { return true } + )* ); + false + } +} + +/// A trait for querying whether a type can be said to "contain" a pair-value. +pub trait ContainsPair { + /// Return `true` if this "contains" the pair-value `(a, b)`. + fn contains(a: &A, b: &B) -> bool; +} + +#[impl_trait_for_tuples::impl_for_tuples(0, 30)] +impl ContainsPair for Tuple { + fn contains(a: &A, b: &B) -> bool { + for_tuples!( #( + if Tuple::contains(a, b) { return true } + )* ); + false + } +} + +/// Converter `struct` to use a `ContainsPair` implementation for a `Contains` bound. +pub struct FromContainsPair(PhantomData); +impl> Contains<(A, B)> for FromContainsPair { + fn contains((ref a, ref b): &(A, B)) -> bool { + CP::contains(a, b) + } +} + /// A [`Contains`] implementation that contains every value. pub enum Everything {} impl Contains for Everything { @@ -32,6 +66,11 @@ impl Contains for Everything { true } } +impl ContainsPair for Everything { + fn contains(_: &A, _: &B) -> bool { + true + } +} /// A [`Contains`] implementation that contains no value. pub enum Nothing {} @@ -40,43 +79,112 @@ impl Contains for Nothing { false } } +impl ContainsPair for Nothing { + fn contains(_: &A, _: &B) -> bool { + false + } +} -#[deprecated = "Use `Everything` instead"] -pub type AllowAll = Everything; -#[deprecated = "Use `Nothing` instead"] -pub type DenyAll = Nothing; -#[deprecated = "Use `Contains` instead"] -pub trait Filter { - fn filter(t: &T) -> bool; +/// A [`Contains`] implementation that contains everything except the values in `Exclude`. +pub struct EverythingBut(PhantomData); +impl> Contains for EverythingBut { + fn contains(t: &T) -> bool { + !Exclude::contains(t) + } } -#[allow(deprecated)] -impl> Filter for C { - fn filter(t: &T) -> bool { - Self::contains(t) +impl> ContainsPair for EverythingBut { + fn contains(a: &A, b: &B) -> bool { + !Exclude::contains(a, b) } } -#[impl_trait_for_tuples::impl_for_tuples(1, 30)] -impl Contains for Tuple { +/// A [`Contains`] implementation that contains all members of `These` excepting any members in +/// `Except`. +pub struct TheseExcept(PhantomData<(These, Except)>); +impl, Except: Contains> Contains for TheseExcept { fn contains(t: &T) -> bool { - for_tuples!( #( - if Tuple::contains(t) { return true } - )* ); - false + These::contains(t) && !Except::contains(t) + } +} +impl, Except: ContainsPair> ContainsPair + for TheseExcept +{ + fn contains(a: &A, b: &B) -> bool { + These::contains(a, b) && !Except::contains(a, b) + } +} + +/// A [`Contains`] implementation which contains all members of `These` which are also members of +/// `Those`. +pub struct InsideBoth(PhantomData<(These, Those)>); +impl, Those: Contains> Contains for InsideBoth { + fn contains(t: &T) -> bool { + These::contains(t) && Those::contains(t) + } +} +impl, Those: ContainsPair> ContainsPair + for InsideBoth +{ + fn contains(a: &A, b: &B) -> bool { + These::contains(a, b) && Those::contains(a, b) } } /// Create a type which implements the `Contains` trait for a particular type with syntax similar /// to `matches!`. #[macro_export] -macro_rules! match_type { - ( pub type $n:ident: impl Contains<$t:ty> = { $phead:pat $( | $ptail:pat )* } ; ) => { +macro_rules! match_types { + ( + pub type $n:ident: impl Contains<$t:ty> = { + $phead:pat_param $( | $ptail:pat )* + }; + $( $rest:tt )* + ) => { pub struct $n; impl $crate::traits::Contains<$t> for $n { fn contains(l: &$t) -> bool { matches!(l, $phead $( | $ptail )* ) } } + $crate::match_types!( $( $rest )* ); + }; + ( + pub type $n:ident: impl ContainsPair<$a:ty, $b:ty> = { + $phead:pat_param $( | $ptail:pat )* + }; + $( $rest:tt )* + ) => { + pub struct $n; + impl $crate::traits::ContainsPair<$a, $b> for $n { + fn contains(a: &$a, b: &$b) -> bool { + matches!((a, b), $phead $( | $ptail )* ) + } + } + $crate::match_types!( $( $rest )* ); + }; + () => {} +} + +/// Create a type which implements the `Contains` trait for a particular type with syntax similar +/// to `matches!`. +#[macro_export] +#[deprecated = "Use `match_types!` instead"] +macro_rules! match_type { + ($( $x:tt )*) => { $crate::match_types!( $( $x )* ); } +} + +#[deprecated = "Use `Everything` instead"] +pub type AllowAll = Everything; +#[deprecated = "Use `Nothing` instead"] +pub type DenyAll = Nothing; +#[deprecated = "Use `Contains` instead"] +pub trait Filter { + fn filter(t: &T) -> bool; +} +#[allow(deprecated)] +impl> Filter for C { + fn filter(t: &T) -> bool { + Self::contains(t) } } @@ -84,12 +192,12 @@ macro_rules! match_type { mod tests { use super::*; - match_type! { + match_types! { pub type OneOrTenToTwenty: impl Contains = { 1 | 10..=20 }; } #[test] - fn match_type_works() { + fn match_types_works() { for i in 0..=255 { assert_eq!(OneOrTenToTwenty::contains(&i), i == 1 || i >= 10 && i <= 20); } diff --git a/frame/support/src/traits/metadata.rs b/frame/support/src/traits/metadata.rs index 14b7e6d7355e..c76c53dbe8b2 100644 --- a/frame/support/src/traits/metadata.rs +++ b/frame/support/src/traits/metadata.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +19,7 @@ use codec::{Decode, Encode}; use sp_runtime::RuntimeDebug; +use sp_std::prelude::*; /// Provides information about the pallet itself and its setup in the runtime. /// @@ -35,6 +36,19 @@ pub trait PalletInfo { fn crate_version() -> Option; } +/// Information regarding an instance of a pallet. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug)] +pub struct PalletInfoData { + /// Index of the pallet as configured in the runtime. + pub index: usize, + /// Name of the pallet as configured in the runtime. + pub name: &'static str, + /// Name of the Rust module containing the pallet. + pub module_name: &'static str, + /// Version of the crate containing the pallet. + pub crate_version: CrateVersion, +} + /// Provides information about the pallet itself and its setup in the runtime. /// /// Declare some information and access the information provided by [`PalletInfo`] for a specific @@ -50,6 +64,49 @@ pub trait PalletInfoAccess { fn crate_version() -> CrateVersion; } +/// Provide information about a bunch of pallets. +pub trait PalletsInfoAccess { + /// The number of pallets' information that this type represents. + /// + /// You probably don't want this function but `infos()` instead. + fn count() -> usize { + 0 + } + + /// Extend the given vector by all of the pallets' information that this type represents. + /// + /// You probably don't want this function but `infos()` instead. + fn accumulate(_accumulator: &mut Vec) {} + + /// All of the pallets' information that this type represents. + fn infos() -> Vec { + let mut result = Vec::with_capacity(Self::count()); + Self::accumulate(&mut result); + result + } +} + +impl PalletsInfoAccess for () {} +impl PalletsInfoAccess for (T,) { + fn count() -> usize { + T::count() + } + fn accumulate(acc: &mut Vec) { + T::accumulate(acc) + } +} + +impl PalletsInfoAccess for (T1, T2) { + fn count() -> usize { + T1::count() + T2::count() + } + fn accumulate(acc: &mut Vec) { + // The AllPallets type tuplises the pallets in reverse order, so we unreverse them here. + T2::accumulate(acc); + T1::accumulate(acc); + } +} + /// The function and pallet name of the Call. #[derive(Clone, Eq, PartialEq, Default, RuntimeDebug)] pub struct CallMetadata { @@ -78,7 +135,7 @@ pub trait GetCallMetadata { } /// The version of a crate. -#[derive(RuntimeDebug, Eq, PartialEq, Encode, Decode, Ord, Clone, Copy, Default)] +#[derive(Debug, Eq, PartialEq, Encode, Decode, Clone, Copy, Default)] pub struct CrateVersion { /// The major version of the crate. pub major: u16, @@ -94,14 +151,17 @@ impl CrateVersion { } } -impl sp_std::cmp::PartialOrd for CrateVersion { - fn partial_cmp(&self, other: &Self) -> Option { - let res = self - .major +impl sp_std::cmp::Ord for CrateVersion { + fn cmp(&self, other: &Self) -> sp_std::cmp::Ordering { + self.major .cmp(&other.major) - .then_with(|| self.minor.cmp(&other.minor).then_with(|| self.patch.cmp(&other.patch))); + .then_with(|| self.minor.cmp(&other.minor).then_with(|| self.patch.cmp(&other.patch))) + } +} - Some(res) +impl sp_std::cmp::PartialOrd for CrateVersion { + fn partial_cmp(&self, other: &Self) -> Option { + Some(::cmp(&self, other)) } } @@ -115,7 +175,7 @@ pub const STORAGE_VERSION_STORAGE_KEY_POSTFIX: &[u8] = b":__STORAGE_VERSION__:"; /// /// Each storage version of a pallet is stored in the state under a fixed key. See /// [`STORAGE_VERSION_STORAGE_KEY_POSTFIX`] for how this key is built. -#[derive(RuntimeDebug, Eq, PartialEq, Encode, Decode, Ord, Clone, Copy, PartialOrd, Default)] +#[derive(Debug, Eq, PartialEq, Encode, Decode, Ord, Clone, Copy, PartialOrd, Default)] pub struct StorageVersion(u16); impl StorageVersion { @@ -203,6 +263,37 @@ pub trait GetStorageVersion { mod tests { use super::*; + struct Pallet1; + impl PalletInfoAccess for Pallet1 { + fn index() -> usize { + 1 + } + fn name() -> &'static str { + "Pallet1" + } + fn module_name() -> &'static str { + "pallet1" + } + fn crate_version() -> CrateVersion { + CrateVersion::new(1, 0, 0) + } + } + struct Pallet2; + impl PalletInfoAccess for Pallet2 { + fn index() -> usize { + 2 + } + fn name() -> &'static str { + "Pallet2" + } + fn module_name() -> &'static str { + "pallet2" + } + fn crate_version() -> CrateVersion { + CrateVersion::new(1, 0, 0) + } + } + #[test] fn check_storage_version_ordering() { let version = StorageVersion::new(1); diff --git a/frame/support/src/traits/misc.rs b/frame/support/src/traits/misc.rs index db6e0321005a..2a0d6f0523e7 100644 --- a/frame/support/src/traits/misc.rs +++ b/frame/support/src/traits/misc.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,10 +18,317 @@ //! Smaller traits used in FRAME which don't need their own file. use crate::dispatch::Parameter; -use codec::{Decode, Encode, EncodeLike, Input, MaxEncodedLen}; +use codec::{CompactLen, Decode, DecodeAll, Encode, EncodeLike, Input, MaxEncodedLen}; use scale_info::{build::Fields, meta_type, Path, Type, TypeInfo, TypeParameter}; +use sp_arithmetic::traits::{CheckedAdd, CheckedMul, CheckedSub, Saturating}; use sp_runtime::{traits::Block as BlockT, DispatchError}; -use sp_std::prelude::*; +use sp_std::{cmp::Ordering, prelude::*}; + +#[doc(hidden)] +pub const DEFENSIVE_OP_PUBLIC_ERROR: &'static str = "a defensive failure has been triggered; please report the block number at https://github.com/paritytech/substrate/issues"; +#[doc(hidden)] +pub const DEFENSIVE_OP_INTERNAL_ERROR: &'static str = "Defensive failure has been triggered!"; + +/// Generic function to mark an execution path as ONLY defensive. +/// +/// Similar to mark a match arm or `if/else` branch as `unreachable!`. +#[macro_export] +macro_rules! defensive { + () => { + frame_support::log::error!( + target: "runtime", + "{}", + $crate::traits::DEFENSIVE_OP_PUBLIC_ERROR + ); + debug_assert!(false, "{}", $crate::traits::DEFENSIVE_OP_INTERNAL_ERROR); + }; + ($error:tt) => { + frame_support::log::error!( + target: "runtime", + "{}: {:?}", + $crate::traits::DEFENSIVE_OP_PUBLIC_ERROR, + $error + ); + debug_assert!(false, "{}: {:?}", $crate::traits::DEFENSIVE_OP_INTERNAL_ERROR, $error); + } +} + +/// Prelude module for all defensive traits to be imported at once. +pub mod defensive_prelude { + pub use super::{Defensive, DefensiveOption, DefensiveResult}; +} + +/// A trait to handle errors and options when you are really sure that a condition must hold, but +/// not brave enough to `expect` on it, or a default fallback value makes more sense. +/// +/// This trait mostly focuses on methods that eventually unwrap the inner value. See +/// [`DefensiveResult`] and [`DefensiveOption`] for methods that specifically apply to the +/// respective types. +/// +/// Each function in this trait will have two side effects, aside from behaving exactly as the name +/// would suggest: +/// +/// 1. It panics on `#[debug_assertions]`, so if the infallible code is reached in any of the tests, +/// you realize. +/// 2. It will log an error using the runtime logging system. This might help you detect such bugs +/// in production as well. Note that the log message, as of now, are not super expressive. Your +/// best shot of fully diagnosing the error would be to infer the block number of which the log +/// message was emitted, then re-execute that block using `check-block` or `try-runtime` +/// subcommands in substrate client. +pub trait Defensive { + /// Exactly the same as `unwrap_or`, but it does the defensive warnings explained in the trait + /// docs. + fn defensive_unwrap_or(self, other: T) -> T; + + /// Exactly the same as `unwrap_or_else`, but it does the defensive warnings explained in the + /// trait docs. + fn defensive_unwrap_or_else T>(self, f: F) -> T; + + /// Exactly the same as `unwrap_or_default`, but it does the defensive warnings explained in the + /// trait docs. + fn defensive_unwrap_or_default(self) -> T + where + T: Default; + + /// Does not alter the inner value at all, but it will log warnings if the inner value is `None` + /// or `Err`. + /// + /// In some ways, this is like `.defensive_map(|x| x)`. + /// + /// This is useful as: + /// ```nocompile + /// if let Some(inner) = maybe_value().defensive() { + /// .. + /// } + /// ``` + fn defensive(self) -> Self; +} + +/// Subset of methods similar to [`Defensive`] that can only work for a `Result`. +pub trait DefensiveResult { + /// Defensively map the error into another return type, but you are really sure that this + /// conversion should never be needed. + fn defensive_map_err F>(self, o: O) -> Result; + + /// Defensively map and unpack the value to something else (`U`), or call the default callback + /// if `Err`, which should never happen. + fn defensive_map_or_else U, F: FnOnce(T) -> U>(self, default: D, f: F) -> U; + + /// Defensively transform this result into an option, discarding the `Err` variant if it + /// happens, which should never happen. + fn defensive_ok(self) -> Option; + + /// Exactly the same as `map`, but it prints the appropriate warnings if the value being mapped + /// is `Err`. + fn defensive_map U>(self, f: F) -> Result; +} + +/// Subset of methods similar to [`Defensive`] that can only work for a `Option`. +pub trait DefensiveOption { + /// Potentially map and unpack the value to something else (`U`), or call the default callback + /// if `None`, which should never happen. + fn defensive_map_or_else U, F: FnOnce(T) -> U>(self, default: D, f: F) -> U; + + /// Defensively transform this option to a result. + fn defensive_ok_or_else E>(self, err: F) -> Result; + + /// Exactly the same as `map`, but it prints the appropriate warnings if the value being mapped + /// is `None`. + fn defensive_map U>(self, f: F) -> Option; +} + +impl Defensive for Option { + fn defensive_unwrap_or(self, or: T) -> T { + match self { + Some(inner) => inner, + None => { + defensive!(); + or + }, + } + } + + fn defensive_unwrap_or_else T>(self, f: F) -> T { + match self { + Some(inner) => inner, + None => { + defensive!(); + f() + }, + } + } + + fn defensive_unwrap_or_default(self) -> T + where + T: Default, + { + match self { + Some(inner) => inner, + None => { + defensive!(); + Default::default() + }, + } + } + + fn defensive(self) -> Self { + match self { + Some(inner) => Some(inner), + None => { + defensive!(); + None + }, + } + } +} + +impl Defensive for Result { + fn defensive_unwrap_or(self, or: T) -> T { + match self { + Ok(inner) => inner, + Err(e) => { + defensive!(e); + or + }, + } + } + + fn defensive_unwrap_or_else T>(self, f: F) -> T { + match self { + Ok(inner) => inner, + Err(e) => { + defensive!(e); + f() + }, + } + } + + fn defensive_unwrap_or_default(self) -> T + where + T: Default, + { + match self { + Ok(inner) => inner, + Err(e) => { + defensive!(e); + Default::default() + }, + } + } + + fn defensive(self) -> Self { + match self { + Ok(inner) => Ok(inner), + Err(e) => { + defensive!(e); + Err(e) + }, + } + } +} + +impl DefensiveResult for Result { + fn defensive_map_err F>(self, o: O) -> Result { + self.map_err(|e| { + defensive!(e); + o(e) + }) + } + + fn defensive_map_or_else U, F: FnOnce(T) -> U>(self, default: D, f: F) -> U { + self.map_or_else( + |e| { + defensive!(e); + default(e) + }, + f, + ) + } + + fn defensive_ok(self) -> Option { + match self { + Ok(inner) => Some(inner), + Err(e) => { + defensive!(e); + None + }, + } + } + + fn defensive_map U>(self, f: F) -> Result { + match self { + Ok(inner) => Ok(f(inner)), + Err(e) => { + defensive!(e); + Err(e) + }, + } + } +} + +impl DefensiveOption for Option { + fn defensive_map_or_else U, F: FnOnce(T) -> U>(self, default: D, f: F) -> U { + self.map_or_else( + || { + defensive!(); + default() + }, + f, + ) + } + + fn defensive_ok_or_else E>(self, err: F) -> Result { + self.ok_or_else(|| { + defensive!(); + err() + }) + } + + fn defensive_map U>(self, f: F) -> Option { + match self { + Some(inner) => Some(f(inner)), + None => { + defensive!(); + None + }, + } + } +} + +/// A variant of [`Defensive`] with the same rationale, for the arithmetic operations where in +/// case an infallible operation fails, it saturates. +pub trait DefensiveSaturating { + /// Add `self` and `other` defensively. + fn defensive_saturating_add(self, other: Self) -> Self; + /// Subtract `other` from `self` defensively. + fn defensive_saturating_sub(self, other: Self) -> Self; + /// Multiply `self` and `other` defensively. + fn defensive_saturating_mul(self, other: Self) -> Self; +} + +// NOTE: A bit unfortunate, since T has to be bound by all the traits needed. Could make it +// `DefensiveSaturating` to mitigate. +impl DefensiveSaturating for T { + fn defensive_saturating_add(self, other: Self) -> Self { + self.checked_add(&other).defensive_unwrap_or_else(|| self.saturating_add(other)) + } + fn defensive_saturating_sub(self, other: Self) -> Self { + self.checked_sub(&other).defensive_unwrap_or_else(|| self.saturating_sub(other)) + } + fn defensive_saturating_mul(self, other: Self) -> Self { + self.checked_mul(&other).defensive_unwrap_or_else(|| self.saturating_mul(other)) + } +} + +/// Try and collect into a collection `C`. +pub trait TryCollect { + type Error; + /// Consume self and try to collect the results into `C`. + /// + /// This is useful in preventing the undesirable `.collect().try_into()` call chain on + /// collections that need to be converted into a bounded type (e.g. `BoundedVec`). + fn try_collect(self) -> Result; +} /// Anything that can have a `::len()` method. pub trait Len { @@ -60,20 +367,34 @@ impl Get for GetDefault { } } -/// Implement `Get` and `Get>` using the given const. -pub struct ConstU32; - -impl Get for ConstU32 { - fn get() -> u32 { - T - } +macro_rules! impl_const_get { + ($name:ident, $t:ty) => { + #[derive($crate::RuntimeDebug)] + pub struct $name; + impl Get<$t> for $name { + fn get() -> $t { + T + } + } + impl Get> for $name { + fn get() -> Option<$t> { + Some(T) + } + } + }; } -impl Get> for ConstU32 { - fn get() -> Option { - Some(T) - } -} +impl_const_get!(ConstBool, bool); +impl_const_get!(ConstU8, u8); +impl_const_get!(ConstU16, u16); +impl_const_get!(ConstU32, u32); +impl_const_get!(ConstU64, u64); +impl_const_get!(ConstU128, u128); +impl_const_get!(ConstI8, i8); +impl_const_get!(ConstI16, i16); +impl_const_get!(ConstI32, i32); +impl_const_get!(ConstI64, i64); +impl_const_get!(ConstI128, i128); /// A type for which some values make sense to be able to drop without further consideration. pub trait TryDrop: Sized { @@ -81,6 +402,12 @@ pub trait TryDrop: Sized { fn try_drop(self) -> Result<(), Self>; } +impl TryDrop for () { + fn try_drop(self) -> Result<(), Self> { + Ok(()) + } +} + /// Return type used when we need to return one of two items, each of the opposite direction or /// sign, with one (`Same`) being of the same type as the `self` or primary argument of the function /// that returned it. @@ -183,7 +510,7 @@ pub trait HandleLifetime { impl HandleLifetime for () {} pub trait Time { - type Moment: sp_arithmetic::traits::AtLeast32Bit + Parameter + Default + Copy; + type Moment: sp_arithmetic::traits::AtLeast32Bit + Parameter + Default + Copy + MaxEncodedLen; fn now() -> Self::Moment; } @@ -289,6 +616,26 @@ pub trait ExecuteBlock { fn execute_block(block: Block); } +/// Something that can compare privileges of two origins. +pub trait PrivilegeCmp { + /// Compare the `left` to the `right` origin. + /// + /// The returned ordering should be from the pov of the `left` origin. + /// + /// Should return `None` when it can not compare the given origins. + fn cmp_privilege(left: &Origin, right: &Origin) -> Option; +} + +/// Implementation of [`PrivilegeCmp`] that only checks for equal origins. +/// +/// This means it will either return [`Ordering::Equal`] or `None`. +pub struct EqualPrivilegeOnly; +impl PrivilegeCmp for EqualPrivilegeOnly { + fn cmp_privilege(left: &Origin, right: &Origin) -> Option { + (left == right).then(|| Ordering::Equal) + } +} + /// Off-chain computation trait. /// /// Implementing this trait on a module allows you to perform long-running tasks @@ -390,6 +737,7 @@ impl, const T: u32> EstimateCallFee for pub struct WrapperOpaque(pub T); impl EncodeLike for WrapperOpaque {} +impl EncodeLike> for WrapperOpaque {} impl Encode for WrapperOpaque { fn size_hint(&self) -> usize { @@ -429,7 +777,7 @@ impl MaxEncodedLen for WrapperOpaque { fn max_encoded_len() -> usize { let t_max_len = T::max_encoded_len(); - // See scale encoding https://substrate.dev/docs/en/knowledgebase/advanced/codec + // See scale encoding https://docs.substrate.io/v3/advanced/scale-codec if t_max_len < 64 { t_max_len + 1 } else if t_max_len < 2usize.pow(14) { @@ -456,6 +804,152 @@ impl TypeInfo for WrapperOpaque { } } +/// A wrapper for any type `T` which implement encode/decode in a way compatible with `Vec`. +/// +/// This type is similar to [`WrapperOpaque`], but it differs in the way it stores the type `T`. +/// While [`WrapperOpaque`] stores the decoded type, the [`WrapperKeepOpaque`] stores the type only +/// in its opaque format, aka as a `Vec`. To access the real type `T` [`Self::try_decode`] needs +/// to be used. +#[derive(Debug, Eq, PartialEq, Default, Clone)] +pub struct WrapperKeepOpaque { + data: Vec, + _phantom: sp_std::marker::PhantomData, +} + +impl WrapperKeepOpaque { + /// Try to decode the wrapped type from the inner `data`. + /// + /// Returns `None` if the decoding failed. + pub fn try_decode(&self) -> Option { + T::decode_all(&mut &self.data[..]).ok() + } + + /// Returns the length of the encoded `T`. + pub fn encoded_len(&self) -> usize { + self.data.len() + } + + /// Returns the encoded data. + pub fn encoded(&self) -> &[u8] { + &self.data + } + + /// Create from the given encoded `data`. + pub fn from_encoded(data: Vec) -> Self { + Self { data, _phantom: sp_std::marker::PhantomData } + } +} + +impl EncodeLike for WrapperKeepOpaque {} +impl EncodeLike> for WrapperKeepOpaque {} + +impl Encode for WrapperKeepOpaque { + fn size_hint(&self) -> usize { + self.data.len() + codec::Compact::::compact_len(&(self.data.len() as u32)) + } + + fn encode_to(&self, dest: &mut O) { + self.data.encode_to(dest); + } + + fn encode(&self) -> Vec { + self.data.encode() + } + + fn using_encoded R>(&self, f: F) -> R { + self.data.using_encoded(f) + } +} + +impl Decode for WrapperKeepOpaque { + fn decode(input: &mut I) -> Result { + Ok(Self { data: Vec::::decode(input)?, _phantom: sp_std::marker::PhantomData }) + } + + fn skip(input: &mut I) -> Result<(), codec::Error> { + >::skip(input) + } +} + +impl MaxEncodedLen for WrapperKeepOpaque { + fn max_encoded_len() -> usize { + WrapperOpaque::::max_encoded_len() + } +} + +impl TypeInfo for WrapperKeepOpaque { + type Identity = Self; + fn type_info() -> Type { + Type::builder() + .path(Path::new("WrapperKeepOpaque", module_path!())) + .type_params(vec![TypeParameter::new("T", Some(meta_type::()))]) + .composite( + Fields::unnamed() + .field(|f| f.compact::()) + .field(|f| f.ty::().type_name("T")), + ) + } +} + +/// A interface for looking up preimages from their hash on chain. +pub trait PreimageProvider { + /// Returns whether a preimage exists for a given hash. + /// + /// A value of `true` implies that `get_preimage` is `Some`. + fn have_preimage(hash: &Hash) -> bool; + + /// Returns the preimage for a given hash. + fn get_preimage(hash: &Hash) -> Option>; + + /// Returns whether a preimage request exists for a given hash. + fn preimage_requested(hash: &Hash) -> bool; + + /// Request that someone report a preimage. Providers use this to optimise the economics for + /// preimage reporting. + fn request_preimage(hash: &Hash); + + /// Cancel a previous preimage request. + fn unrequest_preimage(hash: &Hash); +} + +impl PreimageProvider for () { + fn have_preimage(_: &Hash) -> bool { + false + } + fn get_preimage(_: &Hash) -> Option> { + None + } + fn preimage_requested(_: &Hash) -> bool { + false + } + fn request_preimage(_: &Hash) {} + fn unrequest_preimage(_: &Hash) {} +} + +/// A interface for managing preimages to hashes on chain. +/// +/// Note that this API does not assume any underlying user is calling, and thus +/// does not handle any preimage ownership or fees. Other system level logic that +/// uses this API should implement that on their own side. +pub trait PreimageRecipient: PreimageProvider { + /// Maximum size of a preimage. + type MaxSize: Get; + + /// Store the bytes of a preimage on chain. + fn note_preimage(bytes: crate::BoundedVec); + + /// Clear a previously noted preimage. This is infallible and should be treated more like a + /// hint - if it was not previously noted or if it is now requested, then this will not do + /// anything. + fn unnote_preimage(hash: &Hash); +} + +impl PreimageRecipient for () { + type MaxSize = (); + fn note_preimage(_: crate::BoundedVec) {} + fn unnote_preimage(_: &Hash) {} +} + #[cfg(test)] mod test { use super::*; @@ -488,4 +982,17 @@ mod test { ); assert_eq!(>::max_encoded_len(), 2usize.pow(14) + 4); } + + #[test] + fn test_keep_opaque_wrapper() { + let data = 3u32.encode().encode(); + + let keep_opaque = WrapperKeepOpaque::::decode(&mut &data[..]).unwrap(); + keep_opaque.try_decode().unwrap(); + + let data = WrapperOpaque(50u32).encode(); + let decoded = WrapperKeepOpaque::::decode(&mut &data[..]).unwrap(); + let data = decoded.encode(); + WrapperOpaque::::decode(&mut &data[..]).unwrap(); + } } diff --git a/frame/support/src/traits/randomness.rs b/frame/support/src/traits/randomness.rs index 865893f99b39..d68b95b1dfcf 100644 --- a/frame/support/src/traits/randomness.rs +++ b/frame/support/src/traits/randomness.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/traits/schedule.rs b/frame/support/src/traits/schedule.rs index a4a4f9c03ab1..c2d0d4bc3b81 100644 --- a/frame/support/src/traits/schedule.rs +++ b/frame/support/src/traits/schedule.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,9 +17,10 @@ //! Traits and associated utilities for scheduling dispatchables in FRAME. -use codec::{Codec, Decode, Encode, EncodeLike}; -use sp_runtime::{DispatchError, RuntimeDebug}; -use sp_std::{fmt::Debug, prelude::*}; +use codec::{Codec, Decode, Encode, EncodeLike, MaxEncodedLen}; +use scale_info::TypeInfo; +use sp_runtime::{traits::Saturating, DispatchError, RuntimeDebug}; +use sp_std::{fmt::Debug, prelude::*, result::Result}; /// Information relating to the period of a scheduled task. First item is the length of the /// period and the second is the number of times it should be executed in total before the task @@ -31,7 +32,7 @@ pub type Period = (BlockNumber, u32); pub type Priority = u8; /// The dispatch time of a scheduled task. -#[derive(Encode, Decode, Copy, Clone, PartialEq, Eq, RuntimeDebug)] +#[derive(Encode, Decode, Copy, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub enum DispatchTime { /// At specified block. At(BlockNumber), @@ -39,6 +40,15 @@ pub enum DispatchTime { After(BlockNumber), } +impl DispatchTime { + pub fn evaluate(&self, since: BlockNumber) -> BlockNumber { + match &self { + Self::At(m) => *m, + Self::After(m) => m.saturating_add(since), + } + } +} + /// The highest priority. We invert the value so that normal sorting will place the highest /// priority at the beginning of the list. pub const HIGHEST_PRIORITY: Priority = 0; @@ -48,86 +58,323 @@ pub const HARD_DEADLINE: Priority = 63; /// The lowest priority. Most stuff should be around here. pub const LOWEST_PRIORITY: Priority = 255; -/// A type that can be used as a scheduler. -pub trait Anon { - /// An address which can be used for removing a scheduled task. - type Address: Codec + Clone + Eq + EncodeLike + Debug; - - /// Schedule a dispatch to happen at the beginning of some block in the future. - /// - /// This is not named. - fn schedule( - when: DispatchTime, - maybe_periodic: Option>, - priority: Priority, - origin: Origin, - call: Call, - ) -> Result; - - /// Cancel a scheduled task. If periodic, then it will cancel all further instances of that, - /// also. - /// - /// Will return an error if the `address` is invalid. - /// - /// NOTE: This guaranteed to work only *before* the point that it is due to be executed. - /// If it ends up being delayed beyond the point of execution, then it cannot be cancelled. - /// - /// NOTE2: This will not work to cancel periodic tasks after their initial execution. For - /// that, you must name the task explicitly using the `Named` trait. - fn cancel(address: Self::Address) -> Result<(), ()>; - - /// Reschedule a task. For one-off tasks, this dispatch is guaranteed to succeed - /// only if it is executed *before* the currently scheduled block. For periodic tasks, - /// this dispatch is guaranteed to succeed only before the *initial* execution; for - /// others, use `reschedule_named`. - /// - /// Will return an error if the `address` is invalid. - fn reschedule( - address: Self::Address, - when: DispatchTime, - ) -> Result; - - /// Return the next dispatch time for a given task. - /// - /// Will return an error if the `address` is invalid. - fn next_dispatch_time(address: Self::Address) -> Result; +/// Type representing an encodable value or the hash of the encoding of such a value. +#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub enum MaybeHashed { + /// The value itself. + Value(T), + /// The hash of the encoded value which this value represents. + Hash(Hash), +} + +impl From for MaybeHashed { + fn from(t: T) -> Self { + MaybeHashed::Value(t) + } +} + +/// Error type for `MaybeHashed::lookup`. +#[derive(Clone, Eq, PartialEq, Encode, Decode, RuntimeDebug, TypeInfo, MaxEncodedLen)] +pub enum LookupError { + /// A call of this hash was not known. + Unknown, + /// The preimage for this hash was known but could not be decoded into a `Call`. + BadFormat, +} + +impl MaybeHashed { + pub fn as_value(&self) -> Option<&T> { + match &self { + Self::Value(c) => Some(c), + Self::Hash(_) => None, + } + } + + pub fn as_hash(&self) -> Option<&H> { + match &self { + Self::Value(_) => None, + Self::Hash(h) => Some(h), + } + } + + pub fn ensure_requested>(&self) { + match &self { + Self::Value(_) => (), + Self::Hash(hash) => P::request_preimage(hash), + } + } + + pub fn ensure_unrequested>(&self) { + match &self { + Self::Value(_) => (), + Self::Hash(hash) => P::unrequest_preimage(hash), + } + } + + pub fn resolved>(self) -> (Self, Option) { + match self { + Self::Value(c) => (Self::Value(c), None), + Self::Hash(h) => { + let data = match P::get_preimage(&h) { + Some(p) => p, + None => return (Self::Hash(h), None), + }; + match T::decode(&mut &data[..]) { + Ok(c) => (Self::Value(c), Some(h)), + Err(_) => (Self::Hash(h), None), + } + }, + } + } +} + +pub mod v1 { + use super::*; + + /// A type that can be used as a scheduler. + pub trait Anon { + /// An address which can be used for removing a scheduled task. + type Address: Codec + Clone + Eq + EncodeLike + Debug + TypeInfo; + + /// Schedule a dispatch to happen at the beginning of some block in the future. + /// + /// This is not named. + fn schedule( + when: DispatchTime, + maybe_periodic: Option>, + priority: Priority, + origin: Origin, + call: Call, + ) -> Result; + + /// Cancel a scheduled task. If periodic, then it will cancel all further instances of that, + /// also. + /// + /// Will return an error if the `address` is invalid. + /// + /// NOTE: This guaranteed to work only *before* the point that it is due to be executed. + /// If it ends up being delayed beyond the point of execution, then it cannot be cancelled. + /// + /// NOTE2: This will not work to cancel periodic tasks after their initial execution. For + /// that, you must name the task explicitly using the `Named` trait. + fn cancel(address: Self::Address) -> Result<(), ()>; + + /// Reschedule a task. For one-off tasks, this dispatch is guaranteed to succeed + /// only if it is executed *before* the currently scheduled block. For periodic tasks, + /// this dispatch is guaranteed to succeed only before the *initial* execution; for + /// others, use `reschedule_named`. + /// + /// Will return an error if the `address` is invalid. + fn reschedule( + address: Self::Address, + when: DispatchTime, + ) -> Result; + + /// Return the next dispatch time for a given task. + /// + /// Will return an error if the `address` is invalid. + fn next_dispatch_time(address: Self::Address) -> Result; + } + + /// A type that can be used as a scheduler. + pub trait Named { + /// An address which can be used for removing a scheduled task. + type Address: Codec + Clone + Eq + EncodeLike + sp_std::fmt::Debug; + + /// Schedule a dispatch to happen at the beginning of some block in the future. + /// + /// - `id`: The identity of the task. This must be unique and will return an error if not. + fn schedule_named( + id: Vec, + when: DispatchTime, + maybe_periodic: Option>, + priority: Priority, + origin: Origin, + call: Call, + ) -> Result; + + /// Cancel a scheduled, named task. If periodic, then it will cancel all further instances + /// of that, also. + /// + /// Will return an error if the `id` is invalid. + /// + /// NOTE: This guaranteed to work only *before* the point that it is due to be executed. + /// If it ends up being delayed beyond the point of execution, then it cannot be cancelled. + fn cancel_named(id: Vec) -> Result<(), ()>; + + /// Reschedule a task. For one-off tasks, this dispatch is guaranteed to succeed + /// only if it is executed *before* the currently scheduled block. + fn reschedule_named( + id: Vec, + when: DispatchTime, + ) -> Result; + + /// Return the next dispatch time for a given task. + /// + /// Will return an error if the `id` is invalid. + fn next_dispatch_time(id: Vec) -> Result; + } + + impl Anon for T + where + T: v2::Anon, + { + type Address = T::Address; + + fn schedule( + when: DispatchTime, + maybe_periodic: Option>, + priority: Priority, + origin: Origin, + call: Call, + ) -> Result { + let c = MaybeHashed::::Value(call); + T::schedule(when, maybe_periodic, priority, origin, c) + } + + fn cancel(address: Self::Address) -> Result<(), ()> { + T::cancel(address) + } + + fn reschedule( + address: Self::Address, + when: DispatchTime, + ) -> Result { + T::reschedule(address, when) + } + + fn next_dispatch_time(address: Self::Address) -> Result { + T::next_dispatch_time(address) + } + } + + impl Named for T + where + T: v2::Named, + { + type Address = T::Address; + + fn schedule_named( + id: Vec, + when: DispatchTime, + maybe_periodic: Option>, + priority: Priority, + origin: Origin, + call: Call, + ) -> Result { + let c = MaybeHashed::::Value(call); + T::schedule_named(id, when, maybe_periodic, priority, origin, c) + } + + fn cancel_named(id: Vec) -> Result<(), ()> { + T::cancel_named(id) + } + + fn reschedule_named( + id: Vec, + when: DispatchTime, + ) -> Result { + T::reschedule_named(id, when) + } + + fn next_dispatch_time(id: Vec) -> Result { + T::next_dispatch_time(id) + } + } } -/// A type that can be used as a scheduler. -pub trait Named { - /// An address which can be used for removing a scheduled task. - type Address: Codec + Clone + Eq + EncodeLike + sp_std::fmt::Debug; - - /// Schedule a dispatch to happen at the beginning of some block in the future. - /// - /// - `id`: The identity of the task. This must be unique and will return an error if not. - fn schedule_named( - id: Vec, - when: DispatchTime, - maybe_periodic: Option>, - priority: Priority, - origin: Origin, - call: Call, - ) -> Result; - - /// Cancel a scheduled, named task. If periodic, then it will cancel all further instances - /// of that, also. - /// - /// Will return an error if the `id` is invalid. - /// - /// NOTE: This guaranteed to work only *before* the point that it is due to be executed. - /// If it ends up being delayed beyond the point of execution, then it cannot be cancelled. - fn cancel_named(id: Vec) -> Result<(), ()>; - - /// Reschedule a task. For one-off tasks, this dispatch is guaranteed to succeed - /// only if it is executed *before* the currently scheduled block. - fn reschedule_named( - id: Vec, - when: DispatchTime, - ) -> Result; - - /// Return the next dispatch time for a given task. - /// - /// Will return an error if the `id` is invalid. - fn next_dispatch_time(id: Vec) -> Result; +pub mod v2 { + use super::*; + + /// A type that can be used as a scheduler. + pub trait Anon { + /// An address which can be used for removing a scheduled task. + type Address: Codec + Clone + Eq + EncodeLike + Debug + TypeInfo; + /// A means of expressing a call by the hash of its encoded data. + type Hash; + + /// Schedule a dispatch to happen at the beginning of some block in the future. + /// + /// This is not named. + fn schedule( + when: DispatchTime, + maybe_periodic: Option>, + priority: Priority, + origin: Origin, + call: MaybeHashed, + ) -> Result; + + /// Cancel a scheduled task. If periodic, then it will cancel all further instances of that, + /// also. + /// + /// Will return an error if the `address` is invalid. + /// + /// NOTE: This guaranteed to work only *before* the point that it is due to be executed. + /// If it ends up being delayed beyond the point of execution, then it cannot be cancelled. + /// + /// NOTE2: This will not work to cancel periodic tasks after their initial execution. For + /// that, you must name the task explicitly using the `Named` trait. + fn cancel(address: Self::Address) -> Result<(), ()>; + + /// Reschedule a task. For one-off tasks, this dispatch is guaranteed to succeed + /// only if it is executed *before* the currently scheduled block. For periodic tasks, + /// this dispatch is guaranteed to succeed only before the *initial* execution; for + /// others, use `reschedule_named`. + /// + /// Will return an error if the `address` is invalid. + fn reschedule( + address: Self::Address, + when: DispatchTime, + ) -> Result; + + /// Return the next dispatch time for a given task. + /// + /// Will return an error if the `address` is invalid. + fn next_dispatch_time(address: Self::Address) -> Result; + } + + /// A type that can be used as a scheduler. + pub trait Named { + /// An address which can be used for removing a scheduled task. + type Address: Codec + Clone + Eq + EncodeLike + sp_std::fmt::Debug; + /// A means of expressing a call by the hash of its encoded data. + type Hash; + + /// Schedule a dispatch to happen at the beginning of some block in the future. + /// + /// - `id`: The identity of the task. This must be unique and will return an error if not. + fn schedule_named( + id: Vec, + when: DispatchTime, + maybe_periodic: Option>, + priority: Priority, + origin: Origin, + call: MaybeHashed, + ) -> Result; + + /// Cancel a scheduled, named task. If periodic, then it will cancel all further instances + /// of that, also. + /// + /// Will return an error if the `id` is invalid. + /// + /// NOTE: This guaranteed to work only *before* the point that it is due to be executed. + /// If it ends up being delayed beyond the point of execution, then it cannot be cancelled. + fn cancel_named(id: Vec) -> Result<(), ()>; + + /// Reschedule a task. For one-off tasks, this dispatch is guaranteed to succeed + /// only if it is executed *before* the currently scheduled block. + fn reschedule_named( + id: Vec, + when: DispatchTime, + ) -> Result; + + /// Return the next dispatch time for a given task. + /// + /// Will return an error if the `id` is invalid. + fn next_dispatch_time(id: Vec) -> Result; + } } + +pub use v1::*; + +use super::PreimageProvider; diff --git a/frame/support/src/traits/storage.rs b/frame/support/src/traits/storage.rs index 9a88a3ed4404..e484140cc2fd 100644 --- a/frame/support/src/traits/storage.rs +++ b/frame/support/src/traits/storage.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/traits/stored_map.rs b/frame/support/src/traits/stored_map.rs index 715a5211be43..3c3ff2eb0ed9 100644 --- a/frame/support/src/traits/stored_map.rs +++ b/frame/support/src/traits/stored_map.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,7 +29,9 @@ pub trait StoredMap { fn get(k: &K) -> T; /// Maybe mutate the item only if an `Ok` value is returned from `f`. Do nothing if an `Err` is - /// returned. It is removed or reset to default value if it has been mutated to `None` + /// returned. It is removed or reset to default value if it has been mutated to `None`. + /// `f` will always be called with an option representing if the storage item exists (`Some`) + /// or if the storage item does not exist (`None`), independent of the `QueryType`. fn try_mutate_exists>( k: &K, f: impl FnOnce(&mut Option) -> Result, diff --git a/frame/support/src/traits/tokens.rs b/frame/support/src/traits/tokens.rs index 91a9382d07fc..92f8ce12d912 100644 --- a/frame/support/src/traits/tokens.rs +++ b/frame/support/src/traits/tokens.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/traits/tokens/currency.rs b/frame/support/src/traits/tokens/currency.rs index bf078658477f..d4b5c0c184f8 100644 --- a/frame/support/src/traits/tokens/currency.rs +++ b/frame/support/src/traits/tokens/currency.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,10 @@ use super::{ imbalance::{Imbalance, SignedImbalance}, misc::{Balance, ExistenceRequirement, WithdrawReasons}, }; -use crate::dispatch::{DispatchError, DispatchResult}; +use crate::{ + dispatch::{DispatchError, DispatchResult}, + traits::Get, +}; use codec::MaxEncodedLen; use sp_runtime::traits::MaybeSerializeDeserialize; use sp_std::fmt::Debug; @@ -199,3 +202,100 @@ pub trait Currency { balance: Self::Balance, ) -> SignedImbalance; } + +/// A non-const `Get` implementation parameterised by a `Currency` impl which provides the result +/// of `total_issuance`. +pub struct TotalIssuanceOf, A>(sp_std::marker::PhantomData<(C, A)>); +impl, A> Get for TotalIssuanceOf { + fn get() -> C::Balance { + C::total_issuance() + } +} + +#[cfg(feature = "std")] +impl Currency for () { + type Balance = u32; + type PositiveImbalance = (); + type NegativeImbalance = (); + fn total_balance(_: &AccountId) -> Self::Balance { + 0 + } + fn can_slash(_: &AccountId, _: Self::Balance) -> bool { + true + } + fn total_issuance() -> Self::Balance { + 0 + } + fn minimum_balance() -> Self::Balance { + 0 + } + fn burn(_: Self::Balance) -> Self::PositiveImbalance { + () + } + fn issue(_: Self::Balance) -> Self::NegativeImbalance { + () + } + fn pair(_: Self::Balance) -> (Self::PositiveImbalance, Self::NegativeImbalance) { + ((), ()) + } + fn free_balance(_: &AccountId) -> Self::Balance { + 0 + } + fn ensure_can_withdraw( + _: &AccountId, + _: Self::Balance, + _: WithdrawReasons, + _: Self::Balance, + ) -> DispatchResult { + Ok(()) + } + fn transfer( + _: &AccountId, + _: &AccountId, + _: Self::Balance, + _: ExistenceRequirement, + ) -> DispatchResult { + Ok(()) + } + fn slash(_: &AccountId, _: Self::Balance) -> (Self::NegativeImbalance, Self::Balance) { + ((), 0) + } + fn deposit_into_existing( + _: &AccountId, + _: Self::Balance, + ) -> Result { + Ok(()) + } + fn resolve_into_existing( + _: &AccountId, + _: Self::NegativeImbalance, + ) -> Result<(), Self::NegativeImbalance> { + Ok(()) + } + fn deposit_creating(_: &AccountId, _: Self::Balance) -> Self::PositiveImbalance { + () + } + fn resolve_creating(_: &AccountId, _: Self::NegativeImbalance) {} + fn withdraw( + _: &AccountId, + _: Self::Balance, + _: WithdrawReasons, + _: ExistenceRequirement, + ) -> Result { + Ok(()) + } + fn settle( + _: &AccountId, + _: Self::PositiveImbalance, + _: WithdrawReasons, + _: ExistenceRequirement, + ) -> Result<(), Self::PositiveImbalance> { + Ok(()) + } + fn make_free_balance_be( + _: &AccountId, + _: Self::Balance, + ) -> SignedImbalance { + SignedImbalance::Positive(()) + } +} diff --git a/frame/support/src/traits/tokens/currency/lockable.rs b/frame/support/src/traits/tokens/currency/lockable.rs index 26463864a647..a10edd6e3e87 100644 --- a/frame/support/src/traits/tokens/currency/lockable.rs +++ b/frame/support/src/traits/tokens/currency/lockable.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/traits/tokens/currency/reservable.rs b/frame/support/src/traits/tokens/currency/reservable.rs index 0ca7a93dc7f6..35455aaecdb4 100644 --- a/frame/support/src/traits/tokens/currency/reservable.rs +++ b/frame/support/src/traits/tokens/currency/reservable.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -81,6 +81,33 @@ pub trait ReservableCurrency: Currency { ) -> Result; } +#[cfg(feature = "std")] +impl ReservableCurrency for () { + fn can_reserve(_: &AccountId, _: Self::Balance) -> bool { + true + } + fn slash_reserved(_: &AccountId, _: Self::Balance) -> (Self::NegativeImbalance, Self::Balance) { + ((), 0) + } + fn reserved_balance(_: &AccountId) -> Self::Balance { + 0 + } + fn reserve(_: &AccountId, _: Self::Balance) -> DispatchResult { + Ok(()) + } + fn unreserve(_: &AccountId, _: Self::Balance) -> Self::Balance { + 0 + } + fn repatriate_reserved( + _: &AccountId, + _: &AccountId, + _: Self::Balance, + _: BalanceStatus, + ) -> Result { + Ok(0) + } +} + pub trait NamedReservableCurrency: ReservableCurrency { /// An identifier for a reserve. Used for disambiguating different reserves so that /// they can be individually replaced or removed. diff --git a/frame/support/src/traits/tokens/fungible.rs b/frame/support/src/traits/tokens/fungible.rs index b033236d447b..712103a1e883 100644 --- a/frame/support/src/traits/tokens/fungible.rs +++ b/frame/support/src/traits/tokens/fungible.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/traits/tokens/fungible/balanced.rs b/frame/support/src/traits/tokens/fungible/balanced.rs index 7b33a595a1b5..196f3a35754a 100644 --- a/frame/support/src/traits/tokens/fungible/balanced.rs +++ b/frame/support/src/traits/tokens/fungible/balanced.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/traits/tokens/fungible/imbalance.rs b/frame/support/src/traits/tokens/fungible/imbalance.rs index 362e0c126d99..ca911cf12d44 100644 --- a/frame/support/src/traits/tokens/fungible/imbalance.rs +++ b/frame/support/src/traits/tokens/fungible/imbalance.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/traits/tokens/fungibles.rs b/frame/support/src/traits/tokens/fungibles.rs index 457ec4e8bf20..8e68b36d60c7 100644 --- a/frame/support/src/traits/tokens/fungibles.rs +++ b/frame/support/src/traits/tokens/fungibles.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,8 +23,11 @@ use super::{ }; use crate::dispatch::{DispatchError, DispatchResult}; use sp_runtime::traits::Saturating; +use sp_std::vec::Vec; +pub mod approvals; mod balanced; +pub mod metadata; pub use balanced::{Balanced, Unbalanced}; mod imbalance; pub use imbalance::{CreditOf, DebtOf, HandleImbalanceDrop, Imbalance}; @@ -65,6 +68,18 @@ pub trait Inspect { ) -> WithdrawConsequence; } +/// Trait for reading metadata from a fungible asset. +pub trait InspectMetadata: Inspect { + /// Return the name of an asset. + fn name(asset: &Self::AssetId) -> Vec; + + /// Return the symbol of an asset. + fn symbol(asset: &Self::AssetId) -> Vec; + + /// Return the decimals of an asset. + fn decimals(asset: &Self::AssetId) -> u8; +} + /// Trait for providing a set of named fungible assets which can be created and destroyed. pub trait Mutate: Inspect { /// Attempt to increase the `asset` balance of `who` by `amount`. diff --git a/frame/support/src/traits/tokens/fungibles/approvals.rs b/frame/support/src/traits/tokens/fungibles/approvals.rs new file mode 100644 index 000000000000..7a08f11cf042 --- /dev/null +++ b/frame/support/src/traits/tokens/fungibles/approvals.rs @@ -0,0 +1,43 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Inspect and Mutate traits for Asset approvals + +use crate::dispatch::DispatchResult; +pub trait Inspect: super::Inspect { + // Check the amount approved by an owner to be spent by a delegate + fn allowance(asset: Self::AssetId, owner: &AccountId, delegate: &AccountId) -> Self::Balance; +} + +pub trait Mutate: Inspect { + // Aprove a delegate account to spend an amount of tokens owned by an owner + fn approve( + asset: Self::AssetId, + owner: &AccountId, + delegate: &AccountId, + amount: Self::Balance, + ) -> DispatchResult; + + // Transfer from a delegate account an amount approved by the owner of the asset + fn transfer_from( + asset: Self::AssetId, + owner: &AccountId, + delegate: &AccountId, + dest: &AccountId, + amount: Self::Balance, + ) -> DispatchResult; +} diff --git a/frame/support/src/traits/tokens/fungibles/balanced.rs b/frame/support/src/traits/tokens/fungibles/balanced.rs index 40a65305b87d..e07d45cc4717 100644 --- a/frame/support/src/traits/tokens/fungibles/balanced.rs +++ b/frame/support/src/traits/tokens/fungibles/balanced.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/traits/tokens/fungibles/imbalance.rs b/frame/support/src/traits/tokens/fungibles/imbalance.rs index c44c47164648..61bd4a43064e 100644 --- a/frame/support/src/traits/tokens/fungibles/imbalance.rs +++ b/frame/support/src/traits/tokens/fungibles/imbalance.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/traits/tokens/fungibles/metadata.rs b/frame/support/src/traits/tokens/fungibles/metadata.rs new file mode 100644 index 000000000000..b736ab1489f5 --- /dev/null +++ b/frame/support/src/traits/tokens/fungibles/metadata.rs @@ -0,0 +1,41 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Inspect and Mutate traits for Asset metadata + +use crate::dispatch::DispatchResult; +use sp_std::vec::Vec; + +pub trait Inspect: super::Inspect { + // Get name for an AssetId. + fn name(asset: Self::AssetId) -> Vec; + // Get symbol for an AssetId. + fn symbol(asset: Self::AssetId) -> Vec; + // Get decimals for an AssetId. + fn decimals(asset: Self::AssetId) -> u8; +} + +pub trait Mutate: Inspect { + // Set name, symbol and decimals for a given assetId. + fn set( + asset: Self::AssetId, + from: &AccountId, + name: Vec, + symbol: Vec, + decimals: u8, + ) -> DispatchResult; +} diff --git a/frame/support/src/traits/tokens/imbalance.rs b/frame/support/src/traits/tokens/imbalance.rs index 0f7b38a65efc..d721beb41494 100644 --- a/frame/support/src/traits/tokens/imbalance.rs +++ b/frame/support/src/traits/tokens/imbalance.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -177,3 +177,55 @@ pub trait Imbalance: Sized + TryDrop + Default { /// The raw value of self. fn peek(&self) -> Balance; } + +#[cfg(feature = "std")] +impl Imbalance for () { + type Opposite = (); + fn zero() -> Self { + () + } + fn drop_zero(self) -> Result<(), Self> { + Ok(()) + } + fn split(self, _: Balance) -> (Self, Self) { + ((), ()) + } + fn ration(self, _: u32, _: u32) -> (Self, Self) + where + Balance: From + Saturating + Div, + { + ((), ()) + } + fn split_merge(self, _: Balance, _: (Self, Self)) -> (Self, Self) { + ((), ()) + } + fn ration_merge(self, _: u32, _: u32, _: (Self, Self)) -> (Self, Self) + where + Balance: From + Saturating + Div, + { + ((), ()) + } + fn split_merge_into(self, _: Balance, _: &mut (Self, Self)) {} + fn ration_merge_into(self, _: u32, _: u32, _: &mut (Self, Self)) + where + Balance: From + Saturating + Div, + { + } + fn merge(self, _: Self) -> Self { + () + } + fn merge_into(self, _: &mut Self) {} + fn maybe_merge(self, _: Option) -> Self { + () + } + fn subsume(&mut self, _: Self) {} + fn maybe_subsume(&mut self, _: Option) { + () + } + fn offset(self, _: Self::Opposite) -> SameOrOther { + SameOrOther::None + } + fn peek(&self) -> Balance { + Default::default() + } +} diff --git a/frame/support/src/traits/tokens/imbalance/on_unbalanced.rs b/frame/support/src/traits/tokens/imbalance/on_unbalanced.rs index bc7df0e2acf3..0125254cefc8 100644 --- a/frame/support/src/traits/tokens/imbalance/on_unbalanced.rs +++ b/frame/support/src/traits/tokens/imbalance/on_unbalanced.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/traits/tokens/imbalance/signed_imbalance.rs b/frame/support/src/traits/tokens/imbalance/signed_imbalance.rs index 3e76d069f50e..f969a4363405 100644 --- a/frame/support/src/traits/tokens/imbalance/signed_imbalance.rs +++ b/frame/support/src/traits/tokens/imbalance/signed_imbalance.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/src/traits/tokens/imbalance/split_two_ways.rs b/frame/support/src/traits/tokens/imbalance/split_two_ways.rs index 882b43c2e914..b963895af0de 100644 --- a/frame/support/src/traits/tokens/imbalance/split_two_ways.rs +++ b/frame/support/src/traits/tokens/imbalance/split_two_ways.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,27 +18,26 @@ //! Means for splitting an imbalance into two and hanlding them differently. use super::super::imbalance::{Imbalance, OnUnbalanced}; -use sp_core::u32_trait::Value as U32; use sp_runtime::traits::Saturating; use sp_std::{marker::PhantomData, ops::Div}; /// Split an unbalanced amount two ways between a common divisor. -pub struct SplitTwoWays( - PhantomData<(Balance, Imbalance, Part1, Target1, Part2, Target2)>, +pub struct SplitTwoWays( + PhantomData<(Balance, Imbalance, Target1, Target2)>, ); impl< Balance: From + Saturating + Div, I: Imbalance, - Part1: U32, Target1: OnUnbalanced, - Part2: U32, Target2: OnUnbalanced, - > OnUnbalanced for SplitTwoWays + const PART1: u32, + const PART2: u32, + > OnUnbalanced for SplitTwoWays { fn on_nonzero_unbalanced(amount: I) { - let total: u32 = Part1::VALUE + Part2::VALUE; - let amount1 = amount.peek().saturating_mul(Part1::VALUE.into()) / total.into(); + let total: u32 = PART1 + PART2; + let amount1 = amount.peek().saturating_mul(PART1.into()) / total.into(); let (imb1, imb2) = amount.split(amount1); Target1::on_unbalanced(imb1); Target2::on_unbalanced(imb2); diff --git a/frame/support/src/traits/tokens/misc.rs b/frame/support/src/traits/tokens/misc.rs index 214c28708a19..f30fd02bfe83 100644 --- a/frame/support/src/traits/tokens/misc.rs +++ b/frame/support/src/traits/tokens/misc.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -161,8 +161,8 @@ impl WithdrawReasons { } /// Simple amalgamation trait to collect together properties for an AssetId under one roof. -pub trait AssetId: FullCodec + Copy + Eq + PartialEq + Debug {} -impl AssetId for T {} +pub trait AssetId: FullCodec + Copy + Eq + PartialEq + Debug + scale_info::TypeInfo {} +impl AssetId for T {} /// Simple amalgamation trait to collect together properties for a Balance under one roof. pub trait Balance: diff --git a/frame/support/src/traits/tokens/nonfungible.rs b/frame/support/src/traits/tokens/nonfungible.rs index 821884f6e390..5cf9638131b2 100644 --- a/frame/support/src/traits/tokens/nonfungible.rs +++ b/frame/support/src/traits/tokens/nonfungible.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -85,7 +85,10 @@ pub trait Mutate: Inspect { /// Burn some asset `instance`. /// /// By default, this is not a supported operation. - fn burn_from(_instance: &Self::InstanceId) -> DispatchResult { + fn burn( + _instance: &Self::InstanceId, + _maybe_check_owner: Option<&AccountId>, + ) -> DispatchResult { Err(TokenError::Unsupported.into()) } @@ -166,8 +169,8 @@ impl< fn mint_into(instance: &Self::InstanceId, who: &AccountId) -> DispatchResult { >::mint_into(&A::get(), instance, who) } - fn burn_from(instance: &Self::InstanceId) -> DispatchResult { - >::burn_from(&A::get(), instance) + fn burn(instance: &Self::InstanceId, maybe_check_owner: Option<&AccountId>) -> DispatchResult { + >::burn(&A::get(), instance, maybe_check_owner) } fn set_attribute(instance: &Self::InstanceId, key: &[u8], value: &[u8]) -> DispatchResult { >::set_attribute(&A::get(), instance, key, value) diff --git a/frame/support/src/traits/tokens/nonfungibles.rs b/frame/support/src/traits/tokens/nonfungibles.rs index b5a14761064f..8bd731b20342 100644 --- a/frame/support/src/traits/tokens/nonfungibles.rs +++ b/frame/support/src/traits/tokens/nonfungibles.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -165,7 +165,11 @@ pub trait Mutate: Inspect { /// Burn some asset `instance` of `class`. /// /// By default, this is not a supported operation. - fn burn_from(_class: &Self::ClassId, _instance: &Self::InstanceId) -> DispatchResult { + fn burn( + _class: &Self::ClassId, + _instance: &Self::InstanceId, + _maybe_check_owner: Option<&AccountId>, + ) -> DispatchResult { Err(TokenError::Unsupported.into()) } diff --git a/frame/support/src/traits/validation.rs b/frame/support/src/traits/validation.rs index 11ea5a79f67b..135dd927acbb 100644 --- a/frame/support/src/traits/validation.rs +++ b/frame/support/src/traits/validation.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -77,7 +77,7 @@ pub trait VerifySeal { /// A session handler for specific key type. pub trait OneSessionHandler: BoundToRuntimeAppPublic { /// The key type expected. - type Key: Decode + Default + RuntimeAppPublic; + type Key: Decode + RuntimeAppPublic; /// The given validator set will be used for the genesis session. /// It is guaranteed that the given validator set will also be used @@ -109,7 +109,7 @@ pub trait OneSessionHandler: BoundToRuntimeAppPublic { fn on_before_session_ending() {} /// A validator got disabled. Act accordingly until a new session begins. - fn on_disabled(_validator_index: usize); + fn on_disabled(_validator_index: u32); } /// Something that can estimate at which block the next session rotation will happen (i.e. a new diff --git a/frame/support/src/traits/voting.rs b/frame/support/src/traits/voting.rs index 62c6217ad59b..978c5ce4f6a0 100644 --- a/frame/support/src/traits/voting.rs +++ b/frame/support/src/traits/voting.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,14 @@ //! Traits and associated data structures concerned with voting, and moving between tokens and //! votes. -use sp_arithmetic::traits::{SaturatedConversion, UniqueSaturatedFrom, UniqueSaturatedInto}; +use crate::dispatch::{DispatchError, Parameter}; +use codec::HasCompact; +use sp_arithmetic::{ + traits::{SaturatedConversion, UniqueSaturatedFrom, UniqueSaturatedInto}, + Perbill, +}; +use sp_runtime::traits::Member; +use sp_std::prelude::*; /// A trait similar to `Convert` to convert values from `B` an abstract balance type /// into u64 and back from u128. (This conversion is used in election and other places where complex @@ -87,3 +94,74 @@ impl + UniqueSaturatedFrom> CurrencyToVote B::unique_saturated_from(value) } } + +pub trait VoteTally { + fn ayes(&self) -> Votes; + fn turnout(&self) -> Perbill; + fn approval(&self) -> Perbill; + #[cfg(feature = "runtime-benchmarks")] + fn unanimity() -> Self; + #[cfg(feature = "runtime-benchmarks")] + fn from_requirements(turnout: Perbill, approval: Perbill) -> Self; +} + +pub enum PollStatus { + None, + Ongoing(Tally, Class), + Completed(Moment, bool), +} + +impl PollStatus { + pub fn ensure_ongoing(self) -> Option<(Tally, Class)> { + match self { + Self::Ongoing(t, c) => Some((t, c)), + _ => None, + } + } +} + +pub trait Polling { + type Index: Parameter + Member + Ord + PartialOrd + Copy + HasCompact; + type Votes: Parameter + Member + Ord + PartialOrd + Copy + HasCompact; + type Class: Parameter + Member + Ord + PartialOrd; + type Moment; + + /// Provides a vec of values that `T` may take. + fn classes() -> Vec; + + /// `Some` if the referendum `index` can be voted on, along with the tally and class of + /// referendum. + /// + /// Don't use this if you might mutate - use `try_access_poll` instead. + fn as_ongoing(index: Self::Index) -> Option<(Tally, Self::Class)>; + + fn access_poll( + index: Self::Index, + f: impl FnOnce(PollStatus<&mut Tally, Self::Moment, Self::Class>) -> R, + ) -> R; + + fn try_access_poll( + index: Self::Index, + f: impl FnOnce(PollStatus<&mut Tally, Self::Moment, Self::Class>) -> Result, + ) -> Result; + + /// Create an ongoing majority-carries poll of given class lasting given period for the purpose + /// of benchmarking. + /// + /// May return `Err` if it is impossible. + #[cfg(feature = "runtime-benchmarks")] + fn create_ongoing(class: Self::Class) -> Result; + + /// End the given ongoing poll and return the result. + /// + /// Returns `Err` if `index` is not an ongoing poll. + #[cfg(feature = "runtime-benchmarks")] + fn end_ongoing(index: Self::Index, approved: bool) -> Result<(), ()>; + + /// The maximum amount of ongoing polls within any single class. By default it practically + /// unlimited (`u32::max_value()`). + #[cfg(feature = "runtime-benchmarks")] + fn max_ongoing() -> (Self::Class, u32) { + (Self::classes().into_iter().next().expect("Always one class"), u32::max_value()) + } +} diff --git a/frame/support/src/weights.rs b/frame/support/src/weights.rs index 115470a9bf03..4c3fcaa0fd42 100644 --- a/frame/support/src/weights.rs +++ b/frame/support/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -127,7 +127,15 @@ //! - Ubuntu 19.10 (GNU/Linux 5.3.0-18-generic x86_64) //! - rustc 1.42.0 (b8cedc004 2020-03-09) -use crate::dispatch::{DispatchError, DispatchErrorWithPostInfo, DispatchResultWithPostInfo}; +mod block_weights; +mod extrinsic_weights; +mod paritydb_weights; +mod rocksdb_weights; + +use crate::{ + dispatch::{DispatchError, DispatchErrorWithPostInfo, DispatchResultWithPostInfo}, + traits::Get, +}; use codec::{Decode, Encode}; use scale_info::TypeInfo; #[cfg(feature = "std")] @@ -152,32 +160,23 @@ pub type Weight = u64; /// These constants are specific to FRAME, and the current implementation of its various components. /// For example: FRAME System, FRAME Executive, our FRAME support libraries, etc... pub mod constants { - use super::{RuntimeDbWeight, Weight}; - use crate::parameter_types; + use super::Weight; pub const WEIGHT_PER_SECOND: Weight = 1_000_000_000_000; pub const WEIGHT_PER_MILLIS: Weight = WEIGHT_PER_SECOND / 1000; // 1_000_000_000 pub const WEIGHT_PER_MICROS: Weight = WEIGHT_PER_MILLIS / 1000; // 1_000_000 pub const WEIGHT_PER_NANOS: Weight = WEIGHT_PER_MICROS / 1000; // 1_000 - parameter_types! { - /// Importing a block with 0 txs takes ~5 ms - pub const BlockExecutionWeight: Weight = 5 * WEIGHT_PER_MILLIS; - /// Executing 10,000 System remarks (no-op) txs takes ~1.26 seconds -> ~125 µs per tx - pub const ExtrinsicBaseWeight: Weight = 125 * WEIGHT_PER_MICROS; - /// By default, Substrate uses RocksDB, so this will be the weight used throughout - /// the runtime. - pub const RocksDbWeight: RuntimeDbWeight = RuntimeDbWeight { - read: 25 * WEIGHT_PER_MICROS, // ~25 µs @ 200,000 items - write: 100 * WEIGHT_PER_MICROS, // ~100 µs @ 200,000 items - }; - /// ParityDB can be enabled with a feature flag, but is still experimental. These weights - /// are available for brave runtime engineers who may want to try this out as default. - pub const ParityDbWeight: RuntimeDbWeight = RuntimeDbWeight { - read: 8 * WEIGHT_PER_MICROS, // ~8 µs @ 200,000 items - write: 50 * WEIGHT_PER_MICROS, // ~50 µs @ 200,000 items - }; - } + // Expose the Block and Extrinsic base weights. + pub use super::{ + block_weights::constants::BlockExecutionWeight, + extrinsic_weights::constants::ExtrinsicBaseWeight, + }; + + // Expose the DB weights. + pub use super::{ + paritydb_weights::constants::ParityDbWeight, rocksdb_weights::constants::RocksDbWeight, + }; } /// Means of weighing some particular kind of data (`T`). @@ -287,30 +286,6 @@ impl<'a> OneOrMany for &'a [DispatchClass] { } } -/// Primitives related to priority management of Frame. -pub mod priority { - /// The starting point of all Operational transactions. 3/4 of u64::MAX. - pub const LIMIT: u64 = 13_835_058_055_282_163_711_u64; - - /// Wrapper for priority of different dispatch classes. - /// - /// This only makes sure that any value created for the operational dispatch class is - /// incremented by [`LIMIT`]. - pub enum FrameTransactionPriority { - Normal(u64), - Operational(u64), - } - - impl From for u64 { - fn from(priority: FrameTransactionPriority) -> Self { - match priority { - FrameTransactionPriority::Normal(inner) => inner, - FrameTransactionPriority::Operational(inner) => inner.saturating_add(LIMIT), - } - } - } -} - /// A bundle of static information collected from the `#[weight = $x]` attributes. #[derive(Clone, Copy, Eq, PartialEq, Default, RuntimeDebug, Encode, Decode, TypeInfo)] pub struct DispatchInfo { @@ -737,6 +712,34 @@ where } } +/// Implementor of [`WeightToFeePolynomial`] that uses a constant multiplier. +/// # Example +/// +/// ``` +/// # use frame_support::traits::ConstU128; +/// # use frame_support::weights::ConstantMultiplier; +/// // Results in a multiplier of 10 for each unit of weight (or length) +/// type LengthToFee = ConstantMultiplier::>; +/// ``` +pub struct ConstantMultiplier(sp_std::marker::PhantomData<(T, M)>); + +impl WeightToFeePolynomial for ConstantMultiplier +where + T: BaseArithmetic + From + Copy + Unsigned, + M: Get, +{ + type Balance = T; + + fn polynomial() -> WeightToFeeCoefficients { + smallvec!(WeightToFeeCoefficient { + coeff_integer: M::get(), + coeff_frac: Perbill::zero(), + negative: false, + degree: 1, + }) + } +} + /// A struct holding value for each `DispatchClass`. #[derive(Clone, Eq, PartialEq, Default, RuntimeDebug, Encode, Decode, TypeInfo)] pub struct PerDispatchClass { @@ -1011,4 +1014,13 @@ mod tests { assert_eq!(IdentityFee::::calc(&50), 50); assert_eq!(IdentityFee::::calc(&Weight::max_value()), Balance::max_value()); } + + #[test] + fn constant_fee_works() { + use crate::traits::ConstU128; + assert_eq!(ConstantMultiplier::>::calc(&0), 0); + assert_eq!(ConstantMultiplier::>::calc(&50), 500); + assert_eq!(ConstantMultiplier::>::calc(&16), 16384); + assert_eq!(ConstantMultiplier::>::calc(&2), u128::MAX); + } } diff --git a/frame/support/src/weights/block_weights.rs b/frame/support/src/weights/block_weights.rs new file mode 100644 index 000000000000..4db90f0c0207 --- /dev/null +++ b/frame/support/src/weights/block_weights.rs @@ -0,0 +1,46 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod constants { + use frame_support::{ + parameter_types, + weights::{constants, Weight}, + }; + + parameter_types! { + /// Importing a block with 0 Extrinsics. + pub const BlockExecutionWeight: Weight = 5_000_000 * constants::WEIGHT_PER_NANOS; + } + + #[cfg(test)] + mod test_weights { + use frame_support::weights::constants; + + /// Checks that the weight exists and is sane. + // NOTE: If this test fails but you are sure that the generated values are fine, + // you can delete it. + #[test] + fn sane() { + let w = super::constants::BlockExecutionWeight::get(); + + // At least 100 µs. + assert!(w >= 100 * constants::WEIGHT_PER_MICROS, "Weight should be at least 100 µs."); + // At most 50 ms. + assert!(w <= 50 * constants::WEIGHT_PER_MILLIS, "Weight should be at most 50 ms."); + } + } +} diff --git a/frame/support/src/weights/extrinsic_weights.rs b/frame/support/src/weights/extrinsic_weights.rs new file mode 100644 index 000000000000..158ba99c6a4c --- /dev/null +++ b/frame/support/src/weights/extrinsic_weights.rs @@ -0,0 +1,46 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod constants { + use frame_support::{ + parameter_types, + weights::{constants, Weight}, + }; + + parameter_types! { + /// Executing a NO-OP `System::remarks` Extrinsic. + pub const ExtrinsicBaseWeight: Weight = 125_000 * constants::WEIGHT_PER_NANOS; + } + + #[cfg(test)] + mod test_weights { + use frame_support::weights::constants; + + /// Checks that the weight exists and is sane. + // NOTE: If this test fails but you are sure that the generated values are fine, + // you can delete it. + #[test] + fn sane() { + let w = super::constants::ExtrinsicBaseWeight::get(); + + // At least 10 µs. + assert!(w >= 10 * constants::WEIGHT_PER_MICROS, "Weight should be at least 10 µs."); + // At most 1 ms. + assert!(w <= constants::WEIGHT_PER_MILLIS, "Weight should be at most 1 ms."); + } + } +} diff --git a/frame/support/src/weights/paritydb_weights.rs b/frame/support/src/weights/paritydb_weights.rs new file mode 100644 index 000000000000..572187ba78a9 --- /dev/null +++ b/frame/support/src/weights/paritydb_weights.rs @@ -0,0 +1,63 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod constants { + use frame_support::{ + parameter_types, + weights::{constants, RuntimeDbWeight}, + }; + + parameter_types! { + /// ParityDB can be enabled with a feature flag, but is still experimental. These weights + /// are available for brave runtime engineers who may want to try this out as default. + pub const ParityDbWeight: RuntimeDbWeight = RuntimeDbWeight { + read: 8_000 * constants::WEIGHT_PER_NANOS, + write: 50_000 * constants::WEIGHT_PER_NANOS, + }; + } + + #[cfg(test)] + mod test_db_weights { + use super::constants::ParityDbWeight as W; + use frame_support::weights::constants; + + /// Checks that all weights exist and have sane values. + // NOTE: If this test fails but you are sure that the generated values are fine, + // you can delete it. + #[test] + fn sane() { + // At least 1 µs. + assert!( + W::get().reads(1) >= constants::WEIGHT_PER_MICROS, + "Read weight should be at least 1 µs." + ); + assert!( + W::get().writes(1) >= constants::WEIGHT_PER_MICROS, + "Write weight should be at least 1 µs." + ); + // At most 1 ms. + assert!( + W::get().reads(1) <= constants::WEIGHT_PER_MILLIS, + "Read weight should be at most 1 ms." + ); + assert!( + W::get().writes(1) <= constants::WEIGHT_PER_MILLIS, + "Write weight should be at most 1 ms." + ); + } + } +} diff --git a/frame/support/src/weights/rocksdb_weights.rs b/frame/support/src/weights/rocksdb_weights.rs new file mode 100644 index 000000000000..f37964dcbd82 --- /dev/null +++ b/frame/support/src/weights/rocksdb_weights.rs @@ -0,0 +1,63 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod constants { + use frame_support::{ + parameter_types, + weights::{constants, RuntimeDbWeight}, + }; + + parameter_types! { + /// By default, Substrate uses RocksDB, so this will be the weight used throughout + /// the runtime. + pub const RocksDbWeight: RuntimeDbWeight = RuntimeDbWeight { + read: 25_000 * constants::WEIGHT_PER_NANOS, + write: 100_000 * constants::WEIGHT_PER_NANOS, + }; + } + + #[cfg(test)] + mod test_db_weights { + use super::constants::RocksDbWeight as W; + use frame_support::weights::constants; + + /// Checks that all weights exist and have sane values. + // NOTE: If this test fails but you are sure that the generated values are fine, + // you can delete it. + #[test] + fn sane() { + // At least 1 µs. + assert!( + W::get().reads(1) >= constants::WEIGHT_PER_MICROS, + "Read weight should be at least 1 µs." + ); + assert!( + W::get().writes(1) >= constants::WEIGHT_PER_MICROS, + "Write weight should be at least 1 µs." + ); + // At most 1 ms. + assert!( + W::get().reads(1) <= constants::WEIGHT_PER_MILLIS, + "Read weight should be at most 1 ms." + ); + assert!( + W::get().writes(1) <= constants::WEIGHT_PER_MILLIS, + "Write weight should be at most 1 ms." + ); + } + } +} diff --git a/frame/support/test/Cargo.toml b/frame/support/test/Cargo.toml index 863afceac4a9..383b7cf812b4 100644 --- a/frame/support/test/Cargo.toml +++ b/frame/support/test/Cargo.toml @@ -2,30 +2,30 @@ name = "frame-support-test" version = "3.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" publish = false -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -serde = { version = "1.0.126", default-features = false, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-arithmetic = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/arithmetic" } -sp-io = { version = "4.0.0-dev", path = "../../../primitives/io", default-features = false } -sp-state-machine = { version = "0.10.0-dev", optional = true, path = "../../../primitives/state-machine" } +serde = { version = "1.0.136", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-arithmetic = { version = "5.0.0", default-features = false, path = "../../../primitives/arithmetic" } +sp-io = { version = "6.0.0", path = "../../../primitives/io", default-features = false } +sp-state-machine = { version = "0.12.0", optional = true, path = "../../../primitives/state-machine" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/runtime" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/core" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/std" } -sp-version = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/version" } -trybuild = "1.0.43" -pretty_assertions = "0.6.1" -rustversion = "1.0.0" +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } +sp-core = { version = "6.0.0", default-features = false, path = "../../../primitives/core" } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } +sp-version = { version = "5.0.0", default-features = false, path = "../../../primitives/version" } +trybuild = { version = "1.0.53", features = [ "diff" ] } +pretty_assertions = "1.0.0" +rustversion = "1.0.6" frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" } # The "std" feature for this pallet is never activated on purpose, in order to test construct_runtime error message test-pallet = { package = "frame-support-test-pallet", default-features = false, path = "pallet" } @@ -36,13 +36,15 @@ std = [ "serde/std", "codec/std", "scale-info/std", - "sp-io/std", "frame-support/std", "frame-system/std", "sp-core/std", "sp-std/std", + "sp-io/std", "sp-runtime/std", "sp-state-machine", + "sp-arithmetic/std", + "sp-version/std", ] try-runtime = ["frame-support/try-runtime"] # WARNING: CI only execute pallet test with this feature, @@ -50,3 +52,4 @@ try-runtime = ["frame-support/try-runtime"] conditional-storage = [] # Disable ui tests disable-ui-tests = [] +no-metadata-docs = ["frame-support/no-metadata-docs"] diff --git a/frame/support/test/compile_pass/Cargo.toml b/frame/support/test/compile_pass/Cargo.toml new file mode 100644 index 000000000000..5850d2e5db14 --- /dev/null +++ b/frame/support/test/compile_pass/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "frame-support-test-compile-pass" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +publish = false +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-core = { version = "6.0.0", default-features = false, path = "../../../../primitives/core" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../../primitives/runtime" } +sp-version = { version = "5.0.0", default-features = false, path = "../../../../primitives/version" } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../../" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../system" } + +[features] +default = ["std"] +std = [ + "codec/std", + "scale-info/std", + "sp-core/std", + "sp-runtime/std", + "sp-version/std", + "frame-support/std", + "frame-system/std", +] diff --git a/frame/support/test/compile_pass/src/lib.rs b/frame/support/test/compile_pass/src/lib.rs new file mode 100644 index 000000000000..0c955c749613 --- /dev/null +++ b/frame/support/test/compile_pass/src/lib.rs @@ -0,0 +1,96 @@ +// This file is part of Substrate. + +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#![cfg_attr(not(feature = "std"), no_std)] +// `construct_runtime!` does a lot of recursion and requires us to increase the limit to 256. +#![recursion_limit = "256"] +//! This crate tests that `construct_runtime!` expands the pallet parts +//! correctly even when frame-support is renamed in Cargo.toml + +use frame_support::{ + construct_runtime, parameter_types, + traits::{ConstU16, ConstU32}, +}; +use sp_core::{sr25519, H256}; +use sp_runtime::{ + create_runtime_str, generic, + traits::{BlakeTwo256, IdentityLookup, Verify}, +}; +use sp_version::RuntimeVersion; + +pub const VERSION: RuntimeVersion = RuntimeVersion { + spec_name: create_runtime_str!("frame-support-test-compile-pass"), + impl_name: create_runtime_str!("substrate-frame-support-test-compile-pass-runtime"), + authoring_version: 0, + spec_version: 0, + impl_version: 0, + apis: sp_version::create_apis_vec!([]), + transaction_version: 0, + state_version: 0, +}; + +pub type Signature = sr25519::Signature; +pub type AccountId = ::Signer; +pub type BlockNumber = u64; +pub type Index = u64; + +parameter_types! { + pub const BlockHashCount: BlockNumber = 2400; + pub const Version: RuntimeVersion = VERSION; +} + +impl frame_system::Config for Runtime { + type BaseCallFilter = frame_support::traits::Everything; + type BlockWeights = (); + type BlockLength = (); + type Index = u128; + type Hash = H256; + type Hashing = BlakeTwo256; + type Header = Header; + type Lookup = IdentityLookup; + type BlockHashCount = BlockHashCount; + type Version = Version; + type AccountData = (); + type Origin = Origin; + type BlockNumber = BlockNumber; + type AccountId = AccountId; + type Event = Event; + type PalletInfo = PalletInfo; + type Call = Call; + type DbWeight = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type OnSetCode = (); + type MaxConsumers = ConstU32<16>; + type SystemWeightInfo = (); + type SS58Prefix = ConstU16<0>; +} + +pub type Header = generic::Header; +pub type Block = generic::Block; +pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; + +construct_runtime!( + pub enum Runtime where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic + { + System: frame_system, + } +); diff --git a/frame/support/test/pallet/Cargo.toml b/frame/support/test/pallet/Cargo.toml index 35eb4f34acae..51b74d5ec55f 100644 --- a/frame/support/test/pallet/Cargo.toml +++ b/frame/support/test/pallet/Cargo.toml @@ -2,18 +2,18 @@ name = "frame-support-test-pallet" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" publish = false -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../../" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../system" } diff --git a/frame/support/test/pallet/src/lib.rs b/frame/support/test/pallet/src/lib.rs index f9f94b06a0a5..37678e056f3e 100644 --- a/frame/support/test/pallet/src/lib.rs +++ b/frame/support/test/pallet/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,12 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + +//! Testing pallet macro + +// Ensure docs are propagated properly by the macros. +#![warn(missing_docs)] + pub use pallet::*; #[frame_support::pallet] @@ -29,18 +35,22 @@ pub mod pallet { #[pallet::config] pub trait Config: frame_system::Config {} + /// I'm the documentation + #[pallet::storage] + pub type Value = StorageValue; + #[pallet::genesis_config] + #[cfg_attr(feature = "std", derive(Default))] pub struct GenesisConfig {} - #[cfg(feature = "std")] - impl Default for GenesisConfig { - fn default() -> Self { - Self {} - } - } - #[pallet::genesis_build] impl GenesisBuild for GenesisConfig { fn build(&self) {} } + + #[pallet::error] + pub enum Error { + /// Something failed + Test, + } } diff --git a/frame/support/test/src/lib.rs b/frame/support/test/src/lib.rs index 073f8c9c1935..dd3fbd1f3020 100644 --- a/frame/support/test/src/lib.rs +++ b/frame/support/test/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/src/pallet_version.rs b/frame/support/test/src/pallet_version.rs index bdea3859d65c..096289116c41 100644 --- a/frame/support/test/src/pallet_version.rs +++ b/frame/support/test/src/pallet_version.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/tests/construct_runtime.rs b/frame/support/test/tests/construct_runtime.rs index dd5538370449..804deb08919a 100644 --- a/frame/support/test/tests/construct_runtime.rs +++ b/frame/support/test/tests/construct_runtime.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,7 +27,7 @@ use sp_core::{sr25519, H256}; use sp_runtime::{ generic, traits::{BlakeTwo256, Verify}, - DispatchError, + DispatchError, ModuleError, }; use sp_std::cell::RefCell; @@ -229,6 +229,10 @@ pub type AccountId = ::Signer; pub type BlockNumber = u64; pub type Index = u64; +fn test_pub() -> AccountId { + AccountId::from_raw([0; 32]) +} + impl system::Config for Runtime { type BaseCallFilter = frame_support::traits::Everything; type Hash = H256; @@ -267,127 +271,95 @@ pub type Header = generic::Header; pub type Block = generic::Block; pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; -mod origin_test { - use super::{module3, nested, system, Block, UncheckedExtrinsic}; - use frame_support::traits::{Contains, OriginTrait}; - - impl nested::module3::Config for RuntimeOriginTest {} - impl module3::Config for RuntimeOriginTest {} - - pub struct BaseCallFilter; - impl Contains for BaseCallFilter { - fn contains(c: &Call) -> bool { - match c { - Call::NestedModule3(_) => true, - _ => false, - } - } - } - - impl system::Config for RuntimeOriginTest { - type BaseCallFilter = BaseCallFilter; - type Hash = super::H256; - type Origin = Origin; - type BlockNumber = super::BlockNumber; - type AccountId = u32; - type Event = Event; - type PalletInfo = PalletInfo; - type Call = Call; - type DbWeight = (); - } - - frame_support::construct_runtime!( - pub enum RuntimeOriginTest where - Block = Block, - NodeBlock = Block, - UncheckedExtrinsic = UncheckedExtrinsic - { - System: system::{Pallet, Event, Origin}, - NestedModule3: nested::module3::{Pallet, Origin, Call}, - Module3: module3::{Pallet, Origin, Call}, - } - ); - - #[test] - fn origin_default_filter() { - let accepted_call = nested::module3::Call::fail {}.into(); - let rejected_call = module3::Call::fail {}.into(); - - assert_eq!(Origin::root().filter_call(&accepted_call), true); - assert_eq!(Origin::root().filter_call(&rejected_call), true); - assert_eq!(Origin::none().filter_call(&accepted_call), true); - assert_eq!(Origin::none().filter_call(&rejected_call), false); - assert_eq!(Origin::signed(0).filter_call(&accepted_call), true); - assert_eq!(Origin::signed(0).filter_call(&rejected_call), false); - assert_eq!(Origin::from(Some(0)).filter_call(&accepted_call), true); - assert_eq!(Origin::from(Some(0)).filter_call(&rejected_call), false); - assert_eq!(Origin::from(None).filter_call(&accepted_call), true); - assert_eq!(Origin::from(None).filter_call(&rejected_call), false); - assert_eq!(Origin::from(super::nested::module3::Origin).filter_call(&accepted_call), true); - assert_eq!(Origin::from(super::nested::module3::Origin).filter_call(&rejected_call), false); - - let mut origin = Origin::from(Some(0)); - - origin.add_filter(|c| matches!(c, Call::Module3(_))); - assert_eq!(origin.filter_call(&accepted_call), false); - assert_eq!(origin.filter_call(&rejected_call), false); - - origin.set_caller_from(Origin::root()); - assert!(matches!(origin.caller, OriginCaller::system(super::system::RawOrigin::Root))); - assert_eq!(origin.filter_call(&accepted_call), false); - assert_eq!(origin.filter_call(&rejected_call), false); - - origin.reset_filter(); - assert_eq!(origin.filter_call(&accepted_call), true); - assert_eq!(origin.filter_call(&rejected_call), false); - } -} - #[test] fn check_modules_error_type() { assert_eq!( Module1_1::fail(system::Origin::::Root.into()), - Err(DispatchError::Module { index: 31, error: 0, message: Some("Something") }), + Err(DispatchError::Module(ModuleError { + index: 31, + error: [0; 4], + message: Some("Something") + })), ); assert_eq!( Module2::fail(system::Origin::::Root.into()), - Err(DispatchError::Module { index: 32, error: 0, message: Some("Something") }), + Err(DispatchError::Module(ModuleError { + index: 32, + error: [0; 4], + message: Some("Something") + })), ); assert_eq!( Module1_2::fail(system::Origin::::Root.into()), - Err(DispatchError::Module { index: 33, error: 0, message: Some("Something") }), + Err(DispatchError::Module(ModuleError { + index: 33, + error: [0; 4], + message: Some("Something") + })), ); assert_eq!( NestedModule3::fail(system::Origin::::Root.into()), - Err(DispatchError::Module { index: 34, error: 0, message: Some("Something") }), + Err(DispatchError::Module(ModuleError { + index: 34, + error: [0; 4], + message: Some("Something") + })), ); assert_eq!( Module1_3::fail(system::Origin::::Root.into()), - Err(DispatchError::Module { index: 6, error: 0, message: Some("Something") }), + Err(DispatchError::Module(ModuleError { + index: 6, + error: [0; 4], + message: Some("Something") + })), ); assert_eq!( Module1_4::fail(system::Origin::::Root.into()), - Err(DispatchError::Module { index: 3, error: 0, message: Some("Something") }), + Err(DispatchError::Module(ModuleError { + index: 3, + error: [0; 4], + message: Some("Something") + })), ); assert_eq!( Module1_5::fail(system::Origin::::Root.into()), - Err(DispatchError::Module { index: 4, error: 0, message: Some("Something") }), + Err(DispatchError::Module(ModuleError { + index: 4, + error: [0; 4], + message: Some("Something") + })), ); assert_eq!( Module1_6::fail(system::Origin::::Root.into()), - Err(DispatchError::Module { index: 1, error: 0, message: Some("Something") }), + Err(DispatchError::Module(ModuleError { + index: 1, + error: [0; 4], + message: Some("Something") + })), ); assert_eq!( Module1_7::fail(system::Origin::::Root.into()), - Err(DispatchError::Module { index: 2, error: 0, message: Some("Something") }), + Err(DispatchError::Module(ModuleError { + index: 2, + error: [0; 4], + message: Some("Something") + })), ); assert_eq!( Module1_8::fail(system::Origin::::Root.into()), - Err(DispatchError::Module { index: 12, error: 0, message: Some("Something") }), + Err(DispatchError::Module(ModuleError { + index: 12, + error: [0; 4], + message: Some("Something") + })), ); assert_eq!( Module1_9::fail(system::Origin::::Root.into()), - Err(DispatchError::Module { index: 13, error: 0, message: Some("Something") }), + Err(DispatchError::Module(ModuleError { + index: 13, + error: [0; 4], + message: Some("Something") + })), ); } @@ -439,13 +411,13 @@ fn event_codec() { let event = system::Event::::ExtrinsicSuccess; assert_eq!(Event::from(event).encode()[0], 30); - let event = module1::Event::::A(Default::default()); + let event = module1::Event::::A(test_pub()); assert_eq!(Event::from(event).encode()[0], 31); let event = module2::Event::A; assert_eq!(Event::from(event).encode()[0], 32); - let event = module1::Event::::A(Default::default()); + let event = module1::Event::::A(test_pub()); assert_eq!(Event::from(event).encode()[0], 33); let event = nested::module3::Event::A; @@ -454,19 +426,19 @@ fn event_codec() { let event = module3::Event::A; assert_eq!(Event::from(event).encode()[0], 35); - let event = module1::Event::::A(Default::default()); + let event = module1::Event::::A(test_pub()); assert_eq!(Event::from(event).encode()[0], 4); - let event = module1::Event::::A(Default::default()); + let event = module1::Event::::A(test_pub()); assert_eq!(Event::from(event).encode()[0], 1); - let event = module1::Event::::A(Default::default()); + let event = module1::Event::::A(test_pub()); assert_eq!(Event::from(event).encode()[0], 2); - let event = module1::Event::::A(Default::default()); + let event = module1::Event::::A(test_pub()); assert_eq!(Event::from(event).encode()[0], 12); - let event = module1::Event::::A(Default::default()); + let event = module1::Event::::A(test_pub()); assert_eq!(Event::from(event).encode()[0], 13); } diff --git a/frame/support/test/tests/construct_runtime_ui.rs b/frame/support/test/tests/construct_runtime_ui.rs index ee475e37605e..66636416c1f5 100644 --- a/frame/support/test/tests/construct_runtime_ui.rs +++ b/frame/support/test/tests/construct_runtime_ui.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/tests/construct_runtime_ui/both_use_and_excluded_parts.rs b/frame/support/test/tests/construct_runtime_ui/both_use_and_excluded_parts.rs new file mode 100644 index 000000000000..98cd1f197f61 --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/both_use_and_excluded_parts.rs @@ -0,0 +1,33 @@ +use frame_support::construct_runtime; +use sp_runtime::{generic, traits::BlakeTwo256}; +use sp_core::sr25519; + +#[frame_support::pallet] +mod pallet { + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::pallet] + pub struct Pallet(_); +} + +pub type Signature = sr25519::Signature; +pub type BlockNumber = u64; +pub type Header = generic::Header; +pub type Block = generic::Block; +pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; + +impl pallet::Config for Runtime {} + +construct_runtime! { + pub enum Runtime where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic + { + System: system::{Pallet, Call, Storage, Config, Event}, + Pallet: pallet exclude_parts { Pallet } use_parts { Pallet }, + } +} + +fn main() {} diff --git a/frame/support/test/tests/construct_runtime_ui/both_use_and_excluded_parts.stderr b/frame/support/test/tests/construct_runtime_ui/both_use_and_excluded_parts.stderr new file mode 100644 index 000000000000..608d57d6a97f --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/both_use_and_excluded_parts.stderr @@ -0,0 +1,28 @@ +error: Unexpected tokens, expected one of `=`, `,` + --> $DIR/both_use_and_excluded_parts.rs:29:43 + | +29 | Pallet: pallet exclude_parts { Pallet } use_parts { Pallet }, + | ^^^^^^^^^ + +error[E0412]: cannot find type `Call` in this scope + --> $DIR/both_use_and_excluded_parts.rs:18:64 + | +18 | pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; + | ^^^^ not found in this scope + | +help: consider importing one of these items + | +1 | use crate::pallet::Call; + | +1 | use frame_support_test::Call; + | +1 | use frame_system::Call; + | +1 | use test_pallet::Call; + | + +error[E0412]: cannot find type `Runtime` in this scope + --> $DIR/both_use_and_excluded_parts.rs:20:25 + | +20 | impl pallet::Config for Runtime {} + | ^^^^^^^ not found in this scope diff --git a/frame/support/test/tests/construct_runtime_ui/duplicate_exclude.rs b/frame/support/test/tests/construct_runtime_ui/duplicate_exclude.rs new file mode 100644 index 000000000000..6d21c2a6e170 --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/duplicate_exclude.rs @@ -0,0 +1,13 @@ +use frame_support::construct_runtime; + +construct_runtime! { + pub enum Runtime where + UncheckedExtrinsic = UncheckedExtrinsic, + Block = Block, + NodeBlock = Block, + { + System: frame_system exclude_parts { Call, Call }, + } +} + +fn main() {} diff --git a/frame/support/test/tests/construct_runtime_ui/duplicate_exclude.stderr b/frame/support/test/tests/construct_runtime_ui/duplicate_exclude.stderr new file mode 100644 index 000000000000..75de56076528 --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/duplicate_exclude.stderr @@ -0,0 +1,5 @@ +error: `Call` was already declared before. Please remove the duplicate declaration + --> $DIR/duplicate_exclude.rs:9:46 + | +9 | System: frame_system exclude_parts { Call, Call }, + | ^^^^ diff --git a/frame/support/test/tests/construct_runtime_ui/exclude_missspell.rs b/frame/support/test/tests/construct_runtime_ui/exclude_missspell.rs new file mode 100644 index 000000000000..16cbf1e82cf8 --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/exclude_missspell.rs @@ -0,0 +1,13 @@ +use frame_support::construct_runtime; + +construct_runtime! { + pub enum Runtime where + UncheckedExtrinsic = UncheckedExtrinsic, + Block = Block, + NodeBlock = Block, + { + System: frame_system exclude_part { Call }, + } +} + +fn main() {} diff --git a/frame/support/test/tests/construct_runtime_ui/exclude_missspell.stderr b/frame/support/test/tests/construct_runtime_ui/exclude_missspell.stderr new file mode 100644 index 000000000000..82e6aa6c8e30 --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/exclude_missspell.stderr @@ -0,0 +1,5 @@ +error: Unexpected tokens, expected one of `::$ident` `::{`, `exclude_parts`, `use_parts`, `=`, `,` + --> $DIR/exclude_missspell.rs:9:24 + | +9 | System: frame_system exclude_part { Call }, + | ^^^^^^^^^^^^ diff --git a/frame/support/test/tests/construct_runtime_ui/exclude_undefined_part.rs b/frame/support/test/tests/construct_runtime_ui/exclude_undefined_part.rs new file mode 100644 index 000000000000..51be7e30bd3e --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/exclude_undefined_part.rs @@ -0,0 +1,38 @@ +use frame_support::construct_runtime; +use sp_runtime::{generic, traits::BlakeTwo256}; +use sp_core::sr25519; + +#[frame_support::pallet] +mod pallet { + use frame_support::pallet_prelude::*; + + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::storage] + type Foo = StorageValue; +} + +pub type Signature = sr25519::Signature; +pub type BlockNumber = u64; +pub type Header = generic::Header; +pub type Block = generic::Block; +pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; + +impl pallet::Config for Runtime {} + +construct_runtime! { + pub enum Runtime where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic + { + System: system::{Pallet, Call, Storage, Config, Event}, + Pallet: pallet exclude_parts { Call }, + } +} + +fn main() {} diff --git a/frame/support/test/tests/construct_runtime_ui/exclude_undefined_part.stderr b/frame/support/test/tests/construct_runtime_ui/exclude_undefined_part.stderr new file mode 100644 index 000000000000..4e31cfb75c07 --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/exclude_undefined_part.stderr @@ -0,0 +1,28 @@ +error: Invalid pallet part specified, the pallet `Pallet` doesn't have the `Call` part. Available parts are: `Pallet`, `Storage`. + --> $DIR/exclude_undefined_part.rs:34:34 + | +34 | Pallet: pallet exclude_parts { Call }, + | ^^^^ + +error[E0412]: cannot find type `Call` in this scope + --> $DIR/exclude_undefined_part.rs:23:64 + | +23 | pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; + | ^^^^ not found in this scope + | +help: consider importing one of these items + | +1 | use crate::pallet::Call; + | +1 | use frame_support_test::Call; + | +1 | use frame_system::Call; + | +1 | use test_pallet::Call; + | + +error[E0412]: cannot find type `Runtime` in this scope + --> $DIR/exclude_undefined_part.rs:25:25 + | +25 | impl pallet::Config for Runtime {} + | ^^^^^^^ not found in this scope diff --git a/frame/support/test/tests/construct_runtime_ui/invalid_module_details.stderr b/frame/support/test/tests/construct_runtime_ui/invalid_module_details.stderr index 50505b9130cb..db96b8749ca1 100644 --- a/frame/support/test/tests/construct_runtime_ui/invalid_module_details.stderr +++ b/frame/support/test/tests/construct_runtime_ui/invalid_module_details.stderr @@ -1,5 +1,5 @@ -error: expected one of: identifier, curly braces, `<` - --> $DIR/invalid_module_details.rs:9:19 +error: Unexpected tokens, expected one of `::$ident` `::{`, `exclude_parts`, `use_parts`, `=`, `,` + --> $DIR/invalid_module_details.rs:9:17 | 9 | system: System::(), - | ^^ + | ^^ diff --git a/frame/support/test/tests/construct_runtime_ui/invalid_token_after_module.stderr b/frame/support/test/tests/construct_runtime_ui/invalid_token_after_module.stderr index 3b967f96d7b4..6025de82bd20 100644 --- a/frame/support/test/tests/construct_runtime_ui/invalid_token_after_module.stderr +++ b/frame/support/test/tests/construct_runtime_ui/invalid_token_after_module.stderr @@ -1,4 +1,4 @@ -error: expected `::` +error: Unexpected tokens, expected one of `::$ident` `::{`, `exclude_parts`, `use_parts`, `=`, `,` --> $DIR/invalid_token_after_module.rs:9:18 | 9 | system: System ? diff --git a/frame/support/test/tests/construct_runtime_ui/missing_where_block.stderr b/frame/support/test/tests/construct_runtime_ui/missing_where_block.stderr index 4af672a2610b..c6baf8fc24d0 100644 --- a/frame/support/test/tests/construct_runtime_ui/missing_where_block.stderr +++ b/frame/support/test/tests/construct_runtime_ui/missing_where_block.stderr @@ -1,5 +1,5 @@ error: expected `where` - --> $DIR/missing_where_block.rs:4:19 + --> tests/construct_runtime_ui/missing_where_block.rs:4:19 | 4 | pub enum Runtime {} - | ^^ + | ^ diff --git a/frame/support/test/tests/construct_runtime_ui/missing_where_param.stderr b/frame/support/test/tests/construct_runtime_ui/missing_where_param.stderr index ac7313523c0c..fb7e38b53dcd 100644 --- a/frame/support/test/tests/construct_runtime_ui/missing_where_param.stderr +++ b/frame/support/test/tests/construct_runtime_ui/missing_where_param.stderr @@ -1,5 +1,5 @@ error: Missing associated type for `UncheckedExtrinsic`. Add `UncheckedExtrinsic` = ... to where section. - --> $DIR/missing_where_param.rs:7:2 + --> tests/construct_runtime_ui/missing_where_param.rs:7:2 | 7 | {} - | ^^ + | ^ diff --git a/frame/support/test/tests/construct_runtime_ui/no_std_genesis_config.rs b/frame/support/test/tests/construct_runtime_ui/no_std_genesis_config.rs index 89774eb8a770..c06333795e3c 100644 --- a/frame/support/test/tests/construct_runtime_ui/no_std_genesis_config.rs +++ b/frame/support/test/tests/construct_runtime_ui/no_std_genesis_config.rs @@ -3,20 +3,47 @@ use sp_runtime::{generic, traits::BlakeTwo256}; use sp_core::sr25519; pub type Signature = sr25519::Signature; -pub type BlockNumber = u64; +pub type BlockNumber = u32; pub type Header = generic::Header; pub type Block = generic::Block; pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; impl test_pallet::Config for Runtime {} +impl frame_system::Config for Runtime { + type BaseCallFilter = frame_support::traits::Everything; + type Origin = Origin; + type Index = u64; + type BlockNumber = u32; + type Call = Call; + type Hash = sp_runtime::testing::H256; + type Hashing = sp_runtime::traits::BlakeTwo256; + type AccountId = u64; + type Lookup = sp_runtime::traits::IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = frame_support::traits::ConstU32<250>; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; +} + construct_runtime! { pub enum Runtime where Block = Block, NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: system::{Pallet, Call, Storage, Config, Event}, + System: frame_system::{Pallet, Call, Storage, Config, Event}, Pallet: test_pallet::{Pallet, Config}, } } diff --git a/frame/support/test/tests/construct_runtime_ui/no_std_genesis_config.stderr b/frame/support/test/tests/construct_runtime_ui/no_std_genesis_config.stderr index 3dc7fcda9f18..6d5a48bf0909 100644 --- a/frame/support/test/tests/construct_runtime_ui/no_std_genesis_config.stderr +++ b/frame/support/test/tests/construct_runtime_ui/no_std_genesis_config.stderr @@ -1,73 +1,27 @@ error: `Pallet` does not have the std feature enabled, this will cause the `test_pallet::GenesisConfig` type to be undefined. - --> $DIR/no_std_genesis_config.rs:13:1 + --> tests/construct_runtime_ui/no_std_genesis_config.rs:40:1 | -13 | / construct_runtime! { -14 | | pub enum Runtime where -15 | | Block = Block, -16 | | NodeBlock = Block, +40 | / construct_runtime! { +41 | | pub enum Runtime where +42 | | Block = Block, +43 | | NodeBlock = Block, ... | -21 | | } -22 | | } +48 | | } +49 | | } | |_^ | = note: this error originates in the macro `test_pallet::__substrate_genesis_config_check::is_std_enabled_for_genesis` (in Nightly builds, run with -Z macro-backtrace for more info) -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/no_std_genesis_config.rs:19:11 - | -19 | System: system::{Pallet, Call, Storage, Config, Event}, - | ^^^^^^ use of undeclared crate or module `system` - -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/no_std_genesis_config.rs:13:1 - | -13 | / construct_runtime! { -14 | | pub enum Runtime where -15 | | Block = Block, -16 | | NodeBlock = Block, -... | -21 | | } -22 | | } - | |_^ not found in `system` - | - = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing this enum - | -1 | use frame_system::RawOrigin; - | - -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/no_std_genesis_config.rs:13:1 - | -13 | / construct_runtime! { -14 | | pub enum Runtime where -15 | | Block = Block, -16 | | NodeBlock = Block, -... | -21 | | } -22 | | } - | |_^ not found in `system` - | - = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing one of these items - | -1 | use frame_support_test::Pallet; - | -1 | use frame_system::Pallet; - | -1 | use test_pallet::Pallet; - | - error[E0412]: cannot find type `GenesisConfig` in crate `test_pallet` - --> $DIR/no_std_genesis_config.rs:13:1 + --> tests/construct_runtime_ui/no_std_genesis_config.rs:40:1 | -13 | / construct_runtime! { -14 | | pub enum Runtime where -15 | | Block = Block, -16 | | NodeBlock = Block, +40 | / construct_runtime! { +41 | | pub enum Runtime where +42 | | Block = Block, +43 | | NodeBlock = Block, ... | -21 | | } -22 | | } +48 | | } +49 | | } | |_^ not found in `test_pallet` | = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) @@ -76,13 +30,22 @@ help: consider importing this struct 1 | use frame_system::GenesisConfig; | -error[E0277]: the trait bound `Runtime: frame_system::pallet::Config` is not satisfied - --> $DIR/no_std_genesis_config.rs:11:6 - | -11 | impl test_pallet::Config for Runtime {} - | ^^^^^^^^^^^^^^^^^^^ the trait `frame_system::pallet::Config` is not implemented for `Runtime` - | - ::: $WORKSPACE/frame/support/test/pallet/src/lib.rs - | - | pub trait Config: frame_system::Config {} - | -------------------- required by this bound in `Config` +error[E0283]: type annotations needed + --> tests/construct_runtime_ui/no_std_genesis_config.rs:40:1 + | +40 | / construct_runtime! { +41 | | pub enum Runtime where +42 | | Block = Block, +43 | | NodeBlock = Block, +... | +48 | | } +49 | | } + | |_^ cannot infer type + | + = note: cannot satisfy `_: std::default::Default` +note: required by `std::default::Default::default` + --> $RUST/core/src/default.rs + | + | fn default() -> Self; + | ^^^^^^^^^^^^^^^^^^^^^ + = note: this error originates in the derive macro `Default` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/frame/support/test/tests/construct_runtime_ui/old_unsupported_pallet_decl.rs b/frame/support/test/tests/construct_runtime_ui/old_unsupported_pallet_decl.rs new file mode 100644 index 000000000000..706d444f2359 --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/old_unsupported_pallet_decl.rs @@ -0,0 +1,26 @@ +use frame_support::construct_runtime; + +mod pallet_old { + pub trait Config: frame_system::Config {} + + decl_storage! { + trait Store for Module as Example {} + } + + decl_module! { + pub struct Module for enum Call where origin: T::Origin {} + } + +} +construct_runtime! { + pub enum Runtime where + UncheckedExtrinsic = UncheckedExtrinsic, + Block = Block, + NodeBlock = Block, + { + System: frame_system, + OldPallet: pallet_old, + } +} + +fn main() {} diff --git a/frame/support/test/tests/construct_runtime_ui/old_unsupported_pallet_decl.stderr b/frame/support/test/tests/construct_runtime_ui/old_unsupported_pallet_decl.stderr new file mode 100644 index 000000000000..f8ec07e00106 --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/old_unsupported_pallet_decl.stderr @@ -0,0 +1,31 @@ +error[E0433]: failed to resolve: could not find `tt_default_parts` in `pallet_old` + --> $DIR/old_unsupported_pallet_decl.rs:15:1 + | +15 | / construct_runtime! { +16 | | pub enum Runtime where +17 | | UncheckedExtrinsic = UncheckedExtrinsic, +18 | | Block = Block, +... | +23 | | } +24 | | } + | |_^ could not find `tt_default_parts` in `pallet_old` + | + = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: cannot find macro `decl_storage` in this scope + --> $DIR/old_unsupported_pallet_decl.rs:6:2 + | +6 | decl_storage! { + | ^^^^^^^^^^^^ + | + = note: consider importing this macro: + frame_support::decl_storage + +error: cannot find macro `decl_module` in this scope + --> $DIR/old_unsupported_pallet_decl.rs:10:2 + | +10 | decl_module! { + | ^^^^^^^^^^^ + | + = note: consider importing this macro: + frame_support::decl_module diff --git a/frame/support/test/tests/construct_runtime_ui/pallet_error_too_large.rs b/frame/support/test/tests/construct_runtime_ui/pallet_error_too_large.rs new file mode 100644 index 000000000000..827d8a58af73 --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/pallet_error_too_large.rs @@ -0,0 +1,85 @@ +use frame_support::construct_runtime; +use sp_runtime::{generic, traits::BlakeTwo256}; +use sp_core::sr25519; + +#[frame_support::pallet] +mod pallet { + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::pallet] + pub struct Pallet(core::marker::PhantomData); + + #[pallet::error] + pub enum Error { + MyError(crate::Nested1), + } +} + +#[derive(scale_info::TypeInfo, frame_support::PalletError, codec::Encode, codec::Decode)] +pub enum Nested1 { + Nested2(Nested2) +} + +#[derive(scale_info::TypeInfo, frame_support::PalletError, codec::Encode, codec::Decode)] +pub enum Nested2 { + Nested3(Nested3) +} + +#[derive(scale_info::TypeInfo, frame_support::PalletError, codec::Encode, codec::Decode)] +pub enum Nested3 { + Nested4(Nested4) +} + +#[derive(scale_info::TypeInfo, frame_support::PalletError, codec::Encode, codec::Decode)] +pub enum Nested4 { + Num(u8) +} + +pub type Signature = sr25519::Signature; +pub type BlockNumber = u32; +pub type Header = generic::Header; +pub type Block = generic::Block; +pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; + +impl pallet::Config for Runtime {} + +impl frame_system::Config for Runtime { + type BaseCallFilter = frame_support::traits::Everything; + type Origin = Origin; + type Index = u64; + type BlockNumber = u32; + type Call = Call; + type Hash = sp_runtime::testing::H256; + type Hashing = sp_runtime::traits::BlakeTwo256; + type AccountId = u64; + type Lookup = sp_runtime::traits::IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = frame_support::traits::ConstU32<250>; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; +} + +construct_runtime! { + pub enum Runtime where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic + { + System: frame_system::{Pallet, Call, Storage, Config, Event}, + Pallet: pallet::{Pallet}, + } +} + +fn main() {} diff --git a/frame/support/test/tests/construct_runtime_ui/pallet_error_too_large.stderr b/frame/support/test/tests/construct_runtime_ui/pallet_error_too_large.stderr new file mode 100644 index 000000000000..161873866b6f --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/pallet_error_too_large.stderr @@ -0,0 +1,13 @@ +error[E0080]: evaluation of constant value failed + --> tests/construct_runtime_ui/pallet_error_too_large.rs:74:1 + | +74 | / construct_runtime! { +75 | | pub enum Runtime where +76 | | Block = Block, +77 | | NodeBlock = Block, +... | +82 | | } +83 | | } + | |_^ the evaluated program panicked at 'The maximum encoded size of the error type in the `Pallet` pallet exceeds `MAX_MODULE_ERROR_ENCODED_SIZE`', $DIR/tests/construct_runtime_ui/pallet_error_too_large.rs:74:1 + | + = note: this error originates in the macro `$crate::panic::panic_2021` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_call_part.rs b/frame/support/test/tests/construct_runtime_ui/undefined_call_part.rs index c5b9fcca1f31..1653e830f0b4 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_call_part.rs +++ b/frame/support/test/tests/construct_runtime_ui/undefined_call_part.rs @@ -12,20 +12,47 @@ mod pallet { } pub type Signature = sr25519::Signature; -pub type BlockNumber = u64; +pub type BlockNumber = u32; pub type Header = generic::Header; pub type Block = generic::Block; pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; impl pallet::Config for Runtime {} +impl frame_system::Config for Runtime { + type BaseCallFilter = frame_support::traits::Everything; + type Origin = Origin; + type Index = u64; + type BlockNumber = u32; + type Call = Call; + type Hash = sp_runtime::testing::H256; + type Hashing = sp_runtime::traits::BlakeTwo256; + type AccountId = u64; + type Lookup = sp_runtime::traits::IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = frame_support::traits::ConstU32<250>; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; +} + construct_runtime! { pub enum Runtime where Block = Block, NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: system::{Pallet, Call, Storage, Config, Event}, + System: frame_system::{Pallet, Call, Storage, Config, Event}, Pallet: pallet::{Pallet, Call}, } } diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_call_part.stderr b/frame/support/test/tests/construct_runtime_ui/undefined_call_part.stderr index 2629cf410192..c162a22bb87b 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_call_part.stderr +++ b/frame/support/test/tests/construct_runtime_ui/undefined_call_part.stderr @@ -1,73 +1,16 @@ error: `Pallet` does not have #[pallet::call] defined, perhaps you should remove `Call` from construct_runtime? - --> $DIR/undefined_call_part.rs:5:1 + --> tests/construct_runtime_ui/undefined_call_part.rs:5:1 | 5 | #[frame_support::pallet] | ^^^^^^^^^^^^^^^^^^^^^^^^ ... -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } +57 | | } +58 | | } | |_- in this macro invocation | = note: this error originates in the macro `pallet::__substrate_call_check::is_call_part_defined` (in Nightly builds, run with -Z macro-backtrace for more info) - -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_call_part.rs:28:11 - | -28 | System: system::{Pallet, Call, Storage, Config, Event}, - | ^^^^^^ use of undeclared crate or module `system` - -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_call_part.rs:22:1 - | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, -... | -30 | | } -31 | | } - | |_^ not found in `system` - | - = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing this enum - | -1 | use frame_system::RawOrigin; - | - -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_call_part.rs:22:1 - | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, -... | -30 | | } -31 | | } - | |_^ not found in `system` - | - = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing one of these items - | -1 | use crate::pallet::Pallet; - | -1 | use frame_support_test::Pallet; - | -1 | use frame_system::Pallet; - | -1 | use test_pallet::Pallet; - | - -error[E0277]: the trait bound `Runtime: frame_system::Config` is not satisfied - --> $DIR/undefined_call_part.rs:20:6 - | -8 | pub trait Config: frame_system::Config {} - | -------------------- required by this bound in `pallet::Config` -... -20 | impl pallet::Config for Runtime {} - | ^^^^^^^^^^^^^^ the trait `frame_system::Config` is not implemented for `Runtime` diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_event_part.rs b/frame/support/test/tests/construct_runtime_ui/undefined_event_part.rs index 6aec45f240c9..b8f91cf4bc69 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_event_part.rs +++ b/frame/support/test/tests/construct_runtime_ui/undefined_event_part.rs @@ -12,20 +12,47 @@ mod pallet { } pub type Signature = sr25519::Signature; -pub type BlockNumber = u64; +pub type BlockNumber = u32; pub type Header = generic::Header; pub type Block = generic::Block; pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; impl pallet::Config for Runtime {} +impl frame_system::Config for Runtime { + type BaseCallFilter = frame_support::traits::Everything; + type Origin = Origin; + type Index = u64; + type BlockNumber = u32; + type Call = Call; + type Hash = sp_runtime::testing::H256; + type Hashing = sp_runtime::traits::BlakeTwo256; + type AccountId = u64; + type Lookup = sp_runtime::traits::IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = frame_support::traits::ConstU32<250>; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; +} + construct_runtime! { pub enum Runtime where Block = Block, NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: system::{Pallet, Call, Storage, Config, Event}, + System: frame_system::{Pallet, Call, Storage, Config, Event}, Pallet: pallet::{Pallet, Event}, } } diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_event_part.stderr b/frame/support/test/tests/construct_runtime_ui/undefined_event_part.stderr index af69b79ed1a6..31229f8c93cb 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_event_part.stderr +++ b/frame/support/test/tests/construct_runtime_ui/undefined_event_part.stderr @@ -1,36 +1,30 @@ error: `Pallet` does not have #[pallet::event] defined, perhaps you should remove `Event` from construct_runtime? - --> $DIR/undefined_event_part.rs:5:1 + --> tests/construct_runtime_ui/undefined_event_part.rs:5:1 | 5 | #[frame_support::pallet] | ^^^^^^^^^^^^^^^^^^^^^^^^ ... -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } +57 | | } +58 | | } | |_- in this macro invocation | = note: this error originates in the macro `pallet::__substrate_event_check::is_event_part_defined` (in Nightly builds, run with -Z macro-backtrace for more info) -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_event_part.rs:28:11 - | -28 | System: system::{Pallet, Call, Storage, Config, Event}, - | ^^^^^^ use of undeclared crate or module `system` - error[E0412]: cannot find type `Event` in module `pallet` - --> $DIR/undefined_event_part.rs:22:1 + --> tests/construct_runtime_ui/undefined_event_part.rs:49:1 | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } +57 | | } +58 | | } | |_^ not found in `pallet` | = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) @@ -40,15 +34,15 @@ help: consider importing this enum | error[E0412]: cannot find type `Event` in module `pallet` - --> $DIR/undefined_event_part.rs:22:1 + --> tests/construct_runtime_ui/undefined_event_part.rs:49:1 | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } +57 | | } +58 | | } | |_^ not found in `pallet` | = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) @@ -58,54 +52,3 @@ help: consider importing one of these items | 1 | use frame_system::Event; | - -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_event_part.rs:22:1 - | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, -... | -30 | | } -31 | | } - | |_^ not found in `system` - | - = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing this enum - | -1 | use frame_system::RawOrigin; - | - -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_event_part.rs:22:1 - | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, -... | -30 | | } -31 | | } - | |_^ not found in `system` - | - = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing one of these items - | -1 | use crate::pallet::Pallet; - | -1 | use frame_support_test::Pallet; - | -1 | use frame_system::Pallet; - | -1 | use test_pallet::Pallet; - | - -error[E0277]: the trait bound `Runtime: frame_system::Config` is not satisfied - --> $DIR/undefined_event_part.rs:20:6 - | -8 | pub trait Config: frame_system::Config {} - | -------------------- required by this bound in `pallet::Config` -... -20 | impl pallet::Config for Runtime {} - | ^^^^^^^^^^^^^^ the trait `frame_system::Config` is not implemented for `Runtime` diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_genesis_config_part.rs b/frame/support/test/tests/construct_runtime_ui/undefined_genesis_config_part.rs index 5e08fd96fa1a..a61d545b3279 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_genesis_config_part.rs +++ b/frame/support/test/tests/construct_runtime_ui/undefined_genesis_config_part.rs @@ -12,20 +12,47 @@ mod pallet { } pub type Signature = sr25519::Signature; -pub type BlockNumber = u64; +pub type BlockNumber = u32; pub type Header = generic::Header; pub type Block = generic::Block; pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; impl pallet::Config for Runtime {} +impl frame_system::Config for Runtime { + type BaseCallFilter = frame_support::traits::Everything; + type Origin = Origin; + type Index = u64; + type BlockNumber = u32; + type Call = Call; + type Hash = sp_runtime::testing::H256; + type Hashing = sp_runtime::traits::BlakeTwo256; + type AccountId = u64; + type Lookup = sp_runtime::traits::IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = frame_support::traits::ConstU32<250>; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; +} + construct_runtime! { pub enum Runtime where Block = Block, NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: system::{Pallet, Call, Storage, Config, Event}, + System: frame_system::{Pallet, Call, Storage, Config, Event}, Pallet: pallet::{Pallet, Config}, } } diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_genesis_config_part.stderr b/frame/support/test/tests/construct_runtime_ui/undefined_genesis_config_part.stderr index bfedb921bca4..e8532aa9a064 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_genesis_config_part.stderr +++ b/frame/support/test/tests/construct_runtime_ui/undefined_genesis_config_part.stderr @@ -1,78 +1,30 @@ error: `Pallet` does not have #[pallet::genesis_config] defined, perhaps you should remove `Config` from construct_runtime? - --> $DIR/undefined_genesis_config_part.rs:5:1 + --> tests/construct_runtime_ui/undefined_genesis_config_part.rs:5:1 | 5 | #[frame_support::pallet] | ^^^^^^^^^^^^^^^^^^^^^^^^ ... -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } +57 | | } +58 | | } | |_- in this macro invocation | = note: this error originates in the macro `pallet::__substrate_genesis_config_check::is_genesis_config_defined` (in Nightly builds, run with -Z macro-backtrace for more info) -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_genesis_config_part.rs:28:17 - | -28 | System: system::{Pallet, Call, Storage, Config, Event}, - | ^^^^^^ use of undeclared crate or module `system` - -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_genesis_config_part.rs:22:1 - | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, -... | -30 | | } -31 | | } - | |_^ not found in `system` - | - = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing this enum - | -1 | use frame_system::RawOrigin; - | - -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_genesis_config_part.rs:22:1 - | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, -... | -30 | | } -31 | | } - | |_^ not found in `system` - | - = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing one of these items - | -1 | use crate::pallet::Pallet; - | -1 | use frame_support_test::Pallet; - | -1 | use frame_system::Pallet; - | -1 | use test_pallet::Pallet; - | - error[E0412]: cannot find type `GenesisConfig` in module `pallet` - --> $DIR/undefined_genesis_config_part.rs:22:1 + --> tests/construct_runtime_ui/undefined_genesis_config_part.rs:49:1 | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } +57 | | } +58 | | } | |_^ not found in `pallet` | = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) @@ -81,11 +33,22 @@ help: consider importing this struct 1 | use frame_system::GenesisConfig; | -error[E0277]: the trait bound `Runtime: frame_system::Config` is not satisfied - --> $DIR/undefined_genesis_config_part.rs:20:6 - | -8 | pub trait Config: frame_system::Config {} - | -------------------- required by this bound in `pallet::Config` -... -20 | impl pallet::Config for Runtime {} - | ^^^^^^^^^^^^^^ the trait `frame_system::Config` is not implemented for `Runtime` +error[E0283]: type annotations needed + --> tests/construct_runtime_ui/undefined_genesis_config_part.rs:49:1 + | +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, +... | +57 | | } +58 | | } + | |_^ cannot infer type + | + = note: cannot satisfy `_: std::default::Default` +note: required by `std::default::Default::default` + --> $RUST/core/src/default.rs + | + | fn default() -> Self; + | ^^^^^^^^^^^^^^^^^^^^^ + = note: this error originates in the derive macro `Default` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.rs b/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.rs index 06c36a30f550..6e4764286ab4 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.rs +++ b/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.rs @@ -12,20 +12,47 @@ mod pallet { } pub type Signature = sr25519::Signature; -pub type BlockNumber = u64; +pub type BlockNumber = u32; pub type Header = generic::Header; pub type Block = generic::Block; pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; impl pallet::Config for Runtime {} +impl frame_system::Config for Runtime { + type BaseCallFilter = frame_support::traits::Everything; + type Origin = Origin; + type Index = u64; + type BlockNumber = u32; + type Call = Call; + type Hash = sp_runtime::testing::H256; + type Hashing = sp_runtime::traits::BlakeTwo256; + type AccountId = u64; + type Lookup = sp_runtime::traits::IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = frame_support::traits::ConstU32<250>; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; +} + construct_runtime! { pub enum Runtime where Block = Block, NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: system::{Pallet, Call, Storage, Config, Event}, + System: frame_system::{Pallet, Call, Storage, Config, Event}, Pallet: pallet::{Pallet, Inherent}, } } diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.stderr b/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.stderr index 50dde1108263..9f646469d86a 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.stderr +++ b/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.stderr @@ -1,73 +1,116 @@ error: `Pallet` does not have #[pallet::inherent] defined, perhaps you should remove `Inherent` from construct_runtime? - --> $DIR/undefined_inherent_part.rs:5:1 + --> tests/construct_runtime_ui/undefined_inherent_part.rs:5:1 | 5 | #[frame_support::pallet] | ^^^^^^^^^^^^^^^^^^^^^^^^ ... -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } +57 | | } +58 | | } | |_- in this macro invocation | = note: this error originates in the macro `pallet::__substrate_inherent_check::is_inherent_part_defined` (in Nightly builds, run with -Z macro-backtrace for more info) -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_inherent_part.rs:28:11 +error[E0599]: no function or associated item named `create_inherent` found for struct `pallet::Pallet` in the current scope + --> tests/construct_runtime_ui/undefined_inherent_part.rs:49:1 | -28 | System: system::{Pallet, Call, Storage, Config, Event}, - | ^^^^^^ use of undeclared crate or module `system` - -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_inherent_part.rs:22:1 - | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +11 | pub struct Pallet(_); + | ------------------------ function or associated item `create_inherent` not found for this +... +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } - | |_^ not found in `system` +57 | | } +58 | | } + | |_^ function or associated item not found in `pallet::Pallet` | + = help: items from traits can only be used if the trait is implemented and in scope + = note: the following trait defines an item `create_inherent`, perhaps you need to implement it: + candidate #1: `ProvideInherent` = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing this enum - | -1 | use frame_system::RawOrigin; - | -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_inherent_part.rs:22:1 +error[E0599]: no function or associated item named `is_inherent` found for struct `pallet::Pallet` in the current scope + --> tests/construct_runtime_ui/undefined_inherent_part.rs:49:1 | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +11 | pub struct Pallet(_); + | ------------------------ function or associated item `is_inherent` not found for this +... +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } - | |_^ not found in `system` +57 | | } +58 | | } + | |_^ function or associated item not found in `pallet::Pallet` | + = help: items from traits can only be used if the trait is implemented and in scope + = note: the following trait defines an item `is_inherent`, perhaps you need to implement it: + candidate #1: `ProvideInherent` = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing one of these items - | -1 | use crate::pallet::Pallet; + +error[E0599]: no function or associated item named `check_inherent` found for struct `pallet::Pallet` in the current scope + --> tests/construct_runtime_ui/undefined_inherent_part.rs:49:1 | -1 | use frame_support_test::Pallet; +11 | pub struct Pallet(_); + | ------------------------ function or associated item `check_inherent` not found for this +... +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, +... | +57 | | } +58 | | } + | |_^ function or associated item not found in `pallet::Pallet` | -1 | use frame_system::Pallet; + = help: items from traits can only be used if the trait is implemented and in scope + = note: the following trait defines an item `check_inherent`, perhaps you need to implement it: + candidate #1: `ProvideInherent` + = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0599]: no associated item named `INHERENT_IDENTIFIER` found for struct `pallet::Pallet` in the current scope + --> tests/construct_runtime_ui/undefined_inherent_part.rs:49:1 | -1 | use test_pallet::Pallet; +11 | pub struct Pallet(_); + | ------------------------ associated item `INHERENT_IDENTIFIER` not found for this +... +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, +... | +57 | | } +58 | | } + | |_^ associated item not found in `pallet::Pallet` | + = help: items from traits can only be used if the trait is implemented and in scope + = note: the following trait defines an item `INHERENT_IDENTIFIER`, perhaps you need to implement it: + candidate #1: `ProvideInherent` + = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -error[E0277]: the trait bound `Runtime: frame_system::Config` is not satisfied - --> $DIR/undefined_inherent_part.rs:20:6 +error[E0599]: no function or associated item named `is_inherent_required` found for struct `pallet::Pallet` in the current scope + --> tests/construct_runtime_ui/undefined_inherent_part.rs:49:1 | -8 | pub trait Config: frame_system::Config {} - | -------------------- required by this bound in `pallet::Config` +11 | pub struct Pallet(_); + | ------------------------ function or associated item `is_inherent_required` not found for this ... -20 | impl pallet::Config for Runtime {} - | ^^^^^^^^^^^^^^ the trait `frame_system::Config` is not implemented for `Runtime` +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, +... | +57 | | } +58 | | } + | |_^ function or associated item not found in `pallet::Pallet` + | + = help: items from traits can only be used if the trait is implemented and in scope + = note: the following trait defines an item `is_inherent_required`, perhaps you need to implement it: + candidate #1: `ProvideInherent` + = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_origin_part.rs b/frame/support/test/tests/construct_runtime_ui/undefined_origin_part.rs index bec5c27ec034..9233404a865b 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_origin_part.rs +++ b/frame/support/test/tests/construct_runtime_ui/undefined_origin_part.rs @@ -12,20 +12,47 @@ mod pallet { } pub type Signature = sr25519::Signature; -pub type BlockNumber = u64; +pub type BlockNumber = u32; pub type Header = generic::Header; pub type Block = generic::Block; pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; impl pallet::Config for Runtime {} +impl frame_system::Config for Runtime { + type BaseCallFilter = frame_support::traits::Everything; + type Origin = Origin; + type Index = u64; + type BlockNumber = u32; + type Call = Call; + type Hash = sp_runtime::testing::H256; + type Hashing = sp_runtime::traits::BlakeTwo256; + type AccountId = u64; + type Lookup = sp_runtime::traits::IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = frame_support::traits::ConstU32<250>; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; +} + construct_runtime! { pub enum Runtime where Block = Block, NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: system::{Pallet, Call, Storage, Config, Event}, + System: frame_system::{Pallet, Call, Storage, Config, Event}, Pallet: pallet::{Pallet, Origin}, } } diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_origin_part.stderr b/frame/support/test/tests/construct_runtime_ui/undefined_origin_part.stderr index b5f3ec4d381b..06e845618d44 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_origin_part.stderr +++ b/frame/support/test/tests/construct_runtime_ui/undefined_origin_part.stderr @@ -1,54 +1,30 @@ error: `Pallet` does not have #[pallet::origin] defined, perhaps you should remove `Origin` from construct_runtime? - --> $DIR/undefined_origin_part.rs:5:1 + --> tests/construct_runtime_ui/undefined_origin_part.rs:5:1 | 5 | #[frame_support::pallet] | ^^^^^^^^^^^^^^^^^^^^^^^^ ... -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } +57 | | } +58 | | } | |_- in this macro invocation | = note: this error originates in the macro `pallet::__substrate_origin_check::is_origin_part_defined` (in Nightly builds, run with -Z macro-backtrace for more info) -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_origin_part.rs:28:11 - | -28 | System: system::{Pallet, Call, Storage, Config, Event}, - | ^^^^^^ use of undeclared crate or module `system` - -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_origin_part.rs:22:1 - | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, -... | -30 | | } -31 | | } - | |_^ not found in `system` - | - = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing this enum - | -1 | use frame_system::RawOrigin; - | - error[E0412]: cannot find type `Origin` in module `pallet` - --> $DIR/undefined_origin_part.rs:22:1 + --> tests/construct_runtime_ui/undefined_origin_part.rs:49:1 | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } +57 | | } +58 | | } | |_^ not found in `pallet` | = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) @@ -58,15 +34,15 @@ help: consider importing this type alias | error[E0412]: cannot find type `Origin` in module `pallet` - --> $DIR/undefined_origin_part.rs:22:1 + --> tests/construct_runtime_ui/undefined_origin_part.rs:49:1 | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } +57 | | } +58 | | } | |_^ not found in `pallet` | = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) @@ -77,35 +53,16 @@ help: consider importing one of these items 1 | use frame_system::Origin; | -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_origin_part.rs:22:1 +error[E0282]: type annotations needed + --> tests/construct_runtime_ui/undefined_origin_part.rs:49:1 | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } - | |_^ not found in `system` +57 | | } +58 | | } + | |_^ cannot infer type for type parameter `AccountId` declared on the enum `RawOrigin` | = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing one of these items - | -1 | use crate::pallet::Pallet; - | -1 | use frame_support_test::Pallet; - | -1 | use frame_system::Pallet; - | -1 | use test_pallet::Pallet; - | - -error[E0277]: the trait bound `Runtime: frame_system::Config` is not satisfied - --> $DIR/undefined_origin_part.rs:20:6 - | -8 | pub trait Config: frame_system::Config {} - | -------------------- required by this bound in `pallet::Config` -... -20 | impl pallet::Config for Runtime {} - | ^^^^^^^^^^^^^^ the trait `frame_system::Config` is not implemented for `Runtime` diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_validate_unsigned_part.rs b/frame/support/test/tests/construct_runtime_ui/undefined_validate_unsigned_part.rs index 816f52b91ccc..621683aca375 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_validate_unsigned_part.rs +++ b/frame/support/test/tests/construct_runtime_ui/undefined_validate_unsigned_part.rs @@ -12,20 +12,47 @@ mod pallet { } pub type Signature = sr25519::Signature; -pub type BlockNumber = u64; +pub type BlockNumber = u32; pub type Header = generic::Header; pub type Block = generic::Block; pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; impl pallet::Config for Runtime {} +impl frame_system::Config for Runtime { + type BaseCallFilter = frame_support::traits::Everything; + type Origin = Origin; + type Index = u64; + type BlockNumber = u32; + type Call = Call; + type Hash = sp_runtime::testing::H256; + type Hashing = sp_runtime::traits::BlakeTwo256; + type AccountId = u64; + type Lookup = sp_runtime::traits::IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = frame_support::traits::ConstU32<250>; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; +} + construct_runtime! { pub enum Runtime where Block = Block, NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: system::{Pallet, Call, Storage, Config, Event}, + System: frame_system::{Pallet, Call, Storage, Config, Event}, Pallet: pallet::{Pallet, ValidateUnsigned}, } } diff --git a/frame/support/test/tests/construct_runtime_ui/undefined_validate_unsigned_part.stderr b/frame/support/test/tests/construct_runtime_ui/undefined_validate_unsigned_part.stderr index 12bdce67cf03..94226075d9a4 100644 --- a/frame/support/test/tests/construct_runtime_ui/undefined_validate_unsigned_part.stderr +++ b/frame/support/test/tests/construct_runtime_ui/undefined_validate_unsigned_part.stderr @@ -1,73 +1,67 @@ error: `Pallet` does not have #[pallet::validate_unsigned] defined, perhaps you should remove `ValidateUnsigned` from construct_runtime? - --> $DIR/undefined_validate_unsigned_part.rs:5:1 + --> tests/construct_runtime_ui/undefined_validate_unsigned_part.rs:5:1 | 5 | #[frame_support::pallet] | ^^^^^^^^^^^^^^^^^^^^^^^^ ... -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } +57 | | } +58 | | } | |_- in this macro invocation | = note: this error originates in the macro `pallet::__substrate_validate_unsigned_check::is_validate_unsigned_part_defined` (in Nightly builds, run with -Z macro-backtrace for more info) -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_validate_unsigned_part.rs:28:11 +error[E0599]: no variant or associated item named `Pallet` found for enum `Call` in the current scope + --> tests/construct_runtime_ui/undefined_validate_unsigned_part.rs:56:3 | -28 | System: system::{Pallet, Call, Storage, Config, Event}, - | ^^^^^^ use of undeclared crate or module `system` +49 | construct_runtime! { + | ------------------ variant or associated item `Pallet` not found here +... +56 | Pallet: pallet::{Pallet, ValidateUnsigned}, + | ^^^^^^ variant or associated item not found in `Call` -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_validate_unsigned_part.rs:22:1 +error[E0599]: no function or associated item named `pre_dispatch` found for struct `pallet::Pallet` in the current scope + --> tests/construct_runtime_ui/undefined_validate_unsigned_part.rs:49:1 | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +11 | pub struct Pallet(_); + | ------------------------ function or associated item `pre_dispatch` not found for this +... +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } - | |_^ not found in `system` +57 | | } +58 | | } + | |_^ function or associated item not found in `pallet::Pallet` | + = help: items from traits can only be used if the trait is implemented and in scope + = note: the following traits define an item `pre_dispatch`, perhaps you need to implement one of them: + candidate #1: `SignedExtension` + candidate #2: `ValidateUnsigned` = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing this enum - | -1 | use frame_system::RawOrigin; - | -error[E0433]: failed to resolve: use of undeclared crate or module `system` - --> $DIR/undefined_validate_unsigned_part.rs:22:1 +error[E0599]: no function or associated item named `validate_unsigned` found for struct `pallet::Pallet` in the current scope + --> tests/construct_runtime_ui/undefined_validate_unsigned_part.rs:49:1 | -22 | / construct_runtime! { -23 | | pub enum Runtime where -24 | | Block = Block, -25 | | NodeBlock = Block, +11 | pub struct Pallet(_); + | ------------------------ function or associated item `validate_unsigned` not found for this +... +49 | / construct_runtime! { +50 | | pub enum Runtime where +51 | | Block = Block, +52 | | NodeBlock = Block, ... | -30 | | } -31 | | } - | |_^ not found in `system` +57 | | } +58 | | } + | |_^ function or associated item not found in `pallet::Pallet` | + = help: items from traits can only be used if the trait is implemented and in scope + = note: the following traits define an item `validate_unsigned`, perhaps you need to implement one of them: + candidate #1: `SignedExtension` + candidate #2: `ValidateUnsigned` = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) -help: consider importing one of these items - | -1 | use crate::pallet::Pallet; - | -1 | use frame_support_test::Pallet; - | -1 | use frame_system::Pallet; - | -1 | use test_pallet::Pallet; - | - -error[E0277]: the trait bound `Runtime: frame_system::Config` is not satisfied - --> $DIR/undefined_validate_unsigned_part.rs:20:6 - | -8 | pub trait Config: frame_system::Config {} - | -------------------- required by this bound in `pallet::Config` -... -20 | impl pallet::Config for Runtime {} - | ^^^^^^^^^^^^^^ the trait `frame_system::Config` is not implemented for `Runtime` diff --git a/frame/support/test/tests/construct_runtime_ui/use_undefined_part.rs b/frame/support/test/tests/construct_runtime_ui/use_undefined_part.rs new file mode 100644 index 000000000000..1664dcc42b75 --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/use_undefined_part.rs @@ -0,0 +1,38 @@ +use frame_support::construct_runtime; +use sp_runtime::{generic, traits::BlakeTwo256}; +use sp_core::sr25519; + +#[frame_support::pallet] +mod pallet { + use frame_support::pallet_prelude::*; + + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::storage] + type Foo = StorageValue; +} + +pub type Signature = sr25519::Signature; +pub type BlockNumber = u64; +pub type Header = generic::Header; +pub type Block = generic::Block; +pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; + +impl pallet::Config for Runtime {} + +construct_runtime! { + pub enum Runtime where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic + { + System: system::{Pallet, Call, Storage, Config, Event}, + Pallet: pallet use_parts { Call }, + } +} + +fn main() {} diff --git a/frame/support/test/tests/construct_runtime_ui/use_undefined_part.stderr b/frame/support/test/tests/construct_runtime_ui/use_undefined_part.stderr new file mode 100644 index 000000000000..ed41f0ce673a --- /dev/null +++ b/frame/support/test/tests/construct_runtime_ui/use_undefined_part.stderr @@ -0,0 +1,28 @@ +error: Invalid pallet part specified, the pallet `Pallet` doesn't have the `Call` part. Available parts are: `Pallet`, `Storage`. + --> $DIR/use_undefined_part.rs:34:30 + | +34 | Pallet: pallet use_parts { Call }, + | ^^^^ + +error[E0412]: cannot find type `Call` in this scope + --> $DIR/use_undefined_part.rs:23:64 + | +23 | pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; + | ^^^^ not found in this scope + | +help: consider importing one of these items + | +1 | use crate::pallet::Call; + | +1 | use frame_support_test::Call; + | +1 | use frame_system::Call; + | +1 | use test_pallet::Call; + | + +error[E0412]: cannot find type `Runtime` in this scope + --> $DIR/use_undefined_part.rs:25:25 + | +25 | impl pallet::Config for Runtime {} + | ^^^^^^^ not found in this scope diff --git a/frame/support/test/tests/decl_module_ui.rs b/frame/support/test/tests/decl_module_ui.rs index e84025b9f256..829850ec2d47 100644 --- a/frame/support/test/tests/decl_module_ui.rs +++ b/frame/support/test/tests/decl_module_ui.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/tests/decl_storage.rs b/frame/support/test/tests/decl_storage.rs index 347a3130daa7..be5d70be17f6 100644 --- a/frame/support/test/tests/decl_storage.rs +++ b/frame/support/test/tests/decl_storage.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/tests/decl_storage_ui.rs b/frame/support/test/tests/decl_storage_ui.rs index 400ddfc0f94f..d4db02ad19a0 100644 --- a/frame/support/test/tests/decl_storage_ui.rs +++ b/frame/support/test/tests/decl_storage_ui.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/tests/decl_storage_ui/config_duplicate.rs b/frame/support/test/tests/decl_storage_ui/config_duplicate.rs index 17f80c8c8475..db2cdbdc6549 100644 --- a/frame/support/test/tests/decl_storage_ui/config_duplicate.rs +++ b/frame/support/test/tests/decl_storage_ui/config_duplicate.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/tests/decl_storage_ui/config_get_duplicate.rs b/frame/support/test/tests/decl_storage_ui/config_get_duplicate.rs index fec6aeb64cec..b804bf898038 100644 --- a/frame/support/test/tests/decl_storage_ui/config_get_duplicate.rs +++ b/frame/support/test/tests/decl_storage_ui/config_get_duplicate.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/tests/decl_storage_ui/get_duplicate.rs b/frame/support/test/tests/decl_storage_ui/get_duplicate.rs index 13c57a638bb1..bc03ff6b4a4f 100644 --- a/frame/support/test/tests/decl_storage_ui/get_duplicate.rs +++ b/frame/support/test/tests/decl_storage_ui/get_duplicate.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/tests/derive_no_bound.rs b/frame/support/test/tests/derive_no_bound.rs index 1827844664fa..e1cf539f2ac5 100644 --- a/frame/support/test/tests/derive_no_bound.rs +++ b/frame/support/test/tests/derive_no_bound.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/tests/derive_no_bound_ui.rs b/frame/support/test/tests/derive_no_bound_ui.rs index 22c116931a47..91e530d1d8da 100644 --- a/frame/support/test/tests/derive_no_bound_ui.rs +++ b/frame/support/test/tests/derive_no_bound_ui.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/tests/derive_no_bound_ui/clone.stderr b/frame/support/test/tests/derive_no_bound_ui/clone.stderr index 050b576c8b9e..45428a8728c2 100644 --- a/frame/support/test/tests/derive_no_bound_ui/clone.stderr +++ b/frame/support/test/tests/derive_no_bound_ui/clone.stderr @@ -1,11 +1,11 @@ error[E0277]: the trait bound `::C: Clone` is not satisfied - --> $DIR/clone.rs:7:2 + --> tests/derive_no_bound_ui/clone.rs:7:2 | 7 | c: T::C, | ^ the trait `Clone` is not implemented for `::C` | note: required by `clone` - --> $DIR/clone.rs:121:5 + --> $RUST/core/src/clone.rs | -121 | fn clone(&self) -> Self; + | fn clone(&self) -> Self; | ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/frame/support/test/tests/derive_no_bound_ui/eq.stderr b/frame/support/test/tests/derive_no_bound_ui/eq.stderr index fce13d6f17f0..97ac29f96902 100644 --- a/frame/support/test/tests/derive_no_bound_ui/eq.stderr +++ b/frame/support/test/tests/derive_no_bound_ui/eq.stderr @@ -1,12 +1,12 @@ error[E0277]: can't compare `Foo` with `Foo` - --> $DIR/eq.rs:6:8 + --> tests/derive_no_bound_ui/eq.rs:6:8 | 6 | struct Foo { | ^^^ no implementation for `Foo == Foo` | - ::: $RUST/core/src/cmp.rs + = help: the trait `PartialEq` is not implemented for `Foo` +note: required by a bound in `std::cmp::Eq` + --> $RUST/core/src/cmp.rs | | pub trait Eq: PartialEq { - | --------------- required by this bound in `std::cmp::Eq` - | - = help: the trait `PartialEq` is not implemented for `Foo` + | ^^^^^^^^^^^^^^^ required by this bound in `std::cmp::Eq` diff --git a/frame/support/test/tests/final_keys.rs b/frame/support/test/tests/final_keys.rs index e89f961d893f..c1723c6ad7a1 100644 --- a/frame/support/test/tests/final_keys.rs +++ b/frame/support/test/tests/final_keys.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/tests/genesisconfig.rs b/frame/support/test/tests/genesisconfig.rs index d488e8bfbfaf..367c7236d015 100644 --- a/frame/support/test/tests/genesisconfig.rs +++ b/frame/support/test/tests/genesisconfig.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,5 +40,5 @@ impl Config for Test {} #[test] fn init_genesis_config() { - GenesisConfig:: { t: Default::default() }; + GenesisConfig::::default(); } diff --git a/frame/support/test/tests/instance.rs b/frame/support/test/tests/instance.rs index 809edae14f80..2fe8d0e0f22f 100644 --- a/frame/support/test/tests/instance.rs +++ b/frame/support/test/tests/instance.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,8 +24,7 @@ use frame_support::{ PalletStorageMetadata, StorageEntryMetadata, StorageEntryModifier, StorageEntryType, StorageHasher, }, - parameter_types, - traits::Get, + traits::{ConstU32, Get}, Parameter, StorageDoubleMap, StorageMap, StorageValue, }; use scale_info::TypeInfo; @@ -229,20 +228,16 @@ mod module3 { } } -parameter_types! { - pub const SomeValue: u32 = 100; -} - impl module1::Config for Runtime { type Event = Event; type Origin = Origin; - type SomeParameter = SomeValue; + type SomeParameter = ConstU32<100>; type GenericType = u32; } impl module1::Config for Runtime { type Event = Event; type Origin = Origin; - type SomeParameter = SomeValue; + type SomeParameter = ConstU32<100>; type GenericType = u32; } impl module2::Config for Runtime { diff --git a/frame/support/test/tests/issue2219.rs b/frame/support/test/tests/issue2219.rs index 68ad2a50a21b..d7e3d2cb5b13 100644 --- a/frame/support/test/tests/issue2219.rs +++ b/frame/support/test/tests/issue2219.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -184,7 +184,9 @@ frame_support::construct_runtime!( #[test] fn create_genesis_config() { - GenesisConfig { + let config = GenesisConfig { module: module::GenesisConfig { request_life_time: 0, enable_storage_role: true }, }; + assert_eq!(config.module.request_life_time, 0); + assert!(config.module.enable_storage_role); } diff --git a/frame/support/test/tests/origin.rs b/frame/support/test/tests/origin.rs new file mode 100644 index 000000000000..1def44c15b48 --- /dev/null +++ b/frame/support/test/tests/origin.rs @@ -0,0 +1,214 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Origin tests for construct_runtime macro + +#![recursion_limit = "128"] + +use frame_support::traits::{Contains, OriginTrait}; +use scale_info::TypeInfo; +use sp_core::{sr25519, H256}; +use sp_runtime::{generic, traits::BlakeTwo256}; + +mod system; + +mod nested { + use super::*; + + pub mod module { + use super::*; + + pub trait Config: system::Config {} + + frame_support::decl_module! { + pub struct Module for enum Call + where origin: ::Origin, system=system + { + #[weight = 0] + pub fn fail(_origin) -> frame_support::dispatch::DispatchResult { + Err(Error::::Something.into()) + } + } + } + + #[derive(Clone, PartialEq, Eq, Debug, codec::Encode, codec::Decode, TypeInfo)] + pub struct Origin; + + frame_support::decl_event! { + pub enum Event { + A, + } + } + + frame_support::decl_error! { + pub enum Error for Module { + Something + } + } + + frame_support::decl_storage! { + trait Store for Module as Module {} + add_extra_genesis { + build(|_config| {}) + } + } + } +} + +pub mod module { + use super::*; + + pub trait Config: system::Config {} + + frame_support::decl_module! { + pub struct Module for enum Call + where origin: ::Origin, system=system + { + #[weight = 0] + pub fn fail(_origin) -> frame_support::dispatch::DispatchResult { + Err(Error::::Something.into()) + } + #[weight = 0] + pub fn aux_1(_origin, #[compact] _data: u32) -> frame_support::dispatch::DispatchResult { + unreachable!() + } + #[weight = 0] + pub fn aux_2(_origin, _data: i32, #[compact] _data2: u32) -> frame_support::dispatch::DispatchResult { + unreachable!() + } + #[weight = 0] + fn aux_3(_origin, _data: i32, _data2: String) -> frame_support::dispatch::DispatchResult { + unreachable!() + } + #[weight = 3] + fn aux_4(_origin) -> frame_support::dispatch::DispatchResult { unreachable!() } + #[weight = (5, frame_support::weights::DispatchClass::Operational)] + fn operational(_origin) { unreachable!() } + } + } + + #[derive(Clone, PartialEq, Eq, Debug, codec::Encode, codec::Decode, TypeInfo)] + pub struct Origin(pub core::marker::PhantomData); + + frame_support::decl_event! { + pub enum Event { + A, + } + } + + frame_support::decl_error! { + pub enum Error for Module { + Something + } + } + + frame_support::decl_storage! { + trait Store for Module as Module {} + add_extra_genesis { + build(|_config| {}) + } + } +} + +impl nested::module::Config for RuntimeOriginTest {} +impl module::Config for RuntimeOriginTest {} + +pub struct BaseCallFilter; +impl Contains for BaseCallFilter { + fn contains(c: &Call) -> bool { + match c { + Call::NestedModule(_) => true, + _ => false, + } + } +} + +impl system::Config for RuntimeOriginTest { + type BaseCallFilter = BaseCallFilter; + type Hash = H256; + type Origin = Origin; + type BlockNumber = BlockNumber; + type AccountId = u32; + type Event = Event; + type PalletInfo = PalletInfo; + type Call = Call; + type DbWeight = (); +} + +frame_support::construct_runtime!( + pub enum RuntimeOriginTest where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic + { + System: system::{Pallet, Event, Origin}, + NestedModule: nested::module::{Pallet, Origin, Call}, + Module: module::{Pallet, Origin, Call}, + } +); + +pub type Signature = sr25519::Signature; +pub type BlockNumber = u64; +pub type Header = generic::Header; +pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; +pub type Block = generic::Block; + +#[test] +fn origin_default_filter() { + let accepted_call = nested::module::Call::fail {}.into(); + let rejected_call = module::Call::fail {}.into(); + + assert_eq!(Origin::root().filter_call(&accepted_call), true); + assert_eq!(Origin::root().filter_call(&rejected_call), true); + assert_eq!(Origin::none().filter_call(&accepted_call), true); + assert_eq!(Origin::none().filter_call(&rejected_call), false); + assert_eq!(Origin::signed(0).filter_call(&accepted_call), true); + assert_eq!(Origin::signed(0).filter_call(&rejected_call), false); + assert_eq!(Origin::from(Some(0)).filter_call(&accepted_call), true); + assert_eq!(Origin::from(Some(0)).filter_call(&rejected_call), false); + assert_eq!(Origin::from(None).filter_call(&accepted_call), true); + assert_eq!(Origin::from(None).filter_call(&rejected_call), false); + assert_eq!(Origin::from(nested::module::Origin).filter_call(&accepted_call), true); + assert_eq!(Origin::from(nested::module::Origin).filter_call(&rejected_call), false); + + let mut origin = Origin::from(Some(0)); + origin.add_filter(|c| matches!(c, Call::Module(_))); + assert_eq!(origin.filter_call(&accepted_call), false); + assert_eq!(origin.filter_call(&rejected_call), false); + + // Now test for root origin and filters: + let mut origin = Origin::from(Some(0)); + origin.set_caller_from(Origin::root()); + assert!(matches!(origin.caller, OriginCaller::system(system::RawOrigin::Root))); + + // Root origin bypass all filter. + assert_eq!(origin.filter_call(&accepted_call), true); + assert_eq!(origin.filter_call(&rejected_call), true); + + origin.set_caller_from(Origin::from(Some(0))); + + // Back to another signed origin, the filtered are now effective again + assert_eq!(origin.filter_call(&accepted_call), true); + assert_eq!(origin.filter_call(&rejected_call), false); + + origin.set_caller_from(Origin::root()); + origin.reset_filter(); + + // Root origin bypass all filter, even when they are reset. + assert_eq!(origin.filter_call(&accepted_call), true); + assert_eq!(origin.filter_call(&rejected_call), true); +} diff --git a/frame/support/test/tests/pallet.rs b/frame/support/test/tests/pallet.rs index 25fc2d46d256..83f6a722f93a 100644 --- a/frame/support/test/tests/pallet.rs +++ b/frame/support/test/tests/pallet.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,8 +19,8 @@ use frame_support::{ dispatch::{Parameter, UnfilteredDispatchable}, storage::unhashed, traits::{ - GetCallName, GetStorageVersion, OnFinalize, OnGenesis, OnInitialize, OnRuntimeUpgrade, - PalletInfoAccess, StorageVersion, + ConstU32, GetCallName, GetStorageVersion, OnFinalize, OnGenesis, OnInitialize, + OnRuntimeUpgrade, PalletError, PalletInfoAccess, StorageVersion, }, weights::{DispatchClass, DispatchInfo, GetDispatchInfo, Pays, RuntimeDbWeight}, }; @@ -29,7 +29,7 @@ use sp_io::{ hashing::{blake2_128, twox_128, twox_64}, TestExternalities, }; -use sp_runtime::DispatchError; +use sp_runtime::{DispatchError, ModuleError}; pub struct SomeType1; impl From for u64 { @@ -146,11 +146,16 @@ pub mod pallet { fn some_extra_extra() -> T::AccountId { SomeType1.into() } + + /// Some doc + #[pallet::constant_name(SomeExtraRename)] + fn some_extra_rename() -> T::AccountId { + SomeType1.into() + } } #[pallet::pallet] #[pallet::generate_store(pub(crate) trait Store)] - #[pallet::generate_storage_info] #[pallet::storage_version(STORAGE_VERSION)] pub struct Pallet(_); @@ -160,25 +165,25 @@ pub mod pallet { T::AccountId: From + From + SomeAssociation1, { fn on_initialize(_: BlockNumberFor) -> Weight { - T::AccountId::from(SomeType1); // Test for where clause - T::AccountId::from(SomeType2); // Test for where clause + let _ = T::AccountId::from(SomeType1); // Test for where clause + let _ = T::AccountId::from(SomeType2); // Test for where clause Self::deposit_event(Event::Something(10)); 10 } fn on_finalize(_: BlockNumberFor) { - T::AccountId::from(SomeType1); // Test for where clause - T::AccountId::from(SomeType2); // Test for where clause + let _ = T::AccountId::from(SomeType1); // Test for where clause + let _ = T::AccountId::from(SomeType2); // Test for where clause Self::deposit_event(Event::Something(20)); } fn on_runtime_upgrade() -> Weight { - T::AccountId::from(SomeType1); // Test for where clause - T::AccountId::from(SomeType2); // Test for where clause + let _ = T::AccountId::from(SomeType1); // Test for where clause + let _ = T::AccountId::from(SomeType2); // Test for where clause Self::deposit_event(Event::Something(30)); 30 } fn integrity_test() { - T::AccountId::from(SomeType1); // Test for where clause - T::AccountId::from(SomeType2); // Test for where clause + let _ = T::AccountId::from(SomeType1); // Test for where clause + let _ = T::AccountId::from(SomeType2); // Test for where clause } } @@ -194,8 +199,8 @@ pub mod pallet { #[pallet::compact] _foo: u32, _bar: u32, ) -> DispatchResultWithPostInfo { - T::AccountId::from(SomeType1); // Test for where clause - T::AccountId::from(SomeType3); // Test for where clause + let _ = T::AccountId::from(SomeType1); // Test for where clause + let _ = T::AccountId::from(SomeType3); // Test for where clause let _ = origin; Self::deposit_event(Event::Something(3)); Ok(().into()) @@ -224,9 +229,14 @@ pub mod pallet { } #[pallet::error] + #[derive(PartialEq, Eq)] pub enum Error { /// doc comment put into metadata InsufficientProposersBalance, + Code(u8), + #[codec(skip)] + Skipped(u128), + CompactU8(#[codec(compact)] u8), } #[pallet::event] @@ -256,12 +266,13 @@ pub mod pallet { #[pallet::storage_prefix = "Value2"] pub type RenamedValue = StorageValue; + /// Test some doc #[pallet::type_value] pub fn MyDefault() -> u16 where T::AccountId: From + From + SomeAssociation1, { - T::AccountId::from(SomeType7); // Test where clause works + let _ = T::AccountId::from(SomeType7); // Test where clause works 4u16 } @@ -345,8 +356,8 @@ pub mod pallet { T::AccountId: From + SomeAssociation1 + From, { fn build(&self) { - T::AccountId::from(SomeType1); // Test for where clause - T::AccountId::from(SomeType4); // Test for where clause + let _ = T::AccountId::from(SomeType1); // Test for where clause + let _ = T::AccountId::from(SomeType4); // Test for where clause } } @@ -363,8 +374,8 @@ pub mod pallet { { type Call = Call; fn validate_unsigned(_source: TransactionSource, call: &Self::Call) -> TransactionValidity { - T::AccountId::from(SomeType1); // Test for where clause - T::AccountId::from(SomeType5); // Test for where clause + let _ = T::AccountId::from(SomeType1); // Test for where clause + let _ = T::AccountId::from(SomeType5); // Test for where clause if matches!(call, Call::foo_transactional { .. }) { return Ok(ValidTransaction::default()) } @@ -383,8 +394,8 @@ pub mod pallet { const INHERENT_IDENTIFIER: InherentIdentifier = INHERENT_IDENTIFIER; fn create_inherent(_data: &InherentData) -> Option { - T::AccountId::from(SomeType1); // Test for where clause - T::AccountId::from(SomeType6); // Test for where clause + let _ = T::AccountId::from(SomeType1); // Test for where clause + let _ = T::AccountId::from(SomeType6); // Test for where clause Some(Call::foo_no_post_info {}) } @@ -426,7 +437,7 @@ pub mod pallet { } // Test that a pallet with non generic event and generic genesis_config is correctly handled -// and that a pallet without the attribute generate_storage_info is correctly handled. +// and that a pallet with the attribute without_storage_info is correctly handled. #[frame_support::pallet] pub mod pallet2 { use super::{SomeAssociation1, SomeType1}; @@ -443,12 +454,25 @@ pub mod pallet2 { #[pallet::pallet] #[pallet::generate_store(pub(crate) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); #[pallet::hooks] - impl Hooks> for Pallet where - T::AccountId: From + SomeAssociation1 + impl Hooks> for Pallet + where + T::AccountId: From + SomeAssociation1, { + fn on_initialize(_: BlockNumberFor) -> Weight { + Self::deposit_event(Event::Something(11)); + 0 + } + fn on_finalize(_: BlockNumberFor) { + Self::deposit_event(Event::Something(21)); + } + fn on_runtime_upgrade() -> Weight { + Self::deposit_event(Event::Something(31)); + 0 + } } #[pallet::call] @@ -462,6 +486,7 @@ pub mod pallet2 { CountedStorageMap; #[pallet::event] + #[pallet::generate_deposit(fn deposit_event)] pub enum Event { /// Something Something(u32), @@ -503,11 +528,20 @@ pub mod pallet3 { pub struct Pallet(_); } +#[frame_support::pallet] +pub mod pallet4 { + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::call] + impl Pallet {} +} + frame_support::parameter_types!( - pub const MyGetParam: u32 = 10; - pub const MyGetParam2: u32 = 11; pub const MyGetParam3: u32 = 12; - pub const BlockHashCount: u32 = 250; ); impl frame_system::Config for Runtime { @@ -522,7 +556,7 @@ impl frame_system::Config for Runtime { type Lookup = sp_runtime::traits::IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU32<250>; type BlockWeights = (); type BlockLength = (); type DbWeight = (); @@ -534,11 +568,12 @@ impl frame_system::Config for Runtime { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl pallet::Config for Runtime { type Event = Event; - type MyGetParam = MyGetParam; - type MyGetParam2 = MyGetParam2; + type MyGetParam = ConstU32<10>; + type MyGetParam2 = ConstU32<11>; type MyGetParam3 = MyGetParam3; type Balance = u64; } @@ -547,6 +582,8 @@ impl pallet2::Config for Runtime { type Event = Event; } +impl pallet4::Config for Runtime {} + pub type Header = sp_runtime::generic::Header; pub type Block = sp_runtime::generic::Block; pub type UncheckedExtrinsic = sp_runtime::generic::UncheckedExtrinsic; @@ -557,12 +594,21 @@ frame_support::construct_runtime!( NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: frame_system::{Call, Event}, - Example: pallet::{Pallet, Call, Event, Config, Storage, Inherent, Origin, ValidateUnsigned}, - Example2: pallet2::{Pallet, Call, Event, Config, Storage}, + // Exclude part `Storage` in order not to check its metadata in tests. + System: frame_system exclude_parts { Pallet, Storage }, + Example: pallet, + Example2: pallet2 exclude_parts { Call }, + Example4: pallet4 use_parts { Call }, } ); +// Test that the part `Call` is excluded from Example2 and included in Example4. +fn _ensure_call_is_correctly_excluded_and_included(call: Call) { + match call { + Call::System(_) | Call::Example(_) | Call::Example4(_) => (), + } +} + #[test] fn transactional_works() { TestExternalities::default().execute_with(|| { @@ -613,8 +659,13 @@ fn error_expand() { ); assert_eq!( DispatchError::from(pallet::Error::::InsufficientProposersBalance), - DispatchError::Module { index: 1, error: 0, message: Some("InsufficientProposersBalance") }, + DispatchError::Module(ModuleError { + index: 1, + error: [0, 0, 0, 0], + message: Some("InsufficientProposersBalance") + }), ); + assert_eq!( as PalletError>::MAX_ENCODED_SIZE, 3); } #[test] @@ -840,7 +891,7 @@ fn pallet_expand_deposit_event() { #[test] fn pallet_new_call_variant() { - Call::Example(pallet::Call::new_call_variant_foo(3, 4)); + pallet::Call::::new_call_variant_foo(3, 4); } #[test] @@ -933,10 +984,10 @@ fn pallet_hooks_expand() { TestExternalities::default().execute_with(|| { frame_system::Pallet::::set_block_number(1); - assert_eq!(AllPallets::on_initialize(1), 10); - AllPallets::on_finalize(1); + assert_eq!(AllPalletsWithoutSystem::on_initialize(1), 10); + AllPalletsWithoutSystem::on_finalize(1); - assert_eq!(AllPallets::on_runtime_upgrade(), 30); + assert_eq!(AllPalletsWithoutSystem::on_runtime_upgrade(), 30); assert_eq!( frame_system::Pallet::::events()[0].event, @@ -944,10 +995,62 @@ fn pallet_hooks_expand() { ); assert_eq!( frame_system::Pallet::::events()[1].event, + Event::Example2(pallet2::Event::Something(11)), + ); + assert_eq!( + frame_system::Pallet::::events()[2].event, Event::Example(pallet::Event::Something(20)), ); + assert_eq!( + frame_system::Pallet::::events()[3].event, + Event::Example2(pallet2::Event::Something(21)), + ); + assert_eq!( + frame_system::Pallet::::events()[4].event, + Event::Example(pallet::Event::Something(30)), + ); + assert_eq!( + frame_system::Pallet::::events()[5].event, + Event::Example2(pallet2::Event::Something(31)), + ); + }) +} + +#[test] +fn all_pallets_type_reversed_order_is_correct() { + TestExternalities::default().execute_with(|| { + frame_system::Pallet::::set_block_number(1); + + #[allow(deprecated)] + { + assert_eq!(AllPalletsWithoutSystemReversed::on_initialize(1), 10); + AllPalletsWithoutSystemReversed::on_finalize(1); + + assert_eq!(AllPalletsWithoutSystemReversed::on_runtime_upgrade(), 30); + } + + assert_eq!( + frame_system::Pallet::::events()[0].event, + Event::Example2(pallet2::Event::Something(11)), + ); + assert_eq!( + frame_system::Pallet::::events()[1].event, + Event::Example(pallet::Event::Something(10)), + ); assert_eq!( frame_system::Pallet::::events()[2].event, + Event::Example2(pallet2::Event::Something(21)), + ); + assert_eq!( + frame_system::Pallet::::events()[3].event, + Event::Example(pallet::Event::Something(20)), + ); + assert_eq!( + frame_system::Pallet::::events()[4].event, + Event::Example2(pallet2::Event::Something(31)), + ); + assert_eq!( + frame_system::Pallet::::events()[5].event, Event::Example(pallet::Event::Something(30)), ); }) @@ -989,8 +1092,8 @@ fn migrate_from_pallet_version_to_storage_version() { AllPalletsWithSystem, >(&db_weight); - // 3 pallets, 2 writes and every write costs 5 weight. - assert_eq!(3 * 2 * 5, weight); + // 4 pallets, 2 writes and every write costs 5 weight. + assert_eq!(4 * 2 * 5, weight); // All pallet versions should be removed assert!(sp_io::storage::get(&pallet_version_key(Example::name())).is_none()); @@ -1007,6 +1110,14 @@ fn migrate_from_pallet_version_to_storage_version() { fn metadata() { use frame_support::metadata::*; + fn maybe_docs(doc: Vec<&'static str>) -> Vec<&'static str> { + if cfg!(feature = "no-metadata-docs") { + vec![] + } else { + doc + } + } + let pallets = vec![ PalletMetadata { index: 1, @@ -1176,7 +1287,7 @@ fn metadata() { modifier: StorageEntryModifier::Default, ty: StorageEntryType::Plain(meta_type::()), default: vec![0, 0, 0, 0], - docs: vec!["Counter for the related counted storage map"], + docs: maybe_docs(vec!["Counter for the related counted storage map"]), }, StorageEntryMetadata { name: "Unbounded", @@ -1194,13 +1305,13 @@ fn metadata() { name: "MyGetParam", ty: meta_type::(), value: vec![10, 0, 0, 0], - docs: vec![" Some comment", " Some comment"], + docs: maybe_docs(vec![" Some comment", " Some comment"]), }, PalletConstantMetadata { name: "MyGetParam2", ty: meta_type::(), value: vec![11, 0, 0, 0], - docs: vec![" Some comment", " Some comment"], + docs: maybe_docs(vec![" Some comment", " Some comment"]), }, PalletConstantMetadata { name: "MyGetParam3", @@ -1212,13 +1323,19 @@ fn metadata() { name: "some_extra", ty: meta_type::(), value: vec![100, 0, 0, 0, 0, 0, 0, 0], - docs: vec![" Some doc", " Some doc"], + docs: maybe_docs(vec![" Some doc", " Some doc"]), }, PalletConstantMetadata { name: "some_extra_extra", ty: meta_type::(), value: vec![0, 0, 0, 0, 0, 0, 0, 0], - docs: vec![" Some doc"], + docs: maybe_docs(vec![" Some doc"]), + }, + PalletConstantMetadata { + name: "SomeExtraRename", + ty: meta_type::(), + value: vec![0, 0, 0, 0, 0, 0, 0, 0], + docs: maybe_docs(vec![" Some doc"]), }, ], error: Some(PalletErrorMetadata { ty: meta_type::>() }), @@ -1252,17 +1369,27 @@ fn metadata() { modifier: StorageEntryModifier::Default, ty: StorageEntryType::Plain(meta_type::()), default: vec![0, 0, 0, 0], - docs: vec!["Counter for the related counted storage map"], + docs: maybe_docs(vec!["Counter for the related counted storage map"]), }, ], }), - calls: Some(meta_type::>().into()), + calls: None, event: Some(PalletEventMetadata { ty: meta_type::() }), constants: vec![], error: None, }, ]; + let empty_doc = pallets[0].event.as_ref().unwrap().ty.type_info().docs().is_empty() && + pallets[0].error.as_ref().unwrap().ty.type_info().docs().is_empty() && + pallets[0].calls.as_ref().unwrap().ty.type_info().docs().is_empty(); + + if cfg!(feature = "no-metadata-docs") { + assert!(empty_doc) + } else { + assert!(!empty_doc) + } + let extrinsic = ExtrinsicMetadata { ty: meta_type::(), version: 4, @@ -1463,3 +1590,48 @@ fn test_storage_info() { ], ); } + +#[test] +fn assert_type_all_pallets_reversed_with_system_first_is_correct() { + // Just ensure the 2 types are same. + fn _a(_t: AllPalletsReversedWithSystemFirst) {} + fn _b(t: (System, (Example4, (Example2, (Example,))))) { + _a(t) + } +} + +#[test] +fn assert_type_all_pallets_with_system_is_correct() { + // Just ensure the 2 types are same. + fn _a(_t: AllPalletsWithSystem) {} + fn _b(t: (System, (Example, (Example2, (Example4,))))) { + _a(t) + } +} + +#[test] +fn assert_type_all_pallets_without_system_is_correct() { + // Just ensure the 2 types are same. + fn _a(_t: AllPalletsWithoutSystem) {} + fn _b(t: (Example, (Example2, (Example4,)))) { + _a(t) + } +} + +#[test] +fn assert_type_all_pallets_with_system_reversed_is_correct() { + // Just ensure the 2 types are same. + fn _a(_t: AllPalletsWithSystemReversed) {} + fn _b(t: (Example4, (Example2, (Example, (System,))))) { + _a(t) + } +} + +#[test] +fn assert_type_all_pallets_without_system_reversed_is_correct() { + // Just ensure the 2 types are same. + fn _a(_t: AllPalletsWithoutSystemReversed) {} + fn _b(t: (Example4, (Example2, (Example,)))) { + _a(t) + } +} diff --git a/frame/support/test/tests/pallet_compatibility.rs b/frame/support/test/tests/pallet_compatibility.rs index 4523063252ab..9327f5b6a330 100644 --- a/frame/support/test/tests/pallet_compatibility.rs +++ b/frame/support/test/tests/pallet_compatibility.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +15,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Old macros don't support the flag `no-metadata-docs` so the result differs when the feature is +// activated. +#![cfg(not(feature = "no-metadata-docs"))] + +use frame_support::traits::{ConstU32, ConstU64}; + pub trait SomeAssociation { type A: frame_support::dispatch::Parameter + Default; } @@ -118,6 +124,7 @@ pub mod pallet { } #[pallet::pallet] + #[pallet::without_storage_info] pub struct Pallet(_); #[pallet::hooks] @@ -217,11 +224,6 @@ pub mod pallet { } } -frame_support::parameter_types!( - pub const SomeConst: u64 = 10; - pub const BlockHashCount: u32 = 250; -); - impl frame_system::Config for Runtime { type BaseCallFilter = frame_support::traits::Everything; type Origin = Origin; @@ -234,7 +236,7 @@ impl frame_system::Config for Runtime { type Lookup = sp_runtime::traits::IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU32<250>; type BlockWeights = (); type BlockLength = (); type DbWeight = (); @@ -246,15 +248,16 @@ impl frame_system::Config for Runtime { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl pallet::Config for Runtime { type Event = Event; - type SomeConst = SomeConst; + type SomeConst = ConstU64<10>; type Balance = u64; } impl pallet_old::Config for Runtime { type Event = Event; - type SomeConst = SomeConst; + type SomeConst = ConstU64<10>; type Balance = u64; } diff --git a/frame/support/test/tests/pallet_compatibility_instance.rs b/frame/support/test/tests/pallet_compatibility_instance.rs index 768b9f28d35f..3de45df22367 100644 --- a/frame/support/test/tests/pallet_compatibility_instance.rs +++ b/frame/support/test/tests/pallet_compatibility_instance.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +15,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Old macros don't support the flag `no-metadata-docs` so the result differs when the feature is +// activated. +#![cfg(not(feature = "no-metadata-docs"))] + +use frame_support::traits::{ConstU32, ConstU64}; + mod pallet_old { use frame_support::{ decl_error, decl_event, decl_module, decl_storage, traits::Get, weights::Weight, Parameter, @@ -103,6 +109,7 @@ pub mod pallet { } #[pallet::pallet] + #[pallet::without_storage_info] pub struct Pallet(PhantomData<(T, I)>); #[pallet::hooks] @@ -197,11 +204,6 @@ pub mod pallet { } } -frame_support::parameter_types!( - pub const SomeConst: u64 = 10; - pub const BlockHashCount: u32 = 250; -); - impl frame_system::Config for Runtime { type BlockWeights = (); type BlockLength = (); @@ -217,7 +219,7 @@ impl frame_system::Config for Runtime { type Lookup = sp_runtime::traits::IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU32<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -226,35 +228,36 @@ impl frame_system::Config for Runtime { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl pallet::Config for Runtime { type Event = Event; - type SomeConst = SomeConst; + type SomeConst = ConstU64<10>; type Balance = u64; } impl pallet::Config for Runtime { type Event = Event; - type SomeConst = SomeConst; + type SomeConst = ConstU64<10>; type Balance = u64; } impl pallet::Config for Runtime { type Event = Event; - type SomeConst = SomeConst; + type SomeConst = ConstU64<10>; type Balance = u64; } impl pallet_old::Config for Runtime { type Event = Event; - type SomeConst = SomeConst; + type SomeConst = ConstU64<10>; type Balance = u64; } impl pallet_old::Config for Runtime { type Event = Event; - type SomeConst = SomeConst; + type SomeConst = ConstU64<10>; type Balance = u64; } impl pallet_old::Config for Runtime { type Event = Event; - type SomeConst = SomeConst; + type SomeConst = ConstU64<10>; type Balance = u64; } diff --git a/frame/support/test/tests/pallet_instance.rs b/frame/support/test/tests/pallet_instance.rs index 34586e841421..118794e2fa20 100644 --- a/frame/support/test/tests/pallet_instance.rs +++ b/frame/support/test/tests/pallet_instance.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,14 +18,14 @@ use frame_support::{ dispatch::UnfilteredDispatchable, storage::unhashed, - traits::{GetCallName, OnFinalize, OnGenesis, OnInitialize, OnRuntimeUpgrade}, + traits::{ConstU32, GetCallName, OnFinalize, OnGenesis, OnInitialize, OnRuntimeUpgrade}, weights::{DispatchClass, DispatchInfo, GetDispatchInfo, Pays}, }; use sp_io::{ hashing::{blake2_128, twox_128, twox_64}, TestExternalities, }; -use sp_runtime::DispatchError; +use sp_runtime::{DispatchError, ModuleError}; #[frame_support::pallet] pub mod pallet { @@ -245,11 +245,6 @@ pub mod pallet2 { } } -frame_support::parameter_types!( - pub const MyGetParam: u32 = 10; - pub const BlockHashCount: u32 = 250; -); - impl frame_system::Config for Runtime { type BaseCallFilter = frame_support::traits::Everything; type Origin = Origin; @@ -262,7 +257,7 @@ impl frame_system::Config for Runtime { type Lookup = sp_runtime::traits::IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU32<250>; type BlockWeights = (); type BlockLength = (); type DbWeight = (); @@ -274,15 +269,16 @@ impl frame_system::Config for Runtime { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl pallet::Config for Runtime { type Event = Event; - type MyGetParam = MyGetParam; + type MyGetParam = ConstU32<10>; type Balance = u64; } impl pallet::Config for Runtime { type Event = Event; - type MyGetParam = MyGetParam; + type MyGetParam = ConstU32<10>; type Balance = u64; } impl pallet2::Config for Runtime { @@ -302,13 +298,12 @@ frame_support::construct_runtime!( NodeBlock = Block, UncheckedExtrinsic = UncheckedExtrinsic { - System: frame_system::{Pallet, Call, Event}, - Example: pallet::{Pallet, Call, Event, Config, Storage, Inherent, Origin, ValidateUnsigned}, - Instance1Example: pallet::::{ - Pallet, Call, Event, Config, Storage, Inherent, Origin, ValidateUnsigned - }, - Example2: pallet2::{Pallet, Event, Config, Storage}, - Instance1Example2: pallet2::::{Pallet, Event, Config, Storage}, + // Exclude part `Storage` in order not to check its metadata in tests. + System: frame_system exclude_parts { Storage }, + Example: pallet, + Instance1Example: pallet::, + Example2: pallet2, + Instance1Example2: pallet2::, } ); @@ -346,7 +341,11 @@ fn error_expand() { ); assert_eq!( DispatchError::from(pallet::Error::::InsufficientProposersBalance), - DispatchError::Module { index: 1, error: 0, message: Some("InsufficientProposersBalance") }, + DispatchError::Module(ModuleError { + index: 1, + error: [0; 4], + message: Some("InsufficientProposersBalance") + }), ); assert_eq!( @@ -363,7 +362,11 @@ fn error_expand() { DispatchError::from( pallet::Error::::InsufficientProposersBalance ), - DispatchError::Module { index: 2, error: 0, message: Some("InsufficientProposersBalance") }, + DispatchError::Module(ModuleError { + index: 2, + error: [0; 4], + message: Some("InsufficientProposersBalance") + }), ); } @@ -505,40 +508,81 @@ fn storage_expand() { }); } +#[test] +fn pallet_metadata_expands() { + use frame_support::traits::{CrateVersion, PalletInfoData, PalletsInfoAccess}; + let mut infos = AllPalletsWithSystem::infos(); + infos.sort_by_key(|x| x.index); + assert_eq!( + infos, + vec![ + PalletInfoData { + index: 0, + name: "System", + module_name: "frame_system", + crate_version: CrateVersion { major: 4, minor: 0, patch: 0 }, + }, + PalletInfoData { + index: 1, + name: "Example", + module_name: "pallet", + crate_version: CrateVersion { major: 3, minor: 0, patch: 0 }, + }, + PalletInfoData { + index: 2, + name: "Instance1Example", + module_name: "pallet", + crate_version: CrateVersion { major: 3, minor: 0, patch: 0 }, + }, + PalletInfoData { + index: 3, + name: "Example2", + module_name: "pallet2", + crate_version: CrateVersion { major: 3, minor: 0, patch: 0 }, + }, + PalletInfoData { + index: 4, + name: "Instance1Example2", + module_name: "pallet2", + crate_version: CrateVersion { major: 3, minor: 0, patch: 0 }, + }, + ] + ); +} + #[test] fn pallet_hooks_expand() { TestExternalities::default().execute_with(|| { frame_system::Pallet::::set_block_number(1); - assert_eq!(AllPallets::on_initialize(1), 21); - AllPallets::on_finalize(1); + assert_eq!(AllPalletsWithoutSystem::on_initialize(1), 21); + AllPalletsWithoutSystem::on_finalize(1); - assert_eq!(AllPallets::on_runtime_upgrade(), 61); + assert_eq!(AllPalletsWithoutSystem::on_runtime_upgrade(), 61); - // The order is indeed reversed due to https://github.com/paritytech/substrate/issues/6280 assert_eq!( frame_system::Pallet::::events()[0].event, - Event::Instance1Example(pallet::Event::Something(11)), + Event::Example(pallet::Event::Something(10)), ); assert_eq!( frame_system::Pallet::::events()[1].event, - Event::Example(pallet::Event::Something(10)), + Event::Instance1Example(pallet::Event::Something(11)), ); assert_eq!( frame_system::Pallet::::events()[2].event, - Event::Instance1Example(pallet::Event::Something(21)), + Event::Example(pallet::Event::Something(20)), ); assert_eq!( frame_system::Pallet::::events()[3].event, - Event::Example(pallet::Event::Something(20)), + Event::Instance1Example(pallet::Event::Something(21)), ); assert_eq!( frame_system::Pallet::::events()[4].event, - Event::Instance1Example(pallet::Event::Something(31)), + Event::Example(pallet::Event::Something(30)), ); assert_eq!( frame_system::Pallet::::events()[5].event, - Event::Example(pallet::Event::Something(30)), + Event::Instance1Example(pallet::Event::Something(31)), ); }) } @@ -559,7 +603,7 @@ fn metadata() { let system_pallet_metadata = PalletMetadata { index: 0, name: "System", - storage: None, + storage: None, // The storage metadatas have been excluded. calls: Some(scale_info::meta_type::>().into()), event: Some(PalletEventMetadata { ty: scale_info::meta_type::>(), diff --git a/frame/support/test/tests/pallet_ui.rs b/frame/support/test/tests/pallet_ui.rs index 6f56c1efd6d7..a77d76deff8d 100644 --- a/frame/support/test/tests/pallet_ui.rs +++ b/frame/support/test/tests/pallet_ui.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound.stderr b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound.stderr index 3d1ea1adc986..9701b1bdd06f 100644 --- a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound.stderr +++ b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound.stderr @@ -1,5 +1,5 @@ error[E0277]: `::Bar` doesn't implement `std::fmt::Debug` - --> $DIR/call_argument_invalid_bound.rs:20:36 + --> tests/pallet_ui/call_argument_invalid_bound.rs:20:36 | 20 | pub fn foo(origin: OriginFor, bar: T::Bar) -> DispatchResultWithPostInfo { | ^^^ `::Bar` cannot be formatted using `{:?}` because it doesn't implement `std::fmt::Debug` @@ -9,19 +9,19 @@ error[E0277]: `::Bar` doesn't implement `std::fmt::Debug` = note: required for the cast to the object type `dyn std::fmt::Debug` error[E0277]: the trait bound `::Bar: Clone` is not satisfied - --> $DIR/call_argument_invalid_bound.rs:20:36 + --> tests/pallet_ui/call_argument_invalid_bound.rs:20:36 | 20 | pub fn foo(origin: OriginFor, bar: T::Bar) -> DispatchResultWithPostInfo { | ^^^ the trait `Clone` is not implemented for `::Bar` | note: required by `clone` - --> $DIR/clone.rs:121:5 + --> $RUST/core/src/clone.rs | -121 | fn clone(&self) -> Self; + | fn clone(&self) -> Self; | ^^^^^^^^^^^^^^^^^^^^^^^^ error[E0369]: binary operation `==` cannot be applied to type `&::Bar` - --> $DIR/call_argument_invalid_bound.rs:20:36 + --> tests/pallet_ui/call_argument_invalid_bound.rs:20:36 | 20 | pub fn foo(origin: OriginFor, bar: T::Bar) -> DispatchResultWithPostInfo { | ^^^ @@ -29,4 +29,4 @@ error[E0369]: binary operation `==` cannot be applied to type `&::Bar` doesn't implement `std::fmt::Debug` - --> $DIR/call_argument_invalid_bound_2.rs:20:36 + --> tests/pallet_ui/call_argument_invalid_bound_2.rs:20:36 | 20 | pub fn foo(origin: OriginFor, bar: T::Bar) -> DispatchResultWithPostInfo { | ^^^ `::Bar` cannot be formatted using `{:?}` because it doesn't implement `std::fmt::Debug` @@ -9,19 +9,19 @@ error[E0277]: `::Bar` doesn't implement `std::fmt::Debug` = note: required for the cast to the object type `dyn std::fmt::Debug` error[E0277]: the trait bound `::Bar: Clone` is not satisfied - --> $DIR/call_argument_invalid_bound_2.rs:20:36 + --> tests/pallet_ui/call_argument_invalid_bound_2.rs:20:36 | 20 | pub fn foo(origin: OriginFor, bar: T::Bar) -> DispatchResultWithPostInfo { | ^^^ the trait `Clone` is not implemented for `::Bar` | note: required by `clone` - --> $DIR/clone.rs:121:5 + --> $RUST/core/src/clone.rs | -121 | fn clone(&self) -> Self; + | fn clone(&self) -> Self; | ^^^^^^^^^^^^^^^^^^^^^^^^ error[E0369]: binary operation `==` cannot be applied to type `&::Bar` - --> $DIR/call_argument_invalid_bound_2.rs:20:36 + --> tests/pallet_ui/call_argument_invalid_bound_2.rs:20:36 | 20 | pub fn foo(origin: OriginFor, bar: T::Bar) -> DispatchResultWithPostInfo { | ^^^ @@ -29,30 +29,41 @@ error[E0369]: binary operation `==` cannot be applied to type `&::Bar: WrapperTypeEncode` is not satisfied - --> $DIR/call_argument_invalid_bound_2.rs:20:36 + --> tests/pallet_ui/call_argument_invalid_bound_2.rs:1:1 | -20 | pub fn foo(origin: OriginFor, bar: T::Bar) -> DispatchResultWithPostInfo { - | ^^^ the trait `WrapperTypeEncode` is not implemented for `::Bar` +1 | #[frame_support::pallet] + | ^----------------------- + | | + | _in this procedural macro expansion + | | +2 | | mod pallet { +3 | | use frame_support::pallet_prelude::{Hooks, DispatchResultWithPostInfo}; +4 | | use frame_system::pallet_prelude::{BlockNumberFor, OriginFor}; +... | +16 | | +17 | | #[pallet::call] + | |__________________^ the trait `WrapperTypeEncode` is not implemented for `::Bar` | - ::: $CARGO/parity-scale-codec-2.2.0/src/codec.rs + = note: required because of the requirements on the impl of `Encode` for `::Bar` +note: required by a bound in `encode_to` + --> $CARGO/parity-scale-codec-3.0.0/src/codec.rs | | fn encode_to(&self, dest: &mut T) { - | ------ required by this bound in `encode_to` - | - = note: required because of the requirements on the impl of `Encode` for `::Bar` + | ^^^^^^ required by this bound in `encode_to` + = note: this error originates in the derive macro `frame_support::codec::Encode` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0277]: the trait bound `::Bar: WrapperTypeDecode` is not satisfied - --> $DIR/call_argument_invalid_bound_2.rs:20:36 + --> tests/pallet_ui/call_argument_invalid_bound_2.rs:17:12 | -20 | pub fn foo(origin: OriginFor, bar: T::Bar) -> DispatchResultWithPostInfo { - | ^^^ the trait `WrapperTypeDecode` is not implemented for `::Bar` +17 | #[pallet::call] + | ^^^^ the trait `WrapperTypeDecode` is not implemented for `::Bar` | - ::: $CARGO/parity-scale-codec-2.2.0/src/codec.rs + = note: required because of the requirements on the impl of `Decode` for `::Bar` +note: required by a bound in `parity_scale_codec::Decode::decode` + --> $CARGO/parity-scale-codec-3.0.0/src/codec.rs | | fn decode(input: &mut I) -> Result; - | ----- required by this bound in `pallet::_::_parity_scale_codec::Decode::decode` - | - = note: required because of the requirements on the impl of `Decode` for `::Bar` + | ^^^^^ required by this bound in `parity_scale_codec::Decode::decode` diff --git a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_3.rs b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_3.rs index e7f99d7ca4f2..1cdfb369fead 100644 --- a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_3.rs +++ b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_3.rs @@ -1,8 +1,8 @@ #[frame_support::pallet] mod pallet { - use frame_support::pallet_prelude::{Hooks, DispatchResultWithPostInfo}; + use codec::{Decode, Encode}; + use frame_support::pallet_prelude::{DispatchResultWithPostInfo, Hooks}; use frame_system::pallet_prelude::{BlockNumberFor, OriginFor}; - use codec::{Encode, Decode}; #[pallet::config] pub trait Config: frame_system::Config {} @@ -13,7 +13,7 @@ mod pallet { #[pallet::hooks] impl Hooks> for Pallet {} - #[derive(Encode, Decode, scale_info::TypeInfo)] + #[derive(Encode, Decode, scale_info::TypeInfo, PartialEq, Clone)] struct Bar; #[pallet::call] @@ -25,5 +25,4 @@ mod pallet { } } -fn main() { -} +fn main() {} diff --git a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_3.stderr b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_3.stderr index 144b7e12bd66..c196b28a31ce 100644 --- a/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_3.stderr +++ b/frame/support/test/tests/pallet_ui/call_argument_invalid_bound_3.stderr @@ -1,5 +1,5 @@ error[E0277]: `Bar` doesn't implement `std::fmt::Debug` - --> $DIR/call_argument_invalid_bound_3.rs:22:36 + --> tests/pallet_ui/call_argument_invalid_bound_3.rs:22:36 | 22 | pub fn foo(origin: OriginFor, bar: Bar) -> DispatchResultWithPostInfo { | ^^^ `Bar` cannot be formatted using `{:?}` @@ -8,23 +8,3 @@ error[E0277]: `Bar` doesn't implement `std::fmt::Debug` = note: add `#[derive(Debug)]` to `Bar` or manually `impl std::fmt::Debug for Bar` = note: required because of the requirements on the impl of `std::fmt::Debug` for `&Bar` = note: required for the cast to the object type `dyn std::fmt::Debug` - -error[E0277]: the trait bound `Bar: Clone` is not satisfied - --> $DIR/call_argument_invalid_bound_3.rs:22:36 - | -22 | pub fn foo(origin: OriginFor, bar: Bar) -> DispatchResultWithPostInfo { - | ^^^ the trait `Clone` is not implemented for `Bar` - | -note: required by `clone` - --> $DIR/clone.rs:121:5 - | -121 | fn clone(&self) -> Self; - | ^^^^^^^^^^^^^^^^^^^^^^^^ - -error[E0369]: binary operation `==` cannot be applied to type `&Bar` - --> $DIR/call_argument_invalid_bound_3.rs:22:36 - | -22 | pub fn foo(origin: OriginFor, bar: Bar) -> DispatchResultWithPostInfo { - | ^^^ - | - = note: an implementation of `std::cmp::PartialEq` might be missing for `&Bar` diff --git a/frame/support/test/tests/pallet_ui/error_no_fieldless.rs b/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.rs similarity index 50% rename from frame/support/test/tests/pallet_ui/error_no_fieldless.rs rename to frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.rs index c9d444d6f90d..254d65866774 100644 --- a/frame/support/test/tests/pallet_ui/error_no_fieldless.rs +++ b/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.rs @@ -1,25 +1,19 @@ #[frame_support::pallet] mod pallet { - use frame_support::pallet_prelude::Hooks; - use frame_system::pallet_prelude::BlockNumberFor; - #[pallet::config] pub trait Config: frame_system::Config {} #[pallet::pallet] pub struct Pallet(core::marker::PhantomData); - #[pallet::hooks] - impl Hooks> for Pallet {} - - #[pallet::call] - impl Pallet {} - #[pallet::error] pub enum Error { - U8(u8), + CustomError(crate::MyError), } } +#[derive(scale_info::TypeInfo, codec::Encode, codec::Decode)] +enum MyError {} + fn main() { } diff --git a/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.stderr b/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.stderr new file mode 100644 index 000000000000..2a8149e309ac --- /dev/null +++ b/frame/support/test/tests/pallet_ui/error_does_not_derive_pallet_error.stderr @@ -0,0 +1,12 @@ +error[E0277]: the trait bound `MyError: PalletError` is not satisfied + --> tests/pallet_ui/error_does_not_derive_pallet_error.rs:1:1 + | +1 | #[frame_support::pallet] + | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `PalletError` is not implemented for `MyError` + | +note: required by `MAX_ENCODED_SIZE` + --> $WORKSPACE/frame/support/src/traits/error.rs + | + | const MAX_ENCODED_SIZE: usize; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + = note: this error originates in the derive macro `frame_support::PalletError` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/frame/support/test/tests/pallet_ui/error_no_fieldless.stderr b/frame/support/test/tests/pallet_ui/error_no_fieldless.stderr deleted file mode 100644 index 1d69fbeff9aa..000000000000 --- a/frame/support/test/tests/pallet_ui/error_no_fieldless.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: Invalid pallet::error, unexpected fields, must be `Unit` - --> $DIR/error_no_fieldless.rs:20:5 - | -20 | U8(u8), - | ^^^^ diff --git a/frame/support/test/tests/pallet_ui/event_field_not_member.stderr b/frame/support/test/tests/pallet_ui/event_field_not_member.stderr index bf4c05bb4e5b..ff184db988e3 100644 --- a/frame/support/test/tests/pallet_ui/event_field_not_member.stderr +++ b/frame/support/test/tests/pallet_ui/event_field_not_member.stderr @@ -1,17 +1,17 @@ error[E0277]: the trait bound `::Bar: Clone` is not satisfied - --> $DIR/event_field_not_member.rs:23:7 + --> tests/pallet_ui/event_field_not_member.rs:23:7 | 23 | B { b: T::Bar }, | ^ the trait `Clone` is not implemented for `::Bar` | note: required by `clone` - --> $DIR/clone.rs:121:5 + --> $RUST/core/src/clone.rs | -121 | fn clone(&self) -> Self; + | fn clone(&self) -> Self; | ^^^^^^^^^^^^^^^^^^^^^^^^ error[E0369]: binary operation `==` cannot be applied to type `&::Bar` - --> $DIR/event_field_not_member.rs:23:7 + --> tests/pallet_ui/event_field_not_member.rs:23:7 | 23 | B { b: T::Bar }, | ^ @@ -19,10 +19,10 @@ error[E0369]: binary operation `==` cannot be applied to type `& { - | ^^^^^^^^^^^^^^^^^^^^^ + | +++++++++++++++++++++ error[E0277]: `::Bar` doesn't implement `std::fmt::Debug` - --> $DIR/event_field_not_member.rs:23:7 + --> tests/pallet_ui/event_field_not_member.rs:23:7 | 23 | B { b: T::Bar }, | ^ `::Bar` cannot be formatted using `{:?}` because it doesn't implement `std::fmt::Debug` diff --git a/frame/support/test/tests/pallet_ui/genesis_default_not_satisfied.stderr b/frame/support/test/tests/pallet_ui/genesis_default_not_satisfied.stderr index ad8300b8d89b..e01f66fa3d19 100644 --- a/frame/support/test/tests/pallet_ui/genesis_default_not_satisfied.stderr +++ b/frame/support/test/tests/pallet_ui/genesis_default_not_satisfied.stderr @@ -1,10 +1,11 @@ error[E0277]: the trait bound `pallet::GenesisConfig: std::default::Default` is not satisfied - --> $DIR/genesis_default_not_satisfied.rs:22:18 + --> tests/pallet_ui/genesis_default_not_satisfied.rs:22:18 | 22 | impl GenesisBuild for GenesisConfig {} | ^^^^^^^^^^^^^^^ the trait `std::default::Default` is not implemented for `pallet::GenesisConfig` | - ::: $WORKSPACE/frame/support/src/traits/hooks.rs +note: required by a bound in `GenesisBuild` + --> $WORKSPACE/frame/support/src/traits/hooks.rs | | pub trait GenesisBuild: Default + sp_runtime::traits::MaybeSerializeDeserialize { - | ------- required by this bound in `GenesisBuild` + | ^^^^^^^ required by this bound in `GenesisBuild` diff --git a/frame/support/test/tests/pallet_ui/hooks_invalid_item.stderr b/frame/support/test/tests/pallet_ui/hooks_invalid_item.stderr index ecb57bec37a7..d1a89fbb850e 100644 --- a/frame/support/test/tests/pallet_ui/hooks_invalid_item.stderr +++ b/frame/support/test/tests/pallet_ui/hooks_invalid_item.stderr @@ -12,4 +12,4 @@ note: trait defined here, with 1 generic parameter: `BlockNumber` help: add missing generic argument | 12 | impl Hooks for Pallet {} - | ^^^^^^^^^^^^^^^^^^ + | ~~~~~~~~~~~~~~~~~~ diff --git a/frame/support/test/tests/pallet_ui/pass/error_nested_types.rs b/frame/support/test/tests/pallet_ui/pass/error_nested_types.rs new file mode 100644 index 000000000000..1b6f584af23b --- /dev/null +++ b/frame/support/test/tests/pallet_ui/pass/error_nested_types.rs @@ -0,0 +1,41 @@ +use codec::{Decode, Encode}; +use frame_support::PalletError; + +#[frame_support::pallet] +mod pallet { + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::pallet] + pub struct Pallet(core::marker::PhantomData); + + #[pallet::error] + pub enum Error { + CustomError(crate::MyError), + } +} + +#[derive(Encode, Decode, PalletError, scale_info::TypeInfo)] +pub enum MyError { + Foo, + Bar, + Baz(NestedError), + Struct(MyStruct), + Wrapper(Wrapper), +} + +#[derive(Encode, Decode, PalletError, scale_info::TypeInfo)] +pub enum NestedError { + Quux +} + +#[derive(Encode, Decode, PalletError, scale_info::TypeInfo)] +pub struct MyStruct { + field: u8, +} + +#[derive(Encode, Decode, PalletError, scale_info::TypeInfo)] +pub struct Wrapper(bool); + +fn main() { +} diff --git a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.rs b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.rs index 30b6d651f3b8..fe4682c401fa 100644 --- a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.rs +++ b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.rs @@ -7,6 +7,7 @@ mod pallet { pub trait Config: frame_system::Config {} #[pallet::pallet] + #[pallet::without_storage_info] pub struct Pallet(core::marker::PhantomData); #[pallet::hooks] diff --git a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr index cd3032c49735..35f8bbdbd524 100644 --- a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr +++ b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr @@ -1,51 +1,97 @@ +error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.rs:10:12 + | +10 | #[pallet::without_storage_info] + | ^^^^^^^^^^^^^^^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` + | + = note: required because of the requirements on the impl of `Decode` for `Bar` + = note: required because of the requirements on the impl of `FullCodec` for `Bar` + = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` +note: required by `partial_storage_info` + --> $WORKSPACE/frame/support/src/traits/storage.rs + | + | fn partial_storage_info() -> Vec; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.rs:10:12 + | +10 | #[pallet::without_storage_info] + | ^^^^^^^^^^^^^^^^^^^^ the trait `EncodeLike` is not implemented for `Bar` + | + = note: required because of the requirements on the impl of `FullEncode` for `Bar` + = note: required because of the requirements on the impl of `FullCodec` for `Bar` + = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` +note: required by `partial_storage_info` + --> $WORKSPACE/frame/support/src/traits/storage.rs + | + | fn partial_storage_info() -> Vec; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.rs:10:12 + | +10 | #[pallet::without_storage_info] + | ^^^^^^^^^^^^^^^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` + | + = note: required because of the requirements on the impl of `Encode` for `Bar` + = note: required because of the requirements on the impl of `FullEncode` for `Bar` + = note: required because of the requirements on the impl of `FullCodec` for `Bar` + = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` +note: required by `partial_storage_info` + --> $WORKSPACE/frame/support/src/traits/storage.rs + | + | fn partial_storage_info() -> Vec; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + error[E0277]: the trait bound `Bar: TypeInfo` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen.rs:20:12 + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.rs:21:12 | -20 | #[pallet::storage] +21 | #[pallet::storage] | ^^^^^^^ the trait `TypeInfo` is not implemented for `Bar` | = note: required because of the requirements on the impl of `StaticTypeInfo` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` note: required by `build_metadata` - --> $DIR/mod.rs:113:2 + --> $WORKSPACE/frame/support/src/storage/types/mod.rs | -113 | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); + | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen.rs:20:12 + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.rs:21:12 | -20 | #[pallet::storage] +21 | #[pallet::storage] | ^^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` | = note: required because of the requirements on the impl of `Decode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` note: required by `build_metadata` - --> $DIR/mod.rs:113:2 + --> $WORKSPACE/frame/support/src/storage/types/mod.rs | -113 | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); + | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen.rs:20:12 + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.rs:21:12 | -20 | #[pallet::storage] +21 | #[pallet::storage] | ^^^^^^^ the trait `EncodeLike` is not implemented for `Bar` | = note: required because of the requirements on the impl of `FullEncode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` note: required by `build_metadata` - --> $DIR/mod.rs:113:2 + --> $WORKSPACE/frame/support/src/storage/types/mod.rs | -113 | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); + | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen.rs:20:12 + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.rs:21:12 | -20 | #[pallet::storage] +21 | #[pallet::storage] | ^^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` | = note: required because of the requirements on the impl of `Encode` for `Bar` @@ -53,53 +99,7 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` note: required by `build_metadata` - --> $DIR/mod.rs:113:2 + --> $WORKSPACE/frame/support/src/storage/types/mod.rs | -113 | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); + | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen.rs:9:12 - | -9 | #[pallet::pallet] - | ^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` - | - = note: required because of the requirements on the impl of `Decode` for `Bar` - = note: required because of the requirements on the impl of `FullCodec` for `Bar` - = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` -note: required by `partial_storage_info` - --> $DIR/storage.rs:88:2 - | -88 | fn partial_storage_info() -> Vec; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen.rs:9:12 - | -9 | #[pallet::pallet] - | ^^^^^^ the trait `EncodeLike` is not implemented for `Bar` - | - = note: required because of the requirements on the impl of `FullEncode` for `Bar` - = note: required because of the requirements on the impl of `FullCodec` for `Bar` - = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` -note: required by `partial_storage_info` - --> $DIR/storage.rs:88:2 - | -88 | fn partial_storage_info() -> Vec; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen.rs:9:12 - | -9 | #[pallet::pallet] - | ^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` - | - = note: required because of the requirements on the impl of `Encode` for `Bar` - = note: required because of the requirements on the impl of `FullEncode` for `Bar` - = note: required because of the requirements on the impl of `FullCodec` for `Bar` - = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` -note: required by `partial_storage_info` - --> $DIR/storage.rs:88:2 - | -88 | fn partial_storage_info() -> Vec; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs index ddb19121660d..82512a89fb15 100644 --- a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs +++ b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs @@ -7,6 +7,7 @@ mod pallet { pub trait Config: frame_system::Config {} #[pallet::pallet] + #[pallet::without_storage_info] pub struct Pallet(core::marker::PhantomData); #[pallet::hooks] diff --git a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr index 3d03af836986..b5f250bb8971 100644 --- a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr +++ b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr @@ -1,51 +1,97 @@ +error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:10:12 + | +10 | #[pallet::without_storage_info] + | ^^^^^^^^^^^^^^^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` + | + = note: required because of the requirements on the impl of `Decode` for `Bar` + = note: required because of the requirements on the impl of `FullCodec` for `Bar` + = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` +note: required by `partial_storage_info` + --> $WORKSPACE/frame/support/src/traits/storage.rs + | + | fn partial_storage_info() -> Vec; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:10:12 + | +10 | #[pallet::without_storage_info] + | ^^^^^^^^^^^^^^^^^^^^ the trait `EncodeLike` is not implemented for `Bar` + | + = note: required because of the requirements on the impl of `FullEncode` for `Bar` + = note: required because of the requirements on the impl of `FullCodec` for `Bar` + = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` +note: required by `partial_storage_info` + --> $WORKSPACE/frame/support/src/traits/storage.rs + | + | fn partial_storage_info() -> Vec; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:10:12 + | +10 | #[pallet::without_storage_info] + | ^^^^^^^^^^^^^^^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` + | + = note: required because of the requirements on the impl of `Encode` for `Bar` + = note: required because of the requirements on the impl of `FullEncode` for `Bar` + = note: required because of the requirements on the impl of `FullCodec` for `Bar` + = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` +note: required by `partial_storage_info` + --> $WORKSPACE/frame/support/src/traits/storage.rs + | + | fn partial_storage_info() -> Vec; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + error[E0277]: the trait bound `Bar: TypeInfo` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:20:12 + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:21:12 | -20 | #[pallet::storage] +21 | #[pallet::storage] | ^^^^^^^ the trait `TypeInfo` is not implemented for `Bar` | = note: required because of the requirements on the impl of `StaticTypeInfo` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` note: required by `build_metadata` - --> $DIR/mod.rs:113:2 + --> $WORKSPACE/frame/support/src/storage/types/mod.rs | -113 | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); + | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:20:12 + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:21:12 | -20 | #[pallet::storage] +21 | #[pallet::storage] | ^^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` | = note: required because of the requirements on the impl of `Decode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` note: required by `build_metadata` - --> $DIR/mod.rs:113:2 + --> $WORKSPACE/frame/support/src/storage/types/mod.rs | -113 | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); + | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:20:12 + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:21:12 | -20 | #[pallet::storage] +21 | #[pallet::storage] | ^^^^^^^ the trait `EncodeLike` is not implemented for `Bar` | = note: required because of the requirements on the impl of `FullEncode` for `Bar` = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` note: required by `build_metadata` - --> $DIR/mod.rs:113:2 + --> $WORKSPACE/frame/support/src/storage/types/mod.rs | -113 | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); + | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:20:12 + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:21:12 | -20 | #[pallet::storage] +21 | #[pallet::storage] | ^^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` | = note: required because of the requirements on the impl of `Encode` for `Bar` @@ -53,53 +99,7 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied = note: required because of the requirements on the impl of `FullCodec` for `Bar` = note: required because of the requirements on the impl of `StorageEntryMetadataBuilder` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` note: required by `build_metadata` - --> $DIR/mod.rs:113:2 + --> $WORKSPACE/frame/support/src/storage/types/mod.rs | -113 | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); + | fn build_metadata(doc: Vec<&'static str>, entries: &mut Vec); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:9:12 - | -9 | #[pallet::pallet] - | ^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` - | - = note: required because of the requirements on the impl of `Decode` for `Bar` - = note: required because of the requirements on the impl of `FullCodec` for `Bar` - = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` -note: required by `partial_storage_info` - --> $DIR/storage.rs:88:2 - | -88 | fn partial_storage_info() -> Vec; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:9:12 - | -9 | #[pallet::pallet] - | ^^^^^^ the trait `EncodeLike` is not implemented for `Bar` - | - = note: required because of the requirements on the impl of `FullEncode` for `Bar` - = note: required because of the requirements on the impl of `FullCodec` for `Bar` - = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` -note: required by `partial_storage_info` - --> $DIR/storage.rs:88:2 - | -88 | fn partial_storage_info() -> Vec; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied - --> $DIR/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:9:12 - | -9 | #[pallet::pallet] - | ^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` - | - = note: required because of the requirements on the impl of `Encode` for `Bar` - = note: required because of the requirements on the impl of `FullEncode` for `Bar` - = note: required because of the requirements on the impl of `FullCodec` for `Bar` - = note: required because of the requirements on the impl of `PartialStorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` -note: required by `partial_storage_info` - --> $DIR/storage.rs:88:2 - | -88 | fn partial_storage_info() -> Vec; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied.rs b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied.rs index 76e356610064..4d43e3a17a9e 100644 --- a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied.rs +++ b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied.rs @@ -7,7 +7,6 @@ mod pallet { pub trait Config: frame_system::Config {} #[pallet::pallet] - #[pallet::generate_storage_info] pub struct Pallet(core::marker::PhantomData); #[pallet::hooks] diff --git a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied.stderr b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied.stderr index 0ffb015e36bc..35537cfbc9e0 100644 --- a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied.stderr +++ b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied.stderr @@ -1,12 +1,12 @@ error[E0277]: the trait bound `Bar: MaxEncodedLen` is not satisfied - --> $DIR/storage_info_unsatisfied.rs:10:12 + --> tests/pallet_ui/storage_info_unsatisfied.rs:9:12 | -10 | #[pallet::generate_storage_info] - | ^^^^^^^^^^^^^^^^^^^^^ the trait `MaxEncodedLen` is not implemented for `Bar` +9 | #[pallet::pallet] + | ^^^^^^ the trait `MaxEncodedLen` is not implemented for `Bar` | = note: required because of the requirements on the impl of `StorageInfoTrait` for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` note: required by `storage_info` - --> $DIR/storage.rs:71:2 + --> $WORKSPACE/frame/support/src/traits/storage.rs | -71 | fn storage_info() -> Vec; + | fn storage_info() -> Vec; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.rs b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.rs index c5d773d71611..dd10bc0723fe 100644 --- a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.rs +++ b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.rs @@ -1,13 +1,15 @@ #[frame_support::pallet] mod pallet { - use frame_support::pallet_prelude::{Hooks, StorageNMap, Twox64Concat, NMapKey}; + use frame_support::{ + pallet_prelude::{Hooks, Twox64Concat}, + storage::types::{StorageNMap, Key}, + }; use frame_system::pallet_prelude::BlockNumberFor; #[pallet::config] pub trait Config: frame_system::Config {} #[pallet::pallet] - #[pallet::generate_storage_info] pub struct Pallet(core::marker::PhantomData); #[pallet::hooks] @@ -20,8 +22,7 @@ mod pallet { struct Bar; #[pallet::storage] - type Foo = StorageNMap<_, NMapKey, u32>; + type Foo = StorageNMap<_, Key, u32>; } -fn main() { -} +fn main() {} diff --git a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr index 2b70102fdac2..fb6580bb5a3e 100644 --- a/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr +++ b/frame/support/test/tests/pallet_ui/storage_info_unsatisfied_nmap.stderr @@ -1,13 +1,13 @@ error[E0277]: the trait bound `Bar: MaxEncodedLen` is not satisfied - --> $DIR/storage_info_unsatisfied_nmap.rs:10:12 + --> tests/pallet_ui/storage_info_unsatisfied_nmap.rs:12:12 | -10 | #[pallet::generate_storage_info] - | ^^^^^^^^^^^^^^^^^^^^^ the trait `MaxEncodedLen` is not implemented for `Bar` +12 | #[pallet::pallet] + | ^^^^^^ the trait `MaxEncodedLen` is not implemented for `Bar` | = note: required because of the requirements on the impl of `KeyGeneratorMaxEncodedLen` for `Key` = note: required because of the requirements on the impl of `StorageInfoTrait` for `frame_support::pallet_prelude::StorageNMap<_GeneratedPrefixForStorageFoo, Key, u32>` note: required by `storage_info` - --> $DIR/storage.rs:71:2 + --> $WORKSPACE/frame/support/src/traits/storage.rs | -71 | fn storage_info() -> Vec; + | fn storage_info() -> Vec; | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/frame/support/test/tests/pallet_ui/type_value_forgotten_where_clause.stderr b/frame/support/test/tests/pallet_ui/type_value_forgotten_where_clause.stderr index 85d7342b253d..6288dcd534b6 100644 --- a/frame/support/test/tests/pallet_ui/type_value_forgotten_where_clause.stderr +++ b/frame/support/test/tests/pallet_ui/type_value_forgotten_where_clause.stderr @@ -1,47 +1,53 @@ error[E0277]: the trait bound `::AccountId: From` is not satisfied --> $DIR/type_value_forgotten_where_clause.rs:24:34 | -7 | pub trait Config: frame_system::Config - | ------ required by a bound in this -8 | where ::AccountId: From - | --------- required by this bound in `pallet::Config` -... 24 | #[pallet::type_value] fn Foo() -> u32 { 3u32 } | ^^^^^^ the trait `From` is not implemented for `::AccountId` | +note: required by a bound in `pallet::Config` + --> $DIR/type_value_forgotten_where_clause.rs:8:51 + | +7 | pub trait Config: frame_system::Config + | ------ required by a bound in this +8 | where ::AccountId: From + | ^^^^^^^^^ required by this bound in `pallet::Config` help: consider further restricting the associated type | 24 | #[pallet::type_value] fn Foo() -> u32 where ::AccountId: From { 3u32 } - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +++++++++++++++++++++++++++++++++++++++++++++++++++++++ error[E0277]: the trait bound `::AccountId: From` is not satisfied --> $DIR/type_value_forgotten_where_clause.rs:24:12 | -7 | pub trait Config: frame_system::Config - | ------ required by a bound in this -8 | where ::AccountId: From - | --------- required by this bound in `pallet::Config` -... 24 | #[pallet::type_value] fn Foo() -> u32 { 3u32 } | ^^^^^^^^^^ the trait `From` is not implemented for `::AccountId` | +note: required by a bound in `pallet::Config` + --> $DIR/type_value_forgotten_where_clause.rs:8:51 + | +7 | pub trait Config: frame_system::Config + | ------ required by a bound in this +8 | where ::AccountId: From + | ^^^^^^^^^ required by this bound in `pallet::Config` help: consider further restricting the associated type | 24 | #[pallet::type_value where ::AccountId: From] fn Foo() -> u32 { 3u32 } - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +++++++++++++++++++++++++++++++++++++++++++++++++++++++ error[E0277]: the trait bound `::AccountId: From` is not satisfied --> $DIR/type_value_forgotten_where_clause.rs:24:12 | -7 | pub trait Config: frame_system::Config - | ------ required by a bound in this -8 | where ::AccountId: From - | --------- required by this bound in `pallet::Config` -... 24 | #[pallet::type_value] fn Foo() -> u32 { 3u32 } | ^^^^^^^^^^ the trait `From` is not implemented for `::AccountId` | +note: required by a bound in `pallet::Config` + --> $DIR/type_value_forgotten_where_clause.rs:8:51 + | +7 | pub trait Config: frame_system::Config + | ------ required by a bound in this +8 | where ::AccountId: From + | ^^^^^^^^^ required by this bound in `pallet::Config` help: consider further restricting the associated type | 24 | #[pallet::type_value] fn Foo() -> u32 where ::AccountId: From { 3u32 } - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | +++++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/frame/support/test/tests/pallet_with_name_trait_is_valid.rs b/frame/support/test/tests/pallet_with_name_trait_is_valid.rs index 1c47d13a619f..7ed845466832 100644 --- a/frame/support/test/tests/pallet_with_name_trait_is_valid.rs +++ b/frame/support/test/tests/pallet_with_name_trait_is_valid.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -97,7 +97,7 @@ impl frame_support::inherent::ProvideInherent for Module { mod tests { use crate as pallet_test; - use frame_support::parameter_types; + use frame_support::traits::ConstU64; type SignedExtra = ( frame_system::CheckEra, @@ -124,10 +124,6 @@ mod tests { } ); - parameter_types! { - pub const BlockHashCount: u64 = 250; - } - impl frame_system::Config for Runtime { type BaseCallFilter = frame_support::traits::Everything; type Origin = Origin; @@ -140,7 +136,7 @@ mod tests { type Lookup = sp_runtime::traits::IdentityLookup; type Header = TestHeader; type Event = (); - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type DbWeight = (); type BlockWeights = (); type BlockLength = (); @@ -152,6 +148,7 @@ mod tests { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } impl pallet_test::Trait for Runtime { diff --git a/frame/support/test/tests/reserved_keyword/on_initialize.rs b/frame/support/test/tests/reserved_keyword/on_initialize.rs deleted file mode 100644 index 72d53abfb103..000000000000 --- a/frame/support/test/tests/reserved_keyword/on_initialize.rs +++ /dev/null @@ -1,30 +0,0 @@ -macro_rules! reserved { - ($($reserved:ident)*) => { - $( - mod $reserved { - pub use frame_support::dispatch; - - pub trait Config: frame_support_test::Config {} - - pub mod system { - use frame_support::dispatch; - - pub fn ensure_root(_: R) -> dispatch::DispatchResult { - Ok(()) - } - } - - frame_support::decl_module! { - pub struct Module for enum Call where origin: T::Origin, system=frame_support_test { - #[weight = 0] - fn $reserved(_origin) -> dispatch::DispatchResult { unreachable!() } - } - } - } - )* - } -} - -reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); - -fn main() {} diff --git a/frame/support/test/tests/reserved_keyword/on_initialize.stderr b/frame/support/test/tests/reserved_keyword/on_initialize.stderr deleted file mode 100644 index 84e93fa52c2d..000000000000 --- a/frame/support/test/tests/reserved_keyword/on_initialize.stderr +++ /dev/null @@ -1,39 +0,0 @@ -error: Invalid call fn name: `on_finalize`, name is reserved and doesn't match expected signature, please refer to `decl_module!` documentation to see the appropriate usage, or rename it to an unreserved keyword. - --> $DIR/on_initialize.rs:28:1 - | -28 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: this error originates in the macro `$crate::__check_reserved_fn_name` (in Nightly builds, run with -Z macro-backtrace for more info) - -error: Invalid call fn name: `on_initialize`, name is reserved and doesn't match expected signature, please refer to `decl_module!` documentation to see the appropriate usage, or rename it to an unreserved keyword. - --> $DIR/on_initialize.rs:28:1 - | -28 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: this error originates in the macro `$crate::__check_reserved_fn_name` (in Nightly builds, run with -Z macro-backtrace for more info) - -error: Invalid call fn name: `on_runtime_upgrade`, name is reserved and doesn't match expected signature, please refer to `decl_module!` documentation to see the appropriate usage, or rename it to an unreserved keyword. - --> $DIR/on_initialize.rs:28:1 - | -28 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: this error originates in the macro `$crate::__check_reserved_fn_name` (in Nightly builds, run with -Z macro-backtrace for more info) - -error: Invalid call fn name: `offchain_worker`, name is reserved and doesn't match expected signature, please refer to `decl_module!` documentation to see the appropriate usage, or rename it to an unreserved keyword. - --> $DIR/on_initialize.rs:28:1 - | -28 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: this error originates in the macro `$crate::__check_reserved_fn_name` (in Nightly builds, run with -Z macro-backtrace for more info) - -error: Invalid call fn name: `deposit_event`, name is reserved and doesn't match expected signature, please refer to `decl_module!` documentation to see the appropriate usage, or rename it to an unreserved keyword. - --> $DIR/on_initialize.rs:28:1 - | -28 | reserved!(on_finalize on_initialize on_runtime_upgrade offchain_worker deposit_event); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | - = note: this error originates in the macro `$crate::__check_reserved_fn_name` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/frame/support/test/tests/storage_transaction.rs b/frame/support/test/tests/storage_transaction.rs index 4e97a87377b1..848a91a7f5a8 100644 --- a/frame/support/test/tests/storage_transaction.rs +++ b/frame/support/test/tests/storage_transaction.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,12 +16,13 @@ // limitations under the License. use frame_support::{ - assert_noop, assert_ok, + assert_noop, assert_ok, assert_storage_noop, dispatch::{DispatchError, DispatchResult}, storage::{with_transaction, TransactionOutcome::*}, transactional, StorageMap, StorageValue, }; use sp_io::TestExternalities; +use sp_runtime::TransactionOutcome; use sp_std::result; pub trait Config: frame_support_test::Config {} @@ -67,13 +68,13 @@ fn storage_transaction_basic_commit() { assert_eq!(Value::get(), 0); assert!(!Map::contains_key("val0")); - with_transaction(|| { + assert_ok!(with_transaction(|| -> TransactionOutcome { Value::set(99); Map::insert("val0", 99); assert_eq!(Value::get(), 99); assert_eq!(Map::get("val0"), 99); - Commit(()) - }); + Commit(Ok(())) + })); assert_eq!(Value::get(), 99); assert_eq!(Map::get("val0"), 99); @@ -86,13 +87,26 @@ fn storage_transaction_basic_rollback() { assert_eq!(Value::get(), 0); assert_eq!(Map::get("val0"), 0); - with_transaction(|| { - Value::set(99); - Map::insert("val0", 99); - assert_eq!(Value::get(), 99); - assert_eq!(Map::get("val0"), 99); - Rollback(()) - }); + assert_noop!( + with_transaction(|| -> TransactionOutcome { + Value::set(99); + Map::insert("val0", 99); + assert_eq!(Value::get(), 99); + assert_eq!(Map::get("val0"), 99); + Rollback(Err("revert".into())) + }), + "revert" + ); + + assert_storage_noop!(assert_ok!(with_transaction( + || -> TransactionOutcome { + Value::set(99); + Map::insert("val0", 99); + assert_eq!(Value::get(), 99); + assert_eq!(Map::get("val0"), 99); + Rollback(Ok(())) + } + ))); assert_eq!(Value::get(), 0); assert_eq!(Map::get("val0"), 0); @@ -105,32 +119,35 @@ fn storage_transaction_rollback_then_commit() { Value::set(1); Map::insert("val1", 1); - with_transaction(|| { + assert_ok!(with_transaction(|| -> TransactionOutcome { Value::set(2); Map::insert("val1", 2); Map::insert("val2", 2); - with_transaction(|| { - Value::set(3); - Map::insert("val1", 3); - Map::insert("val2", 3); - Map::insert("val3", 3); + assert_noop!( + with_transaction(|| -> TransactionOutcome { + Value::set(3); + Map::insert("val1", 3); + Map::insert("val2", 3); + Map::insert("val3", 3); - assert_eq!(Value::get(), 3); - assert_eq!(Map::get("val1"), 3); - assert_eq!(Map::get("val2"), 3); - assert_eq!(Map::get("val3"), 3); + assert_eq!(Value::get(), 3); + assert_eq!(Map::get("val1"), 3); + assert_eq!(Map::get("val2"), 3); + assert_eq!(Map::get("val3"), 3); - Rollback(()) - }); + Rollback(Err("revert".into())) + }), + "revert" + ); assert_eq!(Value::get(), 2); assert_eq!(Map::get("val1"), 2); assert_eq!(Map::get("val2"), 2); assert_eq!(Map::get("val3"), 0); - Commit(()) - }); + Commit(Ok(())) + })); assert_eq!(Value::get(), 2); assert_eq!(Map::get("val1"), 2); @@ -145,32 +162,35 @@ fn storage_transaction_commit_then_rollback() { Value::set(1); Map::insert("val1", 1); - with_transaction(|| { - Value::set(2); - Map::insert("val1", 2); - Map::insert("val2", 2); + assert_noop!( + with_transaction(|| -> TransactionOutcome { + Value::set(2); + Map::insert("val1", 2); + Map::insert("val2", 2); + + assert_ok!(with_transaction(|| -> TransactionOutcome { + Value::set(3); + Map::insert("val1", 3); + Map::insert("val2", 3); + Map::insert("val3", 3); - with_transaction(|| { - Value::set(3); - Map::insert("val1", 3); - Map::insert("val2", 3); - Map::insert("val3", 3); + assert_eq!(Value::get(), 3); + assert_eq!(Map::get("val1"), 3); + assert_eq!(Map::get("val2"), 3); + assert_eq!(Map::get("val3"), 3); + + Commit(Ok(())) + })); assert_eq!(Value::get(), 3); assert_eq!(Map::get("val1"), 3); assert_eq!(Map::get("val2"), 3); assert_eq!(Map::get("val3"), 3); - Commit(()) - }); - - assert_eq!(Value::get(), 3); - assert_eq!(Map::get("val1"), 3); - assert_eq!(Map::get("val2"), 3); - assert_eq!(Map::get("val3"), 3); - - Rollback(()) - }); + Rollback(Err("revert".into())) + }), + "revert" + ); assert_eq!(Value::get(), 1); assert_eq!(Map::get("val1"), 1); diff --git a/frame/support/test/tests/system.rs b/frame/support/test/tests/system.rs index 4acc248d25f2..b30fd8d5ec56 100644 --- a/frame/support/test/tests/system.rs +++ b/frame/support/test/tests/system.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -63,27 +63,13 @@ frame_support::decl_error! { TestError, /// Error documentation /// with multiple lines - AnotherError - } -} - -/// Origin for the system module. -#[derive(PartialEq, Eq, Clone, sp_runtime::RuntimeDebug, Encode, Decode, scale_info::TypeInfo)] -pub enum RawOrigin { - Root, - Signed(AccountId), - None, -} - -impl From> for RawOrigin { - fn from(s: Option) -> RawOrigin { - match s { - Some(who) => RawOrigin::Signed(who), - None => RawOrigin::None, - } + AnotherError, + // Required by construct_runtime + CallFiltered, } } +pub use frame_support::dispatch::RawOrigin; pub type Origin = RawOrigin<::AccountId>; #[allow(dead_code)] diff --git a/frame/system/Cargo.toml b/frame/system/Cargo.toml index 389730107b43..e9196c4eb94f 100644 --- a/frame/system/Cargo.toml +++ b/frame/system/Cargo.toml @@ -2,9 +2,9 @@ name = "frame-system" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME system module" readme = "README.md" @@ -13,20 +13,20 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -serde = { version = "1.0.126", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io", default-features = false } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-version = { version = "4.0.0-dev", default-features = false, path = "../../primitives/version" } +serde = { version = "1.0.136", optional = true, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", path = "../../primitives/io", default-features = false } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-version = { version = "5.0.0", default-features = false, path = "../../primitives/version" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } log = { version = "0.4.14", default-features = false } [dev-dependencies] criterion = "0.3.3" -sp-externalities = { version = "0.10.0-dev", path = "../../primitives/externalities" } +sp-externalities = { version = "0.12.0", path = "../../primitives/externalities" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } [features] diff --git a/frame/system/README.md b/frame/system/README.md index 6766c3d73f4d..c22b41e42d79 100644 --- a/frame/system/README.md +++ b/frame/system/README.md @@ -54,21 +54,28 @@ Import the System module and derive your module's configuration trait from the s ### Example - Get extrinsic count and parent hash for the current block ```rust -use frame_support::{decl_module, dispatch}; -use frame_system::{self as system, ensure_signed}; - -pub trait Config: system::Config {} - -decl_module! { - pub struct Module for enum Call where origin: T::Origin { - #[weight = 0] - pub fn system_module_example(origin) -> dispatch::DispatchResult { - let _sender = ensure_signed(origin)?; - let _extrinsic_count = >::extrinsic_count(); - let _parent_hash = >::parent_hash(); - Ok(()) - } - } +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::call] + impl Pallet { + #[pallet::weight(0)] + pub fn system_module_example(origin: OriginFor) -> DispatchResult { + let _sender = ensure_signed(origin)?; + let _extrinsic_count = >::extrinsic_count(); + let _parent_hash = >::parent_hash(); + Ok(()) + } + } } ``` diff --git a/frame/system/benches/bench.rs b/frame/system/benches/bench.rs index c8a9d4eadfea..0bc34fcbc5be 100644 --- a/frame/system/benches/bench.rs +++ b/frame/system/benches/bench.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +16,7 @@ // limitations under the License. use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use frame_support::traits::{ConstU32, ConstU64}; use sp_core::H256; use sp_runtime::{ testing::Header, @@ -58,7 +59,6 @@ frame_support::construct_runtime!( ); frame_support::parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::with_sensible_defaults( 4 * 1024 * 1024, Perbill::from_percent(75), @@ -83,7 +83,7 @@ impl frame_system::Config for Runtime { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -92,6 +92,7 @@ impl frame_system::Config for Runtime { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl module::Config for Runtime { diff --git a/frame/system/benchmarking/Cargo.toml b/frame/system/benchmarking/Cargo.toml index 29bcccfd7d83..c543d5af0412 100644 --- a/frame/system/benchmarking/Cargo.toml +++ b/frame/system/benchmarking/Cargo.toml @@ -2,9 +2,9 @@ name = "frame-system-benchmarking" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME System benchmarking" readme = "README.md" @@ -13,17 +13,17 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../../benchmarking" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/core" } +sp-core = { version = "6.0.0", default-features = false, path = "../../../primitives/core" } [dev-dependencies] -sp-io = { version = "4.0.0-dev", path = "../../../primitives/io" } +sp-io = { version = "6.0.0", path = "../../../primitives/io" } [features] default = ["std"] diff --git a/frame/system/benchmarking/src/lib.rs b/frame/system/benchmarking/src/lib.rs index beb61829bce3..367e6c73c413 100644 --- a/frame/system/benchmarking/src/lib.rs +++ b/frame/system/benchmarking/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,10 +20,10 @@ #![cfg_attr(not(feature = "std"), no_std)] use codec::Encode; -use frame_benchmarking::{benchmarks, impl_benchmark_test_suite, whitelisted_caller}; +use frame_benchmarking::{benchmarks, whitelisted_caller}; use frame_support::{storage, traits::Get, weights::DispatchClass}; -use frame_system::{Call, DigestItemOf, Pallet as System, RawOrigin}; -use sp_core::{storage::well_known_keys, ChangesTrieConfiguration}; +use frame_system::{Call, Pallet as System, RawOrigin}; +use sp_core::storage::well_known_keys; use sp_runtime::traits::Hash; use sp_std::{prelude::*, vec}; @@ -62,23 +62,6 @@ benchmarks! { assert_eq!(current_code.len(), 4_000_000 as usize); } - set_changes_trie_config { - let d = 1000; - - let digest_item = DigestItemOf::::Other(vec![]); - - for i in 0 .. d { - System::::deposit_log(digest_item.clone()); - } - let changes_trie_config = ChangesTrieConfiguration { - digest_interval: d, - digest_levels: d, - }; - }: _(RawOrigin::Root, Some(changes_trie_config)) - verify { - assert_eq!(System::::digest().logs.len(), (d + 1) as usize) - } - #[skip_meta] set_storage { let i in 1 .. 1000; @@ -140,6 +123,6 @@ benchmarks! { verify { assert_eq!(storage::unhashed::get_raw(&last_key), None); } -} -impl_benchmark_test_suite!(Pallet, crate::mock::new_test_ext(), crate::mock::Test); + impl_benchmark_test_suite!(Pallet, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/frame/system/benchmarking/src/mock.rs b/frame/system/benchmarking/src/mock.rs index d828fb22ff5f..08b043ae6274 100644 --- a/frame/system/benchmarking/src/mock.rs +++ b/frame/system/benchmarking/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -62,6 +62,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; } impl crate::Config for Test {} diff --git a/frame/system/rpc/runtime-api/Cargo.toml b/frame/system/rpc/runtime-api/Cargo.toml index fce29612b4d8..e2f85000f5d0 100644 --- a/frame/system/rpc/runtime-api/Cargo.toml +++ b/frame/system/rpc/runtime-api/Cargo.toml @@ -2,9 +2,9 @@ name = "frame-system-rpc-runtime-api" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Runtime API definition required by System RPC extensions." readme = "README.md" @@ -14,7 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../../primitives/api" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } [features] default = ["std"] diff --git a/frame/system/rpc/runtime-api/src/lib.rs b/frame/system/rpc/runtime-api/src/lib.rs index 319883c36d74..6e01bdae2d15 100644 --- a/frame/system/rpc/runtime-api/src/lib.rs +++ b/frame/system/rpc/runtime-api/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/system/src/extensions/check_genesis.rs b/frame/system/src/extensions/check_genesis.rs index 6f409d5d3d4a..a0679b11487f 100644 --- a/frame/system/src/extensions/check_genesis.rs +++ b/frame/system/src/extensions/check_genesis.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,11 +19,16 @@ use crate::{Config, Pallet}; use codec::{Decode, Encode}; use scale_info::TypeInfo; use sp_runtime::{ - traits::{SignedExtension, Zero}, + traits::{DispatchInfoOf, SignedExtension, Zero}, transaction_validity::TransactionValidityError, }; /// Genesis hash check to provide replay protection between different networks. +/// +/// # Transaction Validity +/// +/// Note that while a transaction with invalid `genesis_hash` will fail to be decoded, +/// the extension does not affect any other fields of `TransactionValidity` directly. #[derive(Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] #[scale_info(skip_type_params(T))] pub struct CheckGenesis(sp_std::marker::PhantomData); @@ -57,4 +62,14 @@ impl SignedExtension for CheckGenesis { fn additional_signed(&self) -> Result { Ok(>::block_hash(T::BlockNumber::zero())) } + + fn pre_dispatch( + self, + who: &Self::AccountId, + call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> Result { + self.validate(who, call, info, len).map(|_| ()) + } } diff --git a/frame/system/src/extensions/check_mortality.rs b/frame/system/src/extensions/check_mortality.rs index 69cca765efea..2fb99c9f45e2 100644 --- a/frame/system/src/extensions/check_mortality.rs +++ b/frame/system/src/extensions/check_mortality.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,6 +27,10 @@ use sp_runtime::{ }; /// Check for transaction mortality. +/// +/// # Transaction Validity +/// +/// The extension affects `longevity` of the transaction according to the [`Era`] definition. #[derive(Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] #[scale_info(skip_type_params(T))] pub struct CheckMortality(Era, sp_std::marker::PhantomData); @@ -81,6 +85,16 @@ impl SignedExtension for CheckMortality { Ok(>::block_hash(n)) } } + + fn pre_dispatch( + self, + who: &Self::AccountId, + call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> Result { + self.validate(who, call, info, len).map(|_| ()) + } } #[cfg(test)] diff --git a/frame/system/src/extensions/check_non_zero_sender.rs b/frame/system/src/extensions/check_non_zero_sender.rs new file mode 100644 index 000000000000..f517201fbebc --- /dev/null +++ b/frame/system/src/extensions/check_non_zero_sender.rs @@ -0,0 +1,110 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::Config; +use codec::{Decode, Encode}; +use frame_support::weights::DispatchInfo; +use scale_info::TypeInfo; +use sp_runtime::{ + traits::{DispatchInfoOf, Dispatchable, SignedExtension}, + transaction_validity::{ + InvalidTransaction, TransactionValidity, TransactionValidityError, ValidTransaction, + }, +}; +use sp_std::{marker::PhantomData, prelude::*}; + +/// Check to ensure that the sender is not the zero address. +#[derive(Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] +#[scale_info(skip_type_params(T))] +pub struct CheckNonZeroSender(PhantomData); + +impl sp_std::fmt::Debug for CheckNonZeroSender { + #[cfg(feature = "std")] + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + write!(f, "CheckNonZeroSender") + } + + #[cfg(not(feature = "std"))] + fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + Ok(()) + } +} + +impl CheckNonZeroSender { + /// Create new `SignedExtension` to check runtime version. + pub fn new() -> Self { + Self(sp_std::marker::PhantomData) + } +} + +impl SignedExtension for CheckNonZeroSender +where + T::Call: Dispatchable, +{ + type AccountId = T::AccountId; + type Call = T::Call; + type AdditionalSigned = (); + type Pre = (); + const IDENTIFIER: &'static str = "CheckNonZeroSender"; + + fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { + Ok(()) + } + + fn pre_dispatch( + self, + who: &Self::AccountId, + call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> Result { + self.validate(who, call, info, len).map(|_| ()) + } + + fn validate( + &self, + who: &Self::AccountId, + _call: &Self::Call, + _info: &DispatchInfoOf, + _len: usize, + ) -> TransactionValidity { + if who.using_encoded(|d| d.into_iter().all(|x| *x == 0)) { + return Err(TransactionValidityError::Invalid(InvalidTransaction::BadSigner)) + } + Ok(ValidTransaction::default()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mock::{new_test_ext, Test, CALL}; + use frame_support::{assert_noop, assert_ok}; + + #[test] + fn zero_account_ban_works() { + new_test_ext().execute_with(|| { + let info = DispatchInfo::default(); + let len = 0_usize; + assert_noop!( + CheckNonZeroSender::::new().validate(&0, CALL, &info, len), + InvalidTransaction::BadSigner + ); + assert_ok!(CheckNonZeroSender::::new().validate(&1, CALL, &info, len)); + }) + } +} diff --git a/frame/system/src/extensions/check_nonce.rs b/frame/system/src/extensions/check_nonce.rs index 74be83398421..476aa2fb7478 100644 --- a/frame/system/src/extensions/check_nonce.rs +++ b/frame/system/src/extensions/check_nonce.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,8 +30,11 @@ use sp_std::vec; /// Nonce check and increment to give replay protection for transactions. /// -/// Note that this does not set any priority by default. Make sure that AT LEAST one of the signed -/// extension sets some kind of priority upon validating transactions. +/// # Transaction Validity +/// +/// This extension affects `requires` and `provides` tags of validity, but DOES NOT +/// set the `priority` field. Make sure that AT LEAST one of the signed extension sets +/// some kind of priority upon validating transactions. #[derive(Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] #[scale_info(skip_type_params(T))] pub struct CheckNonce(#[codec(compact)] pub T::Index); diff --git a/frame/system/src/extensions/check_spec_version.rs b/frame/system/src/extensions/check_spec_version.rs index 0217aefae6b9..0280d31f657a 100644 --- a/frame/system/src/extensions/check_spec_version.rs +++ b/frame/system/src/extensions/check_spec_version.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,9 +18,17 @@ use crate::{Config, Pallet}; use codec::{Decode, Encode}; use scale_info::TypeInfo; -use sp_runtime::{traits::SignedExtension, transaction_validity::TransactionValidityError}; +use sp_runtime::{ + traits::{DispatchInfoOf, SignedExtension}, + transaction_validity::TransactionValidityError, +}; /// Ensure the runtime version registered in the transaction is the same as at present. +/// +/// # Transaction Validity +/// +/// The transaction with incorrect `spec_version` are considered invalid. The validity +/// is not affected in any other way. #[derive(Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] #[scale_info(skip_type_params(T))] pub struct CheckSpecVersion(sp_std::marker::PhantomData); @@ -54,4 +62,14 @@ impl SignedExtension for CheckSpecVersion { fn additional_signed(&self) -> Result { Ok(>::runtime_version().spec_version) } + + fn pre_dispatch( + self, + who: &Self::AccountId, + call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> Result { + self.validate(who, call, info, len).map(|_| ()) + } } diff --git a/frame/system/src/extensions/check_tx_version.rs b/frame/system/src/extensions/check_tx_version.rs index 9418d3ff5d93..b92d8978bde0 100644 --- a/frame/system/src/extensions/check_tx_version.rs +++ b/frame/system/src/extensions/check_tx_version.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,9 +18,17 @@ use crate::{Config, Pallet}; use codec::{Decode, Encode}; use scale_info::TypeInfo; -use sp_runtime::{traits::SignedExtension, transaction_validity::TransactionValidityError}; +use sp_runtime::{ + traits::{DispatchInfoOf, SignedExtension}, + transaction_validity::TransactionValidityError, +}; /// Ensure the transaction version registered in the transaction is the same as at present. +/// +/// # Transaction Validity +/// +/// The transaction with incorrect `transaction_version` are considered invalid. The validity +/// is not affected in any other way. #[derive(Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] #[scale_info(skip_type_params(T))] pub struct CheckTxVersion(sp_std::marker::PhantomData); @@ -54,4 +62,13 @@ impl SignedExtension for CheckTxVersion { fn additional_signed(&self) -> Result { Ok(>::runtime_version().transaction_version) } + fn pre_dispatch( + self, + who: &Self::AccountId, + call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> Result { + self.validate(who, call, info, len).map(|_| ()) + } } diff --git a/frame/system/src/extensions/check_weight.rs b/frame/system/src/extensions/check_weight.rs index 92dc7382fa2d..774139054d08 100644 --- a/frame/system/src/extensions/check_weight.rs +++ b/frame/system/src/extensions/check_weight.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,19 +19,21 @@ use crate::{limits::BlockWeights, Config, Pallet}; use codec::{Decode, Encode}; use frame_support::{ traits::Get, - weights::{priority::FrameTransactionPriority, DispatchClass, DispatchInfo, PostDispatchInfo}, + weights::{DispatchClass, DispatchInfo, PostDispatchInfo}, }; use scale_info::TypeInfo; use sp_runtime::{ traits::{DispatchInfoOf, Dispatchable, PostDispatchInfoOf, SignedExtension}, - transaction_validity::{ - InvalidTransaction, TransactionPriority, TransactionValidity, TransactionValidityError, - ValidTransaction, - }, + transaction_validity::{InvalidTransaction, TransactionValidity, TransactionValidityError}, DispatchResult, }; /// Block resource (weight) limit check. +/// +/// # Transaction Validity +/// +/// This extension does not influence any fields of `TransactionValidity` in case the +/// transaction is valid. #[derive(Encode, Decode, Clone, Eq, PartialEq, Default, TypeInfo)] #[scale_info(skip_type_params(T))] pub struct CheckWeight(sp_std::marker::PhantomData); @@ -81,23 +83,6 @@ where } } - /// Get the priority of an extrinsic denoted by `info`. - /// - /// Operational transaction will be given a fixed initial amount to be fairly distinguished from - /// the normal ones. - fn get_priority(info: &DispatchInfoOf) -> TransactionPriority { - match info.class { - // Normal transaction. - DispatchClass::Normal => FrameTransactionPriority::Normal(info.weight.into()).into(), - // Don't use up the whole priority space, to allow things like `tip` to be taken into - // account as well. - DispatchClass::Operational => - FrameTransactionPriority::Operational(info.weight.into()).into(), - // Mandatory extrinsics are only for inherents; never transactions. - DispatchClass::Mandatory => TransactionPriority::min_value(), - } - } - /// Creates new `SignedExtension` to check weight of the extrinsic. pub fn new() -> Self { Self(Default::default()) @@ -130,7 +115,7 @@ where // consumption from causing false negatives. Self::check_extrinsic_weight(info)?; - Ok(ValidTransaction { priority: Self::get_priority(info), ..Default::default() }) + Ok(Default::default()) } } @@ -238,7 +223,7 @@ where } fn post_dispatch( - _pre: Self::Pre, + _pre: Option, info: &DispatchInfoOf, post_info: &PostDispatchInfoOf, _len: usize, @@ -368,13 +353,7 @@ mod tests { }; let len = 0_usize; - assert_eq!( - CheckWeight::::do_validate(&okay, len), - Ok(ValidTransaction { - priority: CheckWeight::::get_priority(&okay), - ..Default::default() - }) - ); + assert_eq!(CheckWeight::::do_validate(&okay, len), Ok(Default::default())); assert_err!( CheckWeight::::do_validate(&max, len), InvalidTransaction::ExhaustsResources @@ -506,30 +485,6 @@ mod tests { }) } - #[test] - fn signed_ext_check_weight_works() { - new_test_ext().execute_with(|| { - let normal = - DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: Pays::Yes }; - let op = DispatchInfo { - weight: 100, - class: DispatchClass::Operational, - pays_fee: Pays::Yes, - }; - let len = 0_usize; - - let priority = CheckWeight::(PhantomData) - .validate(&1, CALL, &normal, len) - .unwrap() - .priority; - assert_eq!(priority, 100); - - let priority = - CheckWeight::(PhantomData).validate(&1, CALL, &op, len).unwrap().priority; - assert_eq!(priority, frame_support::weights::priority::LIMIT + 100); - }) - } - #[test] fn signed_ext_check_weight_block_size_works() { new_test_ext().execute_with(|| { @@ -608,7 +563,13 @@ mod tests { let pre = CheckWeight::(PhantomData).pre_dispatch(&1, CALL, &info, len).unwrap(); assert_eq!(BlockWeight::::get().total(), info.weight + 256); - assert_ok!(CheckWeight::::post_dispatch(pre, &info, &post_info, len, &Ok(()))); + assert_ok!(CheckWeight::::post_dispatch( + Some(pre), + &info, + &post_info, + len, + &Ok(()) + )); assert_eq!(BlockWeight::::get().total(), post_info.actual_weight.unwrap() + 256); }) } @@ -632,7 +593,13 @@ mod tests { info.weight + 128 + block_weights().get(DispatchClass::Normal).base_extrinsic, ); - assert_ok!(CheckWeight::::post_dispatch(pre, &info, &post_info, len, &Ok(()))); + assert_ok!(CheckWeight::::post_dispatch( + Some(pre), + &info, + &post_info, + len, + &Ok(()) + )); assert_eq!( BlockWeight::::get().total(), info.weight + 128 + block_weights().get(DispatchClass::Normal).base_extrinsic, diff --git a/frame/system/src/extensions/mod.rs b/frame/system/src/extensions/mod.rs index 0af9722e475d..9ba73c5f4d4e 100644 --- a/frame/system/src/extensions/mod.rs +++ b/frame/system/src/extensions/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,6 +17,7 @@ pub mod check_genesis; pub mod check_mortality; +pub mod check_non_zero_sender; pub mod check_nonce; pub mod check_spec_version; pub mod check_tx_version; diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index 2e7f26eef16f..9b30e7b45227 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -73,7 +73,7 @@ use sp_runtime::{ CheckEqual, Dispatchable, Hash, Lookup, LookupError, MaybeDisplay, MaybeMallocSizeOf, MaybeSerializeDeserialize, Member, One, Saturating, SimpleBitOps, StaticLookup, Zero, }, - DispatchError, Either, Perbill, RuntimeDebug, + DispatchError, Perbill, RuntimeDebug, }; #[cfg(any(feature = "std", test))] use sp_std::map; @@ -85,8 +85,8 @@ use frame_support::{ dispatch::{DispatchResult, DispatchResultWithPostInfo}, storage, traits::{ - Contains, EnsureOrigin, Get, HandleLifetime, OnKilledAccount, OnNewAccount, OriginTrait, - PalletInfo, SortedMembers, StoredMap, + ConstU32, Contains, EnsureOrigin, Get, HandleLifetime, OnKilledAccount, OnNewAccount, + OriginTrait, PalletInfo, SortedMembers, StoredMap, }, weights::{ extract_actual_weight, DispatchClass, DispatchInfo, PerDispatchClass, RuntimeDbWeight, @@ -95,7 +95,7 @@ use frame_support::{ Parameter, }; use scale_info::TypeInfo; -use sp_core::{storage::well_known_keys, ChangesTrieConfiguration}; +use sp_core::storage::well_known_keys; #[cfg(feature = "std")] use frame_support::traits::GenesisBuild; @@ -114,23 +114,33 @@ pub mod mocking; mod tests; pub mod weights; +pub mod migrations; + pub use extensions::{ - check_genesis::CheckGenesis, check_mortality::CheckMortality, check_nonce::CheckNonce, + check_genesis::CheckGenesis, check_mortality::CheckMortality, + check_non_zero_sender::CheckNonZeroSender, check_nonce::CheckNonce, check_spec_version::CheckSpecVersion, check_tx_version::CheckTxVersion, check_weight::CheckWeight, }; // Backward compatible re-export. pub use extensions::check_mortality::CheckMortality as CheckEra; +pub use frame_support::dispatch::RawOrigin; pub use weights::WeightInfo; /// Compute the trie root of a list of extrinsics. +/// +/// The merkle proof is using the same trie as runtime state with +/// `state_version` 0. pub fn extrinsics_root(extrinsics: &[E]) -> H::Output { extrinsics_data_root::(extrinsics.iter().map(codec::Encode::encode).collect()) } /// Compute the trie root of a list of extrinsics. +/// +/// The merkle proof is using the same trie as runtime state with +/// `state_version` 0. pub fn extrinsics_data_root(xts: Vec>) -> H::Output { - H::ordered_trie_root(xts) + H::ordered_trie_root(xts, sp_core::storage::StateVersion::V0) } /// An object to track the currently used extrinsic weight in a block. @@ -151,6 +161,36 @@ impl SetCode for () { } } +/// Numeric limits over the ability to add a consumer ref using `inc_consumers`. +pub trait ConsumerLimits { + /// The number of consumers over which `inc_consumers` will cease to work. + fn max_consumers() -> RefCount; + /// The maximum number of additional consumers expected to be over be added at once using + /// `inc_consumers_without_limit`. + /// + /// Note: This is not enforced and it's up to the chain's author to ensure this reflects the + /// actual situation. + fn max_overflow() -> RefCount; +} + +impl ConsumerLimits for ConstU32 { + fn max_consumers() -> RefCount { + Z + } + fn max_overflow() -> RefCount { + Z + } +} + +impl, MaxOverflow: Get> ConsumerLimits for (MaxNormal, MaxOverflow) { + fn max_consumers() -> RefCount { + MaxNormal::get() + } + fn max_overflow() -> RefCount { + MaxOverflow::get() + } +} + #[frame_support::pallet] pub mod pallet { use crate::{self as frame_system, pallet_prelude::*, *}; @@ -235,7 +275,6 @@ pub mod pallet { + Debug + MaybeDisplay + Ord - + Default + MaxEncodedLen; /// Converting trait to take a source type and convert to `AccountId`. @@ -301,27 +340,23 @@ pub mod pallet { /// What to do if the runtime wants to change the code to something new. /// /// The default (`()`) implementation is responsible for setting the correct storage - /// entry and emitting corresponding event and log item. (see [`update_code_in_storage`]). + /// entry and emitting corresponding event and log item. (see + /// [`Pallet::update_code_in_storage`]). /// It's unlikely that this needs to be customized, unless you are writing a parachain using /// `Cumulus`, where the actual code change is deferred. type OnSetCode: SetCode; + + /// The maximum number of consumers allowed on a single account. + type MaxConsumers: ConsumerLimits; } #[pallet::pallet] #[pallet::generate_store(pub (super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); #[pallet::hooks] impl Hooks> for Pallet { - fn on_runtime_upgrade() -> frame_support::weights::Weight { - if !UpgradedToTripleRefCount::::get() { - UpgradedToTripleRefCount::::put(true); - migrations::migrate_to_triple_ref_count::() - } else { - 0 - } - } - fn integrity_test() { T::BlockWeights::get().validate().expect("The weights are invalid."); } @@ -345,19 +380,11 @@ pub mod pallet { /// # #[pallet::weight(T::SystemWeightInfo::remark(_remark.len() as u32))] pub fn remark(origin: OriginFor, _remark: Vec) -> DispatchResultWithPostInfo { - ensure_signed(origin)?; + ensure_signed_or_root(origin)?; Ok(().into()) } /// Set the number of pages in the WebAssembly environment's heap. - /// - /// # - /// - `O(1)` - /// - 1 storage write. - /// - Base Weight: 1.405 µs - /// - 1 write to HEAP_PAGES - /// - 1 digest item - /// # #[pallet::weight((T::SystemWeightInfo::set_heap_pages(), DispatchClass::Operational))] pub fn set_heap_pages(origin: OriginFor, pages: u64) -> DispatchResultWithPostInfo { ensure_root(origin)?; @@ -405,45 +432,7 @@ pub mod pallet { Ok(().into()) } - /// Set the new changes trie configuration. - /// - /// # - /// - `O(1)` - /// - 1 storage write or delete (codec `O(1)`). - /// - 1 call to `deposit_log`: Uses `append` API, so O(1) - /// - Base Weight: 7.218 µs - /// - DB Weight: - /// - Writes: Changes Trie, System Digest - /// # - #[pallet::weight((T::SystemWeightInfo::set_changes_trie_config(), DispatchClass::Operational))] - pub fn set_changes_trie_config( - origin: OriginFor, - changes_trie_config: Option, - ) -> DispatchResultWithPostInfo { - ensure_root(origin)?; - match changes_trie_config.clone() { - Some(changes_trie_config) => storage::unhashed::put_raw( - well_known_keys::CHANGES_TRIE_CONFIG, - &changes_trie_config.encode(), - ), - None => storage::unhashed::kill(well_known_keys::CHANGES_TRIE_CONFIG), - } - - let log = generic::DigestItem::ChangesTrieSignal( - generic::ChangesTrieSignal::NewConfiguration(changes_trie_config), - ); - Self::deposit_log(log.into()); - Ok(().into()) - } - /// Set some items of storage. - /// - /// # - /// - `O(I)` where `I` length of `items` - /// - `I` storage writes (`O(1)`). - /// - Base Weight: 0.568 * i µs - /// - Writes: Number of items - /// # #[pallet::weight(( T::SystemWeightInfo::set_storage(items.len() as u32), DispatchClass::Operational, @@ -460,13 +449,6 @@ pub mod pallet { } /// Kill some items from storage. - /// - /// # - /// - `O(IK)` where `I` length of `keys` and `K` length of one key - /// - `I` storage deletions. - /// - Base Weight: .378 * i µs - /// - Writes: Number of items - /// # #[pallet::weight(( T::SystemWeightInfo::kill_storage(keys.len() as u32), DispatchClass::Operational, @@ -483,13 +465,6 @@ pub mod pallet { /// /// **NOTE:** We rely on the Root origin to provide us the number of subkeys under /// the prefix we are removing to accurately calculate the weight of this function. - /// - /// # - /// - `O(P)` where `P` amount of keys with prefix `prefix` - /// - `P` storage deletions. - /// - Base Weight: 0.834 * P µs - /// - Writes: Number of subkeys + 1 - /// # #[pallet::weight(( T::SystemWeightInfo::kill_prefix(_subkeys.saturating_add(1)), DispatchClass::Operational, @@ -505,11 +480,6 @@ pub mod pallet { } /// Make some on-chain remark and emit event. - /// - /// # - /// - `O(b)` where b is the length of the remark. - /// - 1 event. - /// # #[pallet::weight(T::SystemWeightInfo::remark_with_event(remark.len() as u32))] pub fn remark_with_event( origin: OriginFor, @@ -517,7 +487,7 @@ pub mod pallet { ) -> DispatchResultWithPostInfo { let who = ensure_signed(origin)?; let hash = T::Hashing::hash(&remark[..]); - Self::deposit_event(Event::Remarked(who, hash)); + Self::deposit_event(Event::Remarked { sender: who, hash }); Ok(().into()) } } @@ -525,18 +495,18 @@ pub mod pallet { /// Event for the System pallet. #[pallet::event] pub enum Event { - /// An extrinsic completed successfully. \[info\] - ExtrinsicSuccess(DispatchInfo), - /// An extrinsic failed. \[error, info\] - ExtrinsicFailed(DispatchError, DispatchInfo), + /// An extrinsic completed successfully. + ExtrinsicSuccess { dispatch_info: DispatchInfo }, + /// An extrinsic failed. + ExtrinsicFailed { dispatch_error: DispatchError, dispatch_info: DispatchInfo }, /// `:code` was updated. CodeUpdated, - /// A new \[account\] was created. - NewAccount(T::AccountId), - /// An \[account\] was reaped. - KilledAccount(T::AccountId), - /// On on-chain remark happened. \[origin, remark_hash\] - Remarked(T::AccountId, T::Hash), + /// A new account was created. + NewAccount { account: T::AccountId }, + /// An account was reaped. + KilledAccount { account: T::AccountId }, + /// On on-chain remark happened. + Remarked { sender: T::AccountId, hash: T::Hash }, } /// Old name generated by `decl_event`. @@ -560,6 +530,8 @@ pub mod pallet { NonDefaultComposite, /// There is a non-zero reference count preventing the account from being purged. NonZeroRefCount, + /// The origin filter prevent the call to be dispatched. + CallFiltered, } /// Exposed trait-generic origin type. @@ -615,7 +587,7 @@ pub mod pallet { /// Digest of the current block, also part of the block header. #[pallet::storage] #[pallet::getter(fn digest)] - pub(super) type Digest = StorageValue<_, DigestOf, ValueQuery>; + pub(super) type Digest = StorageValue<_, generic::Digest, ValueQuery>; /// Events deposited for the current block. /// @@ -662,20 +634,13 @@ pub mod pallet { #[pallet::storage] pub(super) type ExecutionPhase = StorageValue<_, Phase>; + #[cfg_attr(feature = "std", derive(Default))] #[pallet::genesis_config] pub struct GenesisConfig { - pub changes_trie_config: Option, #[serde(with = "sp_core::bytes")] pub code: Vec, } - #[cfg(feature = "std")] - impl Default for GenesisConfig { - fn default() -> Self { - Self { changes_trie_config: Default::default(), code: Default::default() } - } - } - #[pallet::genesis_build] impl GenesisBuild for GenesisConfig { fn build(&self) { @@ -687,56 +652,10 @@ pub mod pallet { sp_io::storage::set(well_known_keys::CODE, &self.code); sp_io::storage::set(well_known_keys::EXTRINSIC_INDEX, &0u32.encode()); - if let Some(ref changes_trie_config) = self.changes_trie_config { - sp_io::storage::set( - well_known_keys::CHANGES_TRIE_CONFIG, - &changes_trie_config.encode(), - ); - } } } } -pub mod migrations { - use super::*; - - #[allow(dead_code)] - /// Migrate from unique `u8` reference counting to triple `u32` reference counting. - pub fn migrate_all() -> frame_support::weights::Weight { - Account::::translate::<(T::Index, u8, T::AccountData), _>(|_key, (nonce, rc, data)| { - Some(AccountInfo { - nonce, - consumers: rc as RefCount, - providers: 1, - sufficients: 0, - data, - }) - }); - T::BlockWeights::get().max_block - } - - #[allow(dead_code)] - /// Migrate from unique `u32` reference counting to triple `u32` reference counting. - pub fn migrate_to_dual_ref_count() -> frame_support::weights::Weight { - Account::::translate::<(T::Index, RefCount, T::AccountData), _>( - |_key, (nonce, consumers, data)| { - Some(AccountInfo { nonce, consumers, providers: 1, sufficients: 0, data }) - }, - ); - T::BlockWeights::get().max_block - } - - /// Migrate from dual `u32` reference counting to triple `u32` reference counting. - pub fn migrate_to_triple_ref_count() -> frame_support::weights::Weight { - Account::::translate::<(T::Index, RefCount, RefCount, T::AccountData), _>( - |_key, (nonce, consumers, providers, data)| { - Some(AccountInfo { nonce, consumers, providers, sufficients: 0, data }) - }, - ); - T::BlockWeights::get().max_block - } -} - #[cfg(feature = "std")] impl GenesisConfig { /// Direct implementation of `GenesisBuild::build_storage`. @@ -757,9 +676,6 @@ impl GenesisConfig { } } -pub type DigestOf = generic::Digest<::Hash>; -pub type DigestItemOf = generic::DigestItem<::Hash>; - pub type Key = Vec; pub type KeyValue = (Vec, Vec); @@ -793,28 +709,6 @@ pub struct EventRecord { pub topics: Vec, } -/// Origin for the System pallet. -#[derive(PartialEq, Eq, Clone, RuntimeDebug, Encode, Decode, TypeInfo)] -pub enum RawOrigin { - /// The system itself ordained this dispatch to happen: this is the highest privilege level. - Root, - /// It is signed by some public key and we provide the `AccountId`. - Signed(AccountId), - /// It is signed by nobody, can be either: - /// * included and agreed upon by the validators anyway, - /// * or unsigned transaction validated by a pallet. - None, -} - -impl From> for RawOrigin { - fn from(s: Option) -> RawOrigin { - match s { - Some(who) => RawOrigin::Signed(who), - None => RawOrigin::None, - } - } -} - // Create a Hash with 69 for each byte, // only used to build genesis config. #[cfg(feature = "std")] @@ -895,7 +789,7 @@ impl, O>> + From>, Acco } pub struct EnsureSigned(sp_std::marker::PhantomData); -impl, O>> + From>, AccountId: Default> +impl, O>> + From>, AccountId: Decode> EnsureOrigin for EnsureSigned { type Success = AccountId; @@ -908,7 +802,10 @@ impl, O>> + From>, Acco #[cfg(feature = "runtime-benchmarks")] fn successful_origin() -> O { - O::from(RawOrigin::Signed(Default::default())) + let zero_account_id = + AccountId::decode(&mut sp_runtime::traits::TrailingZeroInput::zeroes()) + .expect("infinite length input; no invalid inputs for type; qed"); + O::from(RawOrigin::Signed(zero_account_id)) } } @@ -916,7 +813,7 @@ pub struct EnsureSignedBy(sp_std::marker::PhantomData<(Who, Acco impl< O: Into, O>> + From>, Who: SortedMembers, - AccountId: PartialEq + Clone + Ord + Default, + AccountId: PartialEq + Clone + Ord + Decode, > EnsureOrigin for EnsureSignedBy { type Success = AccountId; @@ -929,10 +826,13 @@ impl< #[cfg(feature = "runtime-benchmarks")] fn successful_origin() -> O { + let zero_account_id = + AccountId::decode(&mut sp_runtime::traits::TrailingZeroInput::zeroes()) + .expect("infinite length input; no invalid inputs for type; qed"); let members = Who::sorted_members(); let first_member = match members.get(0) { Some(account) => account.clone(), - None => Default::default(), + None => zero_account_id, }; O::from(RawOrigin::Signed(first_member.clone())) } @@ -969,29 +869,6 @@ impl EnsureOrigin for EnsureNever { } } -/// The "OR gate" implementation of `EnsureOrigin`. -/// -/// Origin check will pass if `L` or `R` origin check passes. `L` is tested first. -pub struct EnsureOneOf(sp_std::marker::PhantomData<(AccountId, L, R)>); -impl< - AccountId, - O: Into, O>> + From>, - L: EnsureOrigin, - R: EnsureOrigin, - > EnsureOrigin for EnsureOneOf -{ - type Success = Either; - fn try_origin(o: O) -> Result { - L::try_origin(o) - .map_or_else(|o| R::try_origin(o).map(|o| Either::Right(o)), |o| Ok(Either::Left(o))) - } - - #[cfg(feature = "runtime-benchmarks")] - fn successful_origin() -> O { - L::successful_origin() - } -} - /// Ensure that the origin `o` represents a signed extrinsic (i.e. transaction). /// Returns `Ok` with the account that signed the extrinsic or an `Err` otherwise. pub fn ensure_signed(o: OuterOrigin) -> Result @@ -1004,6 +881,22 @@ where } } +/// Ensure that the origin `o` represents either a signed extrinsic (i.e. transaction) or the root. +/// Returns `Ok` with the account that signed the extrinsic, `None` if it was root, or an `Err` +/// otherwise. +pub fn ensure_signed_or_root( + o: OuterOrigin, +) -> Result, BadOrigin> +where + OuterOrigin: Into, OuterOrigin>>, +{ + match o.into() { + Ok(RawOrigin::Root) => Ok(None), + Ok(RawOrigin::Signed(t)) => Ok(Some(t)), + _ => Err(BadOrigin), + } +} + /// Ensure that the origin `o` represents the root. Returns `Ok` or an `Err` otherwise. pub fn ensure_root(o: OuterOrigin) -> Result<(), BadOrigin> where @@ -1026,27 +919,6 @@ where } } -/// A type of block initialization to perform. -pub enum InitKind { - /// Leave inspectable storage entries in state. - /// - /// i.e. `Events` are not being reset. - /// Should only be used for off-chain calls, - /// regular block execution should clear those. - Inspection, - - /// Reset also inspectable storage entries. - /// - /// This should be used for regular block execution. - Full, -} - -impl Default for InitKind { - fn default() -> Self { - InitKind::Full - } -} - /// Reference status; can be either referenced or unreferenced. #[derive(RuntimeDebug)] pub enum RefStatus { @@ -1239,8 +1111,27 @@ impl Pallet { /// Increment the reference counter on an account. /// - /// The account `who`'s `providers` must be non-zero or this will return an error. + /// The account `who`'s `providers` must be non-zero and the current number of consumers must + /// be less than `MaxConsumers::max_consumers()` or this will return an error. pub fn inc_consumers(who: &T::AccountId) -> Result<(), DispatchError> { + Account::::try_mutate(who, |a| { + if a.providers > 0 { + if a.consumers < T::MaxConsumers::max_consumers() { + a.consumers = a.consumers.saturating_add(1); + Ok(()) + } else { + Err(DispatchError::TooManyConsumers) + } + } else { + Err(DispatchError::NoProviders) + } + }) + } + + /// Increment the reference counter on an account, ignoring the `MaxConsumers` limits. + /// + /// The account `who`'s `providers` must be non-zero or this will return an error. + pub fn inc_consumers_without_limit(who: &T::AccountId) -> Result<(), DispatchError> { Account::::try_mutate(who, |a| { if a.providers > 0 { a.consumers = a.consumers.saturating_add(1); @@ -1284,7 +1175,8 @@ impl Pallet { /// True if the account has at least one provider reference. pub fn can_inc_consumer(who: &T::AccountId) -> bool { - Account::::get(who).providers > 0 + let a = Account::::get(who); + a.providers > 0 && a.consumers < T::MaxConsumers::max_consumers() } /// Deposits an event into this block's event record. @@ -1364,12 +1256,7 @@ impl Pallet { } /// Start the execution of a particular block. - pub fn initialize( - number: &T::BlockNumber, - parent_hash: &T::Hash, - digest: &DigestOf, - kind: InitKind, - ) { + pub fn initialize(number: &T::BlockNumber, parent_hash: &T::Hash, digest: &generic::Digest) { // populate environment ExecutionPhase::::put(Phase::Initialization); storage::unhashed::put(well_known_keys::EXTRINSIC_INDEX, &0u32); @@ -1380,18 +1267,45 @@ impl Pallet { // Remove previous block data from storage BlockWeight::::kill(); - - // Kill inspectable storage entries in state when `InitKind::Full`. - if let InitKind::Full = kind { - >::kill(); - EventCount::::kill(); - >::remove_all(None); - } } /// Remove temporary "environment" entries in storage, compute the storage root and return the /// resulting header for this block. pub fn finalize() -> T::Header { + log::debug!( + target: "runtime::system", + "[{:?}] length: {} (normal {}%, op: {}%, mandatory {}%) / normal weight: {} ({}%) \ + / op weight {} ({}%) / mandatory weight {} ({}%)", + Self::block_number(), + Self::all_extrinsics_len(), + sp_runtime::Percent::from_rational( + Self::all_extrinsics_len(), + *T::BlockLength::get().max.get(DispatchClass::Normal) + ).deconstruct(), + sp_runtime::Percent::from_rational( + Self::all_extrinsics_len(), + *T::BlockLength::get().max.get(DispatchClass::Operational) + ).deconstruct(), + sp_runtime::Percent::from_rational( + Self::all_extrinsics_len(), + *T::BlockLength::get().max.get(DispatchClass::Mandatory) + ).deconstruct(), + Self::block_weight().get(DispatchClass::Normal), + sp_runtime::Percent::from_rational( + *Self::block_weight().get(DispatchClass::Normal), + T::BlockWeights::get().get(DispatchClass::Normal).max_total.unwrap_or(Bounded::max_value()) + ).deconstruct(), + Self::block_weight().get(DispatchClass::Operational), + sp_runtime::Percent::from_rational( + *Self::block_weight().get(DispatchClass::Operational), + T::BlockWeights::get().get(DispatchClass::Operational).max_total.unwrap_or(Bounded::max_value()) + ).deconstruct(), + Self::block_weight().get(DispatchClass::Mandatory), + sp_runtime::Percent::from_rational( + *Self::block_weight().get(DispatchClass::Mandatory), + T::BlockWeights::get().get(DispatchClass::Mandatory).max_total.unwrap_or(Bounded::max_value()) + ).deconstruct(), + ); ExecutionPhase::::kill(); AllExtrinsicsLen::::kill(); @@ -1407,7 +1321,7 @@ impl Pallet { // stay to be inspected by the client and will be cleared by `Self::initialize`. let number = >::get(); let parent_hash = >::get(); - let mut digest = >::get(); + let digest = >::get(); let extrinsics = (0..ExtrinsicCount::::take().unwrap_or_default()) .map(ExtrinsicData::::take) @@ -1423,19 +1337,9 @@ impl Pallet { >::remove(to_remove); } - let storage_root = T::Hash::decode(&mut &sp_io::storage::root()[..]) + let version = T::Version::get().state_version(); + let storage_root = T::Hash::decode(&mut &sp_io::storage::root(version)[..]) .expect("Node is configured to use the same hash; qed"); - let storage_changes_root = sp_io::storage::changes_root(&parent_hash.encode()); - - // we can't compute changes trie root earlier && put it to the Digest - // because it will include all currently existing temporaries. - if let Some(storage_changes_root) = storage_changes_root { - let item = generic::DigestItem::ChangesTrieRoot( - T::Hash::decode(&mut &storage_changes_root[..]) - .expect("Node is configured to use the same hash; qed"), - ); - digest.push(item); - } ::new( number, @@ -1452,7 +1356,7 @@ impl Pallet { /// - `O(1)` /// - 1 storage write (codec `O(1)`) /// # - pub fn deposit_log(item: DigestItemOf) { + pub fn deposit_log(item: generic::DigestItem) { >::append(item); } @@ -1516,9 +1420,10 @@ impl Pallet { AllExtrinsicsLen::::put(len as u32); } - /// Reset events. Can be used as an alternative to - /// `initialize` for tests that don't need to bother with the other environment entries. - #[cfg(any(feature = "std", feature = "runtime-benchmarks", test))] + /// Reset events. + /// + /// This needs to be used in prior calling [`initialize`](Self::initialize) for each new block + /// to clear events from previous block. pub fn reset_events() { >::kill(); EventCount::::kill(); @@ -1564,7 +1469,7 @@ impl Pallet { pub fn note_applied_extrinsic(r: &DispatchResultWithPostInfo, mut info: DispatchInfo) { info.weight = extract_actual_weight(r, &info); Self::deposit_event(match r { - Ok(_) => Event::ExtrinsicSuccess(info), + Ok(_) => Event::ExtrinsicSuccess { dispatch_info: info }, Err(err) => { log::trace!( target: "runtime::system", @@ -1572,7 +1477,7 @@ impl Pallet { Self::block_number(), err, ); - Event::ExtrinsicFailed(err.error, info) + Event::ExtrinsicFailed { dispatch_error: err.error, dispatch_info: info } }, }); @@ -1600,13 +1505,13 @@ impl Pallet { /// An account is being created. pub fn on_created_account(who: T::AccountId, _a: &mut AccountInfo) { T::OnNewAccount::on_new_account(&who); - Self::deposit_event(Event::NewAccount(who)); + Self::deposit_event(Event::NewAccount { account: who }); } /// Do anything that needs to be done after an account has been killed. fn on_killed_account(who: T::AccountId) { T::OnKilledAccount::on_killed_account(&who); - Self::deposit_event(Event::KilledAccount(who)); + Self::deposit_event(Event::KilledAccount { account: who }); } /// Determine whether or not it is possible to update the code. @@ -1749,7 +1654,7 @@ impl Lookup for ChainContext { /// Prelude to be used alongside pallet macro, for ease of use. pub mod pallet_prelude { - pub use crate::{ensure_none, ensure_root, ensure_signed}; + pub use crate::{ensure_none, ensure_root, ensure_signed, ensure_signed_or_root}; /// Type alias for the `Origin` associated type of system config. pub type OriginFor = ::Origin; diff --git a/frame/system/src/limits.rs b/frame/system/src/limits.rs index 687fb6f3dd36..4942a5dace7d 100644 --- a/frame/system/src/limits.rs +++ b/frame/system/src/limits.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/system/src/migrations/mod.rs b/frame/system/src/migrations/mod.rs new file mode 100644 index 000000000000..358ba55b7c81 --- /dev/null +++ b/frame/system/src/migrations/mod.rs @@ -0,0 +1,139 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Migrate the reference counting state. + +use codec::{Decode, Encode, FullCodec}; +use frame_support::{ + pallet_prelude::ValueQuery, traits::PalletInfoAccess, weights::Weight, Blake2_128Concat, + RuntimeDebug, +}; +use sp_std::prelude::*; + +/// Type used to encode the number of references an account has. +type RefCount = u32; + +/// Information of an account. +#[derive(Clone, Eq, PartialEq, Default, RuntimeDebug, Encode, Decode)] +struct AccountInfo { + nonce: Index, + consumers: RefCount, + providers: RefCount, + sufficients: RefCount, + data: AccountData, +} + +/// Trait to implement to give information about types used for migration +pub trait V2ToV3 { + /// The system pallet. + type Pallet: 'static + PalletInfoAccess; + + /// System config account id + type AccountId: 'static + FullCodec; + + /// System config index + type Index: 'static + FullCodec + Copy; + + /// System config account data + type AccountData: 'static + FullCodec; +} + +// ### Warning +// +// The call below is only valid because the name System is enforced +// at runtime construction level for the system pallet. +frame_support::generate_storage_alias!( + System, UpgradedToU32RefCount => Value< + bool, + ValueQuery + > +); + +// ### Warning +// +// The call below is only valid because the name System is enforced +// at runtime construction level for the system pallet. +frame_support::generate_storage_alias!( + System, UpgradedToTripleRefCount => Value< + bool, + ValueQuery + > +); + +// ### Warning +// +// The call below is only valid because the name System is enforced +// at runtime construction level for the system pallet. +frame_support::generate_storage_alias!( + System, Account => Map< + (Blake2_128Concat, T::AccountId), + AccountInfo + > +); + +/// Migrate from unique `u8` reference counting to triple `u32` reference counting. +pub fn migrate_from_single_u8_to_triple_ref_count() -> Weight { + let mut translated: usize = 0; + >::translate::<(T::Index, u8, T::AccountData), _>(|_key, (nonce, rc, data)| { + translated = translated + 1; + Some(AccountInfo { nonce, consumers: rc as RefCount, providers: 1, sufficients: 0, data }) + }); + log::info!( + target: "runtime::system", + "Applied migration from single u8 to triple reference counting to {:?} elements.", + translated + ); + ::put(true); + ::put(true); + Weight::max_value() +} + +/// Migrate from unique `u32` reference counting to triple `u32` reference counting. +pub fn migrate_from_single_to_triple_ref_count() -> Weight { + let mut translated: usize = 0; + >::translate::<(T::Index, RefCount, T::AccountData), _>( + |_key, (nonce, consumers, data)| { + translated = translated + 1; + Some(AccountInfo { nonce, consumers, providers: 1, sufficients: 0, data }) + }, + ); + log::info!( + target: "runtime::system", + "Applied migration from single to triple reference counting to {:?} elements.", + translated + ); + ::put(true); + Weight::max_value() +} + +/// Migrate from dual `u32` reference counting to triple `u32` reference counting. +pub fn migrate_from_dual_to_triple_ref_count() -> Weight { + let mut translated: usize = 0; + >::translate::<(T::Index, RefCount, RefCount, T::AccountData), _>( + |_key, (nonce, consumers, providers, data)| { + translated = translated + 1; + Some(AccountInfo { nonce, consumers, providers, sufficients: 0, data }) + }, + ); + log::info!( + target: "runtime::system", + "Applied migration from dual to triple reference counting to {:?} elements.", + translated + ); + ::put(true); + Weight::max_value() +} diff --git a/frame/system/src/mock.rs b/frame/system/src/mock.rs index 9dd35691cab8..f3f542aa83a9 100644 --- a/frame/system/src/mock.rs +++ b/frame/system/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,10 @@ // limitations under the License. use crate::{self as frame_system, *}; -use frame_support::parameter_types; +use frame_support::{ + parameter_types, + traits::{ConstU32, ConstU64}, +}; use sp_core::H256; use sp_runtime::{ testing::Header, @@ -42,7 +45,6 @@ const NORMAL_DISPATCH_RATIO: Perbill = Perbill::from_percent(75); const MAX_BLOCK_WEIGHT: Weight = 1024; parameter_types! { - pub const BlockHashCount: u64 = 10; pub Version: RuntimeVersion = RuntimeVersion { spec_name: sp_version::create_runtime_str!("test"), impl_name: sp_version::create_runtime_str!("system-test"), @@ -51,6 +53,7 @@ parameter_types! { impl_version: 1, apis: sp_version::create_apis_vec!([]), transaction_version: 1, + state_version: 1, }; pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { read: 10, @@ -101,7 +104,7 @@ impl Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<10>; type DbWeight = DbWeight; type Version = Version; type PalletInfo = PalletInfo; @@ -111,6 +114,7 @@ impl Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } pub type SysEvent = frame_system::Event; diff --git a/frame/system/src/mocking.rs b/frame/system/src/mocking.rs index 7e6026b72618..ccb63f9bb236 100644 --- a/frame/system/src/mocking.rs +++ b/frame/system/src/mocking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/system/src/offchain.rs b/frame/system/src/offchain.rs index ed758a2556b7..bf52ab8e3791 100644 --- a/frame/system/src/offchain.rs +++ b/frame/system/src/offchain.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -62,11 +62,7 @@ use sp_runtime::{ app_crypto::RuntimeAppPublic, traits::{Extrinsic as ExtrinsicT, IdentifyAccount, One}, }; -use sp_std::{ - collections::btree_set::BTreeSet, - convert::{TryFrom, TryInto}, - prelude::{Box, Vec}, -}; +use sp_std::{collections::btree_set::BTreeSet, prelude::*}; /// Marker struct used to flag using all supported keys to sign a payload. pub struct ForAll {} diff --git a/frame/system/src/tests.rs b/frame/system/src/tests.rs index a4dd3403f2c3..0facd796b2a0 100644 --- a/frame/system/src/tests.rs +++ b/frame/system/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -154,7 +154,8 @@ fn provider_required_to_support_consumer() { #[test] fn deposit_event_should_work() { new_test_ext().execute_with(|| { - System::initialize(&1, &[0u8; 32].into(), &Default::default(), InitKind::Full); + System::reset_events(); + System::initialize(&1, &[0u8; 32].into(), &Default::default()); System::note_finished_extrinsics(); System::deposit_event(SysEvent::CodeUpdated); System::finalize(); @@ -167,45 +168,46 @@ fn deposit_event_should_work() { }] ); - System::initialize(&2, &[0u8; 32].into(), &Default::default(), InitKind::Full); - System::deposit_event(SysEvent::NewAccount(32)); + System::reset_events(); + System::initialize(&2, &[0u8; 32].into(), &Default::default()); + System::deposit_event(SysEvent::NewAccount { account: 32 }); System::note_finished_initialize(); - System::deposit_event(SysEvent::KilledAccount(42)); + System::deposit_event(SysEvent::KilledAccount { account: 42 }); System::note_applied_extrinsic(&Ok(().into()), Default::default()); System::note_applied_extrinsic(&Err(DispatchError::BadOrigin.into()), Default::default()); System::note_finished_extrinsics(); - System::deposit_event(SysEvent::NewAccount(3)); + System::deposit_event(SysEvent::NewAccount { account: 3 }); System::finalize(); assert_eq!( System::events(), vec![ EventRecord { phase: Phase::Initialization, - event: SysEvent::NewAccount(32).into(), + event: SysEvent::NewAccount { account: 32 }.into(), topics: vec![], }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: SysEvent::KilledAccount(42).into(), + event: SysEvent::KilledAccount { account: 42 }.into(), topics: vec![] }, EventRecord { phase: Phase::ApplyExtrinsic(0), - event: SysEvent::ExtrinsicSuccess(Default::default()).into(), + event: SysEvent::ExtrinsicSuccess { dispatch_info: Default::default() }.into(), topics: vec![] }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: SysEvent::ExtrinsicFailed( - DispatchError::BadOrigin.into(), - Default::default() - ) + event: SysEvent::ExtrinsicFailed { + dispatch_error: DispatchError::BadOrigin.into(), + dispatch_info: Default::default() + } .into(), topics: vec![] }, EventRecord { phase: Phase::Finalization, - event: SysEvent::NewAccount(3).into(), + event: SysEvent::NewAccount { account: 3 }.into(), topics: vec![] }, ] @@ -216,7 +218,8 @@ fn deposit_event_should_work() { #[test] fn deposit_event_uses_actual_weight() { new_test_ext().execute_with(|| { - System::initialize(&1, &[0u8; 32].into(), &Default::default(), InitKind::Full); + System::reset_events(); + System::initialize(&1, &[0u8; 32].into(), &Default::default()); System::note_finished_initialize(); let pre_info = DispatchInfo { weight: 1000, ..Default::default() }; @@ -234,37 +237,34 @@ fn deposit_event_uses_actual_weight() { vec![ EventRecord { phase: Phase::ApplyExtrinsic(0), - event: SysEvent::ExtrinsicSuccess(DispatchInfo { - weight: 300, - ..Default::default() - },) + event: SysEvent::ExtrinsicSuccess { + dispatch_info: DispatchInfo { weight: 300, ..Default::default() }, + } .into(), topics: vec![] }, EventRecord { phase: Phase::ApplyExtrinsic(1), - event: SysEvent::ExtrinsicSuccess(DispatchInfo { - weight: 1000, - ..Default::default() - },) + event: SysEvent::ExtrinsicSuccess { + dispatch_info: DispatchInfo { weight: 1000, ..Default::default() }, + } .into(), topics: vec![] }, EventRecord { phase: Phase::ApplyExtrinsic(2), - event: SysEvent::ExtrinsicSuccess(DispatchInfo { - weight: 1000, - ..Default::default() - },) + event: SysEvent::ExtrinsicSuccess { + dispatch_info: DispatchInfo { weight: 1000, ..Default::default() }, + } .into(), topics: vec![] }, EventRecord { phase: Phase::ApplyExtrinsic(3), - event: SysEvent::ExtrinsicFailed( - DispatchError::BadOrigin.into(), - DispatchInfo { weight: 999, ..Default::default() }, - ) + event: SysEvent::ExtrinsicFailed { + dispatch_error: DispatchError::BadOrigin.into(), + dispatch_info: DispatchInfo { weight: 999, ..Default::default() }, + } .into(), topics: vec![] }, @@ -278,15 +278,16 @@ fn deposit_event_topics() { new_test_ext().execute_with(|| { const BLOCK_NUMBER: u64 = 1; - System::initialize(&BLOCK_NUMBER, &[0u8; 32].into(), &Default::default(), InitKind::Full); + System::reset_events(); + System::initialize(&BLOCK_NUMBER, &[0u8; 32].into(), &Default::default()); System::note_finished_extrinsics(); let topics = vec![H256::repeat_byte(1), H256::repeat_byte(2), H256::repeat_byte(3)]; // We deposit a few events with different sets of topics. - System::deposit_event_indexed(&topics[0..3], SysEvent::NewAccount(1).into()); - System::deposit_event_indexed(&topics[0..1], SysEvent::NewAccount(2).into()); - System::deposit_event_indexed(&topics[1..2], SysEvent::NewAccount(3).into()); + System::deposit_event_indexed(&topics[0..3], SysEvent::NewAccount { account: 1 }.into()); + System::deposit_event_indexed(&topics[0..1], SysEvent::NewAccount { account: 2 }.into()); + System::deposit_event_indexed(&topics[1..2], SysEvent::NewAccount { account: 3 }.into()); System::finalize(); @@ -296,17 +297,17 @@ fn deposit_event_topics() { vec![ EventRecord { phase: Phase::Finalization, - event: SysEvent::NewAccount(1).into(), + event: SysEvent::NewAccount { account: 1 }.into(), topics: topics[0..3].to_vec(), }, EventRecord { phase: Phase::Finalization, - event: SysEvent::NewAccount(2).into(), + event: SysEvent::NewAccount { account: 2 }.into(), topics: topics[0..1].to_vec(), }, EventRecord { phase: Phase::Finalization, - event: SysEvent::NewAccount(3).into(), + event: SysEvent::NewAccount { account: 3 }.into(), topics: topics[1..2].to_vec(), } ] @@ -336,7 +337,8 @@ fn prunes_block_hash_mappings() { new_test_ext().execute_with(|| { // simulate import of 15 blocks for n in 1..=15 { - System::initialize(&n, &[n as u8 - 1; 32].into(), &Default::default(), InitKind::Full); + System::reset_events(); + System::initialize(&n, &[n as u8 - 1; 32].into(), &Default::default()); System::finalize(); } @@ -464,21 +466,11 @@ fn events_not_emitted_during_genesis() { }); } -#[test] -fn ensure_one_of_works() { - fn ensure_root_or_signed(o: RawOrigin) -> Result, Origin> { - EnsureOneOf::, EnsureSigned>::try_origin(o.into()) - } - - assert_eq!(ensure_root_or_signed(RawOrigin::Root).unwrap(), Either::Left(())); - assert_eq!(ensure_root_or_signed(RawOrigin::Signed(0)).unwrap(), Either::Right(0)); - assert!(ensure_root_or_signed(RawOrigin::None).is_err()); -} - #[test] fn extrinsics_root_is_calculated_correctly() { new_test_ext().execute_with(|| { - System::initialize(&1, &[0u8; 32].into(), &Default::default(), InitKind::Full); + System::reset_events(); + System::initialize(&1, &[0u8; 32].into(), &Default::default()); System::note_finished_initialize(); System::note_extrinsic(vec![1]); System::note_applied_extrinsic(&Ok(().into()), Default::default()); @@ -495,8 +487,32 @@ fn extrinsics_root_is_calculated_correctly() { #[test] fn runtime_updated_digest_emitted_when_heap_pages_changed() { new_test_ext().execute_with(|| { - System::initialize(&1, &[0u8; 32].into(), &Default::default(), InitKind::Full); + System::reset_events(); + System::initialize(&1, &[0u8; 32].into(), &Default::default()); System::set_heap_pages(RawOrigin::Root.into(), 5).unwrap(); assert_runtime_updated_digest(1); }); } + +#[test] +fn ensure_signed_stuff_works() { + struct Members; + impl SortedMembers for Members { + fn sorted_members() -> Vec { + (0..10).collect() + } + } + + let signed_origin = Origin::signed(0u64); + assert_ok!(EnsureSigned::try_origin(signed_origin.clone())); + assert_ok!(EnsureSignedBy::::try_origin(signed_origin)); + + #[cfg(feature = "runtime-benchmarks")] + { + let successful_origin: Origin = EnsureSigned::successful_origin(); + assert_ok!(EnsureSigned::try_origin(successful_origin)); + + let successful_origin: Origin = EnsureSignedBy::::successful_origin(); + assert_ok!(EnsureSignedBy::::try_origin(successful_origin)); + } +} diff --git a/frame/system/src/weights.rs b/frame/system/src/weights.rs index 281d26375c81..a016fe6e7d03 100644 --- a/frame/system/src/weights.rs +++ b/frame/system/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for frame_system //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/system/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -48,7 +49,6 @@ pub trait WeightInfo { fn remark(b: u32, ) -> Weight; fn remark_with_event(b: u32, ) -> Weight; fn set_heap_pages() -> Weight; - fn set_changes_trie_config() -> Weight; fn set_storage(i: u32, ) -> Weight; fn kill_storage(i: u32, ) -> Weight; fn kill_prefix(p: u32, ) -> Weight; @@ -57,25 +57,18 @@ pub trait WeightInfo { /// Weights for frame_system using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { - fn remark(b: u32, ) -> Weight { - (574_000 as Weight) - // Standard Error: 0 - .saturating_add((1_000 as Weight).saturating_mul(b as Weight)) + fn remark(_b: u32, ) -> Weight { + (0 as Weight) } fn remark_with_event(b: u32, ) -> Weight { (0 as Weight) // Standard Error: 0 - .saturating_add((2_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((1_000 as Weight).saturating_mul(b as Weight)) } + // Storage: System Digest (r:1 w:1) // Storage: unknown [0x3a686561707061676573] (r:0 w:1) fn set_heap_pages() -> Weight { - (1_891_000 as Weight) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) - } - // Storage: System Digest (r:1 w:1) - // Storage: unknown [0x3a6368616e6765735f74726965] (r:0 w:1) - fn set_changes_trie_config() -> Weight { - (7_370_000 as Weight) + (2_864_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -83,46 +76,39 @@ impl WeightInfo for SubstrateWeight { fn set_storage(i: u32, ) -> Weight { (0 as Weight) // Standard Error: 0 - .saturating_add((848_000 as Weight).saturating_mul(i as Weight)) + .saturating_add((389_000 as Weight).saturating_mul(i as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(i as Weight))) } // Storage: Skipped Metadata (r:0 w:0) fn kill_storage(i: u32, ) -> Weight { - (308_000 as Weight) + (0 as Weight) // Standard Error: 0 - .saturating_add((559_000 as Weight).saturating_mul(i as Weight)) + .saturating_add((285_000 as Weight).saturating_mul(i as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(i as Weight))) } // Storage: Skipped Metadata (r:0 w:0) fn kill_prefix(p: u32, ) -> Weight { - (7_616_000 as Weight) - // Standard Error: 1_000 - .saturating_add((783_000 as Weight).saturating_mul(p as Weight)) + (753_000 as Weight) + // Standard Error: 0 + .saturating_add((630_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(p as Weight))) } } // For backwards compatibility and tests impl WeightInfo for () { - fn remark(b: u32, ) -> Weight { - (574_000 as Weight) - // Standard Error: 0 - .saturating_add((1_000 as Weight).saturating_mul(b as Weight)) + fn remark(_b: u32, ) -> Weight { + (0 as Weight) } fn remark_with_event(b: u32, ) -> Weight { (0 as Weight) // Standard Error: 0 - .saturating_add((2_000 as Weight).saturating_mul(b as Weight)) + .saturating_add((1_000 as Weight).saturating_mul(b as Weight)) } + // Storage: System Digest (r:1 w:1) // Storage: unknown [0x3a686561707061676573] (r:0 w:1) fn set_heap_pages() -> Weight { - (1_891_000 as Weight) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) - } - // Storage: System Digest (r:1 w:1) - // Storage: unknown [0x3a6368616e6765735f74726965] (r:0 w:1) - fn set_changes_trie_config() -> Weight { - (7_370_000 as Weight) + (2_864_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -130,21 +116,21 @@ impl WeightInfo for () { fn set_storage(i: u32, ) -> Weight { (0 as Weight) // Standard Error: 0 - .saturating_add((848_000 as Weight).saturating_mul(i as Weight)) + .saturating_add((389_000 as Weight).saturating_mul(i as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(i as Weight))) } // Storage: Skipped Metadata (r:0 w:0) fn kill_storage(i: u32, ) -> Weight { - (308_000 as Weight) + (0 as Weight) // Standard Error: 0 - .saturating_add((559_000 as Weight).saturating_mul(i as Weight)) + .saturating_add((285_000 as Weight).saturating_mul(i as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(i as Weight))) } // Storage: Skipped Metadata (r:0 w:0) fn kill_prefix(p: u32, ) -> Weight { - (7_616_000 as Weight) - // Standard Error: 1_000 - .saturating_add((783_000 as Weight).saturating_mul(p as Weight)) + (753_000 as Weight) + // Standard Error: 0 + .saturating_add((630_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(p as Weight))) } } diff --git a/frame/timestamp/Cargo.toml b/frame/timestamp/Cargo.toml index 1c95c4782b5c..72d632faf858 100644 --- a/frame/timestamp/Cargo.toml +++ b/frame/timestamp/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-timestamp" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME Timestamp Module" documentation = "https://docs.rs/pallet-timestamp" @@ -15,11 +15,11 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.2.0", default-features = false, features = ["derive", "max-encoded-len"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io", optional = true } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io", optional = true } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../primitives/inherents" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } @@ -28,8 +28,8 @@ sp-timestamp = { version = "4.0.0-dev", default-features = false, path = "../../ log = { version = "0.4.14", default-features = false } [dev-dependencies] -sp-io ={ version = "4.0.0-dev", path = "../../primitives/io" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-io ={ version = "6.0.0", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } [features] default = ["std"] @@ -45,5 +45,5 @@ std = [ "sp-timestamp/std", "log/std", ] -runtime-benchmarks = ["frame-benchmarking", "sp-io"] +runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks", "sp-io"] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/timestamp/README.md b/frame/timestamp/README.md index 5f8388b04f82..1546377ee674 100644 --- a/frame/timestamp/README.md +++ b/frame/timestamp/README.md @@ -45,20 +45,29 @@ trait from the timestamp trait. ### Get current timestamp ```rust -use frame_support::{decl_module, dispatch}; -use frame_system::ensure_signed; - -pub trait Config: timestamp::Config {} - -decl_module! { - pub struct Module for enum Call where origin: T::Origin { - #[weight = 0] - pub fn get_time(origin) -> dispatch::DispatchResult { - let _sender = ensure_signed(origin)?; - let _now = >::get(); - Ok(()) - } - } +use pallet_timestamp::{self as timestamp}; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config + timestamp::Config {} + + #[pallet::call] + impl Pallet { + #[pallet::weight(0)] + pub fn get_time(origin: OriginFor) -> DispatchResult { + let _sender = ensure_signed(origin)?; + let _now = >::get(); + Ok(()) + } + } } ``` diff --git a/frame/timestamp/src/benchmarking.rs b/frame/timestamp/src/benchmarking.rs index 97ddd4cddd63..0da71dbdd1a5 100644 --- a/frame/timestamp/src/benchmarking.rs +++ b/frame/timestamp/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ #![cfg(feature = "runtime-benchmarks")] use super::*; -use frame_benchmarking::{benchmarks, impl_benchmark_test_suite, TrackedStorageKey}; +use frame_benchmarking::{benchmarks, TrackedStorageKey}; use frame_support::{ensure, traits::OnFinalize}; use frame_system::RawOrigin; @@ -55,6 +55,6 @@ benchmarks! { verify { ensure!(!DidUpdate::::exists(), "Time was not removed."); } -} -impl_benchmark_test_suite!(Timestamp, crate::tests::new_test_ext(), crate::tests::Test); + impl_benchmark_test_suite!(Timestamp, crate::tests::new_test_ext(), crate::tests::Test); +} diff --git a/frame/timestamp/src/lib.rs b/frame/timestamp/src/lib.rs index 153606bedbac..d8bfb405e5c7 100644 --- a/frame/timestamp/src/lib.rs +++ b/frame/timestamp/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -64,18 +64,26 @@ //! ### Get current timestamp //! //! ``` -//! use frame_support::{decl_module, dispatch}; -//! # use pallet_timestamp as timestamp; -//! use frame_system::ensure_signed; +//! use pallet_timestamp::{self as timestamp}; //! -//! pub trait Config: timestamp::Config {} +//! #[frame_support::pallet] +//! pub mod pallet { +//! use super::*; +//! use frame_support::pallet_prelude::*; +//! use frame_system::pallet_prelude::*; //! -//! decl_module! { -//! pub struct Module for enum Call where origin: T::Origin { -//! #[weight = 0] -//! pub fn get_time(origin) -> dispatch::DispatchResult { +//! #[pallet::pallet] +//! pub struct Pallet(_); +//! +//! #[pallet::config] +//! pub trait Config: frame_system::Config + timestamp::Config {} +//! +//! #[pallet::call] +//! impl Pallet { +//! #[pallet::weight(0)] +//! pub fn get_time(origin: OriginFor) -> DispatchResult { //! let _sender = ensure_signed(origin)?; -//! let _now = >::get(); +//! let _now = >::get(); //! Ok(()) //! } //! } @@ -140,7 +148,6 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] - #[pallet::generate_storage_info] pub struct Pallet(PhantomData); /// Current time for the current block. @@ -308,7 +315,10 @@ mod tests { use super::*; use crate as pallet_timestamp; - use frame_support::{assert_ok, parameter_types}; + use frame_support::{ + assert_ok, parameter_types, + traits::{ConstU32, ConstU64}, + }; use sp_core::H256; use sp_io::TestExternalities; use sp_runtime::{ @@ -336,7 +346,6 @@ mod tests { ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -355,7 +364,7 @@ mod tests { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = (); @@ -364,14 +373,13 @@ mod tests { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } - parameter_types! { - pub const MinimumPeriod: u64 = 5; - } + impl Config for Test { type Moment = u64; type OnTimestampSet = (); - type MinimumPeriod = MinimumPeriod; + type MinimumPeriod = ConstU64<5>; type WeightInfo = (); } diff --git a/frame/timestamp/src/weights.rs b/frame/timestamp/src/weights.rs index b4e7370ee761..33d7d6a4b9e3 100644 --- a/frame/timestamp/src/weights.rs +++ b/frame/timestamp/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_timestamp //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/timestamp/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -55,12 +56,12 @@ impl WeightInfo for SubstrateWeight { // Storage: Timestamp Now (r:1 w:1) // Storage: Babe CurrentSlot (r:1 w:0) fn set() -> Weight { - (10_391_000 as Weight) + (5_247_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn on_finalize() -> Weight { - (4_843_000 as Weight) + (2_604_000 as Weight) } } @@ -69,11 +70,11 @@ impl WeightInfo for () { // Storage: Timestamp Now (r:1 w:1) // Storage: Babe CurrentSlot (r:1 w:0) fn set() -> Weight { - (10_391_000 as Weight) + (5_247_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn on_finalize() -> Weight { - (4_843_000 as Weight) + (2_604_000 as Weight) } } diff --git a/frame/tips/Cargo.toml b/frame/tips/Cargo.toml index 8ca395e1c541..dc65d5c28d1c 100644 --- a/frame/tips/Cargo.toml +++ b/frame/tips/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-tips" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet to manage tips" readme = "README.md" @@ -13,15 +13,15 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } log = { version = "0.4.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.126", features = ["derive"], optional = true } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +serde = { version = "1.0.136", features = ["derive"], optional = true } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } @@ -30,7 +30,7 @@ pallet-treasury = { version = "4.0.0-dev", default-features = false, path = "../ frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-storage = { version = "4.0.0-dev", path = "../../primitives/storage" } +sp-storage = { version = "6.0.0", path = "../../primitives/storage" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } [features] @@ -50,7 +50,7 @@ std = [ "pallet-treasury/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", ] diff --git a/frame/tips/src/benchmarking.rs b/frame/tips/src/benchmarking.rs index 5e0812185521..7abee6192f2e 100644 --- a/frame/tips/src/benchmarking.rs +++ b/frame/tips/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,7 @@ #![cfg(feature = "runtime-benchmarks")] -use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite, whitelisted_caller}; +use frame_benchmarking::{account, benchmarks, whitelisted_caller}; use frame_support::ensure; use frame_system::RawOrigin; use sp_runtime::traits::Saturating; @@ -84,12 +84,9 @@ fn setup_pot_account() { let _ = T::Currency::make_free_balance_be(&pot_account, value); } -const MAX_BYTES: u32 = 16384; -const MAX_TIPPERS: u32 = 100; - benchmarks! { report_awesome { - let r in 0 .. MAX_BYTES; + let r in 0 .. T::MaximumReasonLength::get(); let (caller, reason, awesome_person) = setup_awesome::(r); // Whitelist caller account from further DB operations. let caller_key = frame_system::Account::::hashed_key_for(&caller); @@ -97,7 +94,7 @@ benchmarks! { }: _(RawOrigin::Signed(caller), reason, awesome_person) retract_tip { - let r = MAX_BYTES; + let r = T::MaximumReasonLength::get(); let (caller, reason, awesome_person) = setup_awesome::(r); TipsMod::::report_awesome( RawOrigin::Signed(caller.clone()).into(), @@ -112,8 +109,8 @@ benchmarks! { }: _(RawOrigin::Signed(caller), hash) tip_new { - let r in 0 .. MAX_BYTES; - let t in 1 .. MAX_TIPPERS; + let r in 0 .. T::MaximumReasonLength::get(); + let t in 1 .. T::Tippers::max_len() as u32; let (caller, reason, beneficiary, value) = setup_tip::(r, t)?; // Whitelist caller account from further DB operations. @@ -122,7 +119,7 @@ benchmarks! { }: _(RawOrigin::Signed(caller), reason, beneficiary, value) tip { - let t in 1 .. MAX_TIPPERS; + let t in 1 .. T::Tippers::max_len() as u32; let (member, reason, beneficiary, value) = setup_tip::(0, t)?; let value = T::Currency::minimum_balance().saturating_mul(100u32.into()); TipsMod::::tip_new( @@ -142,7 +139,7 @@ benchmarks! { }: _(RawOrigin::Signed(caller), hash, value) close_tip { - let t in 1 .. MAX_TIPPERS; + let t in 1 .. T::Tippers::max_len() as u32; // Make sure pot is funded setup_pot_account::(); @@ -171,7 +168,7 @@ benchmarks! { }: _(RawOrigin::Signed(caller), hash) slash_tip { - let t in 1 .. MAX_TIPPERS; + let t in 1 .. T::Tippers::max_len() as u32; // Make sure pot is funded setup_pot_account::(); @@ -190,6 +187,6 @@ benchmarks! { let hash = T::Hashing::hash_of(&(&reason_hash, &beneficiary)); ensure!(Tips::::contains_key(hash), "tip does not exist"); }: _(RawOrigin::Root, hash) -} -impl_benchmark_test_suite!(TipsMod, crate::tests::new_test_ext(), crate::tests::Test); + impl_benchmark_test_suite!(TipsMod, crate::tests::new_test_ext(), crate::tests::Test); +} diff --git a/frame/tips/src/lib.rs b/frame/tips/src/lib.rs index f4a4edb7b399..ae320629dfc3 100644 --- a/frame/tips/src/lib.rs +++ b/frame/tips/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -61,7 +61,7 @@ pub mod migrations; pub mod weights; use sp_runtime::{ - traits::{AccountIdConversion, BadOrigin, Hash, Zero}, + traits::{AccountIdConversion, BadOrigin, Hash, TrailingZeroInput, Zero}, Percent, RuntimeDebug, }; use sp_std::prelude::*; @@ -120,6 +120,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] #[pallet::storage_version(STORAGE_VERSION)] + #[pallet::without_storage_info] pub struct Pallet(_); #[pallet::config] @@ -128,6 +129,8 @@ pub mod pallet { type Event: From> + IsType<::Event>; /// Maximum acceptable reason length. + /// + /// Benchmarks depend on this value, be sure to update weights file when changing this value #[pallet::constant] type MaximumReasonLength: Get; @@ -150,7 +153,8 @@ pub mod pallet { /// Origin from which tippers must come. /// /// `ContainsLengthBound::max_len` must be cost free (i.e. no storage read or heavy - /// operation). + /// operation). Benchmarks depend on the value of `ContainsLengthBound::max_len` be sure to + /// update weights file when altering this method. type Tippers: SortedMembers + ContainsLengthBound; /// Weight information for extrinsics in this pallet. @@ -179,16 +183,16 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// A new tip suggestion has been opened. \[tip_hash\] - NewTip(T::Hash), - /// A tip suggestion has reached threshold and is closing. \[tip_hash\] - TipClosing(T::Hash), - /// A tip suggestion has been closed. \[tip_hash, who, payout\] - TipClosed(T::Hash, T::AccountId, BalanceOf), - /// A tip suggestion has been retracted. \[tip_hash\] - TipRetracted(T::Hash), - /// A tip suggestion has been slashed. \[tip_hash, finder, deposit\] - TipSlashed(T::Hash, T::AccountId, BalanceOf), + /// A new tip suggestion has been opened. + NewTip { tip_hash: T::Hash }, + /// A tip suggestion has reached threshold and is closing. + TipClosing { tip_hash: T::Hash }, + /// A tip suggestion has been closed. + TipClosed { tip_hash: T::Hash, who: T::AccountId, payout: BalanceOf }, + /// A tip suggestion has been retracted. + TipRetracted { tip_hash: T::Hash }, + /// A tip suggestion has been slashed. + TipSlashed { tip_hash: T::Hash, finder: T::AccountId, deposit: BalanceOf }, } /// Old name generated by `decl_event`. @@ -265,7 +269,7 @@ pub mod pallet { finders_fee: true, }; Tips::::insert(&hash, tip); - Self::deposit_event(Event::NewTip(hash)); + Self::deposit_event(Event::NewTip { tip_hash: hash }); Ok(()) } @@ -300,7 +304,7 @@ pub mod pallet { let err_amount = T::Currency::unreserve(&who, tip.deposit); debug_assert!(err_amount.is_zero()); } - Self::deposit_event(Event::TipRetracted(hash)); + Self::deposit_event(Event::TipRetracted { tip_hash: hash }); Ok(()) } @@ -340,7 +344,7 @@ pub mod pallet { let hash = T::Hashing::hash_of(&(&reason_hash, &who)); Reasons::::insert(&reason_hash, &reason); - Self::deposit_event(Event::NewTip(hash.clone())); + Self::deposit_event(Event::NewTip { tip_hash: hash.clone() }); let tips = vec![(tipper.clone(), tip_value)]; let tip = OpenTip { reason: reason_hash, @@ -390,7 +394,7 @@ pub mod pallet { let mut tip = Tips::::get(hash).ok_or(Error::::UnknownTip)?; if Self::insert_tip_and_check_closing(&mut tip, tipper, tip_value) { - Self::deposit_event(Event::TipClosing(hash.clone())); + Self::deposit_event(Event::TipClosing { tip_hash: hash.clone() }); } Tips::::insert(&hash, tip); Ok(()) @@ -449,7 +453,11 @@ pub mod pallet { T::OnSlash::on_unbalanced(imbalance); } Reasons::::remove(&tip.reason); - Self::deposit_event(Event::TipSlashed(hash, tip.finder, tip.deposit)); + Self::deposit_event(Event::TipSlashed { + tip_hash: hash, + finder: tip.finder, + deposit: tip.deposit, + }); Ok(()) } } @@ -544,7 +552,7 @@ impl Pallet { // same as above: best-effort only. let res = T::Currency::transfer(&treasury, &tip.who, payout, KeepAlive); debug_assert!(res.is_ok()); - Self::deposit_event(Event::TipClosed(hash, tip.who, payout)); + Self::deposit_event(Event::TipClosed { tip_hash: hash, who: tip.who, payout }); } pub fn migrate_retract_tip_for_tip_new(module: &[u8], item: &[u8]) { @@ -573,6 +581,9 @@ impl Pallet { use frame_support::{migration::storage_key_iter, Twox64Concat}; + let zero_account = T::AccountId::decode(&mut TrailingZeroInput::new(&[][..])) + .expect("infinite input; qed"); + for (hash, old_tip) in storage_key_iter::< T::Hash, OldOpenTip, T::BlockNumber, T::Hash>, @@ -582,7 +593,7 @@ impl Pallet { { let (finder, deposit, finders_fee) = match old_tip.finder { Some((finder, deposit)) => (finder, deposit, true), - None => (T::AccountId::default(), Zero::zero(), false), + None => (zero_account.clone(), Zero::zero(), false), }; let new_tip = OpenTip { reason: old_tip.reason, diff --git a/frame/tips/src/migrations/mod.rs b/frame/tips/src/migrations/mod.rs index 81139120da1c..719bb2f86fdd 100644 --- a/frame/tips/src/migrations/mod.rs +++ b/frame/tips/src/migrations/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/tips/src/migrations/v4.rs b/frame/tips/src/migrations/v4.rs index 69df1d08d2c8..34f7a43ec12d 100644 --- a/frame/tips/src/migrations/v4.rs +++ b/frame/tips/src/migrations/v4.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/tips/src/tests.rs b/frame/tips/src/tests.rs index 7ea80d78c553..0c58a949958f 100644 --- a/frame/tips/src/tests.rs +++ b/frame/tips/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,8 +30,12 @@ use sp_runtime::{ use sp_storage::Storage; use frame_support::{ - assert_noop, assert_ok, pallet_prelude::GenesisBuild, parameter_types, - storage::StoragePrefixedMap, traits::SortedMembers, weights::Weight, PalletId, + assert_noop, assert_ok, + pallet_prelude::GenesisBuild, + parameter_types, + storage::StoragePrefixedMap, + traits::{ConstU32, ConstU64, SortedMembers}, + PalletId, }; use super::*; @@ -54,9 +58,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; - pub const MaximumBlockWeight: Weight = 1024; - pub const MaximumBlockLength: u32 = 2 * 1024; pub const AvailableBlockRatio: Perbill = Perbill::one(); } impl frame_system::Config for Test { @@ -74,7 +75,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -83,10 +84,9 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } -parameter_types! { - pub const ExistentialDeposit: u64 = 1; -} + impl pallet_balances::Config for Test { type MaxLocks = (); type MaxReserves = (); @@ -94,7 +94,7 @@ impl pallet_balances::Config for Test { type Balance = u64; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } @@ -125,13 +125,8 @@ impl ContainsLengthBound for TenToFourteen { } parameter_types! { pub const ProposalBond: Permill = Permill::from_percent(5); - pub const ProposalBondMinimum: u64 = 1; - pub const SpendPeriod: u64 = 2; pub const Burn: Permill = Permill::from_percent(50); - pub const DataDepositPerByte: u64 = 1; pub const TreasuryPalletId: PalletId = PalletId(*b"py/trsry"); - pub const MaximumReasonLength: u32 = 16384; - pub const MaxApprovals: u32 = 100; } impl pallet_treasury::Config for Test { type PalletId = TreasuryPalletId; @@ -141,26 +136,25 @@ impl pallet_treasury::Config for Test { type Event = Event; type OnSlash = (); type ProposalBond = ProposalBond; - type ProposalBondMinimum = ProposalBondMinimum; - type SpendPeriod = SpendPeriod; + type ProposalBondMinimum = ConstU64<1>; + type ProposalBondMaximum = (); + type SpendPeriod = ConstU64<2>; type Burn = Burn; type BurnDestination = (); // Just gets burned. type WeightInfo = (); type SpendFunds = (); - type MaxApprovals = MaxApprovals; + type MaxApprovals = ConstU32<100>; } parameter_types! { - pub const TipCountdown: u64 = 1; pub const TipFindersFee: Percent = Percent::from_percent(20); - pub const TipReportDepositBase: u64 = 1; } impl Config for Test { - type MaximumReasonLength = MaximumReasonLength; + type MaximumReasonLength = ConstU32<16384>; type Tippers = TenToFourteen; - type TipCountdown = TipCountdown; + type TipCountdown = ConstU64<1>; type TipFindersFee = TipFindersFee; - type TipReportDepositBase = TipReportDepositBase; - type DataDepositPerByte = DataDepositPerByte; + type TipReportDepositBase = ConstU64<1>; + type DataDepositPerByte = ConstU64<1>; type Event = Event; type WeightInfo = (); } @@ -267,7 +261,7 @@ fn close_tip_works() { let h = tip_hash(); - assert_eq!(last_event(), TipEvent::NewTip(h)); + assert_eq!(last_event(), TipEvent::NewTip { tip_hash: h }); assert_ok!(Tips::tip(Origin::signed(11), h.clone(), 10)); @@ -275,7 +269,7 @@ fn close_tip_works() { assert_ok!(Tips::tip(Origin::signed(12), h.clone(), 10)); - assert_eq!(last_event(), TipEvent::TipClosing(h)); + assert_eq!(last_event(), TipEvent::TipClosing { tip_hash: h }); assert_noop!(Tips::close_tip(Origin::signed(0), h.into()), Error::::Premature); @@ -284,7 +278,7 @@ fn close_tip_works() { assert_ok!(Tips::close_tip(Origin::signed(0), h.into())); assert_eq!(Balances::free_balance(3), 10); - assert_eq!(last_event(), TipEvent::TipClosed(h, 3, 10)); + assert_eq!(last_event(), TipEvent::TipClosed { tip_hash: h, who: 3, payout: 10 }); assert_noop!(Tips::close_tip(Origin::signed(100), h.into()), Error::::UnknownTip); }); @@ -306,14 +300,14 @@ fn slash_tip_works() { assert_eq!(Balances::free_balance(0), 88); let h = tip_hash(); - assert_eq!(last_event(), TipEvent::NewTip(h)); + assert_eq!(last_event(), TipEvent::NewTip { tip_hash: h }); // can't remove from any origin assert_noop!(Tips::slash_tip(Origin::signed(0), h.clone()), BadOrigin); // can remove from root. assert_ok!(Tips::slash_tip(Origin::root(), h.clone())); - assert_eq!(last_event(), TipEvent::TipSlashed(h, 0, 12)); + assert_eq!(last_event(), TipEvent::TipSlashed { tip_hash: h, finder: 0, deposit: 12 }); // tipper slashed assert_eq!(Balances::reserved_balance(0), 0); diff --git a/frame/tips/src/weights.rs b/frame/tips/src/weights.rs index 3376afb06617..2ba15a0c2fcc 100644 --- a/frame/tips/src/weights.rs +++ b/frame/tips/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_tips //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/tips/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -56,60 +57,60 @@ pub trait WeightInfo { /// Weights for pallet_tips using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { - // Storage: Treasury Reasons (r:1 w:1) - // Storage: Treasury Tips (r:1 w:1) + // Storage: Tips Reasons (r:1 w:1) + // Storage: Tips Tips (r:1 w:1) fn report_awesome(r: u32, ) -> Weight { - (50_921_000 as Weight) + (25_262_000 as Weight) // Standard Error: 0 .saturating_add((2_000 as Weight).saturating_mul(r as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } - // Storage: Treasury Tips (r:1 w:1) - // Storage: Treasury Reasons (r:0 w:1) + // Storage: Tips Tips (r:1 w:1) + // Storage: Tips Reasons (r:0 w:1) fn retract_tip() -> Weight { - (46_352_000 as Weight) + (24_162_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Elections Members (r:1 w:0) - // Storage: Treasury Reasons (r:1 w:1) - // Storage: Treasury Tips (r:0 w:1) + // Storage: Tips Reasons (r:1 w:1) + // Storage: Tips Tips (r:0 w:1) fn tip_new(r: u32, t: u32, ) -> Weight { - (33_338_000 as Weight) + (16_435_000 as Weight) // Standard Error: 0 .saturating_add((2_000 as Weight).saturating_mul(r as Weight)) - // Standard Error: 0 - .saturating_add((115_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 4_000 + .saturating_add((231_000 as Weight).saturating_mul(t as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Elections Members (r:1 w:0) - // Storage: Treasury Tips (r:1 w:1) + // Storage: Tips Tips (r:1 w:1) fn tip(t: u32, ) -> Weight { - (22_702_000 as Weight) - // Standard Error: 0 - .saturating_add((538_000 as Weight).saturating_mul(t as Weight)) + (10_427_000 as Weight) + // Standard Error: 7_000 + .saturating_add((507_000 as Weight).saturating_mul(t as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } - // Storage: Treasury Tips (r:1 w:1) + // Storage: Tips Tips (r:1 w:1) // Storage: Elections Members (r:1 w:0) // Storage: System Account (r:1 w:1) - // Storage: Treasury Reasons (r:0 w:1) + // Storage: Tips Reasons (r:0 w:1) fn close_tip(t: u32, ) -> Weight { - (84_094_000 as Weight) - // Standard Error: 0 - .saturating_add((283_000 as Weight).saturating_mul(t as Weight)) + (40_901_000 as Weight) + // Standard Error: 10_000 + .saturating_add((281_000 as Weight).saturating_mul(t as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } - // Storage: Treasury Tips (r:1 w:1) - // Storage: Treasury Reasons (r:0 w:1) + // Storage: Tips Tips (r:1 w:1) + // Storage: Tips Reasons (r:0 w:1) fn slash_tip(t: u32, ) -> Weight { - (24_891_000 as Weight) - // Standard Error: 0 - .saturating_add((6_000 as Weight).saturating_mul(t as Weight)) + (14_636_000 as Weight) + // Standard Error: 4_000 + .saturating_add((29_000 as Weight).saturating_mul(t as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -117,60 +118,60 @@ impl WeightInfo for SubstrateWeight { // For backwards compatibility and tests impl WeightInfo for () { - // Storage: Treasury Reasons (r:1 w:1) - // Storage: Treasury Tips (r:1 w:1) + // Storage: Tips Reasons (r:1 w:1) + // Storage: Tips Tips (r:1 w:1) fn report_awesome(r: u32, ) -> Weight { - (50_921_000 as Weight) + (25_262_000 as Weight) // Standard Error: 0 .saturating_add((2_000 as Weight).saturating_mul(r as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } - // Storage: Treasury Tips (r:1 w:1) - // Storage: Treasury Reasons (r:0 w:1) + // Storage: Tips Tips (r:1 w:1) + // Storage: Tips Reasons (r:0 w:1) fn retract_tip() -> Weight { - (46_352_000 as Weight) + (24_162_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Elections Members (r:1 w:0) - // Storage: Treasury Reasons (r:1 w:1) - // Storage: Treasury Tips (r:0 w:1) + // Storage: Tips Reasons (r:1 w:1) + // Storage: Tips Tips (r:0 w:1) fn tip_new(r: u32, t: u32, ) -> Weight { - (33_338_000 as Weight) + (16_435_000 as Weight) // Standard Error: 0 .saturating_add((2_000 as Weight).saturating_mul(r as Weight)) - // Standard Error: 0 - .saturating_add((115_000 as Weight).saturating_mul(t as Weight)) + // Standard Error: 4_000 + .saturating_add((231_000 as Weight).saturating_mul(t as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Elections Members (r:1 w:0) - // Storage: Treasury Tips (r:1 w:1) + // Storage: Tips Tips (r:1 w:1) fn tip(t: u32, ) -> Weight { - (22_702_000 as Weight) - // Standard Error: 0 - .saturating_add((538_000 as Weight).saturating_mul(t as Weight)) + (10_427_000 as Weight) + // Standard Error: 7_000 + .saturating_add((507_000 as Weight).saturating_mul(t as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } - // Storage: Treasury Tips (r:1 w:1) + // Storage: Tips Tips (r:1 w:1) // Storage: Elections Members (r:1 w:0) // Storage: System Account (r:1 w:1) - // Storage: Treasury Reasons (r:0 w:1) + // Storage: Tips Reasons (r:0 w:1) fn close_tip(t: u32, ) -> Weight { - (84_094_000 as Weight) - // Standard Error: 0 - .saturating_add((283_000 as Weight).saturating_mul(t as Weight)) + (40_901_000 as Weight) + // Standard Error: 10_000 + .saturating_add((281_000 as Weight).saturating_mul(t as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } - // Storage: Treasury Tips (r:1 w:1) - // Storage: Treasury Reasons (r:0 w:1) + // Storage: Tips Tips (r:1 w:1) + // Storage: Tips Reasons (r:0 w:1) fn slash_tip(t: u32, ) -> Weight { - (24_891_000 as Weight) - // Standard Error: 0 - .saturating_add((6_000 as Weight).saturating_mul(t as Weight)) + (14_636_000 as Weight) + // Standard Error: 4_000 + .saturating_add((29_000 as Weight).saturating_mul(t as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } diff --git a/frame/transaction-payment/Cargo.toml b/frame/transaction-payment/Cargo.toml index 1dfeb0c2baa4..d37a98deecb2 100644 --- a/frame/transaction-payment/Cargo.toml +++ b/frame/transaction-payment/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-transaction-payment" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet to manage transaction payments" readme = "README.md" @@ -13,23 +13,23 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.126", optional = true } -smallvec = "1.7.0" +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +serde = { version = "1.0.136", optional = true } +smallvec = "1.8.0" -sp-core = { version = "4.0.0-dev", path = "../../primitives/core", default-features = false } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io", default-features = false } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } +sp-core = { version = "6.0.0", path = "../../primitives/core", default-features = false } +sp-io = { version = "6.0.0", path = "../../primitives/io", default-features = false } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } [dev-dependencies] -serde_json = "1.0.68" +serde_json = "1.0.79" pallet-balances = { version = "4.0.0-dev", path = "../balances" } [features] diff --git a/frame/transaction-payment/asset-tx-payment/Cargo.toml b/frame/transaction-payment/asset-tx-payment/Cargo.toml new file mode 100644 index 000000000000..0c2bcb730aa7 --- /dev/null +++ b/frame/transaction-payment/asset-tx-payment/Cargo.toml @@ -0,0 +1,55 @@ +[package] +name = "pallet-asset-tx-payment" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "pallet to manage transaction payments in assets" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +# Substrate dependencies +sp-core = { version = "6.0.0", default-features = false, path = "../../../primitives/core" } +sp-io = { version = "6.0.0", default-features = false, path = "../../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } + +frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" } +pallet-transaction-payment = { version = "4.0.0-dev", default-features = false, path = ".." } + +# Other dependencies +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +serde = { version = "1.0.136", optional = true } + +[dev-dependencies] +smallvec = "1.8.0" +serde_json = "1.0.79" + +sp-storage = { version = "6.0.0", default-features = false, path = "../../../primitives/storage" } + +pallet-assets = { version = "4.0.0-dev", path = "../../assets" } +pallet-authorship = { version = "4.0.0-dev", path = "../../authorship" } +pallet-balances = { version = "4.0.0-dev", path = "../../balances" } + + +[features] +default = ["std"] +std = [ + "serde", + "codec/std", + "sp-std/std", + "sp-runtime/std", + "frame-support/std", + "frame-system/std", + "sp-io/std", + "sp-core/std", + "pallet-transaction-payment/std", +] +try-runtime = ["frame-support/try-runtime"] diff --git a/frame/transaction-payment/asset-tx-payment/README.md b/frame/transaction-payment/asset-tx-payment/README.md new file mode 100644 index 000000000000..fc860347d85f --- /dev/null +++ b/frame/transaction-payment/asset-tx-payment/README.md @@ -0,0 +1,21 @@ +# pallet-asset-tx-payment + +## Asset Transaction Payment Pallet + +This pallet allows runtimes that include it to pay for transactions in assets other than the +native token of the chain. + +### Overview +It does this by extending transactions to include an optional `AssetId` that specifies the asset +to be used for payment (defaulting to the native token on `None`). It expects an +[`OnChargeAssetTransaction`] implementation analogously to [`pallet-transaction-payment`]. The +included [`FungiblesAdapter`] (implementing [`OnChargeAssetTransaction`]) determines the fee +amount by converting the fee calculated by [`pallet-transaction-payment`] into the desired +asset. + +### Integration +This pallet wraps FRAME's transaction payment pallet and functions as a replacement. This means +you should include both pallets in your `construct_runtime` macro, but only include this +pallet's [`SignedExtension`] ([`ChargeAssetTxPayment`]). + +License: Apache-2.0 diff --git a/frame/transaction-payment/asset-tx-payment/src/lib.rs b/frame/transaction-payment/asset-tx-payment/src/lib.rs new file mode 100644 index 000000000000..83801c44d357 --- /dev/null +++ b/frame/transaction-payment/asset-tx-payment/src/lib.rs @@ -0,0 +1,289 @@ +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Asset Transaction Payment Pallet +//! +//! This pallet allows runtimes that include it to pay for transactions in assets other than the +//! main token of the chain. +//! +//! ## Overview + +//! It does this by extending transactions to include an optional `AssetId` that specifies the asset +//! to be used for payment (defaulting to the native token on `None`). It expects an +//! [`OnChargeAssetTransaction`] implementation analogously to [`pallet-transaction-payment`]. The +//! included [`FungiblesAdapter`] (implementing [`OnChargeAssetTransaction`]) determines the fee +//! amount by converting the fee calculated by [`pallet-transaction-payment`] into the desired +//! asset. +//! +//! ## Integration + +//! This pallet wraps FRAME's transaction payment pallet and functions as a replacement. This means +//! you should include both pallets in your `construct_runtime` macro, but only include this +//! pallet's [`SignedExtension`] ([`ChargeAssetTxPayment`]). + +#![cfg_attr(not(feature = "std"), no_std)] + +use sp_std::prelude::*; + +use codec::{Decode, Encode}; +use frame_support::{ + dispatch::DispatchResult, + traits::{ + tokens::{ + fungibles::{Balanced, CreditOf, Inspect}, + WithdrawConsequence, + }, + IsType, + }, + weights::{DispatchInfo, PostDispatchInfo}, + DefaultNoBound, +}; +use pallet_transaction_payment::OnChargeTransaction; +use scale_info::TypeInfo; +use sp_runtime::{ + traits::{DispatchInfoOf, Dispatchable, PostDispatchInfoOf, SignedExtension, Zero}, + transaction_validity::{ + InvalidTransaction, TransactionValidity, TransactionValidityError, ValidTransaction, + }, + FixedPointOperand, +}; + +#[cfg(test)] +mod tests; + +mod payment; +pub use payment::*; + +// Type aliases used for interaction with `OnChargeTransaction`. +pub(crate) type OnChargeTransactionOf = + ::OnChargeTransaction; +// Balance type alias. +pub(crate) type BalanceOf = as OnChargeTransaction>::Balance; +// Liquity info type alias. +pub(crate) type LiquidityInfoOf = + as OnChargeTransaction>::LiquidityInfo; + +// Type alias used for interaction with fungibles (assets). +// Balance type alias. +pub(crate) type AssetBalanceOf = + <::Fungibles as Inspect<::AccountId>>::Balance; +/// Asset id type alias. +pub(crate) type AssetIdOf = + <::Fungibles as Inspect<::AccountId>>::AssetId; + +// Type aliases used for interaction with `OnChargeAssetTransaction`. +// Balance type alias. +pub(crate) type ChargeAssetBalanceOf = + <::OnChargeAssetTransaction as OnChargeAssetTransaction>::Balance; +// Asset id type alias. +pub(crate) type ChargeAssetIdOf = + <::OnChargeAssetTransaction as OnChargeAssetTransaction>::AssetId; +// Liquity info type alias. +pub(crate) type ChargeAssetLiquidityOf = + <::OnChargeAssetTransaction as OnChargeAssetTransaction>::LiquidityInfo; + +/// Used to pass the initial payment info from pre- to post-dispatch. +#[derive(Encode, Decode, DefaultNoBound, TypeInfo)] +pub enum InitialPayment { + /// No initial fee was payed. + Nothing, + /// The initial fee was payed in the native currency. + Native(LiquidityInfoOf), + /// The initial fee was payed in an asset. + Asset(CreditOf), +} + +pub use pallet::*; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + + #[pallet::config] + pub trait Config: frame_system::Config + pallet_transaction_payment::Config { + /// The fungibles instance used to pay for transactions in assets. + type Fungibles: Balanced; + /// The actual transaction charging logic that charges the fees. + type OnChargeAssetTransaction: OnChargeAssetTransaction; + } + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(_); +} + +/// Require the transactor pay for themselves and maybe include a tip to gain additional priority +/// in the queue. Allows paying via both `Currency` as well as `fungibles::Balanced`. +/// +/// Wraps the transaction logic in [`pallet_transaction_payment`] and extends it with assets. +/// An asset id of `None` falls back to the underlying transaction payment via the native currency. +#[derive(Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] +#[scale_info(skip_type_params(T))] +pub struct ChargeAssetTxPayment { + #[codec(compact)] + tip: BalanceOf, + asset_id: Option>, +} + +impl ChargeAssetTxPayment +where + T::Call: Dispatchable, + AssetBalanceOf: Send + Sync + FixedPointOperand, + BalanceOf: Send + Sync + FixedPointOperand + IsType>, + ChargeAssetIdOf: Send + Sync, + CreditOf: IsType>, +{ + /// Utility constructor. Used only in client/factory code. + pub fn from(tip: BalanceOf, asset_id: Option>) -> Self { + Self { tip, asset_id } + } + + /// Fee withdrawal logic that dispatches to either `OnChargeAssetTransaction` or + /// `OnChargeTransaction`. + fn withdraw_fee( + &self, + who: &T::AccountId, + call: &T::Call, + info: &DispatchInfoOf, + len: usize, + ) -> Result<(BalanceOf, InitialPayment), TransactionValidityError> { + let fee = pallet_transaction_payment::Pallet::::compute_fee(len as u32, info, self.tip); + debug_assert!(self.tip <= fee, "tip should be included in the computed fee"); + if fee.is_zero() { + Ok((fee, InitialPayment::Nothing)) + } else if let Some(asset_id) = self.asset_id { + T::OnChargeAssetTransaction::withdraw_fee( + who, + call, + info, + asset_id, + fee.into(), + self.tip.into(), + ) + .map(|i| (fee, InitialPayment::Asset(i.into()))) + } else { + as OnChargeTransaction>::withdraw_fee( + who, call, info, fee, self.tip, + ) + .map(|i| (fee, InitialPayment::Native(i))) + .map_err(|_| -> TransactionValidityError { InvalidTransaction::Payment.into() }) + } + } +} + +impl sp_std::fmt::Debug for ChargeAssetTxPayment { + #[cfg(feature = "std")] + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + write!(f, "ChargeAssetTxPayment<{:?}, {:?}>", self.tip, self.asset_id.encode()) + } + #[cfg(not(feature = "std"))] + fn fmt(&self, _: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + Ok(()) + } +} + +impl SignedExtension for ChargeAssetTxPayment +where + T::Call: Dispatchable, + AssetBalanceOf: Send + Sync + FixedPointOperand, + BalanceOf: Send + Sync + From + FixedPointOperand + IsType>, + ChargeAssetIdOf: Send + Sync, + CreditOf: IsType>, +{ + const IDENTIFIER: &'static str = "ChargeAssetTxPayment"; + type AccountId = T::AccountId; + type Call = T::Call; + type AdditionalSigned = (); + type Pre = ( + // tip + BalanceOf, + // who paid the fee + Self::AccountId, + // imbalance resulting from withdrawing the fee + InitialPayment, + ); + + fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { + Ok(()) + } + + fn validate( + &self, + who: &Self::AccountId, + call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> TransactionValidity { + use pallet_transaction_payment::ChargeTransactionPayment; + let (fee, _) = self.withdraw_fee(who, call, info, len)?; + let priority = ChargeTransactionPayment::::get_priority(info, len, self.tip, fee); + Ok(ValidTransaction { priority, ..Default::default() }) + } + + fn pre_dispatch( + self, + who: &Self::AccountId, + call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> Result { + let (_fee, initial_payment) = self.withdraw_fee(who, call, info, len)?; + Ok((self.tip, who.clone(), initial_payment)) + } + + fn post_dispatch( + pre: Option, + info: &DispatchInfoOf, + post_info: &PostDispatchInfoOf, + len: usize, + result: &DispatchResult, + ) -> Result<(), TransactionValidityError> { + if let Some((tip, who, initial_payment)) = pre { + match initial_payment { + InitialPayment::Native(already_withdrawn) => { + pallet_transaction_payment::ChargeTransactionPayment::::post_dispatch( + Some((tip, who, already_withdrawn)), + info, + post_info, + len, + result, + )?; + }, + InitialPayment::Asset(already_withdrawn) => { + let actual_fee = pallet_transaction_payment::Pallet::::compute_actual_fee( + len as u32, info, post_info, tip, + ); + T::OnChargeAssetTransaction::correct_and_deposit_fee( + &who, + info, + post_info, + actual_fee.into(), + tip.into(), + already_withdrawn.into(), + )?; + }, + InitialPayment::Nothing => { + // `actual_fee` should be zero here for any signed extrinsic. It would be + // non-zero here in case of unsigned extrinsics as they don't pay fees but + // `compute_actual_fee` is not aware of them. In both cases it's fine to just + // move ahead without adjusting the fee, though, so we do nothing. + debug_assert!(tip.is_zero(), "tip should be zero if initial fee was zero."); + }, + } + } + + Ok(()) + } +} diff --git a/frame/transaction-payment/asset-tx-payment/src/payment.rs b/frame/transaction-payment/asset-tx-payment/src/payment.rs new file mode 100644 index 000000000000..9eafc43fc256 --- /dev/null +++ b/frame/transaction-payment/asset-tx-payment/src/payment.rs @@ -0,0 +1,168 @@ +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +///! Traits and default implementation for paying transaction fees in assets. +use super::*; +use crate::Config; + +use codec::FullCodec; +use frame_support::{ + traits::{ + fungibles::{Balanced, CreditOf, Inspect}, + tokens::BalanceConversion, + }, + unsigned::TransactionValidityError, +}; +use scale_info::TypeInfo; +use sp_runtime::{ + traits::{ + AtLeast32BitUnsigned, DispatchInfoOf, MaybeSerializeDeserialize, One, PostDispatchInfoOf, + }, + transaction_validity::InvalidTransaction, +}; +use sp_std::{fmt::Debug, marker::PhantomData}; + +/// Handle withdrawing, refunding and depositing of transaction fees. +pub trait OnChargeAssetTransaction { + /// The underlying integer type in which fees are calculated. + type Balance: AtLeast32BitUnsigned + + FullCodec + + Copy + + MaybeSerializeDeserialize + + Debug + + Default + + TypeInfo; + /// The type used to identify the assets used for transaction payment. + type AssetId: FullCodec + Copy + MaybeSerializeDeserialize + Debug + Default + Eq + TypeInfo; + /// The type used to store the intermediate values between pre- and post-dispatch. + type LiquidityInfo; + + /// Before the transaction is executed the payment of the transaction fees needs to be secured. + /// + /// Note: The `fee` already includes the `tip`. + fn withdraw_fee( + who: &T::AccountId, + call: &T::Call, + dispatch_info: &DispatchInfoOf, + asset_id: Self::AssetId, + fee: Self::Balance, + tip: Self::Balance, + ) -> Result; + + /// After the transaction was executed the actual fee can be calculated. + /// This function should refund any overpaid fees and optionally deposit + /// the corrected amount. + /// + /// Note: The `fee` already includes the `tip`. + fn correct_and_deposit_fee( + who: &T::AccountId, + dispatch_info: &DispatchInfoOf, + post_info: &PostDispatchInfoOf, + corrected_fee: Self::Balance, + tip: Self::Balance, + already_withdrawn: Self::LiquidityInfo, + ) -> Result<(), TransactionValidityError>; +} + +/// Allows specifying what to do with the withdrawn asset fees. +pub trait HandleCredit> { + /// Implement to determine what to do with the withdrawn asset fees. + /// Default for `CreditOf` from the assets pallet is to burn and + /// decrease total issuance. + fn handle_credit(credit: CreditOf); +} + +/// Default implementation that just drops the credit according to the `OnDrop` in the underlying +/// imbalance type. +impl> HandleCredit for () { + fn handle_credit(_credit: CreditOf) {} +} + +/// Implements the asset transaction for a balance to asset converter (implementing +/// [`BalanceConversion`]) and a credit handler (implementing [`HandleCredit`]). +/// +/// The credit handler is given the complete fee in terms of the asset used for the transaction. +pub struct FungiblesAdapter(PhantomData<(CON, HC)>); + +/// Default implementation for a runtime instantiating this pallet, a balance to asset converter and +/// a credit handler. +impl OnChargeAssetTransaction for FungiblesAdapter +where + T: Config, + CON: BalanceConversion, AssetIdOf, AssetBalanceOf>, + HC: HandleCredit, + AssetIdOf: FullCodec + Copy + MaybeSerializeDeserialize + Debug + Default + Eq + TypeInfo, +{ + type Balance = BalanceOf; + type AssetId = AssetIdOf; + type LiquidityInfo = CreditOf; + + /// Withdraw the predicted fee from the transaction origin. + /// + /// Note: The `fee` already includes the `tip`. + fn withdraw_fee( + who: &T::AccountId, + _call: &T::Call, + _info: &DispatchInfoOf, + asset_id: Self::AssetId, + fee: Self::Balance, + _tip: Self::Balance, + ) -> Result { + // We don't know the precision of the underlying asset. Because the converted fee could be + // less than one (e.g. 0.5) but gets rounded down by integer division we introduce a minimum + // fee. + let min_converted_fee = if fee.is_zero() { Zero::zero() } else { One::one() }; + let converted_fee = CON::to_asset_balance(fee, asset_id) + .map_err(|_| TransactionValidityError::from(InvalidTransaction::Payment))? + .max(min_converted_fee); + let can_withdraw = >::can_withdraw( + asset_id.into(), + who, + converted_fee, + ); + if !matches!(can_withdraw, WithdrawConsequence::Success) { + return Err(InvalidTransaction::Payment.into()) + } + >::withdraw(asset_id.into(), who, converted_fee) + .map_err(|_| TransactionValidityError::from(InvalidTransaction::Payment)) + } + + /// Hand the fee and the tip over to the `[HandleCredit]` implementation. + /// Since the predicted fee might have been too high, parts of the fee may be refunded. + /// + /// Note: The `corrected_fee` already includes the `tip`. + fn correct_and_deposit_fee( + who: &T::AccountId, + _dispatch_info: &DispatchInfoOf, + _post_info: &PostDispatchInfoOf, + corrected_fee: Self::Balance, + _tip: Self::Balance, + paid: Self::LiquidityInfo, + ) -> Result<(), TransactionValidityError> { + let min_converted_fee = if corrected_fee.is_zero() { Zero::zero() } else { One::one() }; + // Convert the corrected fee into the asset used for payment. + let converted_fee = CON::to_asset_balance(corrected_fee, paid.asset().into()) + .map_err(|_| -> TransactionValidityError { InvalidTransaction::Payment.into() })? + .max(min_converted_fee); + // Calculate how much refund we should return. + let (final_fee, refund) = paid.split(converted_fee); + // Refund to the account that paid the fees. If this fails, the account might have dropped + // below the existential balance. In that case we don't refund anything. + let _ = >::resolve(who, refund); + // Handle the final fee, e.g. by transferring to the block author or burning. + HC::handle_credit(final_fee); + Ok(()) + } +} diff --git a/frame/transaction-payment/asset-tx-payment/src/tests.rs b/frame/transaction-payment/asset-tx-payment/src/tests.rs new file mode 100644 index 000000000000..d72a288ac7a3 --- /dev/null +++ b/frame/transaction-payment/asset-tx-payment/src/tests.rs @@ -0,0 +1,733 @@ +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate as pallet_asset_tx_payment; + +use frame_support::{ + assert_ok, + pallet_prelude::*, + parameter_types, + traits::{fungibles::Mutate, ConstU32, ConstU64, ConstU8, FindAuthor}, + weights::{ + DispatchClass, DispatchInfo, PostDispatchInfo, Weight, WeightToFeeCoefficient, + WeightToFeeCoefficients, WeightToFeePolynomial, + }, + ConsensusEngineId, +}; +use frame_system as system; +use frame_system::EnsureRoot; +use pallet_balances::Call as BalancesCall; +use pallet_transaction_payment::CurrencyAdapter; +use smallvec::smallvec; +use sp_core::H256; +use sp_runtime::{ + testing::Header, + traits::{BlakeTwo256, ConvertInto, IdentityLookup, StaticLookup}, + Perbill, +}; +use std::cell::RefCell; + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; +type Balance = u64; +type AccountId = u64; + +frame_support::construct_runtime!( + pub enum Runtime where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: system::{Pallet, Call, Config, Storage, Event}, + Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, + TransactionPayment: pallet_transaction_payment::{Pallet, Storage}, + Assets: pallet_assets::{Pallet, Call, Storage, Event}, + Authorship: pallet_authorship::{Pallet, Call, Storage}, + AssetTxPayment: pallet_asset_tx_payment::{Pallet}, + } +); + +const CALL: &::Call = + &Call::Balances(BalancesCall::transfer { dest: 2, value: 69 }); + +thread_local! { + static EXTRINSIC_BASE_WEIGHT: RefCell = RefCell::new(0); +} + +pub struct BlockWeights; +impl Get for BlockWeights { + fn get() -> frame_system::limits::BlockWeights { + frame_system::limits::BlockWeights::builder() + .base_block(0) + .for_class(DispatchClass::all(), |weights| { + weights.base_extrinsic = EXTRINSIC_BASE_WEIGHT.with(|v| *v.borrow()).into(); + }) + .for_class(DispatchClass::non_mandatory(), |weights| { + weights.max_total = 1024.into(); + }) + .build_or_panic() + } +} + +parameter_types! { + pub static TransactionByteFee: u64 = 1; + pub static WeightToFee: u64 = 1; +} + +impl frame_system::Config for Runtime { + type BaseCallFilter = frame_support::traits::Everything; + type BlockWeights = BlockWeights; + type BlockLength = (); + type DbWeight = (); + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Call = Call; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = AccountId; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = ConstU64<250>; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = ConstU32<16>; +} + +parameter_types! { + pub const ExistentialDeposit: u64 = 10; +} + +impl pallet_balances::Config for Runtime { + type Balance = Balance; + type Event = Event; + type DustRemoval = (); + type ExistentialDeposit = ConstU64<10>; + type AccountStore = System; + type MaxLocks = (); + type WeightInfo = (); + type MaxReserves = ConstU32<50>; + type ReserveIdentifier = [u8; 8]; +} + +impl WeightToFeePolynomial for WeightToFee { + type Balance = u64; + + fn polynomial() -> WeightToFeeCoefficients { + smallvec![WeightToFeeCoefficient { + degree: 1, + coeff_frac: Perbill::zero(), + coeff_integer: WEIGHT_TO_FEE.with(|v| *v.borrow()), + negative: false, + }] + } +} + +impl pallet_transaction_payment::Config for Runtime { + type OnChargeTransaction = CurrencyAdapter; + type WeightToFee = WeightToFee; + type LengthToFee = WeightToFee; + type FeeMultiplierUpdate = (); + type OperationalFeeMultiplier = ConstU8<5>; +} + +impl pallet_assets::Config for Runtime { + type Event = Event; + type Balance = Balance; + type AssetId = u32; + type Currency = Balances; + type ForceOrigin = EnsureRoot; + type AssetDeposit = ConstU64<2>; + type AssetAccountDeposit = ConstU64<2>; + type MetadataDepositBase = ConstU64<0>; + type MetadataDepositPerByte = ConstU64<0>; + type ApprovalDeposit = ConstU64<0>; + type StringLimit = ConstU32<20>; + type Freezer = (); + type Extra = (); + type WeightInfo = (); +} + +pub struct HardcodedAuthor; +const BLOCK_AUTHOR: AccountId = 1234; +impl FindAuthor for HardcodedAuthor { + fn find_author<'a, I>(_: I) -> Option + where + I: 'a + IntoIterator, + { + Some(BLOCK_AUTHOR) + } +} + +impl pallet_authorship::Config for Runtime { + type FindAuthor = HardcodedAuthor; + type UncleGenerations = (); + type FilterUncle = (); + type EventHandler = (); +} + +pub struct CreditToBlockAuthor; +impl HandleCredit for CreditToBlockAuthor { + fn handle_credit(credit: CreditOf) { + if let Some(author) = pallet_authorship::Pallet::::author() { + // What to do in case paying the author fails (e.g. because `fee < min_balance`) + // default: drop the result which will trigger the `OnDrop` of the imbalance. + let _ = >::resolve(&author, credit); + } + } +} + +impl Config for Runtime { + type Fungibles = Assets; + type OnChargeAssetTransaction = FungiblesAdapter< + pallet_assets::BalanceToAssetBalance, + CreditToBlockAuthor, + >; +} + +pub struct ExtBuilder { + balance_factor: u64, + base_weight: u64, + byte_fee: u64, + weight_to_fee: u64, +} + +impl Default for ExtBuilder { + fn default() -> Self { + Self { balance_factor: 1, base_weight: 0, byte_fee: 1, weight_to_fee: 1 } + } +} + +impl ExtBuilder { + pub fn base_weight(mut self, base_weight: u64) -> Self { + self.base_weight = base_weight; + self + } + pub fn balance_factor(mut self, factor: u64) -> Self { + self.balance_factor = factor; + self + } + fn set_constants(&self) { + EXTRINSIC_BASE_WEIGHT.with(|v| *v.borrow_mut() = self.base_weight); + TRANSACTION_BYTE_FEE.with(|v| *v.borrow_mut() = self.byte_fee); + WEIGHT_TO_FEE.with(|v| *v.borrow_mut() = self.weight_to_fee); + } + pub fn build(self) -> sp_io::TestExternalities { + self.set_constants(); + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + pallet_balances::GenesisConfig:: { + balances: if self.balance_factor > 0 { + vec![ + (1, 10 * self.balance_factor), + (2, 20 * self.balance_factor), + (3, 30 * self.balance_factor), + (4, 40 * self.balance_factor), + (5, 50 * self.balance_factor), + (6, 60 * self.balance_factor), + ] + } else { + vec![] + }, + } + .assimilate_storage(&mut t) + .unwrap(); + t.into() + } +} + +/// create a transaction info struct from weight. Handy to avoid building the whole struct. +pub fn info_from_weight(w: Weight) -> DispatchInfo { + // pays_fee: Pays::Yes -- class: DispatchClass::Normal + DispatchInfo { weight: w, ..Default::default() } +} + +fn post_info_from_weight(w: Weight) -> PostDispatchInfo { + PostDispatchInfo { actual_weight: Some(w), pays_fee: Default::default() } +} + +fn info_from_pays(p: Pays) -> DispatchInfo { + DispatchInfo { pays_fee: p, ..Default::default() } +} + +fn post_info_from_pays(p: Pays) -> PostDispatchInfo { + PostDispatchInfo { actual_weight: None, pays_fee: p } +} + +fn default_post_info() -> PostDispatchInfo { + PostDispatchInfo { actual_weight: None, pays_fee: Default::default() } +} + +#[test] +fn transaction_payment_in_native_possible() { + let balance_factor = 100; + ExtBuilder::default() + .balance_factor(balance_factor) + .base_weight(5) + .build() + .execute_with(|| { + let len = 10; + let pre = ChargeAssetTxPayment::::from(0, None) + .pre_dispatch(&1, CALL, &info_from_weight(5), len) + .unwrap(); + let initial_balance = 10 * balance_factor; + assert_eq!(Balances::free_balance(1), initial_balance - 5 - 5 - 10); + + assert_ok!(ChargeAssetTxPayment::::post_dispatch( + Some(pre), + &info_from_weight(5), + &default_post_info(), + len, + &Ok(()) + )); + assert_eq!(Balances::free_balance(1), initial_balance - 5 - 5 - 10); + + let pre = ChargeAssetTxPayment::::from(5 /* tipped */, None) + .pre_dispatch(&2, CALL, &info_from_weight(100), len) + .unwrap(); + let initial_balance_for_2 = 20 * balance_factor; + assert_eq!(Balances::free_balance(2), initial_balance_for_2 - 5 - 10 - 100 - 5); + + assert_ok!(ChargeAssetTxPayment::::post_dispatch( + Some(pre), + &info_from_weight(100), + &post_info_from_weight(50), + len, + &Ok(()) + )); + assert_eq!(Balances::free_balance(2), initial_balance_for_2 - 5 - 10 - 50 - 5); + }); +} + +#[test] +fn transaction_payment_in_asset_possible() { + let base_weight = 5; + let balance_factor = 100; + ExtBuilder::default() + .balance_factor(balance_factor) + .base_weight(base_weight) + .build() + .execute_with(|| { + // create the asset + let asset_id = 1; + let min_balance = 2; + assert_ok!(Assets::force_create( + Origin::root(), + asset_id, + 42, /* owner */ + true, /* is_sufficient */ + min_balance + )); + + // mint into the caller account + let caller = 1; + let beneficiary = ::Lookup::unlookup(caller); + let balance = 100; + assert_ok!(Assets::mint_into(asset_id, &beneficiary, balance)); + assert_eq!(Assets::balance(asset_id, caller), balance); + let weight = 5; + let len = 10; + // we convert the from weight to fee based on the ratio between asset min balance and + // existential deposit + let fee = (base_weight + weight + len as u64) * min_balance / ExistentialDeposit::get(); + let pre = ChargeAssetTxPayment::::from(0, Some(asset_id)) + .pre_dispatch(&caller, CALL, &info_from_weight(weight), len) + .unwrap(); + // assert that native balance is not used + assert_eq!(Balances::free_balance(caller), 10 * balance_factor); + // check that fee was charged in the given asset + assert_eq!(Assets::balance(asset_id, caller), balance - fee); + assert_eq!(Assets::balance(asset_id, BLOCK_AUTHOR), 0); + + assert_ok!(ChargeAssetTxPayment::::post_dispatch( + Some(pre), + &info_from_weight(weight), + &default_post_info(), + len, + &Ok(()) + )); + assert_eq!(Assets::balance(asset_id, caller), balance - fee); + // check that the block author gets rewarded + assert_eq!(Assets::balance(asset_id, BLOCK_AUTHOR), fee); + }); +} + +#[test] +fn transaction_payment_without_fee() { + let base_weight = 5; + let balance_factor = 100; + ExtBuilder::default() + .balance_factor(balance_factor) + .base_weight(base_weight) + .build() + .execute_with(|| { + // create the asset + let asset_id = 1; + let min_balance = 2; + assert_ok!(Assets::force_create( + Origin::root(), + asset_id, + 42, /* owner */ + true, /* is_sufficient */ + min_balance + )); + + // mint into the caller account + let caller = 1; + let beneficiary = ::Lookup::unlookup(caller); + let balance = 100; + assert_ok!(Assets::mint_into(asset_id, &beneficiary, balance)); + assert_eq!(Assets::balance(asset_id, caller), balance); + let weight = 5; + let len = 10; + // we convert the from weight to fee based on the ratio between asset min balance and + // existential deposit + let fee = (base_weight + weight + len as u64) * min_balance / ExistentialDeposit::get(); + let pre = ChargeAssetTxPayment::::from(0, Some(asset_id)) + .pre_dispatch(&caller, CALL, &info_from_weight(weight), len) + .unwrap(); + // assert that native balance is not used + assert_eq!(Balances::free_balance(caller), 10 * balance_factor); + // check that fee was charged in the given asset + assert_eq!(Assets::balance(asset_id, caller), balance - fee); + assert_eq!(Assets::balance(asset_id, BLOCK_AUTHOR), 0); + + assert_ok!(ChargeAssetTxPayment::::post_dispatch( + Some(pre), + &info_from_weight(weight), + &post_info_from_pays(Pays::No), + len, + &Ok(()) + )); + // caller should be refunded + assert_eq!(Assets::balance(asset_id, caller), balance); + // check that the block author did not get rewarded + assert_eq!(Assets::balance(asset_id, BLOCK_AUTHOR), 0); + }); +} + +#[test] +fn asset_transaction_payment_with_tip_and_refund() { + let base_weight = 5; + ExtBuilder::default() + .balance_factor(100) + .base_weight(base_weight) + .build() + .execute_with(|| { + // create the asset + let asset_id = 1; + let min_balance = 2; + assert_ok!(Assets::force_create( + Origin::root(), + asset_id, + 42, /* owner */ + true, /* is_sufficient */ + min_balance + )); + + // mint into the caller account + let caller = 2; + let beneficiary = ::Lookup::unlookup(caller); + let balance = 1000; + assert_ok!(Assets::mint_into(asset_id, &beneficiary, balance)); + assert_eq!(Assets::balance(asset_id, caller), balance); + let weight = 100; + let tip = 5; + let len = 10; + // we convert the from weight to fee based on the ratio between asset min balance and + // existential deposit + let fee_with_tip = + (base_weight + weight + len as u64 + tip) * min_balance / ExistentialDeposit::get(); + let pre = ChargeAssetTxPayment::::from(tip, Some(asset_id)) + .pre_dispatch(&caller, CALL, &info_from_weight(weight), len) + .unwrap(); + assert_eq!(Assets::balance(asset_id, caller), balance - fee_with_tip); + + let final_weight = 50; + assert_ok!(ChargeAssetTxPayment::::post_dispatch( + Some(pre), + &info_from_weight(weight), + &post_info_from_weight(final_weight), + len, + &Ok(()) + )); + let final_fee = + fee_with_tip - (weight - final_weight) * min_balance / ExistentialDeposit::get(); + assert_eq!(Assets::balance(asset_id, caller), balance - (final_fee)); + assert_eq!(Assets::balance(asset_id, BLOCK_AUTHOR), final_fee); + }); +} + +#[test] +fn payment_from_account_with_only_assets() { + let base_weight = 5; + ExtBuilder::default() + .balance_factor(100) + .base_weight(base_weight) + .build() + .execute_with(|| { + // create the asset + let asset_id = 1; + let min_balance = 2; + assert_ok!(Assets::force_create( + Origin::root(), + asset_id, + 42, /* owner */ + true, /* is_sufficient */ + min_balance + )); + + // mint into the caller account + let caller = 333; + let beneficiary = ::Lookup::unlookup(caller); + let balance = 100; + assert_ok!(Assets::mint_into(asset_id, &beneficiary, balance)); + assert_eq!(Assets::balance(asset_id, caller), balance); + // assert that native balance is not necessary + assert_eq!(Balances::free_balance(caller), 0); + let weight = 5; + let len = 10; + // we convert the from weight to fee based on the ratio between asset min balance and + // existential deposit + let fee = (base_weight + weight + len as u64) * min_balance / ExistentialDeposit::get(); + let pre = ChargeAssetTxPayment::::from(0, Some(asset_id)) + .pre_dispatch(&caller, CALL, &info_from_weight(weight), len) + .unwrap(); + assert_eq!(Balances::free_balance(caller), 0); + // check that fee was charged in the given asset + assert_eq!(Assets::balance(asset_id, caller), balance - fee); + + assert_ok!(ChargeAssetTxPayment::::post_dispatch( + Some(pre), + &info_from_weight(weight), + &default_post_info(), + len, + &Ok(()) + )); + assert_eq!(Assets::balance(asset_id, caller), balance - fee); + assert_eq!(Balances::free_balance(caller), 0); + }); +} + +#[test] +fn payment_only_with_existing_sufficient_asset() { + let base_weight = 5; + ExtBuilder::default() + .balance_factor(100) + .base_weight(base_weight) + .build() + .execute_with(|| { + let asset_id = 1; + let caller = 1; + let weight = 5; + let len = 10; + // pre_dispatch fails for non-existent asset + assert!(ChargeAssetTxPayment::::from(0, Some(asset_id)) + .pre_dispatch(&caller, CALL, &info_from_weight(weight), len) + .is_err()); + + // create the non-sufficient asset + let min_balance = 2; + assert_ok!(Assets::force_create( + Origin::root(), + asset_id, + 42, /* owner */ + false, /* is_sufficient */ + min_balance + )); + // pre_dispatch fails for non-sufficient asset + assert!(ChargeAssetTxPayment::::from(0, Some(asset_id)) + .pre_dispatch(&caller, CALL, &info_from_weight(weight), len) + .is_err()); + }); +} + +#[test] +fn converted_fee_is_never_zero_if_input_fee_is_not() { + let base_weight = 1; + ExtBuilder::default() + .balance_factor(100) + .base_weight(base_weight) + .build() + .execute_with(|| { + // create the asset + let asset_id = 1; + let min_balance = 1; + assert_ok!(Assets::force_create( + Origin::root(), + asset_id, + 42, /* owner */ + true, /* is_sufficient */ + min_balance + )); + + // mint into the caller account + let caller = 333; + let beneficiary = ::Lookup::unlookup(caller); + let balance = 100; + assert_ok!(Assets::mint_into(asset_id, &beneficiary, balance)); + assert_eq!(Assets::balance(asset_id, caller), balance); + let weight = 1; + let len = 1; + // we convert the from weight to fee based on the ratio between asset min balance and + // existential deposit + let fee = (base_weight + weight + len as u64) * min_balance / ExistentialDeposit::get(); + // naive fee calculation would round down to zero + assert_eq!(fee, 0); + { + let pre = ChargeAssetTxPayment::::from(0, Some(asset_id)) + .pre_dispatch(&caller, CALL, &info_from_pays(Pays::No), len) + .unwrap(); + // `Pays::No` still implies no fees + assert_eq!(Assets::balance(asset_id, caller), balance); + + assert_ok!(ChargeAssetTxPayment::::post_dispatch( + Some(pre), + &info_from_pays(Pays::No), + &post_info_from_pays(Pays::No), + len, + &Ok(()) + )); + assert_eq!(Assets::balance(asset_id, caller), balance); + } + let pre = ChargeAssetTxPayment::::from(0, Some(asset_id)) + .pre_dispatch(&caller, CALL, &info_from_weight(weight), len) + .unwrap(); + // check that at least one coin was charged in the given asset + assert_eq!(Assets::balance(asset_id, caller), balance - 1); + + assert_ok!(ChargeAssetTxPayment::::post_dispatch( + Some(pre), + &info_from_weight(weight), + &default_post_info(), + len, + &Ok(()) + )); + assert_eq!(Assets::balance(asset_id, caller), balance - 1); + }); +} + +#[test] +fn post_dispatch_fee_is_zero_if_pre_dispatch_fee_is_zero() { + let base_weight = 1; + ExtBuilder::default() + .balance_factor(100) + .base_weight(base_weight) + .build() + .execute_with(|| { + // create the asset + let asset_id = 1; + let min_balance = 100; + assert_ok!(Assets::force_create( + Origin::root(), + asset_id, + 42, /* owner */ + true, /* is_sufficient */ + min_balance + )); + + // mint into the caller account + let caller = 333; + let beneficiary = ::Lookup::unlookup(caller); + let balance = 100; + assert_ok!(Assets::mint_into(asset_id, &beneficiary, balance)); + assert_eq!(Assets::balance(asset_id, caller), balance); + let weight = 1; + let len = 1; + // we convert the from weight to fee based on the ratio between asset min balance and + // existential deposit + let fee = (base_weight + weight + len as u64) * min_balance / ExistentialDeposit::get(); + // calculated fee is greater than 0 + assert!(fee > 0); + let pre = ChargeAssetTxPayment::::from(0, Some(asset_id)) + .pre_dispatch(&caller, CALL, &info_from_pays(Pays::No), len) + .unwrap(); + // `Pays::No` implies no pre-dispatch fees + assert_eq!(Assets::balance(asset_id, caller), balance); + let (_tip, _who, initial_payment) = ⪯ + let not_paying = match initial_payment { + &InitialPayment::Nothing => true, + _ => false, + }; + assert!(not_paying, "initial payment should be Nothing if we pass Pays::No"); + + // `Pays::Yes` on post-dispatch does not mean we pay (we never charge more than the + // initial fee) + assert_ok!(ChargeAssetTxPayment::::post_dispatch( + Some(pre), + &info_from_pays(Pays::No), + &post_info_from_pays(Pays::Yes), + len, + &Ok(()) + )); + assert_eq!(Assets::balance(asset_id, caller), balance); + }); +} + +#[test] +fn post_dispatch_fee_is_zero_if_unsigned_pre_dispatch_fee_is_zero() { + let base_weight = 1; + ExtBuilder::default() + .balance_factor(100) + .base_weight(base_weight) + .build() + .execute_with(|| { + // create the asset + let asset_id = 1; + let min_balance = 100; + assert_ok!(Assets::force_create( + Origin::root(), + asset_id, + 42, /* owner */ + true, /* is_sufficient */ + min_balance + )); + + // mint into the caller account + let caller = 333; + let beneficiary = ::Lookup::unlookup(caller); + let balance = 100; + assert_ok!(Assets::mint_into(asset_id, &beneficiary, balance)); + assert_eq!(Assets::balance(asset_id, caller), balance); + let weight = 1; + let len = 1; + ChargeAssetTxPayment::::pre_dispatch_unsigned( + CALL, + &info_from_weight(weight), + len, + ) + .unwrap(); + + assert_eq!(Assets::balance(asset_id, caller), balance); + + // `Pays::Yes` on post-dispatch does not mean we pay (we never charge more than the + // initial fee) + assert_ok!(ChargeAssetTxPayment::::post_dispatch( + None, + &info_from_weight(weight), + &post_info_from_pays(Pays::Yes), + len, + &Ok(()) + )); + assert_eq!(Assets::balance(asset_id, caller), balance); + }); +} diff --git a/frame/transaction-payment/rpc/Cargo.toml b/frame/transaction-payment/rpc/Cargo.toml index 3858c41a3876..64d7007dfe6c 100644 --- a/frame/transaction-payment/rpc/Cargo.toml +++ b/frame/transaction-payment/rpc/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-transaction-payment-rpc" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "RPC interface for the transaction payment pallet." readme = "README.md" @@ -13,14 +13,14 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0" } +codec = { package = "parity-scale-codec", version = "3.0.0" } jsonrpc-core = "18.0.0" jsonrpc-core-client = "18.0.0" jsonrpc-derive = "18.0.0" sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-rpc = { version = "4.0.0-dev", path = "../../../primitives/rpc" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-rpc = { version = "6.0.0", path = "../../../primitives/rpc" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } pallet-transaction-payment-rpc-runtime-api = { version = "4.0.0-dev", path = "./runtime-api" } diff --git a/frame/transaction-payment/rpc/runtime-api/Cargo.toml b/frame/transaction-payment/rpc/runtime-api/Cargo.toml index 2f78f2439c60..d057361e1f5f 100644 --- a/frame/transaction-payment/rpc/runtime-api/Cargo.toml +++ b/frame/transaction-payment/rpc/runtime-api/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-transaction-payment-rpc-runtime-api" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "RPC runtime API for transaction payment FRAME pallet" readme = "README.md" @@ -13,9 +13,9 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../../primitives/api" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../../../primitives/runtime" } pallet-transaction-payment = { version = "4.0.0-dev", default-features = false, path = "../../../transaction-payment" } [features] diff --git a/frame/transaction-payment/rpc/runtime-api/src/lib.rs b/frame/transaction-payment/rpc/runtime-api/src/lib.rs index 696550d3ef04..5a0c70138db2 100644 --- a/frame/transaction-payment/rpc/runtime-api/src/lib.rs +++ b/frame/transaction-payment/rpc/runtime-api/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/transaction-payment/rpc/src/lib.rs b/frame/transaction-payment/rpc/src/lib.rs index 945156d12a6a..29d94fa26010 100644 --- a/frame/transaction-payment/rpc/src/lib.rs +++ b/frame/transaction-payment/rpc/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,7 +31,7 @@ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, MaybeDisplay}, }; -use std::{convert::TryInto, sync::Arc}; +use std::sync::Arc; #[rpc] pub trait TransactionPaymentApi { @@ -103,7 +103,7 @@ where api.query_info(&at, uxt, encoded_len).map_err(|e| RpcError { code: ErrorCode::ServerError(Error::RuntimeError.into()), message: "Unable to query dispatch info.".into(), - data: Some(format!("{:?}", e).into()), + data: Some(e.to_string().into()), }) } @@ -127,7 +127,7 @@ where let fee_details = api.query_fee_details(&at, uxt, encoded_len).map_err(|e| RpcError { code: ErrorCode::ServerError(Error::RuntimeError.into()), message: "Unable to query fee details.".into(), - data: Some(format!("{:?}", e).into()), + data: Some(e.to_string().into()), })?; let try_into_rpc_balance = |value: Balance| { diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs index 11dbcc010f67..1462faaa0706 100644 --- a/frame/transaction-payment/src/lib.rs +++ b/frame/transaction-payment/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -47,13 +47,13 @@ #![cfg_attr(not(feature = "std"), no_std)] -use codec::{Decode, Encode}; +use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; use sp_runtime::{ traits::{ - Convert, DispatchInfoOf, Dispatchable, PostDispatchInfoOf, SaturatedConversion, Saturating, - SignedExtension, Zero, + Convert, DispatchInfoOf, Dispatchable, One, PostDispatchInfoOf, SaturatedConversion, + Saturating, SignedExtension, Zero, }, transaction_validity::{ TransactionPriority, TransactionValidity, TransactionValidityError, ValidTransaction, @@ -127,7 +127,7 @@ type BalanceOf = <::OnChargeTransaction as OnChargeTransaction +/// pub struct TargetedFeeAdjustment(sp_std::marker::PhantomData<(T, S, V, M)>); /// Something that can convert the current multiplier to the next one. @@ -226,7 +226,7 @@ where } /// Storage releases of the pallet. -#[derive(Encode, Decode, Clone, Copy, PartialEq, Eq, RuntimeDebug, TypeInfo)] +#[derive(Encode, Decode, Clone, Copy, PartialEq, Eq, RuntimeDebug, TypeInfo, MaxEncodedLen)] enum Releases { /// Original version of the pallet. V1Ancient, @@ -260,25 +260,53 @@ pub mod pallet { /// might be refunded. In the end the fees can be deposited. type OnChargeTransaction: OnChargeTransaction; - /// The fee to be paid for making a transaction; the per-byte portion. + /// A fee mulitplier for `Operational` extrinsics to compute "virtual tip" to boost their + /// `priority` + /// + /// This value is multipled by the `final_fee` to obtain a "virtual tip" that is later + /// added to a tip component in regular `priority` calculations. + /// It means that a `Normal` transaction can front-run a similarly-sized `Operational` + /// extrinsic (with no tip), by including a tip value greater than the virtual tip. + /// + /// ```rust,ignore + /// // For `Normal` + /// let priority = priority_calc(tip); + /// + /// // For `Operational` + /// let virtual_tip = (inclusion_fee + tip) * OperationalFeeMultiplier; + /// let priority = priority_calc(tip + virtual_tip); + /// ``` + /// + /// Note that since we use `final_fee` the multiplier applies also to the regular `tip` + /// sent with the transaction. So, not only does the transaction get a priority bump based + /// on the `inclusion_fee`, but we also amplify the impact of tips applied to `Operational` + /// transactions. #[pallet::constant] - type TransactionByteFee: Get>; + type OperationalFeeMultiplier: Get; /// Convert a weight value into a deductible fee based on the currency type. type WeightToFee: WeightToFeePolynomial>; + /// Convert a length value into a deductible fee based on the currency type. + type LengthToFee: WeightToFeePolynomial>; + /// Update the multiplier of the next block, based on the previous block's weight. type FeeMultiplierUpdate: MultiplierUpdate; } #[pallet::extra_constants] impl Pallet { - // TODO: rename to snake case after https://github.com/paritytech/substrate/issues/8826 fixed. - #[allow(non_snake_case)] + #[pallet::constant_name(WeightToFee)] /// The polynomial that is applied in order to derive fee from weight. - fn WeightToFee() -> Vec>> { + fn weight_to_fee_polynomial() -> Vec>> { T::WeightToFee::polynomial().to_vec() } + + /// The polynomial that is applied in order to derive fee from length. + #[pallet::constant_name(LengthToFee)] + fn length_to_fee_polynomial() -> Vec>> { + T::LengthToFee::polynomial().to_vec() + } } #[pallet::type_value] @@ -323,10 +351,9 @@ pub mod pallet { // given weight == u64, we build multipliers from `diff` of two weight values, which can // at most be maximum block weight. Make sure that this can fit in a multiplier without // loss. - use sp_std::convert::TryInto; assert!( ::max_value() >= - Multiplier::checked_from_integer( + Multiplier::checked_from_integer::( T::BlockWeights::get().max_block.try_into().unwrap() ) .unwrap(), @@ -379,7 +406,7 @@ where /// /// All dispatchables must be annotated with weight and will have some fee info. This function /// always returns. - pub fn query_info( + pub fn query_info( unchecked_extrinsic: Extrinsic, len: u32, ) -> RuntimeDispatchInfo> @@ -393,14 +420,20 @@ where // a very very little potential gain in the future. let dispatch_info = ::get_dispatch_info(&unchecked_extrinsic); - let partial_fee = Self::compute_fee(len, &dispatch_info, 0u32.into()); + let partial_fee = if unchecked_extrinsic.is_signed().unwrap_or(false) { + Self::compute_fee(len, &dispatch_info, 0u32.into()) + } else { + // Unsigned extrinsics have no partial fee. + 0u32.into() + }; + let DispatchInfo { weight, class, .. } = dispatch_info; RuntimeDispatchInfo { weight, class, partial_fee } } /// Query the detailed fee of a given `call`. - pub fn query_fee_details( + pub fn query_fee_details( unchecked_extrinsic: Extrinsic, len: u32, ) -> FeeDetails> @@ -408,7 +441,15 @@ where T::Call: Dispatchable, { let dispatch_info = ::get_dispatch_info(&unchecked_extrinsic); - Self::compute_fee_details(len, &dispatch_info, 0u32.into()) + + let tip = 0u32.into(); + + if unchecked_extrinsic.is_signed().unwrap_or(false) { + Self::compute_fee_details(len, &dispatch_info, tip) + } else { + // Unsigned extrinsics have no inclusion fee. + FeeDetails { inclusion_fee: None, tip } + } } /// Compute the final fee value for a particular transaction. @@ -474,25 +515,18 @@ where class: DispatchClass, ) -> FeeDetails> { if pays_fee == Pays::Yes { - let len = >::from(len); - let per_byte = T::TransactionByteFee::get(); - - // length fee. this is not adjusted. - let fixed_len_fee = per_byte.saturating_mul(len); - // the adjustable part of the fee. let unadjusted_weight_fee = Self::weight_to_fee(weight); let multiplier = Self::next_fee_multiplier(); // final adjusted weight fee. let adjusted_weight_fee = multiplier.saturating_mul_int(unadjusted_weight_fee); + // length fee. this is adjusted via `LengthToFee`. + let len_fee = Self::length_to_fee(len); + let base_fee = Self::weight_to_fee(T::BlockWeights::get().get(class).base_extrinsic); FeeDetails { - inclusion_fee: Some(InclusionFee { - base_fee, - len_fee: fixed_len_fee, - adjusted_weight_fee, - }), + inclusion_fee: Some(InclusionFee { base_fee, len_fee, adjusted_weight_fee }), tip, } } else { @@ -500,6 +534,10 @@ where } } + fn length_to_fee(length: u32) -> BalanceOf { + T::LengthToFee::calc(&(length as Weight)) + } + fn weight_to_fee(weight: Weight) -> BalanceOf { // cap the weight to the maximum defined in runtime, otherwise it will be the // `Bounded` maximum of its data type, which is not desired. @@ -525,6 +563,14 @@ where /// Require the transactor pay for themselves and maybe include a tip to gain additional priority /// in the queue. +/// +/// # Transaction Validity +/// +/// This extension sets the `priority` field of `TransactionValidity` depending on the amount +/// of tip being paid per weight unit. +/// +/// Operational transactions will receive an additional priority bump, so that they are normally +/// considered before regular transactions. #[derive(Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] #[scale_info(skip_type_params(T))] pub struct ChargeTransactionPayment(#[codec(compact)] BalanceOf); @@ -566,27 +612,73 @@ where .map(|i| (fee, i)) } - /// Get an appropriate priority for a transaction with the given length and info. + /// Get an appropriate priority for a transaction with the given `DispatchInfo`, encoded length + /// and user-included tip. /// - /// This will try and optimise the `fee/weight` `fee/length`, whichever is consuming more of the - /// maximum corresponding limit. + /// The priority is based on the amount of `tip` the user is willing to pay per unit of either + /// `weight` or `length`, depending which one is more limitting. For `Operational` extrinsics + /// we add a "virtual tip" to the calculations. /// - /// For example, if a transaction consumed 1/4th of the block length and half of the weight, its - /// final priority is `fee * min(2, 4) = fee * 2`. If it consumed `1/4th` of the block length - /// and the entire block weight `(1/1)`, its priority is `fee * min(1, 4) = fee * 1`. This means - /// that the transaction which consumes more resources (either length or weight) with the same - /// `fee` ends up having lower priority. - fn get_priority( - len: usize, + /// The formula should simply be `tip / bounded_{weight|length}`, but since we are using + /// integer division, we have no guarantees it's going to give results in any reasonable + /// range (might simply end up being zero). Hence we use a scaling factor: + /// `tip * (max_block_{weight|length} / bounded_{weight|length})`, since given current + /// state of-the-art blockchains, number of per-block transactions is expected to be in a + /// range reasonable enough to not saturate the `Balance` type while multiplying by the tip. + pub fn get_priority( info: &DispatchInfoOf, + len: usize, + tip: BalanceOf, final_fee: BalanceOf, ) -> TransactionPriority { - let weight_saturation = T::BlockWeights::get().max_block / info.weight.max(1); - let max_block_length = *T::BlockLength::get().max.get(DispatchClass::Normal); - let len_saturation = max_block_length as u64 / (len as u64).max(1); - let coefficient: BalanceOf = - weight_saturation.min(len_saturation).saturated_into::>(); - final_fee.saturating_mul(coefficient).saturated_into::() + // Calculate how many such extrinsics we could fit into an empty block and take + // the limitting factor. + let max_block_weight = T::BlockWeights::get().max_block; + let max_block_length = *T::BlockLength::get().max.get(info.class) as u64; + + let bounded_weight = info.weight.max(1).min(max_block_weight); + let bounded_length = (len as u64).max(1).min(max_block_length); + + let max_tx_per_block_weight = max_block_weight / bounded_weight; + let max_tx_per_block_length = max_block_length / bounded_length; + // Given our current knowledge this value is going to be in a reasonable range - i.e. + // less than 10^9 (2^30), so multiplying by the `tip` value is unlikely to overflow the + // balance type. We still use saturating ops obviously, but the point is to end up with some + // `priority` distribution instead of having all transactions saturate the priority. + let max_tx_per_block = max_tx_per_block_length + .min(max_tx_per_block_weight) + .saturated_into::>(); + let max_reward = |val: BalanceOf| val.saturating_mul(max_tx_per_block); + + // To distribute no-tip transactions a little bit, we increase the tip value by one. + // This means that given two transactions without a tip, smaller one will be preferred. + let tip = tip.saturating_add(One::one()); + let scaled_tip = max_reward(tip); + + match info.class { + DispatchClass::Normal => { + // For normal class we simply take the `tip_per_weight`. + scaled_tip + }, + DispatchClass::Mandatory => { + // Mandatory extrinsics should be prohibited (e.g. by the [`CheckWeight`] + // extensions), but just to be safe let's return the same priority as `Normal` here. + scaled_tip + }, + DispatchClass::Operational => { + // A "virtual tip" value added to an `Operational` extrinsic. + // This value should be kept high enough to allow `Operational` extrinsics + // to get in even during congestion period, but at the same time low + // enough to prevent a possible spam attack by sending invalid operational + // extrinsics which push away regular transactions from the pool. + let fee_multiplier = T::OperationalFeeMultiplier::get().saturated_into(); + let virtual_tip = final_fee.saturating_mul(fee_multiplier); + let scaled_virtual_tip = max_reward(virtual_tip); + + scaled_tip.saturating_add(scaled_virtual_tip) + }, + } + .saturated_into::() } } @@ -613,7 +705,7 @@ where type Pre = ( // tip BalanceOf, - // who paid the fee + // who paid the fee - this is an option to allow for a Default impl. Self::AccountId, // imbalance resulting from withdrawing the fee <::OnChargeTransaction as OnChargeTransaction>::LiquidityInfo, @@ -629,8 +721,12 @@ where info: &DispatchInfoOf, len: usize, ) -> TransactionValidity { - let (fee, _) = self.withdraw_fee(who, call, info, len)?; - Ok(ValidTransaction { priority: Self::get_priority(len, info, fee), ..Default::default() }) + let (final_fee, _) = self.withdraw_fee(who, call, info, len)?; + let tip = self.0; + Ok(ValidTransaction { + priority: Self::get_priority(info, len, tip, final_fee), + ..Default::default() + }) } fn pre_dispatch( @@ -645,17 +741,18 @@ where } fn post_dispatch( - pre: Self::Pre, + maybe_pre: Option, info: &DispatchInfoOf, post_info: &PostDispatchInfoOf, len: usize, _result: &DispatchResult, ) -> Result<(), TransactionValidityError> { - let (tip, who, imbalance) = pre; - let actual_fee = Pallet::::compute_actual_fee(len as u32, info, post_info, tip); - T::OnChargeTransaction::correct_and_deposit_fee( - &who, info, post_info, actual_fee, tip, imbalance, - )?; + if let Some((tip, who, imbalance)) = maybe_pre { + let actual_fee = Pallet::::compute_actual_fee(len as u32, info, post_info, tip); + T::OnChargeTransaction::correct_and_deposit_fee( + &who, info, post_info, actual_fee, tip, imbalance, + )?; + } Ok(()) } } @@ -693,7 +790,7 @@ mod tests { use frame_support::{ assert_noop, assert_ok, parameter_types, - traits::{Currency, Imbalance, OnUnbalanced}, + traits::{ConstU32, ConstU64, Currency, Imbalance, OnUnbalanced}, weights::{ DispatchClass, DispatchInfo, GetDispatchInfo, PostDispatchInfo, Weight, WeightToFeeCoefficient, WeightToFeeCoefficients, WeightToFeePolynomial, @@ -740,9 +837,9 @@ mod tests { } parameter_types! { - pub const BlockHashCount: u64 = 250; - pub static TransactionByteFee: u64 = 1; pub static WeightToFee: u64 = 1; + pub static TransactionByteFee: u64 = 1; + pub static OperationalFeeMultiplier: u8 = 5; } impl frame_system::Config for Runtime { @@ -760,7 +857,7 @@ mod tests { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -769,17 +866,14 @@ mod tests { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); - } - - parameter_types! { - pub const ExistentialDeposit: u64 = 1; + type MaxConsumers = ConstU32<16>; } impl pallet_balances::Config for Runtime { type Balance = u64; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type MaxLocks = (); type MaxReserves = (); @@ -800,6 +894,19 @@ mod tests { } } + impl WeightToFeePolynomial for TransactionByteFee { + type Balance = u64; + + fn polynomial() -> WeightToFeeCoefficients { + smallvec![WeightToFeeCoefficient { + degree: 1, + coeff_frac: Perbill::zero(), + coeff_integer: TRANSACTION_BYTE_FEE.with(|v| *v.borrow()), + negative: false, + }] + } + } + thread_local! { static TIP_UNBALANCED_AMOUNT: RefCell = RefCell::new(0); static FEE_UNBALANCED_AMOUNT: RefCell = RefCell::new(0); @@ -821,8 +928,9 @@ mod tests { impl Config for Runtime { type OnChargeTransaction = CurrencyAdapter; - type TransactionByteFee = TransactionByteFee; + type OperationalFeeMultiplier = OperationalFeeMultiplier; type WeightToFee = WeightToFee; + type LengthToFee = TransactionByteFee; type FeeMultiplierUpdate = (); } @@ -916,7 +1024,7 @@ mod tests { assert_eq!(Balances::free_balance(1), 100 - 5 - 5 - 10); assert_ok!(ChargeTransactionPayment::::post_dispatch( - pre, + Some(pre), &info_from_weight(5), &default_post_info(), len, @@ -934,7 +1042,7 @@ mod tests { assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 100 - 5); assert_ok!(ChargeTransactionPayment::::post_dispatch( - pre, + Some(pre), &info_from_weight(100), &post_info_from_weight(50), len, @@ -963,7 +1071,7 @@ mod tests { assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 150 - 5); assert_ok!(ChargeTransactionPayment::::post_dispatch( - pre, + Some(pre), &info_from_weight(100), &post_info_from_weight(50), len, @@ -1057,20 +1165,24 @@ mod tests { } #[test] - fn query_info_works() { + fn query_info_and_fee_details_works() { let call = Call::Balances(BalancesCall::transfer { dest: 2, value: 69 }); let origin = 111111; let extra = (); - let xt = TestXt::new(call, Some((origin, extra))); + let xt = TestXt::new(call.clone(), Some((origin, extra))); let info = xt.get_dispatch_info(); let ext = xt.encode(); let len = ext.len() as u32; + + let unsigned_xt = TestXt::<_, ()>::new(call, None); + let unsigned_xt_info = unsigned_xt.get_dispatch_info(); + ExtBuilder::default().base_weight(5).weight_fee(2).build().execute_with(|| { // all fees should be x1.5 >::put(Multiplier::saturating_from_rational(3, 2)); assert_eq!( - TransactionPayment::query_info(xt, len), + TransactionPayment::query_info(xt.clone(), len), RuntimeDispatchInfo { weight: info.weight, class: info.class, @@ -1079,6 +1191,33 @@ mod tests { + info.weight.min(BlockWeights::get().max_block) as u64 * 2 * 3 / 2 /* weight */ }, ); + + assert_eq!( + TransactionPayment::query_info(unsigned_xt.clone(), len), + RuntimeDispatchInfo { + weight: unsigned_xt_info.weight, + class: unsigned_xt_info.class, + partial_fee: 0, + }, + ); + + assert_eq!( + TransactionPayment::query_fee_details(xt, len), + FeeDetails { + inclusion_fee: Some(InclusionFee { + base_fee: 5 * 2, + len_fee: len as u64, + adjusted_weight_fee: info.weight.min(BlockWeights::get().max_block) as u64 * + 2 * 3 / 2 + }), + tip: 0, + }, + ); + + assert_eq!( + TransactionPayment::query_fee_details(unsigned_xt, len), + FeeDetails { inclusion_fee: None, tip: 0 }, + ); }); } @@ -1227,7 +1366,7 @@ mod tests { assert_eq!(Balances::free_balance(2), 0); assert_ok!(ChargeTransactionPayment::::post_dispatch( - pre, + Some(pre), &info_from_weight(100), &post_info_from_weight(50), len, @@ -1235,11 +1374,15 @@ mod tests { )); assert_eq!(Balances::free_balance(2), 0); // Transfer Event - System::assert_has_event(Event::Balances(pallet_balances::Event::Transfer( - 2, 3, 80, - ))); + System::assert_has_event(Event::Balances(pallet_balances::Event::Transfer { + from: 2, + to: 3, + amount: 80, + })); // Killed Event - System::assert_has_event(Event::System(system::Event::KilledAccount(2))); + System::assert_has_event(Event::System(system::Event::KilledAccount { + account: 2, + })); }); } @@ -1257,7 +1400,7 @@ mod tests { assert_eq!(Balances::free_balance(2), 200 - 5 - 10 - 100 - 5); assert_ok!(ChargeTransactionPayment::::post_dispatch( - pre, + Some(pre), &info_from_weight(100), &post_info_from_weight(101), len, @@ -1285,7 +1428,7 @@ mod tests { .unwrap(); assert_eq!(Balances::total_balance(&user), 0); assert_ok!(ChargeTransactionPayment::::post_dispatch( - pre, + Some(pre), &dispatch_info, &default_post_info(), len, @@ -1317,7 +1460,7 @@ mod tests { .unwrap(); ChargeTransactionPayment::::post_dispatch( - pre, + Some(pre), &info, &post_info, len, @@ -1335,6 +1478,119 @@ mod tests { }); } + #[test] + fn should_alter_operational_priority() { + let tip = 5; + let len = 10; + + ExtBuilder::default().balance_factor(100).build().execute_with(|| { + let normal = + DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: Pays::Yes }; + let priority = ChargeTransactionPayment::(tip) + .validate(&2, CALL, &normal, len) + .unwrap() + .priority; + + assert_eq!(priority, 60); + + let priority = ChargeTransactionPayment::(2 * tip) + .validate(&2, CALL, &normal, len) + .unwrap() + .priority; + + assert_eq!(priority, 110); + }); + + ExtBuilder::default().balance_factor(100).build().execute_with(|| { + let op = DispatchInfo { + weight: 100, + class: DispatchClass::Operational, + pays_fee: Pays::Yes, + }; + let priority = ChargeTransactionPayment::(tip) + .validate(&2, CALL, &op, len) + .unwrap() + .priority; + assert_eq!(priority, 5810); + + let priority = ChargeTransactionPayment::(2 * tip) + .validate(&2, CALL, &op, len) + .unwrap() + .priority; + assert_eq!(priority, 6110); + }); + } + + #[test] + fn no_tip_has_some_priority() { + let tip = 0; + let len = 10; + + ExtBuilder::default().balance_factor(100).build().execute_with(|| { + let normal = + DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: Pays::Yes }; + let priority = ChargeTransactionPayment::(tip) + .validate(&2, CALL, &normal, len) + .unwrap() + .priority; + + assert_eq!(priority, 10); + }); + + ExtBuilder::default().balance_factor(100).build().execute_with(|| { + let op = DispatchInfo { + weight: 100, + class: DispatchClass::Operational, + pays_fee: Pays::Yes, + }; + let priority = ChargeTransactionPayment::(tip) + .validate(&2, CALL, &op, len) + .unwrap() + .priority; + assert_eq!(priority, 5510); + }); + } + + #[test] + fn higher_tip_have_higher_priority() { + let get_priorities = |tip: u64| { + let mut priority1 = 0; + let mut priority2 = 0; + let len = 10; + ExtBuilder::default().balance_factor(100).build().execute_with(|| { + let normal = + DispatchInfo { weight: 100, class: DispatchClass::Normal, pays_fee: Pays::Yes }; + priority1 = ChargeTransactionPayment::(tip) + .validate(&2, CALL, &normal, len) + .unwrap() + .priority; + }); + + ExtBuilder::default().balance_factor(100).build().execute_with(|| { + let op = DispatchInfo { + weight: 100, + class: DispatchClass::Operational, + pays_fee: Pays::Yes, + }; + priority2 = ChargeTransactionPayment::(tip) + .validate(&2, CALL, &op, len) + .unwrap() + .priority; + }); + + (priority1, priority2) + }; + + let mut prev_priorities = get_priorities(0); + + for tip in 1..3 { + let priorities = get_priorities(tip); + assert!(prev_priorities.0 < priorities.0); + assert!(prev_priorities.1 < priorities.1); + prev_priorities = priorities; + } + } + #[test] fn post_info_can_change_pays_fee() { ExtBuilder::default() @@ -1355,7 +1611,7 @@ mod tests { .unwrap(); ChargeTransactionPayment::::post_dispatch( - pre, + Some(pre), &info, &post_info, len, diff --git a/frame/transaction-payment/src/payment.rs b/frame/transaction-payment/src/payment.rs index 58e6ef63109a..5b4a61310279 100644 --- a/frame/transaction-payment/src/payment.rs +++ b/frame/transaction-payment/src/payment.rs @@ -12,7 +12,7 @@ use sp_runtime::{ use sp_std::{fmt::Debug, marker::PhantomData}; use frame_support::{ - traits::{Currency, ExistenceRequirement, Get, Imbalance, OnUnbalanced, WithdrawReasons}, + traits::{Currency, ExistenceRequirement, Imbalance, OnUnbalanced, WithdrawReasons}, unsigned::TransactionValidityError, }; @@ -73,7 +73,6 @@ pub struct CurrencyAdapter(PhantomData<(C, OU)>); impl OnChargeTransaction for CurrencyAdapter where T: Config, - T::TransactionByteFee: Get<::AccountId>>::Balance>, C: Currency<::AccountId>, C::PositiveImbalance: Imbalance< ::AccountId>>::Balance, diff --git a/frame/transaction-payment/src/types.rs b/frame/transaction-payment/src/types.rs index 3ce5bcf890bd..3faebfed4894 100644 --- a/frame/transaction-payment/src/types.rs +++ b/frame/transaction-payment/src/types.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/transaction-storage/Cargo.toml b/frame/transaction-storage/Cargo.toml index a4ebd5cfbc87..eec505708ed7 100644 --- a/frame/transaction-storage/Cargo.toml +++ b/frame/transaction-storage/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-transaction-storage" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Unlicense" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Storage chain pallet" readme = "README.md" @@ -13,27 +13,27 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -serde = { version = "1.0.126", optional = true } -hex-literal = { version = "0.3.1", optional = true } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +serde = { version = "1.0.136", optional = true } +hex-literal = { version = "0.3.4", optional = true } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../balances" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../primitives/inherents" } sp-transaction-storage-proof = { version = "4.0.0-dev", default-features = false, path = "../../primitives/transaction-storage-proof" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] sp-transaction-storage-proof = { version = "4.0.0-dev", default-features = true, path = "../../primitives/transaction-storage-proof" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core", default-features = false } +sp-core = { version = "6.0.0", path = "../../primitives/core", default-features = false } [features] default = ["std"] -runtime-benchmarks = ["frame-benchmarking", "hex-literal"] +runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks", "hex-literal"] std = [ "serde", "codec/std", diff --git a/frame/transaction-storage/src/benchmarking.rs b/frame/transaction-storage/src/benchmarking.rs index d5da6a42b46f..285b5cba7ad2 100644 --- a/frame/transaction-storage/src/benchmarking.rs +++ b/frame/transaction-storage/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ #![cfg(feature = "runtime-benchmarks")] use super::*; -use frame_benchmarking::{benchmarks, impl_benchmark_test_suite, whitelisted_caller}; +use frame_benchmarking::{benchmarks, whitelisted_caller}; use frame_support::traits::{Currency, OnFinalize, OnInitialize}; use frame_system::{EventRecord, Pallet as System, RawOrigin}; use sp_runtime::traits::{Bounded, One, Zero}; @@ -29,54 +29,77 @@ use sp_transaction_storage_proof::TransactionStorageProof; use crate::Pallet as TransactionStorage; +// Proof generated from max size storage: +// ``` +// let mut transactions = Vec::new(); +// let tx_size = DEFAULT_MAX_TRANSACTION_SIZE; +// for _ in 0..DEFAULT_MAX_BLOCK_TRANSACTIONS { +// transactions.push(vec![0; tx_size]); +// } +// let hash = vec![0; 32]; +// build_proof(hash.as_slice(), transactions).unwrap().encode() +// ``` +// while hardforcing target chunk key in `build_proof` to [22, 21, 1, 0]. const PROOF: &[u8] = &hex_literal::hex!( " - 0104000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 000000000000000000000000000000014cd0780ffff80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe8 - 7d12a3662c4c0080e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb - 13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2 - f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f - 1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f - 3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a47 - 8e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cf - f93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e31 - 6a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f - 53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c8 - 0e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4cbd05807777809a5d7a720ce5f9d9a012 - fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bcbf8a3dc2f6b9e957d129e610c06d411e11743062dc1cf - 3ac289390ae4c8008592aa2d915f52941036afbe72bac4ebe7ce186c4ddc53f118e0ddd4decd8cc809a5d7a720ce5f9d9 - a012fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bcbf8a3dc2f6b9e957d129e610c06d411e11743062d - c1cf3ac289390ae4c00809a5d7a720ce5f9d9a012fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bcbf8a - 3dc2f6b9e957d129e610c06d411e11743062dc1cf3ac289390ae4c8008592aa2d915f52941036afbe72bac4ebe7ce186c - 4ddc53f118e0ddd4decd8cc809a5d7a720ce5f9d9a012fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bc - bf8a3dc2f6b9e957d129e610c06d411e11743062dc1cf3ac289390ae4c8008592aa2d915f52941036afbe72bac4ebe7ce - 186c4ddc53f118e0ddd4decd8cccd0780ffff8081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb0 - 3bdb31008081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253 - 515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa139 - 8e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5 - f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3a - a1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2b - a8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f32 - 2d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa - 9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f0 - 2f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b82 - 5bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb31cd0780ffff80b4f23ac50c8e67d9b280f2b31a - 5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd1885 - 44c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2 - b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd - 188544c5f9b0080b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9 - b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84 - d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e - 67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977aca - ac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac5 - 0c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b89297 - 7acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b104401 - 0000 -" + 0104000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000 + 00000000000000000000000000000000000000000000000000000000000014cd0780ffff8030 + 2eb0a6d2f63b834d15f1e729d1c1004657e3048cf206d697eeb153f61a30ba0080302eb0a6d2 + f63b834d15f1e729d1c1004657e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15 + f1e729d1c1004657e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c1 + 004657e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c1004657e304 + 8cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c1004657e3048cf206d697 + eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c1004657e3048cf206d697eeb153f61a + 30ba80302eb0a6d2f63b834d15f1e729d1c1004657e3048cf206d697eeb153f61a30ba80302e + b0a6d2f63b834d15f1e729d1c1004657e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b + 834d15f1e729d1c1004657e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e7 + 29d1c1004657e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c10046 + 57e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c1004657e3048cf2 + 06d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c1004657e3048cf206d697eeb1 + 53f61a30ba80302eb0a6d2f63b834d15f1e729d1c1004657e3048cf206d697eeb153f61a30ba + bd058077778010fd81bc1359802f0b871aeb95e4410a8ec92b93af10ea767a2027cf4734e8de + 808da338e6b722f7bf2051901bd5bccee5e71d5cf6b1faff338ad7120b0256c28380221ce17f + 19117affa96e077905fe48a99723a065969c638593b7d9ab57b538438010fd81bc1359802f0b + 871aeb95e4410a8ec92b93af10ea767a2027cf4734e8de808da338e6b722f7bf2051901bd5bc + cee5e71d5cf6b1faff338ad7120b0256c283008010fd81bc1359802f0b871aeb95e4410a8ec9 + 2b93af10ea767a2027cf4734e8de808da338e6b722f7bf2051901bd5bccee5e71d5cf6b1faff + 338ad7120b0256c28380221ce17f19117affa96e077905fe48a99723a065969c638593b7d9ab + 57b538438010fd81bc1359802f0b871aeb95e4410a8ec92b93af10ea767a2027cf4734e8de80 + 8da338e6b722f7bf2051901bd5bccee5e71d5cf6b1faff338ad7120b0256c28380221ce17f19 + 117affa96e077905fe48a99723a065969c638593b7d9ab57b53843cd0780ffff804509f59593 + fd47b1a97189127ba65a5649cfb0346637f9836e155eaf891a939c00804509f59593fd47b1a9 + 7189127ba65a5649cfb0346637f9836e155eaf891a939c804509f59593fd47b1a97189127ba6 + 5a5649cfb0346637f9836e155eaf891a939c804509f59593fd47b1a97189127ba65a5649cfb0 + 346637f9836e155eaf891a939c804509f59593fd47b1a97189127ba65a5649cfb0346637f983 + 6e155eaf891a939c804509f59593fd47b1a97189127ba65a5649cfb0346637f9836e155eaf89 + 1a939c804509f59593fd47b1a97189127ba65a5649cfb0346637f9836e155eaf891a939c8045 + 09f59593fd47b1a97189127ba65a5649cfb0346637f9836e155eaf891a939c804509f59593fd + 47b1a97189127ba65a5649cfb0346637f9836e155eaf891a939c804509f59593fd47b1a97189 + 127ba65a5649cfb0346637f9836e155eaf891a939c804509f59593fd47b1a97189127ba65a56 + 49cfb0346637f9836e155eaf891a939c804509f59593fd47b1a97189127ba65a5649cfb03466 + 37f9836e155eaf891a939c804509f59593fd47b1a97189127ba65a5649cfb0346637f9836e15 + 5eaf891a939c804509f59593fd47b1a97189127ba65a5649cfb0346637f9836e155eaf891a93 + 9c804509f59593fd47b1a97189127ba65a5649cfb0346637f9836e155eaf891a939ccd0780ff + ff8078916e776c64ccea05e958559f015c082d9d06feafa3610fc44a5b2ef543cb818078916e + 776c64ccea05e958559f015c082d9d06feafa3610fc44a5b2ef543cb818078916e776c64ccea + 05e958559f015c082d9d06feafa3610fc44a5b2ef543cb818078916e776c64ccea05e958559f + 015c082d9d06feafa3610fc44a5b2ef543cb818078916e776c64ccea05e958559f015c082d9d + 06feafa3610fc44a5b2ef543cb81008078916e776c64ccea05e958559f015c082d9d06feafa3 + 610fc44a5b2ef543cb818078916e776c64ccea05e958559f015c082d9d06feafa3610fc44a5b + 2ef543cb818078916e776c64ccea05e958559f015c082d9d06feafa3610fc44a5b2ef543cb81 + 8078916e776c64ccea05e958559f015c082d9d06feafa3610fc44a5b2ef543cb818078916e77 + 6c64ccea05e958559f015c082d9d06feafa3610fc44a5b2ef543cb818078916e776c64ccea05 + e958559f015c082d9d06feafa3610fc44a5b2ef543cb818078916e776c64ccea05e958559f01 + 5c082d9d06feafa3610fc44a5b2ef543cb818078916e776c64ccea05e958559f015c082d9d06 + feafa3610fc44a5b2ef543cb818078916e776c64ccea05e958559f015c082d9d06feafa3610f + c44a5b2ef543cb818078916e776c64ccea05e958559f015c082d9d06feafa3610fc44a5b2ef5 + 43cb811044010000 + " ); type BalanceOf = @@ -109,7 +132,7 @@ benchmarks! { }: _(RawOrigin::Signed(caller.clone()), vec![0u8; l as usize]) verify { assert!(!BlockTransactions::::get().is_empty()); - assert_last_event::(Event::Stored(0).into()); + assert_last_event::(Event::Stored { index: 0 }.into()); } renew { @@ -122,7 +145,7 @@ benchmarks! { run_to_block::(1u32.into()); }: _(RawOrigin::Signed(caller.clone()), T::BlockNumber::zero(), 0) verify { - assert_last_event::(Event::Renewed(0).into()); + assert_last_event::(Event::Renewed { index: 0 }.into()); } check_proof_max { @@ -136,13 +159,12 @@ benchmarks! { )?; } run_to_block::(StoragePeriod::::get() + T::BlockNumber::one()); - let random_hash = [0u8]; let mut encoded_proof = PROOF; let proof = TransactionStorageProof::decode(&mut encoded_proof).unwrap(); }: check_proof(RawOrigin::None, proof) verify { assert_last_event::(Event::ProofChecked.into()); } -} -impl_benchmark_test_suite!(TransactionStorage, crate::mock::new_test_ext(), crate::mock::Test); + impl_benchmark_test_suite!(TransactionStorage, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index bc31199d9039..d95a60b49512 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -129,6 +129,7 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] + #[pallet::without_storage_info] pub struct Pallet(_); #[pallet::hooks] @@ -188,7 +189,7 @@ pub mod pallet { // Chunk data and compute storage root let chunk_count = num_chunks(data.len() as u32); let chunks = data.chunks(CHUNK_SIZE).map(|c| c.to_vec()).collect(); - let root = sp_io::trie::blake2_256_ordered_root(chunks); + let root = sp_io::trie::blake2_256_ordered_root(chunks, sp_runtime::StateVersion::V1); let content_hash = sp_io::hashing::blake2_256(&data); let extrinsic_index = >::extrinsic_index() @@ -210,7 +211,7 @@ pub mod pallet { }); Ok(()) })?; - Self::deposit_event(Event::Stored(index)); + Self::deposit_event(Event::Stored { index }); Ok(()) } @@ -251,7 +252,7 @@ pub mod pallet { }); Ok(()) })?; - Self::deposit_event(Event::Renewed(index)); + Self::deposit_event(Event::Renewed { index }); Ok(().into()) } @@ -300,6 +301,7 @@ pub mod pallet { &proof.proof, &encode_index(chunk_index), &proof.chunk, + sp_runtime::StateVersion::V1, ), Error::::InvalidProof ); @@ -313,9 +315,9 @@ pub mod pallet { #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { /// Stored data under specified index. - Stored(u32), + Stored { index: u32 }, /// Renewed data under specified index. - Renewed(u32), + Renewed { index: u32 }, /// Storage proof was successfully checked. ProofChecked, } diff --git a/frame/transaction-storage/src/mock.rs b/frame/transaction-storage/src/mock.rs index 38d14129d76e..753d4baaf00e 100644 --- a/frame/transaction-storage/src/mock.rs +++ b/frame/transaction-storage/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,10 +19,7 @@ use crate as pallet_transaction_storage; use crate::TransactionStorageProof; -use frame_support::{ - parameter_types, - traits::{OnFinalize, OnInitialize}, -}; +use frame_support::traits::{ConstU16, ConstU32, ConstU64, OnFinalize, OnInitialize}; use sp_core::H256; use sp_runtime::{ testing::Header, @@ -48,11 +45,6 @@ frame_support::construct_runtime!( } ); -parameter_types! { - pub const BlockHashCount: u64 = 250; - pub const SS58Prefix: u8 = 42; -} - impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; type BlockWeights = (); @@ -67,7 +59,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type DbWeight = (); type Version = (); type PalletInfo = PalletInfo; @@ -75,19 +67,16 @@ impl frame_system::Config for Test { type OnNewAccount = (); type OnKilledAccount = (); type SystemWeightInfo = (); - type SS58Prefix = SS58Prefix; + type SS58Prefix = ConstU16<42>; type OnSetCode = (); -} - -parameter_types! { - pub const ExistentialDeposit: u64 = 1; + type MaxConsumers = ConstU32<16>; } impl pallet_balances::Config for Test { type Balance = u64; type DustRemoval = (); type Event = Event; - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); type MaxLocks = (); diff --git a/frame/transaction-storage/src/tests.rs b/frame/transaction-storage/src/tests.rs index c443f51ffb50..8825890ae67a 100644 --- a/frame/transaction-storage/src/tests.rs +++ b/frame/transaction-storage/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/transaction-storage/src/weights.rs b/frame/transaction-storage/src/weights.rs index 104b18d3f92c..a7033da7b80c 100644 --- a/frame/transaction-storage/src/weights.rs +++ b/frame/transaction-storage/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_transaction_storage //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/transaction-storage/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -62,7 +63,7 @@ impl WeightInfo for SubstrateWeight { fn store(l: u32, ) -> Weight { (0 as Weight) // Standard Error: 0 - .saturating_add((8_000 as Weight).saturating_mul(l as Weight)) + .saturating_add((5_000 as Weight).saturating_mul(l as Weight)) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -73,7 +74,7 @@ impl WeightInfo for SubstrateWeight { // Storage: TransactionStorage BlockTransactions (r:1 w:1) // Storage: TransactionStorage MaxBlockTransactions (r:1 w:0) fn renew() -> Weight { - (67_532_000 as Weight) + (41_286_000 as Weight) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -83,7 +84,7 @@ impl WeightInfo for SubstrateWeight { // Storage: System ParentHash (r:1 w:0) // Storage: TransactionStorage Transactions (r:1 w:0) fn check_proof_max() -> Weight { - (182_886_000 as Weight) + (136_957_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } @@ -100,7 +101,7 @@ impl WeightInfo for () { fn store(l: u32, ) -> Weight { (0 as Weight) // Standard Error: 0 - .saturating_add((8_000 as Weight).saturating_mul(l as Weight)) + .saturating_add((5_000 as Weight).saturating_mul(l as Weight)) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -111,7 +112,7 @@ impl WeightInfo for () { // Storage: TransactionStorage BlockTransactions (r:1 w:1) // Storage: TransactionStorage MaxBlockTransactions (r:1 w:0) fn renew() -> Weight { - (67_532_000 as Weight) + (41_286_000 as Weight) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } @@ -121,7 +122,7 @@ impl WeightInfo for () { // Storage: System ParentHash (r:1 w:0) // Storage: TransactionStorage Transactions (r:1 w:0) fn check_proof_max() -> Weight { - (182_886_000 as Weight) + (136_957_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } diff --git a/frame/treasury/Cargo.toml b/frame/treasury/Cargo.toml index b2991f3febca..85745c6c99fd 100644 --- a/frame/treasury/Cargo.toml +++ b/frame/treasury/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-treasury" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet to manage treasury" readme = "README.md" @@ -13,16 +13,16 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.2.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", "max-encoded-len", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.126", features = ["derive"], optional = true } -impl-trait-for-tuples = "0.2.1" +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +serde = { version = "1.0.136", features = ["derive"], optional = true } +impl-trait-for-tuples = "0.2.2" -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } @@ -31,8 +31,8 @@ pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../ [dev-dependencies] -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } [features] default = ["std"] @@ -47,7 +47,7 @@ std = [ "pallet-balances/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", ] diff --git a/frame/treasury/src/benchmarking.rs b/frame/treasury/src/benchmarking.rs index 2fe0bad704f2..a0dd58ee3d42 100644 --- a/frame/treasury/src/benchmarking.rs +++ b/frame/treasury/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use super::{Pallet as Treasury, *}; -use frame_benchmarking::{account, benchmarks_instance_pallet, impl_benchmark_test_suite}; +use frame_benchmarking::{account, benchmarks_instance_pallet}; use frame_support::{ensure, traits::OnInitialize}; use frame_system::RawOrigin; @@ -94,6 +94,6 @@ benchmarks_instance_pallet! { }: { Treasury::::on_initialize(T::BlockNumber::zero()); } -} -impl_benchmark_test_suite!(Treasury, crate::tests::new_test_ext(), crate::tests::Test); + impl_benchmark_test_suite!(Treasury, crate::tests::new_test_ext(), crate::tests::Test); +} diff --git a/frame/treasury/src/lib.rs b/frame/treasury/src/lib.rs index 646baa99b99b..81fca5243afa 100644 --- a/frame/treasury/src/lib.rs +++ b/frame/treasury/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -139,7 +139,6 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] - #[pallet::generate_storage_info] pub struct Pallet(PhantomData<(T, I)>); #[pallet::config] @@ -168,6 +167,10 @@ pub mod pallet { #[pallet::constant] type ProposalBondMinimum: Get>; + /// Maximum amount of funds that should be placed in a deposit for making a proposal. + #[pallet::constant] + type ProposalBondMaximum: Get>>; + /// Period between successive spends. #[pallet::constant] type SpendPeriod: Get; @@ -190,6 +193,8 @@ pub mod pallet { type SpendFunds: SpendFunds; /// The maximum number of approvals that can wait in the spending queue. + /// + /// NOTE: This parameter is also used within the Bounties Pallet extension if enabled. #[pallet::constant] type MaxApprovals: Get; } @@ -255,21 +260,20 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event, I: 'static = ()> { - /// New proposal. \[proposal_index\] - Proposed(ProposalIndex), - /// We have ended a spend period and will now allocate funds. \[budget_remaining\] - Spending(BalanceOf), - /// Some funds have been allocated. \[proposal_index, award, beneficiary\] - Awarded(ProposalIndex, BalanceOf, T::AccountId), - /// A proposal was rejected; funds were slashed. \[proposal_index, slashed\] - Rejected(ProposalIndex, BalanceOf), - /// Some of our funds have been burnt. \[burn\] - Burnt(BalanceOf), + /// New proposal. + Proposed { proposal_index: ProposalIndex }, + /// We have ended a spend period and will now allocate funds. + Spending { budget_remaining: BalanceOf }, + /// Some funds have been allocated. + Awarded { proposal_index: ProposalIndex, award: BalanceOf, account: T::AccountId }, + /// A proposal was rejected; funds were slashed. + Rejected { proposal_index: ProposalIndex, slashed: BalanceOf }, + /// Some of our funds have been burnt. + Burnt { burnt_funds: BalanceOf }, /// Spending has finished; this is the amount that rolls over until next spend. - /// \[budget_remaining\] - Rollover(BalanceOf), - /// Some funds have been deposited. \[deposit\] - Deposit(BalanceOf), + Rollover { rollover_balance: BalanceOf }, + /// Some funds have been deposited. + Deposit { value: BalanceOf }, } /// Old name generated by `decl_event`. @@ -334,7 +338,7 @@ pub mod pallet { >::put(c + 1); >::insert(c, Proposal { proposer, value, beneficiary, bond }); - Self::deposit_event(Event::Proposed(c)); + Self::deposit_event(Event::Proposed { proposal_index: c }); Ok(()) } @@ -360,7 +364,10 @@ pub mod pallet { let imbalance = T::Currency::slash_reserved(&proposal.proposer, value).0; T::OnSlash::on_unbalanced(imbalance); - Self::deposit_event(Event::::Rejected(proposal_id, value)); + Self::deposit_event(Event::::Rejected { + proposal_index: proposal_id, + slashed: value, + }); Ok(()) } @@ -402,7 +409,11 @@ impl, I: 'static> Pallet { /// The needed bond for a proposal whose spend is `value`. fn calculate_bond(value: BalanceOf) -> BalanceOf { - T::ProposalBondMinimum::get().max(T::ProposalBond::get() * value) + let mut r = T::ProposalBondMinimum::get().max(T::ProposalBond::get() * value); + if let Some(m) = T::ProposalBondMaximum::get() { + r = r.min(m); + } + r } /// Spend some money! returns number of approvals before spend. @@ -410,7 +421,7 @@ impl, I: 'static> Pallet { let mut total_weight: Weight = Zero::zero(); let mut budget_remaining = Self::pot(); - Self::deposit_event(Event::Spending(budget_remaining)); + Self::deposit_event(Event::Spending { budget_remaining }); let account_id = Self::account_id(); let mut missed_any = false; @@ -431,7 +442,11 @@ impl, I: 'static> Pallet { // provide the allocation. imbalance.subsume(T::Currency::deposit_creating(&p.beneficiary, p.value)); - Self::deposit_event(Event::Awarded(index, p.value, p.beneficiary)); + Self::deposit_event(Event::Awarded { + proposal_index: index, + award: p.value, + account: p.beneficiary, + }); false } else { missed_any = true; @@ -462,7 +477,7 @@ impl, I: 'static> Pallet { let (debit, credit) = T::Currency::pair(burn); imbalance.subsume(debit); T::BurnDestination::on_unbalanced(credit); - Self::deposit_event(Event::Burnt(burn)) + Self::deposit_event(Event::Burnt { burnt_funds: burn }) } // Must never be an error, but better to be safe. @@ -477,7 +492,7 @@ impl, I: 'static> Pallet { drop(problem); } - Self::deposit_event(Event::Rollover(budget_remaining)); + Self::deposit_event(Event::Rollover { rollover_balance: budget_remaining }); total_weight } @@ -498,6 +513,6 @@ impl, I: 'static> OnUnbalanced> for Palle // Must resolve into existing but better to be safe. let _ = T::Currency::resolve_creating(&Self::account_id(), amount); - Self::deposit_event(Event::Deposit(numeric_amount)); + Self::deposit_event(Event::Deposit { value: numeric_amount }); } } diff --git a/frame/treasury/src/tests.rs b/frame/treasury/src/tests.rs index 534661b2773b..26189f520149 100644 --- a/frame/treasury/src/tests.rs +++ b/frame/treasury/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,7 +28,10 @@ use sp_runtime::{ }; use frame_support::{ - assert_noop, assert_ok, pallet_prelude::GenesisBuild, parameter_types, traits::OnInitialize, + assert_noop, assert_ok, + pallet_prelude::GenesisBuild, + parameter_types, + traits::{ConstU32, ConstU64, OnInitialize}, PalletId, }; @@ -51,7 +54,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -70,7 +72,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -79,9 +81,7 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); -} -parameter_types! { - pub const ExistentialDeposit: u64 = 1; + type MaxConsumers = ConstU32<16>; } impl pallet_balances::Config for Test { type MaxLocks = (); @@ -90,7 +90,7 @@ impl pallet_balances::Config for Test { type Balance = u64; type Event = Event; type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } @@ -99,8 +99,6 @@ thread_local! { } parameter_types! { pub const ProposalBond: Permill = Permill::from_percent(5); - pub const ProposalBondMinimum: u64 = 1; - pub const SpendPeriod: u64 = 2; pub const Burn: Permill = Permill::from_percent(50); pub const TreasuryPalletId: PalletId = PalletId(*b"py/trsry"); pub const BountyUpdatePeriod: u32 = 20; @@ -116,8 +114,9 @@ impl Config for Test { type Event = Event; type OnSlash = (); type ProposalBond = ProposalBond; - type ProposalBondMinimum = ProposalBondMinimum; - type SpendPeriod = SpendPeriod; + type ProposalBondMinimum = ConstU64<1>; + type ProposalBondMaximum = (); + type SpendPeriod = ConstU64<2>; type Burn = Burn; type BurnDestination = (); // Just gets burned. type WeightInfo = (); diff --git a/frame/treasury/src/weights.rs b/frame/treasury/src/weights.rs index 126c8a176626..dcbf5983fa65 100644 --- a/frame/treasury/src/weights.rs +++ b/frame/treasury/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_treasury //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/treasury/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -57,34 +58,34 @@ impl WeightInfo for SubstrateWeight { // Storage: Treasury ProposalCount (r:1 w:1) // Storage: Treasury Proposals (r:0 w:1) fn propose_spend() -> Weight { - (41_567_000 as Weight) + (21_673_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Treasury Proposals (r:1 w:1) // Storage: System Account (r:1 w:1) fn reject_proposal() -> Weight { - (38_993_000 as Weight) + (25_353_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Treasury Proposals (r:1 w:0) // Storage: Treasury Approvals (r:1 w:1) fn approve_proposal(p: u32, ) -> Weight { - (13_543_000 as Weight) + (8_164_000 as Weight) // Standard Error: 1_000 - .saturating_add((55_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((57_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Treasury Approvals (r:1 w:1) - // Storage: Treasury BountyApprovals (r:1 w:1) + // Storage: Bounties BountyApprovals (r:1 w:1) // Storage: Treasury Proposals (r:2 w:2) // Storage: System Account (r:4 w:4) fn on_initialize_proposals(p: u32, ) -> Weight { - (51_708_000 as Weight) + (20_762_000 as Weight) // Standard Error: 21_000 - .saturating_add((57_926_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((26_835_000 as Weight).saturating_mul(p as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(p as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) @@ -97,34 +98,34 @@ impl WeightInfo for () { // Storage: Treasury ProposalCount (r:1 w:1) // Storage: Treasury Proposals (r:0 w:1) fn propose_spend() -> Weight { - (41_567_000 as Weight) + (21_673_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Treasury Proposals (r:1 w:1) // Storage: System Account (r:1 w:1) fn reject_proposal() -> Weight { - (38_993_000 as Weight) + (25_353_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Treasury Proposals (r:1 w:0) // Storage: Treasury Approvals (r:1 w:1) fn approve_proposal(p: u32, ) -> Weight { - (13_543_000 as Weight) + (8_164_000 as Weight) // Standard Error: 1_000 - .saturating_add((55_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((57_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Treasury Approvals (r:1 w:1) - // Storage: Treasury BountyApprovals (r:1 w:1) + // Storage: Bounties BountyApprovals (r:1 w:1) // Storage: Treasury Proposals (r:2 w:2) // Storage: System Account (r:4 w:4) fn on_initialize_proposals(p: u32, ) -> Weight { - (51_708_000 as Weight) + (20_762_000 as Weight) // Standard Error: 21_000 - .saturating_add((57_926_000 as Weight).saturating_mul(p as Weight)) + .saturating_add((26_835_000 as Weight).saturating_mul(p as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(p as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) diff --git a/frame/try-runtime/Cargo.toml b/frame/try-runtime/Cargo.toml index 0ff534767607..e40b92b8e98d 100644 --- a/frame/try-runtime/Cargo.toml +++ b/frame/try-runtime/Cargo.toml @@ -2,9 +2,9 @@ name = "frame-try-runtime" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for democracy" readme = "README.md" @@ -14,8 +14,8 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sp-api = { version = "4.0.0-dev", path = "../../primitives/api", default-features = false } -sp-std = { version = "4.0.0-dev", path = "../../primitives/std" , default-features = false } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" , default-features = false } +sp-std = { version = "4.0.0", path = "../../primitives/std" , default-features = false } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" , default-features = false } frame-support = { version = "4.0.0-dev", path = "../support", default-features = false } diff --git a/frame/try-runtime/src/lib.rs b/frame/try-runtime/src/lib.rs index 754fc1d2a330..bf08112bfc37 100644 --- a/frame/try-runtime/src/lib.rs +++ b/frame/try-runtime/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/uniques/Cargo.toml b/frame/uniques/Cargo.toml index 4f664ecc2b6a..b0abb814eb5a 100644 --- a/frame/uniques/Cargo.toml +++ b/frame/uniques/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-uniques" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME NFT asset management pallet" readme = "README.md" @@ -13,18 +13,19 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } +log = { version = "0.4.14", default-features = false } [dev-dependencies] -sp-std = { version = "4.0.0-dev", path = "../../primitives/std" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } +sp-std = { version = "4.0.0", path = "../../primitives/std" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } [features] @@ -39,7 +40,7 @@ std = [ "frame-benchmarking/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "frame-system/runtime-benchmarks", ] diff --git a/frame/uniques/README.md b/frame/uniques/README.md index b924e338452f..8d6859d55e23 100644 --- a/frame/uniques/README.md +++ b/frame/uniques/README.md @@ -73,6 +73,6 @@ and its associated variants for documentation on each function. * [`System`](https://docs.rs/frame-system/latest/frame_system/) * [`Support`](https://docs.rs/frame-support/latest/frame_support/) -* [`Assets`](https://docs.rs/pallet-assets/latest/pallet_assetss/) +* [`Assets`](https://docs.rs/pallet-assets/latest/pallet_assets/) License: Apache-2.0 diff --git a/frame/uniques/src/benchmarking.rs b/frame/uniques/src/benchmarking.rs index 5c777dc961e9..19d3dfac6e5a 100644 --- a/frame/uniques/src/benchmarking.rs +++ b/frame/uniques/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,8 +21,7 @@ use super::*; use frame_benchmarking::{ - account, benchmarks_instance_pallet, impl_benchmark_test_suite, whitelist_account, - whitelisted_caller, + account, benchmarks_instance_pallet, whitelist_account, whitelisted_caller, }; use frame_support::{ dispatch::UnfilteredDispatchable, @@ -31,7 +30,7 @@ use frame_support::{ }; use frame_system::RawOrigin as SystemOrigin; use sp_runtime::traits::Bounded; -use sp_std::{convert::TryInto, prelude::*}; +use sp_std::prelude::*; use crate::Pallet as Uniques; @@ -41,12 +40,13 @@ fn create_class, I: 'static>( ) -> (T::ClassId, T::AccountId, ::Source) { let caller: T::AccountId = whitelisted_caller(); let caller_lookup = T::Lookup::unlookup(caller.clone()); - let class = Default::default(); + let class = T::Helper::class(0); T::Currency::make_free_balance_be(&caller, DepositBalanceOf::::max_value()); - assert!(Uniques::::create( - SystemOrigin::Signed(caller.clone()).into(), + assert!(Uniques::::force_create( + SystemOrigin::Root.into(), class, caller_lookup.clone(), + false, ) .is_ok()); (class, caller, caller_lookup) @@ -54,14 +54,14 @@ fn create_class, I: 'static>( fn add_class_metadata, I: 'static>( ) -> (T::AccountId, ::Source) { - let caller = Class::::get(T::ClassId::default()).unwrap().owner; + let caller = Class::::get(T::Helper::class(0)).unwrap().owner; if caller != whitelisted_caller() { whitelist_account!(caller); } let caller_lookup = T::Lookup::unlookup(caller.clone()); assert!(Uniques::::set_class_metadata( SystemOrigin::Signed(caller.clone()).into(), - Default::default(), + T::Helper::class(0), vec![0; T::StringLimit::get() as usize].try_into().unwrap(), false, ) @@ -72,15 +72,15 @@ fn add_class_metadata, I: 'static>( fn mint_instance, I: 'static>( index: u16, ) -> (T::InstanceId, T::AccountId, ::Source) { - let caller = Class::::get(T::ClassId::default()).unwrap().admin; + let caller = Class::::get(T::Helper::class(0)).unwrap().admin; if caller != whitelisted_caller() { whitelist_account!(caller); } let caller_lookup = T::Lookup::unlookup(caller.clone()); - let instance = index.into(); + let instance = T::Helper::instance(index); assert!(Uniques::::mint( SystemOrigin::Signed(caller.clone()).into(), - Default::default(), + T::Helper::class(0), instance, caller_lookup.clone(), ) @@ -91,14 +91,14 @@ fn mint_instance, I: 'static>( fn add_instance_metadata, I: 'static>( instance: T::InstanceId, ) -> (T::AccountId, ::Source) { - let caller = Class::::get(T::ClassId::default()).unwrap().owner; + let caller = Class::::get(T::Helper::class(0)).unwrap().owner; if caller != whitelisted_caller() { whitelist_account!(caller); } let caller_lookup = T::Lookup::unlookup(caller.clone()); assert!(Uniques::::set_metadata( SystemOrigin::Signed(caller.clone()).into(), - Default::default(), + T::Helper::class(0), instance, vec![0; T::StringLimit::get() as usize].try_into().unwrap(), false, @@ -110,7 +110,7 @@ fn add_instance_metadata, I: 'static>( fn add_instance_attribute, I: 'static>( instance: T::InstanceId, ) -> (BoundedVec, T::AccountId, ::Source) { - let caller = Class::::get(T::ClassId::default()).unwrap().owner; + let caller = Class::::get(T::Helper::class(0)).unwrap().owner; if caller != whitelisted_caller() { whitelist_account!(caller); } @@ -118,7 +118,7 @@ fn add_instance_attribute, I: 'static>( let key: BoundedVec<_, _> = vec![0; T::KeyLimit::get() as usize].try_into().unwrap(); assert!(Uniques::::set_attribute( SystemOrigin::Signed(caller.clone()).into(), - Default::default(), + T::Helper::class(0), Some(instance), key.clone(), vec![0; T::ValueLimit::get() as usize].try_into().unwrap(), @@ -137,20 +137,24 @@ fn assert_last_event, I: 'static>(generic_event: >:: benchmarks_instance_pallet! { create { - let caller: T::AccountId = whitelisted_caller(); - let caller_lookup = T::Lookup::unlookup(caller.clone()); + let class = T::Helper::class(0); + let origin = T::CreateOrigin::successful_origin(&class); + let caller = T::CreateOrigin::ensure_origin(origin.clone(), &class).unwrap(); + whitelist_account!(caller); + let admin = T::Lookup::unlookup(caller.clone()); T::Currency::make_free_balance_be(&caller, DepositBalanceOf::::max_value()); - }: _(SystemOrigin::Signed(caller.clone()), Default::default(), caller_lookup) + let call = Call::::create { class, admin }; + }: { call.dispatch_bypass_filter(origin)? } verify { - assert_last_event::(Event::Created(Default::default(), caller.clone(), caller).into()); + assert_last_event::(Event::Created { class: T::Helper::class(0), creator: caller.clone(), owner: caller }.into()); } force_create { let caller: T::AccountId = whitelisted_caller(); let caller_lookup = T::Lookup::unlookup(caller.clone()); - }: _(SystemOrigin::Root, Default::default(), caller_lookup, true) + }: _(SystemOrigin::Root, T::Helper::class(0), caller_lookup, true) verify { - assert_last_event::(Event::ForceCreated(Default::default(), caller).into()); + assert_last_event::(Event::ForceCreated { class: T::Helper::class(0), owner: caller }.into()); } destroy { @@ -164,23 +168,23 @@ benchmarks_instance_pallet! { mint_instance::(i as u16); } for i in 0..m { - add_instance_metadata::((i as u16).into()); + add_instance_metadata::(T::Helper::instance(i as u16)); } for i in 0..a { - add_instance_attribute::((i as u16).into()); + add_instance_attribute::(T::Helper::instance(i as u16)); } let witness = Class::::get(class).unwrap().destroy_witness(); }: _(SystemOrigin::Signed(caller), class, witness) verify { - assert_last_event::(Event::Destroyed(class).into()); + assert_last_event::(Event::Destroyed { class: class }.into()); } mint { let (class, caller, caller_lookup) = create_class::(); - let instance = Default::default(); + let instance = T::Helper::instance(0); }: _(SystemOrigin::Signed(caller.clone()), class, instance, caller_lookup) verify { - assert_last_event::(Event::Issued(class, instance, caller).into()); + assert_last_event::(Event::Issued { class, instance, owner: caller }.into()); } burn { @@ -188,18 +192,18 @@ benchmarks_instance_pallet! { let (instance, ..) = mint_instance::(0); }: _(SystemOrigin::Signed(caller.clone()), class, instance, Some(caller_lookup)) verify { - assert_last_event::(Event::Burned(class, instance, caller).into()); + assert_last_event::(Event::Burned { class, instance, owner: caller }.into()); } transfer { let (class, caller, caller_lookup) = create_class::(); - let (instance, ..) = mint_instance::(Default::default()); + let (instance, ..) = mint_instance::(0); let target: T::AccountId = account("target", 0, SEED); let target_lookup = T::Lookup::unlookup(target.clone()); }: _(SystemOrigin::Signed(caller.clone()), class, instance, target_lookup) verify { - assert_last_event::(Event::Transferred(class, instance, caller, target).into()); + assert_last_event::(Event::Transferred { class, instance, from: caller, to: target }.into()); } redeposit { @@ -218,20 +222,20 @@ benchmarks_instance_pallet! { )?; }: _(SystemOrigin::Signed(caller.clone()), class, instances.clone()) verify { - assert_last_event::(Event::Redeposited(class, instances).into()); + assert_last_event::(Event::Redeposited { class, successful_instances: instances }.into()); } freeze { let (class, caller, caller_lookup) = create_class::(); - let (instance, ..) = mint_instance::(Default::default()); - }: _(SystemOrigin::Signed(caller.clone()), Default::default(), Default::default()) + let (instance, ..) = mint_instance::(0); + }: _(SystemOrigin::Signed(caller.clone()), T::Helper::class(0), T::Helper::instance(0)) verify { - assert_last_event::(Event::Frozen(Default::default(), Default::default()).into()); + assert_last_event::(Event::Frozen { class: T::Helper::class(0), instance: T::Helper::instance(0) }.into()); } thaw { let (class, caller, caller_lookup) = create_class::(); - let (instance, ..) = mint_instance::(Default::default()); + let (instance, ..) = mint_instance::(0); Uniques::::freeze( SystemOrigin::Signed(caller.clone()).into(), class, @@ -239,14 +243,14 @@ benchmarks_instance_pallet! { )?; }: _(SystemOrigin::Signed(caller.clone()), class, instance) verify { - assert_last_event::(Event::Thawed(class, instance).into()); + assert_last_event::(Event::Thawed { class, instance }.into()); } freeze_class { let (class, caller, caller_lookup) = create_class::(); }: _(SystemOrigin::Signed(caller.clone()), class) verify { - assert_last_event::(Event::ClassFrozen(class).into()); + assert_last_event::(Event::ClassFrozen { class }.into()); } thaw_class { @@ -255,7 +259,7 @@ benchmarks_instance_pallet! { Uniques::::freeze_class(origin, class)?; }: _(SystemOrigin::Signed(caller.clone()), class) verify { - assert_last_event::(Event::ClassThawed(class).into()); + assert_last_event::(Event::ClassThawed { class }.into()); } transfer_ownership { @@ -263,9 +267,11 @@ benchmarks_instance_pallet! { let target: T::AccountId = account("target", 0, SEED); let target_lookup = T::Lookup::unlookup(target.clone()); T::Currency::make_free_balance_be(&target, T::Currency::minimum_balance()); + let origin = SystemOrigin::Signed(target.clone()).into(); + Uniques::::set_accept_ownership(origin, Some(class.clone()))?; }: _(SystemOrigin::Signed(caller), class, target_lookup) verify { - assert_last_event::(Event::OwnerChanged(class, target).into()); + assert_last_event::(Event::OwnerChanged { class, new_owner: target }.into()); } set_team { @@ -273,14 +279,14 @@ benchmarks_instance_pallet! { let target0 = T::Lookup::unlookup(account("target", 0, SEED)); let target1 = T::Lookup::unlookup(account("target", 1, SEED)); let target2 = T::Lookup::unlookup(account("target", 2, SEED)); - }: _(SystemOrigin::Signed(caller), Default::default(), target0.clone(), target1.clone(), target2.clone()) + }: _(SystemOrigin::Signed(caller), class, target0.clone(), target1.clone(), target2.clone()) verify { - assert_last_event::(Event::TeamChanged( + assert_last_event::(Event::TeamChanged{ class, - account("target", 0, SEED), - account("target", 1, SEED), - account("target", 2, SEED), - ).into()); + issuer: account("target", 0, SEED), + admin: account("target", 1, SEED), + freezer: account("target", 2, SEED), + }.into()); } force_asset_status { @@ -297,7 +303,7 @@ benchmarks_instance_pallet! { }; }: { call.dispatch_bypass_filter(origin)? } verify { - assert_last_event::(Event::AssetStatusChanged(class).into()); + assert_last_event::(Event::AssetStatusChanged { class }.into()); } set_attribute { @@ -309,7 +315,7 @@ benchmarks_instance_pallet! { add_instance_metadata::(instance); }: _(SystemOrigin::Signed(caller), class, Some(instance), key.clone(), value.clone()) verify { - assert_last_event::(Event::AttributeSet(class, Some(instance), key, value).into()); + assert_last_event::(Event::AttributeSet { class, maybe_instance: Some(instance), key, value }.into()); } clear_attribute { @@ -319,7 +325,7 @@ benchmarks_instance_pallet! { let (key, ..) = add_instance_attribute::(instance); }: _(SystemOrigin::Signed(caller), class, Some(instance), key.clone()) verify { - assert_last_event::(Event::AttributeCleared(class, Some(instance), key).into()); + assert_last_event::(Event::AttributeCleared { class, maybe_instance: Some(instance), key }.into()); } set_metadata { @@ -329,7 +335,7 @@ benchmarks_instance_pallet! { let (instance, ..) = mint_instance::(0); }: _(SystemOrigin::Signed(caller), class, instance, data.clone(), false) verify { - assert_last_event::(Event::MetadataSet(class, instance, data, false).into()); + assert_last_event::(Event::MetadataSet { class, instance, data, is_frozen: false }.into()); } clear_metadata { @@ -338,7 +344,7 @@ benchmarks_instance_pallet! { add_instance_metadata::(instance); }: _(SystemOrigin::Signed(caller), class, instance) verify { - assert_last_event::(Event::MetadataCleared(class, instance).into()); + assert_last_event::(Event::MetadataCleared { class, instance }.into()); } set_class_metadata { @@ -347,7 +353,7 @@ benchmarks_instance_pallet! { let (class, caller, _) = create_class::(); }: _(SystemOrigin::Signed(caller), class, data.clone(), false) verify { - assert_last_event::(Event::ClassMetadataSet(class, data, false).into()); + assert_last_event::(Event::ClassMetadataSet { class, data, is_frozen: false }.into()); } clear_class_metadata { @@ -355,7 +361,7 @@ benchmarks_instance_pallet! { add_class_metadata::(); }: _(SystemOrigin::Signed(caller), class) verify { - assert_last_event::(Event::ClassMetadataCleared(class).into()); + assert_last_event::(Event::ClassMetadataCleared { class }.into()); } approve_transfer { @@ -365,7 +371,7 @@ benchmarks_instance_pallet! { let delegate_lookup = T::Lookup::unlookup(delegate.clone()); }: _(SystemOrigin::Signed(caller.clone()), class, instance, delegate_lookup) verify { - assert_last_event::(Event::ApprovedTransfer(class, instance, caller, delegate).into()); + assert_last_event::(Event::ApprovedTransfer { class, instance, owner: caller, delegate }.into()); } cancel_approval { @@ -377,8 +383,20 @@ benchmarks_instance_pallet! { Uniques::::approve_transfer(origin, class, instance, delegate_lookup.clone())?; }: _(SystemOrigin::Signed(caller.clone()), class, instance, Some(delegate_lookup)) verify { - assert_last_event::(Event::ApprovalCancelled(class, instance, caller, delegate).into()); + assert_last_event::(Event::ApprovalCancelled { class, instance, owner: caller, delegate }.into()); + } + + set_accept_ownership { + let caller: T::AccountId = whitelisted_caller(); + T::Currency::make_free_balance_be(&caller, DepositBalanceOf::::max_value()); + let class = T::Helper::class(0); + }: _(SystemOrigin::Signed(caller.clone()), Some(class.clone())) + verify { + assert_last_event::(Event::OwnershipAcceptanceChanged { + who: caller, + maybe_class: Some(class), + }.into()); } -} -impl_benchmark_test_suite!(Uniques, crate::mock::new_test_ext(), crate::mock::Test); + impl_benchmark_test_suite!(Uniques, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/frame/uniques/src/functions.rs b/frame/uniques/src/functions.rs index 68acf7f1879f..40c436bd56b4 100644 --- a/frame/uniques/src/functions.rs +++ b/frame/uniques/src/functions.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,7 @@ use frame_support::{ensure, traits::Get}; use sp_runtime::{DispatchError, DispatchResult}; impl, I: 'static> Pallet { - pub(crate) fn do_transfer( + pub fn do_transfer( class: T::ClassId, instance: T::InstanceId, dest: T::AccountId, @@ -31,10 +31,11 @@ impl, I: 'static> Pallet { &mut InstanceDetailsFor, ) -> DispatchResult, ) -> DispatchResult { - let class_details = Class::::get(&class).ok_or(Error::::Unknown)?; + let class_details = Class::::get(&class).ok_or(Error::::UnknownClass)?; ensure!(!class_details.is_frozen, Error::::Frozen); - let mut details = Asset::::get(&class, &instance).ok_or(Error::::Unknown)?; + let mut details = + Asset::::get(&class, &instance).ok_or(Error::::UnknownClass)?; ensure!(!details.is_frozen, Error::::Frozen); with_details(&class_details, &mut details)?; @@ -44,11 +45,16 @@ impl, I: 'static> Pallet { details.owner = dest; Asset::::insert(&class, &instance, &details); - Self::deposit_event(Event::Transferred(class, instance, origin, details.owner)); + Self::deposit_event(Event::Transferred { + class, + instance, + from: origin, + to: details.owner, + }); Ok(()) } - pub(super) fn do_create_class( + pub fn do_create_class( class: T::ClassId, owner: T::AccountId, admin: T::AccountId, @@ -76,17 +82,18 @@ impl, I: 'static> Pallet { }, ); + ClassAccount::::insert(&owner, &class, ()); Self::deposit_event(event); Ok(()) } - pub(super) fn do_destroy_class( + pub fn do_destroy_class( class: T::ClassId, witness: DestroyWitness, maybe_check_owner: Option, ) -> Result { Class::::try_mutate_exists(class, |maybe_details| { - let class_details = maybe_details.take().ok_or(Error::::Unknown)?; + let class_details = maybe_details.take().ok_or(Error::::UnknownClass)?; if let Some(check_owner) = maybe_check_owner { ensure!(class_details.owner == check_owner, Error::::NoPermission); } @@ -103,9 +110,10 @@ impl, I: 'static> Pallet { InstanceMetadataOf::::remove_prefix(&class, None); ClassMetadataOf::::remove(&class); Attribute::::remove_prefix((&class,), None); + ClassAccount::::remove(&class_details.owner, &class); T::Currency::unreserve(&class_details.owner, class_details.total_deposit); - Self::deposit_event(Event::Destroyed(class)); + Self::deposit_event(Event::Destroyed { class }); Ok(DestroyWitness { instances: class_details.instances, @@ -115,7 +123,7 @@ impl, I: 'static> Pallet { }) } - pub(super) fn do_mint( + pub fn do_mint( class: T::ClassId, instance: T::InstanceId, owner: T::AccountId, @@ -124,7 +132,7 @@ impl, I: 'static> Pallet { ensure!(!Asset::::contains_key(class, instance), Error::::AlreadyExists); Class::::try_mutate(&class, |maybe_class_details| -> DispatchResult { - let class_details = maybe_class_details.as_mut().ok_or(Error::::Unknown)?; + let class_details = maybe_class_details.as_mut().ok_or(Error::::UnknownClass)?; with_details(&class_details)?; @@ -146,11 +154,11 @@ impl, I: 'static> Pallet { Ok(()) })?; - Self::deposit_event(Event::Issued(class, instance, owner)); + Self::deposit_event(Event::Issued { class, instance, owner }); Ok(()) } - pub(super) fn do_burn( + pub fn do_burn( class: T::ClassId, instance: T::InstanceId, with_details: impl FnOnce(&ClassDetailsFor, &InstanceDetailsFor) -> DispatchResult, @@ -158,9 +166,10 @@ impl, I: 'static> Pallet { let owner = Class::::try_mutate( &class, |maybe_class_details| -> Result { - let class_details = maybe_class_details.as_mut().ok_or(Error::::Unknown)?; + let class_details = + maybe_class_details.as_mut().ok_or(Error::::UnknownClass)?; let details = - Asset::::get(&class, &instance).ok_or(Error::::Unknown)?; + Asset::::get(&class, &instance).ok_or(Error::::UnknownClass)?; with_details(&class_details, &details)?; // Return the deposit. @@ -174,7 +183,7 @@ impl, I: 'static> Pallet { Asset::::remove(&class, &instance); Account::::remove((&owner, &class, &instance)); - Self::deposit_event(Event::Burned(class, instance, owner)); + Self::deposit_event(Event::Burned { class, instance, owner }); Ok(()) } } diff --git a/frame/uniques/src/impl_nonfungibles.rs b/frame/uniques/src/impl_nonfungibles.rs index e68d2d4deecd..89b95fb77048 100644 --- a/frame/uniques/src/impl_nonfungibles.rs +++ b/frame/uniques/src/impl_nonfungibles.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,7 +23,7 @@ use frame_support::{ BoundedSlice, }; use sp_runtime::{DispatchError, DispatchResult}; -use sp_std::convert::TryFrom; +use sp_std::prelude::*; impl, I: 'static> Inspect<::AccountId> for Pallet { type InstanceId = T::InstanceId; @@ -98,7 +98,7 @@ impl, I: 'static> Create<::AccountId> for Pallet admin.clone(), T::ClassDeposit::get(), false, - Event::Created(class.clone(), who.clone(), admin.clone()), + Event::Created { class: class.clone(), creator: who.clone(), owner: admin.clone() }, ) } } @@ -128,8 +128,19 @@ impl, I: 'static> Mutate<::AccountId> for Pallet Self::do_mint(class.clone(), instance.clone(), who.clone(), |_| Ok(())) } - fn burn_from(class: &Self::ClassId, instance: &Self::InstanceId) -> DispatchResult { - Self::do_burn(class.clone(), instance.clone(), |_, _| Ok(())) + fn burn( + class: &Self::ClassId, + instance: &Self::InstanceId, + maybe_check_owner: Option<&T::AccountId>, + ) -> DispatchResult { + Self::do_burn(class.clone(), instance.clone(), |_, d| { + if let Some(check_owner) = maybe_check_owner { + if &d.owner != check_owner { + Err(Error::::NoPermission)?; + } + } + Ok(()) + }) } } diff --git a/frame/uniques/src/lib.rs b/frame/uniques/src/lib.rs index 1bf220e4a787..1e1482545419 100644 --- a/frame/uniques/src/lib.rs +++ b/frame/uniques/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,15 +33,18 @@ mod benchmarking; pub mod mock; #[cfg(test)] mod tests; -pub mod weights; mod functions; mod impl_nonfungibles; mod types; -pub use types::*; -use codec::{Decode, Encode, HasCompact}; -use frame_support::traits::{BalanceStatus::Reserved, Currency, ReservableCurrency}; +pub mod migration; +pub mod weights; + +use codec::{Decode, Encode}; +use frame_support::traits::{ + BalanceStatus::Reserved, Currency, EnsureOriginWithArg, ReservableCurrency, +}; use frame_system::Config as SystemConfig; use sp_runtime::{ traits::{Saturating, StaticLookup, Zero}, @@ -50,6 +53,7 @@ use sp_runtime::{ use sp_std::prelude::*; pub use pallet::*; +pub use types::*; pub use weights::WeightInfo; #[frame_support::pallet] @@ -62,6 +66,21 @@ pub mod pallet { #[pallet::generate_store(pub(super) trait Store)] pub struct Pallet(_); + #[cfg(feature = "runtime-benchmarks")] + pub trait BenchmarkHelper { + fn class(i: u16) -> ClassId; + fn instance(i: u16) -> InstanceId; + } + #[cfg(feature = "runtime-benchmarks")] + impl, InstanceId: From> BenchmarkHelper for () { + fn class(i: u16) -> ClassId { + i.into() + } + fn instance(i: u16) -> InstanceId { + i.into() + } + } + #[pallet::config] /// The module configuration trait. pub trait Config: frame_system::Config { @@ -69,10 +88,10 @@ pub mod pallet { type Event: From> + IsType<::Event>; /// Identifier for the class of asset. - type ClassId: Member + Parameter + Default + Copy + HasCompact; + type ClassId: Member + Parameter + MaxEncodedLen + Copy; /// The type used to identify a unique asset within an asset class. - type InstanceId: Member + Parameter + Default + Copy + HasCompact + From; + type InstanceId: Member + Parameter + MaxEncodedLen + Copy; /// The currency mechanism, used for paying for reserves. type Currency: ReservableCurrency; @@ -81,6 +100,14 @@ pub mod pallet { /// attributes. type ForceOrigin: EnsureOrigin; + /// Standard class creation is only allowed if the origin attempting it and the class are + /// in this set. + type CreateOrigin: EnsureOriginWithArg< + Success = Self::AccountId, + Self::Origin, + Self::ClassId, + >; + /// The basic amount of funds that must be reserved for an asset class. #[pallet::constant] type ClassDeposit: Get>; @@ -114,6 +141,10 @@ pub mod pallet { #[pallet::constant] type ValueLimit: Get; + #[cfg(feature = "runtime-benchmarks")] + /// A set of helper functions for benchmarking. + type Helper: BenchmarkHelper; + /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; } @@ -127,6 +158,11 @@ pub mod pallet { ClassDetails>, >; + #[pallet::storage] + /// The class, if any, of which an account is willing to take ownership. + pub(super) type OwnershipAcceptance, I: 'static = ()> = + StorageMap<_, Blake2_128Concat, T::AccountId, T::ClassId>; + #[pallet::storage] /// The assets held by any given account; set out this way so that assets owned by a single /// account can be enumerated. @@ -141,6 +177,19 @@ pub mod pallet { OptionQuery, >; + #[pallet::storage] + /// The classes owned by any given account; set out this way so that classes owned by a single + /// account can be enumerated. + pub(super) type ClassAccount, I: 'static = ()> = StorageDoubleMap< + _, + Blake2_128Concat, + T::AccountId, + Blake2_128Concat, + T::ClassId, + (), + OptionQuery, + >; + #[pallet::storage] /// The assets in existence and their ownership details. pub(super) type Asset, I: 'static = ()> = StorageDoubleMap< @@ -191,63 +240,92 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event, I: 'static = ()> { - /// An asset class was created. \[ class, creator, owner \] - Created(T::ClassId, T::AccountId, T::AccountId), - /// An asset class was force-created. \[ class, owner \] - ForceCreated(T::ClassId, T::AccountId), - /// An asset `class` was destroyed. \[ class \] - Destroyed(T::ClassId), - /// An asset `instance` was issued. \[ class, instance, owner \] - Issued(T::ClassId, T::InstanceId, T::AccountId), - /// An asset `instance` was transferred. \[ class, instance, from, to \] - Transferred(T::ClassId, T::InstanceId, T::AccountId, T::AccountId), - /// An asset `instance` was destroyed. \[ class, instance, owner \] - Burned(T::ClassId, T::InstanceId, T::AccountId), - /// Some asset `instance` was frozen. \[ class, instance \] - Frozen(T::ClassId, T::InstanceId), - /// Some asset `instance` was thawed. \[ class, instance \] - Thawed(T::ClassId, T::InstanceId), - /// Some asset `class` was frozen. \[ class \] - ClassFrozen(T::ClassId), - /// Some asset `class` was thawed. \[ class \] - ClassThawed(T::ClassId), - /// The owner changed \[ class, new_owner \] - OwnerChanged(T::ClassId, T::AccountId), - /// The management team changed \[ class, issuer, admin, freezer \] - TeamChanged(T::ClassId, T::AccountId, T::AccountId, T::AccountId), + /// An asset class was created. + Created { class: T::ClassId, creator: T::AccountId, owner: T::AccountId }, + /// An asset class was force-created. + ForceCreated { class: T::ClassId, owner: T::AccountId }, + /// An asset `class` was destroyed. + Destroyed { class: T::ClassId }, + /// An asset `instance` was issued. + Issued { class: T::ClassId, instance: T::InstanceId, owner: T::AccountId }, + /// An asset `instance` was transferred. + Transferred { + class: T::ClassId, + instance: T::InstanceId, + from: T::AccountId, + to: T::AccountId, + }, + /// An asset `instance` was destroyed. + Burned { class: T::ClassId, instance: T::InstanceId, owner: T::AccountId }, + /// Some asset `instance` was frozen. + Frozen { class: T::ClassId, instance: T::InstanceId }, + /// Some asset `instance` was thawed. + Thawed { class: T::ClassId, instance: T::InstanceId }, + /// Some asset `class` was frozen. + ClassFrozen { class: T::ClassId }, + /// Some asset `class` was thawed. + ClassThawed { class: T::ClassId }, + /// The owner changed. + OwnerChanged { class: T::ClassId, new_owner: T::AccountId }, + /// The management team changed. + TeamChanged { + class: T::ClassId, + issuer: T::AccountId, + admin: T::AccountId, + freezer: T::AccountId, + }, /// An `instance` of an asset `class` has been approved by the `owner` for transfer by a /// `delegate`. - /// \[ class, instance, owner, delegate \] - ApprovedTransfer(T::ClassId, T::InstanceId, T::AccountId, T::AccountId), + ApprovedTransfer { + class: T::ClassId, + instance: T::InstanceId, + owner: T::AccountId, + delegate: T::AccountId, + }, /// An approval for a `delegate` account to transfer the `instance` of an asset `class` was /// cancelled by its `owner`. - /// \[ class, instance, owner, delegate \] - ApprovalCancelled(T::ClassId, T::InstanceId, T::AccountId, T::AccountId), + ApprovalCancelled { + class: T::ClassId, + instance: T::InstanceId, + owner: T::AccountId, + delegate: T::AccountId, + }, /// An asset `class` has had its attributes changed by the `Force` origin. - /// \[ class \] - AssetStatusChanged(T::ClassId), - /// New metadata has been set for an asset class. \[ class, data, is_frozen \] - ClassMetadataSet(T::ClassId, BoundedVec, bool), - /// Metadata has been cleared for an asset class. \[ class \] - ClassMetadataCleared(T::ClassId), + AssetStatusChanged { class: T::ClassId }, + /// New metadata has been set for an asset class. + ClassMetadataSet { + class: T::ClassId, + data: BoundedVec, + is_frozen: bool, + }, + /// Metadata has been cleared for an asset class. + ClassMetadataCleared { class: T::ClassId }, /// New metadata has been set for an asset instance. - /// \[ class, instance, data, is_frozen \] - MetadataSet(T::ClassId, T::InstanceId, BoundedVec, bool), - /// Metadata has been cleared for an asset instance. \[ class, instance \] - MetadataCleared(T::ClassId, T::InstanceId), - /// Metadata has been cleared for an asset instance. \[ class, successful_instances \] - Redeposited(T::ClassId, Vec), + MetadataSet { + class: T::ClassId, + instance: T::InstanceId, + data: BoundedVec, + is_frozen: bool, + }, + /// Metadata has been cleared for an asset instance. + MetadataCleared { class: T::ClassId, instance: T::InstanceId }, + /// Metadata has been cleared for an asset instance. + Redeposited { class: T::ClassId, successful_instances: Vec }, /// New attribute metadata has been set for an asset class or instance. - /// \[ class, maybe_instance, key, value \] - AttributeSet( - T::ClassId, - Option, - BoundedVec, - BoundedVec, - ), + AttributeSet { + class: T::ClassId, + maybe_instance: Option, + key: BoundedVec, + value: BoundedVec, + }, /// Attribute metadata has been cleared for an asset class or instance. - /// \[ class, maybe_instance, key, maybe_value \] - AttributeCleared(T::ClassId, Option, BoundedVec), + AttributeCleared { + class: T::ClassId, + maybe_instance: Option, + key: BoundedVec, + }, + /// Ownership acceptance has changed for an account. + OwnershipAcceptanceChanged { who: T::AccountId, maybe_class: Option }, } #[pallet::error] @@ -255,7 +333,7 @@ pub mod pallet { /// The signing account has no permission to do the operation. NoPermission, /// The given asset ID is unknown. - Unknown, + UnknownClass, /// The asset instance ID has already been used for an asset. AlreadyExists, /// The owner turned out to be different to what was expected. @@ -272,16 +350,20 @@ pub mod pallet { NoDelegate, /// No approval exists that would allow the transfer. Unapproved, + /// The named owner has not signed ownership of the class is acceptable. + Unaccepted, } - #[pallet::hooks] - impl, I: 'static> Hooks> for Pallet {} - impl, I: 'static> Pallet { /// Get the owner of the asset instance, if the asset exists. pub fn owner(class: T::ClassId, instance: T::InstanceId) -> Option { Asset::::get(class, instance).map(|i| i.owner) } + + /// Get the owner of the asset instance, if the asset exists. + pub fn class_owner(class: T::ClassId) -> Option { + Class::::get(class).map(|i| i.owner) + } } #[pallet::call] @@ -305,10 +387,10 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::create())] pub fn create( origin: OriginFor, - #[pallet::compact] class: T::ClassId, + class: T::ClassId, admin: ::Source, ) -> DispatchResult { - let owner = ensure_signed(origin)?; + let owner = T::CreateOrigin::ensure_origin(origin, &class)?; let admin = T::Lookup::lookup(admin)?; Self::do_create_class( @@ -317,7 +399,7 @@ pub mod pallet { admin.clone(), T::ClassDeposit::get(), false, - Event::Created(class, owner, admin), + Event::Created { class, creator: owner, owner: admin }, ) } @@ -340,7 +422,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::force_create())] pub fn force_create( origin: OriginFor, - #[pallet::compact] class: T::ClassId, + class: T::ClassId, owner: ::Source, free_holding: bool, ) -> DispatchResult { @@ -353,7 +435,7 @@ pub mod pallet { owner.clone(), Zero::zero(), free_holding, - Event::ForceCreated(class, owner), + Event::ForceCreated { class, owner }, ) } @@ -379,7 +461,7 @@ pub mod pallet { ))] pub fn destroy( origin: OriginFor, - #[pallet::compact] class: T::ClassId, + class: T::ClassId, witness: DestroyWitness, ) -> DispatchResultWithPostInfo { let maybe_check_owner = match T::ForceOrigin::try_origin(origin) { @@ -410,8 +492,8 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::mint())] pub fn mint( origin: OriginFor, - #[pallet::compact] class: T::ClassId, - #[pallet::compact] instance: T::InstanceId, + class: T::ClassId, + instance: T::InstanceId, owner: ::Source, ) -> DispatchResult { let origin = ensure_signed(origin)?; @@ -439,8 +521,8 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::burn())] pub fn burn( origin: OriginFor, - #[pallet::compact] class: T::ClassId, - #[pallet::compact] instance: T::InstanceId, + class: T::ClassId, + instance: T::InstanceId, check_owner: Option<::Source>, ) -> DispatchResult { let origin = ensure_signed(origin)?; @@ -475,8 +557,8 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::transfer())] pub fn transfer( origin: OriginFor, - #[pallet::compact] class: T::ClassId, - #[pallet::compact] instance: T::InstanceId, + class: T::ClassId, + instance: T::InstanceId, dest: ::Source, ) -> DispatchResult { let origin = ensure_signed(origin)?; @@ -511,12 +593,13 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::redeposit(instances.len() as u32))] pub fn redeposit( origin: OriginFor, - #[pallet::compact] class: T::ClassId, + class: T::ClassId, instances: Vec, ) -> DispatchResult { let origin = ensure_signed(origin)?; - let mut class_details = Class::::get(&class).ok_or(Error::::Unknown)?; + let mut class_details = + Class::::get(&class).ok_or(Error::::UnknownClass)?; ensure!(class_details.owner == origin, Error::::NoPermission); let deposit = match class_details.free_holding { true => Zero::zero(), @@ -549,7 +632,10 @@ pub mod pallet { } Class::::insert(&class, &class_details); - Self::deposit_event(Event::::Redeposited(class, successful)); + Self::deposit_event(Event::::Redeposited { + class, + successful_instances: successful, + }); Ok(()) } @@ -567,20 +653,20 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::freeze())] pub fn freeze( origin: OriginFor, - #[pallet::compact] class: T::ClassId, - #[pallet::compact] instance: T::InstanceId, + class: T::ClassId, + instance: T::InstanceId, ) -> DispatchResult { let origin = ensure_signed(origin)?; let mut details = - Asset::::get(&class, &instance).ok_or(Error::::Unknown)?; - let class_details = Class::::get(&class).ok_or(Error::::Unknown)?; + Asset::::get(&class, &instance).ok_or(Error::::UnknownClass)?; + let class_details = Class::::get(&class).ok_or(Error::::UnknownClass)?; ensure!(class_details.freezer == origin, Error::::NoPermission); details.is_frozen = true; Asset::::insert(&class, &instance, &details); - Self::deposit_event(Event::::Frozen(class, instance)); + Self::deposit_event(Event::::Frozen { class, instance }); Ok(()) } @@ -597,20 +683,20 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::thaw())] pub fn thaw( origin: OriginFor, - #[pallet::compact] class: T::ClassId, - #[pallet::compact] instance: T::InstanceId, + class: T::ClassId, + instance: T::InstanceId, ) -> DispatchResult { let origin = ensure_signed(origin)?; let mut details = - Asset::::get(&class, &instance).ok_or(Error::::Unknown)?; - let class_details = Class::::get(&class).ok_or(Error::::Unknown)?; + Asset::::get(&class, &instance).ok_or(Error::::UnknownClass)?; + let class_details = Class::::get(&class).ok_or(Error::::UnknownClass)?; ensure!(class_details.admin == origin, Error::::NoPermission); details.is_frozen = false; Asset::::insert(&class, &instance, &details); - Self::deposit_event(Event::::Thawed(class, instance)); + Self::deposit_event(Event::::Thawed { class, instance }); Ok(()) } @@ -624,19 +710,16 @@ pub mod pallet { /// /// Weight: `O(1)` #[pallet::weight(T::WeightInfo::freeze_class())] - pub fn freeze_class( - origin: OriginFor, - #[pallet::compact] class: T::ClassId, - ) -> DispatchResult { + pub fn freeze_class(origin: OriginFor, class: T::ClassId) -> DispatchResult { let origin = ensure_signed(origin)?; Class::::try_mutate(class, |maybe_details| { - let details = maybe_details.as_mut().ok_or(Error::::Unknown)?; + let details = maybe_details.as_mut().ok_or(Error::::UnknownClass)?; ensure!(&origin == &details.freezer, Error::::NoPermission); details.is_frozen = true; - Self::deposit_event(Event::::ClassFrozen(class)); + Self::deposit_event(Event::::ClassFrozen { class }); Ok(()) }) } @@ -651,19 +734,16 @@ pub mod pallet { /// /// Weight: `O(1)` #[pallet::weight(T::WeightInfo::thaw_class())] - pub fn thaw_class( - origin: OriginFor, - #[pallet::compact] class: T::ClassId, - ) -> DispatchResult { + pub fn thaw_class(origin: OriginFor, class: T::ClassId) -> DispatchResult { let origin = ensure_signed(origin)?; Class::::try_mutate(class, |maybe_details| { - let details = maybe_details.as_mut().ok_or(Error::::Unknown)?; + let details = maybe_details.as_mut().ok_or(Error::::UnknownClass)?; ensure!(&origin == &details.admin, Error::::NoPermission); details.is_frozen = false; - Self::deposit_event(Event::::ClassThawed(class)); + Self::deposit_event(Event::::ClassThawed { class }); Ok(()) }) } @@ -673,7 +753,8 @@ pub mod pallet { /// Origin must be Signed and the sender should be the Owner of the asset `class`. /// /// - `class`: The asset class whose owner should be changed. - /// - `owner`: The new Owner of this asset class. + /// - `owner`: The new Owner of this asset class. They must have called + /// `set_accept_ownership` with `class` in order for this operation to succeed. /// /// Emits `OwnerChanged`. /// @@ -681,14 +762,17 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::transfer_ownership())] pub fn transfer_ownership( origin: OriginFor, - #[pallet::compact] class: T::ClassId, + class: T::ClassId, owner: ::Source, ) -> DispatchResult { let origin = ensure_signed(origin)?; let owner = T::Lookup::lookup(owner)?; + let acceptable_class = OwnershipAcceptance::::get(&owner); + ensure!(acceptable_class.as_ref() == Some(&class), Error::::Unaccepted); + Class::::try_mutate(class, |maybe_details| { - let details = maybe_details.as_mut().ok_or(Error::::Unknown)?; + let details = maybe_details.as_mut().ok_or(Error::::UnknownClass)?; ensure!(&origin == &details.owner, Error::::NoPermission); if details.owner == owner { return Ok(()) @@ -701,9 +785,12 @@ pub mod pallet { details.total_deposit, Reserved, )?; + ClassAccount::::remove(&details.owner, &class); + ClassAccount::::insert(&owner, &class, ()); details.owner = owner.clone(); + OwnershipAcceptance::::remove(&owner); - Self::deposit_event(Event::OwnerChanged(class, owner)); + Self::deposit_event(Event::OwnerChanged { class, new_owner: owner }); Ok(()) }) } @@ -723,7 +810,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::set_team())] pub fn set_team( origin: OriginFor, - #[pallet::compact] class: T::ClassId, + class: T::ClassId, issuer: ::Source, admin: ::Source, freezer: ::Source, @@ -734,14 +821,14 @@ pub mod pallet { let freezer = T::Lookup::lookup(freezer)?; Class::::try_mutate(class, |maybe_details| { - let details = maybe_details.as_mut().ok_or(Error::::Unknown)?; + let details = maybe_details.as_mut().ok_or(Error::::UnknownClass)?; ensure!(&origin == &details.owner, Error::::NoPermission); details.issuer = issuer.clone(); details.admin = admin.clone(); details.freezer = freezer.clone(); - Self::deposit_event(Event::TeamChanged(class, issuer, admin, freezer)); + Self::deposit_event(Event::TeamChanged { class, issuer, admin, freezer }); Ok(()) }) } @@ -760,8 +847,8 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::approve_transfer())] pub fn approve_transfer( origin: OriginFor, - #[pallet::compact] class: T::ClassId, - #[pallet::compact] instance: T::InstanceId, + class: T::ClassId, + instance: T::InstanceId, delegate: ::Source, ) -> DispatchResult { let maybe_check: Option = T::ForceOrigin::try_origin(origin) @@ -770,9 +857,9 @@ pub mod pallet { let delegate = T::Lookup::lookup(delegate)?; - let class_details = Class::::get(&class).ok_or(Error::::Unknown)?; + let class_details = Class::::get(&class).ok_or(Error::::UnknownClass)?; let mut details = - Asset::::get(&class, &instance).ok_or(Error::::Unknown)?; + Asset::::get(&class, &instance).ok_or(Error::::UnknownClass)?; if let Some(check) = maybe_check { let permitted = &check == &class_details.admin || &check == &details.owner; @@ -783,7 +870,12 @@ pub mod pallet { Asset::::insert(&class, &instance, &details); let delegate = details.approved.expect("set as Some above; qed"); - Self::deposit_event(Event::ApprovedTransfer(class, instance, details.owner, delegate)); + Self::deposit_event(Event::ApprovedTransfer { + class, + instance, + owner: details.owner, + delegate, + }); Ok(()) } @@ -807,17 +899,17 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::cancel_approval())] pub fn cancel_approval( origin: OriginFor, - #[pallet::compact] class: T::ClassId, - #[pallet::compact] instance: T::InstanceId, + class: T::ClassId, + instance: T::InstanceId, maybe_check_delegate: Option<::Source>, ) -> DispatchResult { let maybe_check: Option = T::ForceOrigin::try_origin(origin) .map(|_| None) .or_else(|origin| ensure_signed(origin).map(Some).map_err(DispatchError::from))?; - let class_details = Class::::get(&class).ok_or(Error::::Unknown)?; + let class_details = Class::::get(&class).ok_or(Error::::UnknownClass)?; let mut details = - Asset::::get(&class, &instance).ok_or(Error::::Unknown)?; + Asset::::get(&class, &instance).ok_or(Error::::UnknownClass)?; if let Some(check) = maybe_check { let permitted = &check == &class_details.admin || &check == &details.owner; ensure!(permitted, Error::::NoPermission); @@ -829,7 +921,12 @@ pub mod pallet { } Asset::::insert(&class, &instance, &details); - Self::deposit_event(Event::ApprovalCancelled(class, instance, details.owner, old)); + Self::deposit_event(Event::ApprovalCancelled { + class, + instance, + owner: details.owner, + delegate: old, + }); Ok(()) } @@ -854,7 +951,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::force_asset_status())] pub fn force_asset_status( origin: OriginFor, - #[pallet::compact] class: T::ClassId, + class: T::ClassId, owner: ::Source, issuer: ::Source, admin: ::Source, @@ -865,16 +962,20 @@ pub mod pallet { T::ForceOrigin::ensure_origin(origin)?; Class::::try_mutate(class, |maybe_asset| { - let mut asset = maybe_asset.take().ok_or(Error::::Unknown)?; - asset.owner = T::Lookup::lookup(owner)?; + let mut asset = maybe_asset.take().ok_or(Error::::UnknownClass)?; + let old_owner = asset.owner; + let new_owner = T::Lookup::lookup(owner)?; + asset.owner = new_owner.clone(); asset.issuer = T::Lookup::lookup(issuer)?; asset.admin = T::Lookup::lookup(admin)?; asset.freezer = T::Lookup::lookup(freezer)?; asset.free_holding = free_holding; asset.is_frozen = is_frozen; *maybe_asset = Some(asset); + ClassAccount::::remove(&old_owner, &class); + ClassAccount::::insert(&new_owner, &class, ()); - Self::deposit_event(Event::AssetStatusChanged(class)); + Self::deposit_event(Event::AssetStatusChanged { class }); Ok(()) }) } @@ -899,7 +1000,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::set_attribute())] pub fn set_attribute( origin: OriginFor, - #[pallet::compact] class: T::ClassId, + class: T::ClassId, maybe_instance: Option, key: BoundedVec, value: BoundedVec, @@ -908,7 +1009,8 @@ pub mod pallet { .map(|_| None) .or_else(|origin| ensure_signed(origin).map(Some))?; - let mut class_details = Class::::get(&class).ok_or(Error::::Unknown)?; + let mut class_details = + Class::::get(&class).ok_or(Error::::UnknownClass)?; if let Some(check_owner) = &maybe_check_owner { ensure!(check_owner == &class_details.owner, Error::::NoPermission); } @@ -940,31 +1042,28 @@ pub mod pallet { Attribute::::insert((&class, maybe_instance, &key), (&value, deposit)); Class::::insert(class, &class_details); - Self::deposit_event(Event::AttributeSet(class, maybe_instance, key, value)); + Self::deposit_event(Event::AttributeSet { class, maybe_instance, key, value }); Ok(()) } - /// Set an attribute for an asset class or instance. + /// Clear an attribute for an asset class or instance. /// /// Origin must be either `ForceOrigin` or Signed and the sender should be the Owner of the /// asset `class`. /// - /// If the origin is Signed, then funds of signer are reserved according to the formula: - /// `MetadataDepositBase + DepositPerByte * (key.len + value.len)` taking into - /// account any already reserved funds. + /// Any deposit is freed for the asset class owner. /// - /// - `class`: The identifier of the asset class whose instance's metadata to set. - /// - `instance`: The identifier of the asset instance whose metadata to set. + /// - `class`: The identifier of the asset class whose instance's metadata to clear. + /// - `maybe_instance`: The identifier of the asset instance whose metadata to clear. /// - `key`: The key of the attribute. - /// - `value`: The value to which to set the attribute. /// - /// Emits `AttributeSet`. + /// Emits `AttributeCleared`. /// /// Weight: `O(1)` #[pallet::weight(T::WeightInfo::clear_attribute())] pub fn clear_attribute( origin: OriginFor, - #[pallet::compact] class: T::ClassId, + class: T::ClassId, maybe_instance: Option, key: BoundedVec, ) -> DispatchResult { @@ -972,7 +1071,8 @@ pub mod pallet { .map(|_| None) .or_else(|origin| ensure_signed(origin).map(Some))?; - let mut class_details = Class::::get(&class).ok_or(Error::::Unknown)?; + let mut class_details = + Class::::get(&class).ok_or(Error::::UnknownClass)?; if let Some(check_owner) = &maybe_check_owner { ensure!(check_owner == &class_details.owner, Error::::NoPermission); } @@ -988,7 +1088,7 @@ pub mod pallet { class_details.total_deposit.saturating_reduce(deposit); T::Currency::unreserve(&class_details.owner, deposit); Class::::insert(class, &class_details); - Self::deposit_event(Event::AttributeCleared(class, maybe_instance, key)); + Self::deposit_event(Event::AttributeCleared { class, maybe_instance, key }); } Ok(()) } @@ -1013,8 +1113,8 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::set_metadata())] pub fn set_metadata( origin: OriginFor, - #[pallet::compact] class: T::ClassId, - #[pallet::compact] instance: T::InstanceId, + class: T::ClassId, + instance: T::InstanceId, data: BoundedVec, is_frozen: bool, ) -> DispatchResult { @@ -1022,7 +1122,8 @@ pub mod pallet { .map(|_| None) .or_else(|origin| ensure_signed(origin).map(Some))?; - let mut class_details = Class::::get(&class).ok_or(Error::::Unknown)?; + let mut class_details = + Class::::get(&class).ok_or(Error::::UnknownClass)?; if let Some(check_owner) = &maybe_check_owner { ensure!(check_owner == &class_details.owner, Error::::NoPermission); @@ -1053,7 +1154,7 @@ pub mod pallet { *metadata = Some(InstanceMetadata { deposit, data: data.clone(), is_frozen }); Class::::insert(&class, &class_details); - Self::deposit_event(Event::MetadataSet(class, instance, data, is_frozen)); + Self::deposit_event(Event::MetadataSet { class, instance, data, is_frozen }); Ok(()) }) } @@ -1074,14 +1175,15 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::clear_metadata())] pub fn clear_metadata( origin: OriginFor, - #[pallet::compact] class: T::ClassId, - #[pallet::compact] instance: T::InstanceId, + class: T::ClassId, + instance: T::InstanceId, ) -> DispatchResult { let maybe_check_owner = T::ForceOrigin::try_origin(origin) .map(|_| None) .or_else(|origin| ensure_signed(origin).map(Some))?; - let mut class_details = Class::::get(&class).ok_or(Error::::Unknown)?; + let mut class_details = + Class::::get(&class).ok_or(Error::::UnknownClass)?; if let Some(check_owner) = &maybe_check_owner { ensure!(check_owner == &class_details.owner, Error::::NoPermission); } @@ -1093,12 +1195,12 @@ pub mod pallet { if metadata.is_some() { class_details.instance_metadatas.saturating_dec(); } - let deposit = metadata.take().ok_or(Error::::Unknown)?.deposit; + let deposit = metadata.take().ok_or(Error::::UnknownClass)?.deposit; T::Currency::unreserve(&class_details.owner, deposit); class_details.total_deposit.saturating_reduce(deposit); Class::::insert(&class, &class_details); - Self::deposit_event(Event::MetadataCleared(class, instance)); + Self::deposit_event(Event::MetadataCleared { class, instance }); Ok(()) }) } @@ -1122,7 +1224,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::set_class_metadata())] pub fn set_class_metadata( origin: OriginFor, - #[pallet::compact] class: T::ClassId, + class: T::ClassId, data: BoundedVec, is_frozen: bool, ) -> DispatchResult { @@ -1130,7 +1232,7 @@ pub mod pallet { .map(|_| None) .or_else(|origin| ensure_signed(origin).map(Some))?; - let mut details = Class::::get(&class).ok_or(Error::::Unknown)?; + let mut details = Class::::get(&class).ok_or(Error::::UnknownClass)?; if let Some(check_owner) = &maybe_check_owner { ensure!(check_owner == &details.owner, Error::::NoPermission); } @@ -1158,7 +1260,7 @@ pub mod pallet { *metadata = Some(ClassMetadata { deposit, data: data.clone(), is_frozen }); - Self::deposit_event(Event::ClassMetadataSet(class, data, is_frozen)); + Self::deposit_event(Event::ClassMetadataSet { class, data, is_frozen }); Ok(()) }) } @@ -1176,15 +1278,12 @@ pub mod pallet { /// /// Weight: `O(1)` #[pallet::weight(T::WeightInfo::clear_class_metadata())] - pub fn clear_class_metadata( - origin: OriginFor, - #[pallet::compact] class: T::ClassId, - ) -> DispatchResult { + pub fn clear_class_metadata(origin: OriginFor, class: T::ClassId) -> DispatchResult { let maybe_check_owner = T::ForceOrigin::try_origin(origin) .map(|_| None) .or_else(|origin| ensure_signed(origin).map(Some))?; - let details = Class::::get(&class).ok_or(Error::::Unknown)?; + let details = Class::::get(&class).ok_or(Error::::UnknownClass)?; if let Some(check_owner) = &maybe_check_owner { ensure!(check_owner == &details.owner, Error::::NoPermission); } @@ -1193,11 +1292,46 @@ pub mod pallet { let was_frozen = metadata.as_ref().map_or(false, |m| m.is_frozen); ensure!(maybe_check_owner.is_none() || !was_frozen, Error::::Frozen); - let deposit = metadata.take().ok_or(Error::::Unknown)?.deposit; + let deposit = metadata.take().ok_or(Error::::UnknownClass)?.deposit; T::Currency::unreserve(&details.owner, deposit); - Self::deposit_event(Event::ClassMetadataCleared(class)); + Self::deposit_event(Event::ClassMetadataCleared { class }); Ok(()) }) } + + /// Set (or reset) the acceptance of ownership for a particular account. + /// + /// Origin must be `Signed` and if `maybe_class` is `Some`, then the signer must have a + /// provider reference. + /// + /// - `maybe_class`: The identifier of the asset class whose ownership the signer is willing + /// to accept, or if `None`, an indication that the signer is willing to accept no + /// ownership transferal. + /// + /// Emits `OwnershipAcceptanceChanged`. + #[pallet::weight(T::WeightInfo::set_accept_ownership())] + pub fn set_accept_ownership( + origin: OriginFor, + maybe_class: Option, + ) -> DispatchResult { + let who = ensure_signed(origin)?; + let old = OwnershipAcceptance::::get(&who); + match (old.is_some(), maybe_class.is_some()) { + (false, true) => { + frame_system::Pallet::::inc_consumers(&who)?; + }, + (true, false) => { + frame_system::Pallet::::dec_consumers(&who); + }, + _ => {}, + } + if let Some(class) = maybe_class.as_ref() { + OwnershipAcceptance::::insert(&who, class); + } else { + OwnershipAcceptance::::remove(&who); + } + Self::deposit_event(Event::OwnershipAcceptanceChanged { who, maybe_class }); + Ok(()) + } } } diff --git a/frame/uniques/src/migration.rs b/frame/uniques/src/migration.rs new file mode 100644 index 000000000000..2bacfc8f43b6 --- /dev/null +++ b/frame/uniques/src/migration.rs @@ -0,0 +1,57 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Various pieces of common functionality. +use super::*; +use frame_support::{ + traits::{Get, GetStorageVersion, PalletInfoAccess, StorageVersion}, + weights::Weight, +}; + +/// Migrate the pallet storage to v1. +pub fn migrate_to_v1, I: 'static, P: GetStorageVersion + PalletInfoAccess>( +) -> frame_support::weights::Weight { + let on_chain_storage_version =

::on_chain_storage_version(); + log::info!( + target: "runtime::uniques", + "Running migration storage v1 for uniques with storage version {:?}", + on_chain_storage_version, + ); + + if on_chain_storage_version < 1 { + let mut count = 0; + for (class, detail) in Class::::iter() { + ClassAccount::::insert(&detail.owner, &class, ()); + count += 1; + } + StorageVersion::new(1).put::

(); + log::info!( + target: "runtime::uniques", + "Running migration storage v1 for uniques with storage version {:?} was complete", + on_chain_storage_version, + ); + // calculate and return migration weights + T::DbWeight::get().reads_writes(count as Weight + 1, count as Weight + 1) + } else { + log::warn!( + target: "runtime::uniques", + "Attempted to apply migration to v1 but failed because storage version is {:?}", + on_chain_storage_version, + ); + T::DbWeight::get().reads(1) + } +} diff --git a/frame/uniques/src/mock.rs b/frame/uniques/src/mock.rs index 658e82a5143e..265142443ef4 100644 --- a/frame/uniques/src/mock.rs +++ b/frame/uniques/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,10 @@ use super::*; use crate as pallet_uniques; -use frame_support::{construct_runtime, parameter_types}; +use frame_support::{ + construct_runtime, + traits::{AsEnsureOriginWithArg, ConstU32, ConstU64}, +}; use sp_core::H256; use sp_runtime::{ testing::Header, @@ -42,9 +45,6 @@ construct_runtime!( } ); -parameter_types! { - pub const BlockHashCount: u64 = 250; -} impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; type BlockWeights = (); @@ -59,7 +59,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type DbWeight = (); type Version = (); type PalletInfo = PalletInfo; @@ -69,51 +69,39 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); -} - -parameter_types! { - pub const ExistentialDeposit: u64 = 1; - pub const MaxReserves: u32 = 50; + type MaxConsumers = ConstU32<16>; } impl pallet_balances::Config for Test { type Balance = u64; type DustRemoval = (); type Event = Event; - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); type MaxLocks = (); - type MaxReserves = MaxReserves; + type MaxReserves = ConstU32<50>; type ReserveIdentifier = [u8; 8]; } -parameter_types! { - pub const ClassDeposit: u64 = 2; - pub const InstanceDeposit: u64 = 1; - pub const KeyLimit: u32 = 50; - pub const ValueLimit: u32 = 50; - pub const StringLimit: u32 = 50; - pub const MetadataDepositBase: u64 = 1; - pub const AttributeDepositBase: u64 = 1; - pub const MetadataDepositPerByte: u64 = 1; -} - impl Config for Test { type Event = Event; type ClassId = u32; type InstanceId = u32; type Currency = Balances; + type CreateOrigin = AsEnsureOriginWithArg>; type ForceOrigin = frame_system::EnsureRoot; - type ClassDeposit = ClassDeposit; - type InstanceDeposit = InstanceDeposit; - type MetadataDepositBase = MetadataDepositBase; - type AttributeDepositBase = AttributeDepositBase; - type DepositPerByte = MetadataDepositPerByte; - type StringLimit = StringLimit; - type KeyLimit = KeyLimit; - type ValueLimit = ValueLimit; + type ClassDeposit = ConstU64<2>; + type InstanceDeposit = ConstU64<1>; + type MetadataDepositBase = ConstU64<1>; + type AttributeDepositBase = ConstU64<1>; + type DepositPerByte = ConstU64<1>; + type StringLimit = ConstU32<50>; + type KeyLimit = ConstU32<50>; + type ValueLimit = ConstU32<50>; type WeightInfo = (); + #[cfg(feature = "runtime-benchmarks")] + type Helper = (); } pub(crate) fn new_test_ext() -> sp_io::TestExternalities { diff --git a/frame/uniques/src/tests.rs b/frame/uniques/src/tests.rs index 8a4f978b7f4f..364073ad37cd 100644 --- a/frame/uniques/src/tests.rs +++ b/frame/uniques/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use super::*; use crate::mock::*; use frame_support::{assert_noop, assert_ok, traits::Currency}; use pallet_balances::Error as BalancesError; -use sp_std::convert::TryInto; +use sp_std::prelude::*; fn assets() -> Vec<(u64, u32, u32)> { let mut r: Vec<_> = Account::::iter().map(|x| x.0).collect(); @@ -39,7 +39,7 @@ fn assets() -> Vec<(u64, u32, u32)> { Some(Some(item)) } }) - .filter_map(|item| item) + .flatten() { let details = Class::::get(class).unwrap(); let instances = Asset::::iter_prefix(class).count() as u32; @@ -48,6 +48,15 @@ fn assets() -> Vec<(u64, u32, u32)> { r } +fn classes() -> Vec<(u64, u32)> { + let mut r: Vec<_> = ClassAccount::::iter().map(|x| (x.0, x.1)).collect(); + r.sort(); + let mut s: Vec<_> = Class::::iter().map(|x| (x.1.owner, x.0)).collect(); + s.sort(); + assert_eq!(r, s); + r +} + macro_rules! bvec { ($( $x:tt )*) => { vec![$( $x )*].try_into().unwrap() @@ -73,10 +82,12 @@ fn basic_setup_works() { fn basic_minting_should_work() { new_test_ext().execute_with(|| { assert_ok!(Uniques::force_create(Origin::root(), 0, 1, true)); + assert_eq!(classes(), vec![(1, 0)]); assert_ok!(Uniques::mint(Origin::signed(1), 0, 42, 1)); assert_eq!(assets(), vec![(1, 0, 42)]); assert_ok!(Uniques::force_create(Origin::root(), 1, 2, true)); + assert_eq!(classes(), vec![(1, 0), (2, 1)]); assert_ok!(Uniques::mint(Origin::signed(2), 1, 69, 1)); assert_eq!(assets(), vec![(1, 0, 42), (1, 1, 69)]); }); @@ -88,7 +99,7 @@ fn lifecycle_should_work() { Balances::make_free_balance_be(&1, 100); assert_ok!(Uniques::create(Origin::signed(1), 0, 1)); assert_eq!(Balances::reserved_balance(&1), 2); - + assert_eq!(classes(), vec![(1, 0)]); assert_ok!(Uniques::set_class_metadata(Origin::signed(1), 0, bvec![0, 0], false)); assert_eq!(Balances::reserved_balance(&1), 5); assert!(ClassMetadataOf::::contains_key(0)); @@ -120,6 +131,7 @@ fn lifecycle_should_work() { assert!(!ClassMetadataOf::::contains_key(0)); assert!(!InstanceMetadataOf::::contains_key(0, 42)); assert!(!InstanceMetadataOf::::contains_key(0, 69)); + assert_eq!(classes(), vec![]); assert_eq!(assets(), vec![]); }); } @@ -142,6 +154,7 @@ fn mint_should_work() { assert_ok!(Uniques::force_create(Origin::root(), 0, 1, true)); assert_ok!(Uniques::mint(Origin::signed(1), 0, 42, 1)); assert_eq!(Uniques::owner(0, 42).unwrap(), 1); + assert_eq!(classes(), vec![(1, 0)]); assert_eq!(assets(), vec![(1, 0, 42)]); }); } @@ -183,6 +196,9 @@ fn origin_guards_should_work() { new_test_ext().execute_with(|| { assert_ok!(Uniques::force_create(Origin::root(), 0, 1, true)); assert_ok!(Uniques::mint(Origin::signed(1), 0, 42, 1)); + + Balances::make_free_balance_be(&2, 100); + assert_ok!(Uniques::set_accept_ownership(Origin::signed(2), Some(0))); assert_noop!( Uniques::transfer_ownership(Origin::signed(2), 0, 2), Error::::NoPermission @@ -204,12 +220,21 @@ fn transfer_owner_should_work() { Balances::make_free_balance_be(&2, 100); Balances::make_free_balance_be(&3, 100); assert_ok!(Uniques::create(Origin::signed(1), 0, 1)); + assert_eq!(classes(), vec![(1, 0)]); + assert_noop!( + Uniques::transfer_ownership(Origin::signed(1), 0, 2), + Error::::Unaccepted + ); + assert_ok!(Uniques::set_accept_ownership(Origin::signed(2), Some(0))); assert_ok!(Uniques::transfer_ownership(Origin::signed(1), 0, 2)); + + assert_eq!(classes(), vec![(2, 0)]); assert_eq!(Balances::total_balance(&1), 98); assert_eq!(Balances::total_balance(&2), 102); assert_eq!(Balances::reserved_balance(&1), 0); assert_eq!(Balances::reserved_balance(&2), 2); + assert_ok!(Uniques::set_accept_ownership(Origin::signed(1), Some(0))); assert_noop!( Uniques::transfer_ownership(Origin::signed(1), 0, 1), Error::::NoPermission @@ -219,11 +244,20 @@ fn transfer_owner_should_work() { assert_ok!(Uniques::set_class_metadata(Origin::signed(2), 0, bvec![0u8; 20], false)); assert_ok!(Uniques::mint(Origin::signed(1), 0, 42, 1)); assert_ok!(Uniques::set_metadata(Origin::signed(2), 0, 42, bvec![0u8; 20], false)); + assert_ok!(Uniques::set_accept_ownership(Origin::signed(3), Some(0))); assert_ok!(Uniques::transfer_ownership(Origin::signed(2), 0, 3)); + assert_eq!(classes(), vec![(3, 0)]); assert_eq!(Balances::total_balance(&2), 57); assert_eq!(Balances::total_balance(&3), 145); assert_eq!(Balances::reserved_balance(&2), 0); assert_eq!(Balances::reserved_balance(&3), 45); + + // 2's acceptence from before is reset when it became owner, so it cannot be transfered + // without a fresh acceptance. + assert_noop!( + Uniques::transfer_ownership(Origin::signed(3), 0, 2), + Error::::Unaccepted + ); }); } @@ -247,7 +281,7 @@ fn set_class_metadata_should_work() { // Cannot add metadata to unknown asset assert_noop!( Uniques::set_class_metadata(Origin::signed(1), 0, bvec![0u8; 20], false), - Error::::Unknown, + Error::::UnknownClass, ); assert_ok!(Uniques::force_create(Origin::root(), 0, 1, false)); // Cannot add metadata to unowned asset @@ -291,7 +325,10 @@ fn set_class_metadata_should_work() { Uniques::clear_class_metadata(Origin::signed(2), 0), Error::::NoPermission ); - assert_noop!(Uniques::clear_class_metadata(Origin::signed(1), 1), Error::::Unknown); + assert_noop!( + Uniques::clear_class_metadata(Origin::signed(1), 1), + Error::::UnknownClass + ); assert_ok!(Uniques::clear_class_metadata(Origin::signed(1), 0)); assert!(!ClassMetadataOf::::contains_key(0)); }); @@ -345,7 +382,10 @@ fn set_instance_metadata_should_work() { Uniques::clear_metadata(Origin::signed(2), 0, 42), Error::::NoPermission ); - assert_noop!(Uniques::clear_metadata(Origin::signed(1), 1, 42), Error::::Unknown); + assert_noop!( + Uniques::clear_metadata(Origin::signed(1), 1, 42), + Error::::UnknownClass + ); assert_ok!(Uniques::clear_metadata(Origin::signed(1), 0, 42)); assert!(!InstanceMetadataOf::::contains_key(0, 42)); }); @@ -470,7 +510,7 @@ fn burn_works() { assert_ok!(Uniques::force_create(Origin::root(), 0, 1, false)); assert_ok!(Uniques::set_team(Origin::signed(1), 0, 2, 3, 4)); - assert_noop!(Uniques::burn(Origin::signed(5), 0, 42, Some(5)), Error::::Unknown); + assert_noop!(Uniques::burn(Origin::signed(5), 0, 42, Some(5)), Error::::UnknownClass); assert_ok!(Uniques::mint(Origin::signed(2), 0, 42, 5)); assert_ok!(Uniques::mint(Origin::signed(2), 0, 69, 5)); @@ -509,11 +549,11 @@ fn cancel_approval_works() { assert_ok!(Uniques::approve_transfer(Origin::signed(2), 0, 42, 3)); assert_noop!( Uniques::cancel_approval(Origin::signed(2), 1, 42, None), - Error::::Unknown + Error::::UnknownClass ); assert_noop!( Uniques::cancel_approval(Origin::signed(2), 0, 43, None), - Error::::Unknown + Error::::UnknownClass ); assert_noop!( Uniques::cancel_approval(Origin::signed(3), 0, 42, None), @@ -541,11 +581,11 @@ fn cancel_approval_works_with_admin() { assert_ok!(Uniques::approve_transfer(Origin::signed(2), 0, 42, 3)); assert_noop!( Uniques::cancel_approval(Origin::signed(1), 1, 42, None), - Error::::Unknown + Error::::UnknownClass ); assert_noop!( Uniques::cancel_approval(Origin::signed(1), 0, 43, None), - Error::::Unknown + Error::::UnknownClass ); assert_noop!( Uniques::cancel_approval(Origin::signed(1), 0, 42, Some(4)), @@ -567,8 +607,14 @@ fn cancel_approval_works_with_force() { assert_ok!(Uniques::mint(Origin::signed(1), 0, 42, 2)); assert_ok!(Uniques::approve_transfer(Origin::signed(2), 0, 42, 3)); - assert_noop!(Uniques::cancel_approval(Origin::root(), 1, 42, None), Error::::Unknown); - assert_noop!(Uniques::cancel_approval(Origin::root(), 0, 43, None), Error::::Unknown); + assert_noop!( + Uniques::cancel_approval(Origin::root(), 1, 42, None), + Error::::UnknownClass + ); + assert_noop!( + Uniques::cancel_approval(Origin::root(), 0, 43, None), + Error::::UnknownClass + ); assert_noop!( Uniques::cancel_approval(Origin::root(), 0, 42, Some(4)), Error::::WrongDelegate diff --git a/frame/uniques/src/types.rs b/frame/uniques/src/types.rs index 1e4405aa09c8..b5aee6912fec 100644 --- a/frame/uniques/src/types.rs +++ b/frame/uniques/src/types.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,10 @@ //! Various basic types for use in the assets pallet. use super::*; -use frame_support::{traits::Get, BoundedVec}; +use frame_support::{ + pallet_prelude::{BoundedVec, MaxEncodedLen}, + traits::Get, +}; use scale_info::TypeInfo; pub(super) type DepositBalanceOf = @@ -28,7 +31,7 @@ pub(super) type ClassDetailsFor = pub(super) type InstanceDetailsFor = InstanceDetails<::AccountId, DepositBalanceOf>; -#[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, TypeInfo)] +#[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub struct ClassDetails { /// Can change `owner`, `issuer`, `freezer` and `admin` accounts. pub(super) owner: AccountId, @@ -54,17 +57,17 @@ pub struct ClassDetails { } /// Witness data for the destroy transactions. -#[derive(Copy, Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, TypeInfo)] +#[derive(Copy, Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, TypeInfo, MaxEncodedLen)] pub struct DestroyWitness { /// The total number of outstanding instances of this asset class. #[codec(compact)] - pub(super) instances: u32, + pub instances: u32, /// The total number of outstanding instance metadata of this asset class. #[codec(compact)] - pub(super) instance_metadatas: u32, + pub instance_metadatas: u32, #[codec(compact)] /// The total number of attributes for this asset class. - pub(super) attributes: u32, + pub attributes: u32, } impl ClassDetails { @@ -78,7 +81,7 @@ impl ClassDetails { } /// Information concerning the ownership of a single unique asset. -#[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, Default, TypeInfo)] +#[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, Default, TypeInfo, MaxEncodedLen)] pub struct InstanceDetails { /// The owner of this asset. pub(super) owner: AccountId, @@ -91,8 +94,9 @@ pub struct InstanceDetails { pub(super) deposit: DepositBalance, } -#[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, Default, TypeInfo)] +#[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, Default, TypeInfo, MaxEncodedLen)] #[scale_info(skip_type_params(StringLimit))] +#[codec(mel_bound(DepositBalance: MaxEncodedLen))] pub struct ClassMetadata> { /// The balance deposited for this metadata. /// @@ -106,8 +110,9 @@ pub struct ClassMetadata> { pub(super) is_frozen: bool, } -#[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, Default, TypeInfo)] +#[derive(Clone, Encode, Decode, Eq, PartialEq, RuntimeDebug, Default, TypeInfo, MaxEncodedLen)] #[scale_info(skip_type_params(StringLimit))] +#[codec(mel_bound(DepositBalance: MaxEncodedLen))] pub struct InstanceMetadata> { /// The balance deposited for this metadata. /// diff --git a/frame/uniques/src/weights.rs b/frame/uniques/src/weights.rs index 40d1ddfdc556..eb9067b7133a 100644 --- a/frame/uniques/src/weights.rs +++ b/frame/uniques/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_uniques //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/uniques/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -67,40 +68,44 @@ pub trait WeightInfo { fn clear_class_metadata() -> Weight; fn approve_transfer() -> Weight; fn cancel_approval() -> Weight; + fn set_accept_ownership() -> Weight; } /// Weights for pallet_uniques using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { // Storage: Uniques Class (r:1 w:1) + // Storage: Uniques ClassAccount (r:0 w:1) fn create() -> Weight { - (42_138_000 as Weight) + (24_063_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) + // Storage: Uniques ClassAccount (r:0 w:1) fn force_create() -> Weight { - (22_238_000 as Weight) + (13_017_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques Asset (r:1 w:0) + // Storage: Uniques ClassAccount (r:0 w:1) // Storage: Uniques Attribute (r:0 w:1000) // Storage: Uniques ClassMetadataOf (r:0 w:1) // Storage: Uniques InstanceMetadataOf (r:0 w:1000) // Storage: Uniques Account (r:0 w:20) fn destroy(n: u32, m: u32, a: u32, ) -> Weight { (0 as Weight) - // Standard Error: 12_000 - .saturating_add((16_171_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 12_000 - .saturating_add((1_058_000 as Weight).saturating_mul(m as Weight)) - // Standard Error: 12_000 - .saturating_add((953_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 14_000 + .saturating_add((9_248_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 14_000 + .saturating_add((854_000 as Weight).saturating_mul(m as Weight)) + // Standard Error: 14_000 + .saturating_add((758_000 as Weight).saturating_mul(a as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) .saturating_add(T::DbWeight::get().writes((2 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(m as Weight))) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(a as Weight))) @@ -109,7 +114,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Uniques Class (r:1 w:1) // Storage: Uniques Account (r:0 w:1) fn mint() -> Weight { - (55_359_000 as Weight) + (29_865_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -117,7 +122,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Account (r:0 w:1) fn burn() -> Weight { - (58_254_000 as Weight) + (31_603_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -125,7 +130,7 @@ impl WeightInfo for SubstrateWeight { // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Account (r:0 w:2) fn transfer() -> Weight { - (42_906_000 as Weight) + (23_331_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -133,8 +138,8 @@ impl WeightInfo for SubstrateWeight { // Storage: Uniques Asset (r:100 w:100) fn redeposit(i: u32, ) -> Weight { (0 as Weight) - // Standard Error: 9_000 - .saturating_add((25_237_000 as Weight).saturating_mul(i as Weight)) + // Standard Error: 12_000 + .saturating_add((11_527_000 as Weight).saturating_mul(i as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(i as Weight))) .saturating_add(T::DbWeight::get().writes(1 as Weight)) @@ -143,53 +148,55 @@ impl WeightInfo for SubstrateWeight { // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Class (r:1 w:0) fn freeze() -> Weight { - (30_153_000 as Weight) + (18_617_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Class (r:1 w:0) fn thaw() -> Weight { - (31_212_000 as Weight) + (18_618_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) fn freeze_class() -> Weight { - (22_689_000 as Weight) + (13_570_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) fn thaw_class() -> Weight { - (22_647_000 as Weight) + (13_937_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: System Account (r:1 w:1) + // Storage: Uniques ClassAccount (r:0 w:2) fn transfer_ownership() -> Weight { - (50_902_000 as Weight) + (31_021_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(4 as Weight)) } // Storage: Uniques Class (r:1 w:1) fn set_team() -> Weight { - (23_632_000 as Weight) + (14_739_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) + // Storage: Uniques ClassAccount (r:0 w:1) fn force_asset_status() -> Weight { - (22_508_000 as Weight) + (16_826_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques InstanceMetadataOf (r:1 w:0) // Storage: Uniques Attribute (r:1 w:1) fn set_attribute() -> Weight { - (69_942_000 as Weight) + (37_010_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -197,85 +204,95 @@ impl WeightInfo for SubstrateWeight { // Storage: Uniques InstanceMetadataOf (r:1 w:0) // Storage: Uniques Attribute (r:1 w:1) fn clear_attribute() -> Weight { - (62_314_000 as Weight) + (34_432_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques InstanceMetadataOf (r:1 w:1) fn set_metadata() -> Weight { - (52_647_000 as Weight) + (28_575_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques InstanceMetadataOf (r:1 w:1) fn clear_metadata() -> Weight { - (50_391_000 as Weight) + (28_730_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques ClassMetadataOf (r:1 w:1) fn set_class_metadata() -> Weight { - (50_928_000 as Weight) + (28_225_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques ClassMetadataOf (r:1 w:1) fn clear_class_metadata() -> Weight { - (46_667_000 as Weight) + (26_455_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques Asset (r:1 w:1) fn approve_transfer() -> Weight { - (32_111_000 as Weight) + (19_587_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques Asset (r:1 w:1) fn cancel_approval() -> Weight { - (32_627_000 as Weight) + (19_417_000 as Weight) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } + // Storage: Uniques Class (r:1 w:0) + // Storage: Uniques Asset (r:1 w:1) + fn set_accept_ownership() -> Weight { + (19_417_000 as Weight) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + } } // For backwards compatibility and tests impl WeightInfo for () { // Storage: Uniques Class (r:1 w:1) + // Storage: Uniques ClassAccount (r:0 w:1) fn create() -> Weight { - (42_138_000 as Weight) + (24_063_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) + // Storage: Uniques ClassAccount (r:0 w:1) fn force_create() -> Weight { - (22_238_000 as Weight) + (13_017_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques Asset (r:1 w:0) + // Storage: Uniques ClassAccount (r:0 w:1) // Storage: Uniques Attribute (r:0 w:1000) // Storage: Uniques ClassMetadataOf (r:0 w:1) // Storage: Uniques InstanceMetadataOf (r:0 w:1000) // Storage: Uniques Account (r:0 w:20) fn destroy(n: u32, m: u32, a: u32, ) -> Weight { (0 as Weight) - // Standard Error: 12_000 - .saturating_add((16_171_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 12_000 - .saturating_add((1_058_000 as Weight).saturating_mul(m as Weight)) - // Standard Error: 12_000 - .saturating_add((953_000 as Weight).saturating_mul(a as Weight)) + // Standard Error: 14_000 + .saturating_add((9_248_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 14_000 + .saturating_add((854_000 as Weight).saturating_mul(m as Weight)) + // Standard Error: 14_000 + .saturating_add((758_000 as Weight).saturating_mul(a as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) .saturating_add(RocksDbWeight::get().writes((2 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(m as Weight))) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(a as Weight))) @@ -284,7 +301,7 @@ impl WeightInfo for () { // Storage: Uniques Class (r:1 w:1) // Storage: Uniques Account (r:0 w:1) fn mint() -> Weight { - (55_359_000 as Weight) + (29_865_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -292,7 +309,7 @@ impl WeightInfo for () { // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Account (r:0 w:1) fn burn() -> Weight { - (58_254_000 as Weight) + (31_603_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -300,7 +317,7 @@ impl WeightInfo for () { // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Account (r:0 w:2) fn transfer() -> Weight { - (42_906_000 as Weight) + (23_331_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -308,8 +325,8 @@ impl WeightInfo for () { // Storage: Uniques Asset (r:100 w:100) fn redeposit(i: u32, ) -> Weight { (0 as Weight) - // Standard Error: 9_000 - .saturating_add((25_237_000 as Weight).saturating_mul(i as Weight)) + // Standard Error: 12_000 + .saturating_add((11_527_000 as Weight).saturating_mul(i as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(i as Weight))) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) @@ -318,53 +335,55 @@ impl WeightInfo for () { // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Class (r:1 w:0) fn freeze() -> Weight { - (30_153_000 as Weight) + (18_617_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Asset (r:1 w:1) // Storage: Uniques Class (r:1 w:0) fn thaw() -> Weight { - (31_212_000 as Weight) + (18_618_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) fn freeze_class() -> Weight { - (22_689_000 as Weight) + (13_570_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) fn thaw_class() -> Weight { - (22_647_000 as Weight) + (13_937_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: System Account (r:1 w:1) + // Storage: Uniques ClassAccount (r:0 w:2) fn transfer_ownership() -> Weight { - (50_902_000 as Weight) + (31_021_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } // Storage: Uniques Class (r:1 w:1) fn set_team() -> Weight { - (23_632_000 as Weight) + (14_739_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:1) + // Storage: Uniques ClassAccount (r:0 w:1) fn force_asset_status() -> Weight { - (22_508_000 as Weight) + (16_826_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques InstanceMetadataOf (r:1 w:0) // Storage: Uniques Attribute (r:1 w:1) fn set_attribute() -> Weight { - (69_942_000 as Weight) + (37_010_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -372,50 +391,57 @@ impl WeightInfo for () { // Storage: Uniques InstanceMetadataOf (r:1 w:0) // Storage: Uniques Attribute (r:1 w:1) fn clear_attribute() -> Weight { - (62_314_000 as Weight) + (34_432_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques InstanceMetadataOf (r:1 w:1) fn set_metadata() -> Weight { - (52_647_000 as Weight) + (28_575_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques InstanceMetadataOf (r:1 w:1) fn clear_metadata() -> Weight { - (50_391_000 as Weight) + (28_730_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:1) // Storage: Uniques ClassMetadataOf (r:1 w:1) fn set_class_metadata() -> Weight { - (50_928_000 as Weight) + (28_225_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques ClassMetadataOf (r:1 w:1) fn clear_class_metadata() -> Weight { - (46_667_000 as Weight) + (26_455_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques Asset (r:1 w:1) fn approve_transfer() -> Weight { - (32_111_000 as Weight) + (19_587_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } // Storage: Uniques Class (r:1 w:0) // Storage: Uniques Asset (r:1 w:1) fn cancel_approval() -> Weight { - (32_627_000 as Weight) + (19_417_000 as Weight) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } + // Storage: Uniques Class (r:1 w:0) + // Storage: Uniques Asset (r:1 w:1) + fn set_accept_ownership() -> Weight { + (19_417_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + } } diff --git a/frame/utility/Cargo.toml b/frame/utility/Cargo.toml index b5b8eab9cdbf..6235d1ee1555 100644 --- a/frame/utility/Cargo.toml +++ b/frame/utility/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-utility" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME utilities pallet" readme = "README.md" @@ -13,19 +13,19 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } [dev-dependencies] -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } [features] @@ -40,7 +40,7 @@ std = [ "sp-std/std", ] runtime-benchmarks = [ - "frame-benchmarking", + "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", ] diff --git a/frame/utility/src/benchmarking.rs b/frame/utility/src/benchmarking.rs index 210a6156499c..402128d00580 100644 --- a/frame/utility/src/benchmarking.rs +++ b/frame/utility/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ #![cfg(feature = "runtime-benchmarks")] use super::*; -use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite, whitelisted_caller}; +use frame_benchmarking::{account, benchmarks, whitelisted_caller}; use frame_system::RawOrigin; const SEED: u32 = 0; @@ -30,6 +30,7 @@ fn assert_last_event(generic_event: ::Event) { } benchmarks! { + where_clause { where ::PalletsOrigin: Clone } batch { let c in 0 .. 1000; let mut calls: Vec<::Call> = Vec::new(); @@ -63,6 +64,14 @@ benchmarks! { verify { assert_last_event::(Event::BatchCompleted.into()) } -} -impl_benchmark_test_suite!(Pallet, crate::tests::new_test_ext(), crate::tests::Test); + dispatch_as { + let caller = account("caller", SEED, SEED); + let call = Box::new(frame_system::Call::remark { remark: vec![] }.into()); + let origin: T::Origin = RawOrigin::Signed(caller).into(); + let pallets_origin: ::PalletsOrigin = origin.caller().clone(); + let pallets_origin = Into::::into(pallets_origin.clone()); + }: _(RawOrigin::Root, Box::new(pallets_origin), call) + + impl_benchmark_test_suite!(Pallet, crate::tests::new_test_ext(), crate::tests::Test); +} diff --git a/frame/utility/src/lib.rs b/frame/utility/src/lib.rs index 54de87c4740c..ec48087e2ef4 100644 --- a/frame/utility/src/lib.rs +++ b/frame/utility/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -65,7 +65,7 @@ use frame_support::{ }; use sp_core::TypeId; use sp_io::hashing::blake2_256; -use sp_runtime::traits::Dispatchable; +use sp_runtime::traits::{Dispatchable, TrailingZeroInput}; use sp_std::prelude::*; pub use weights::WeightInfo; @@ -96,6 +96,11 @@ pub mod pallet { + IsSubType> + IsType<::Call>; + /// The caller origin, overarching type of all pallets origins. + type PalletsOrigin: Parameter + + Into<::Origin> + + IsType<<::Origin as frame_support::traits::OriginTrait>::PalletsOrigin>; + /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; } @@ -104,20 +109,29 @@ pub mod pallet { #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { /// Batch of dispatches did not complete fully. Index of first failing dispatch given, as - /// well as the error. \[index, error\] - BatchInterrupted(u32, DispatchError), + /// well as the error. + BatchInterrupted { index: u32, error: DispatchError }, /// Batch of dispatches completed fully with no error. BatchCompleted, /// A single item within a Batch of dispatches has completed with no error. ItemCompleted, + /// A call was dispatched. + DispatchedAs { result: DispatchResult }, } + // Align the call size to 1KB. As we are currently compiling the runtime for native/wasm + // the `size_of` of the `Call` can be different. To ensure that this don't leads to + // mismatches between native/wasm or to different metadata for the same runtime, we + // algin the call size. The value is choosen big enough to hopefully never reach it. + const CALL_ALIGN: u32 = 1024; + #[pallet::extra_constants] impl Pallet { /// The limit on the number of batched calls. fn batched_calls_limit() -> u32 { let allocator_limit = sp_core::MAX_POSSIBLE_ALLOCATION; - let call_size = core::mem::size_of::<::Call>() as u32; + let call_size = ((sp_std::mem::size_of::<::Call>() as u32 + CALL_ALIGN - + 1) / CALL_ALIGN) * CALL_ALIGN; // The margin to take into account vec doubling capacity. let margin_factor = 3; @@ -125,6 +139,18 @@ pub mod pallet { } } + #[pallet::hooks] + impl Hooks> for Pallet { + fn integrity_test() { + // If you hit this error, you need to try to `Box` big dispatchable parameters. + assert!( + sp_std::mem::size_of::<::Call>() as u32 <= CALL_ALIGN, + "Call enum size should be smaller than {} bytes.", + CALL_ALIGN, + ); + } + } + #[pallet::error] pub enum Error { /// Too many calls batched. @@ -191,7 +217,10 @@ pub mod pallet { // Add the weight of this call. weight = weight.saturating_add(extract_actual_weight(&result, &info)); if let Err(e) = result { - Self::deposit_event(Event::BatchInterrupted(index as u32, e.error)); + Self::deposit_event(Event::BatchInterrupted { + index: index as u32, + error: e.error, + }); // Take the weight of this function itself into account. let base_weight = T::WeightInfo::batch(index.saturating_add(1) as u32); // Return the actual used weight + base_weight of this call. @@ -323,6 +352,39 @@ pub mod pallet { let base_weight = T::WeightInfo::batch_all(calls_len as u32); Ok(Some(base_weight + weight).into()) } + + /// Dispatches a function call with a provided origin. + /// + /// The dispatch origin for this call must be _Root_. + /// + /// # + /// - O(1). + /// - Limited storage reads. + /// - One DB write (event). + /// - Weight of derivative `call` execution + T::WeightInfo::dispatch_as(). + /// # + #[pallet::weight({ + let dispatch_info = call.get_dispatch_info(); + ( + T::WeightInfo::dispatch_as() + .saturating_add(dispatch_info.weight), + dispatch_info.class, + ) + })] + pub fn dispatch_as( + origin: OriginFor, + as_origin: Box, + call: Box<::Call>, + ) -> DispatchResult { + ensure_root(origin)?; + + let res = call.dispatch_bypass_filter((*as_origin).into()); + + Self::deposit_event(Event::DispatchedAs { + result: res.map(|_| ()).map_err(|e| e.error), + }); + Ok(()) + } } } @@ -338,6 +400,7 @@ impl Pallet { /// Derive a derivative account ID from the owner account and the sub-account index. pub fn derivative_account_id(who: T::AccountId, index: u16) -> T::AccountId { let entropy = (b"modlpy/utilisuba", who, index).using_encoded(blake2_256); - T::AccountId::decode(&mut &entropy[..]).unwrap_or_default() + Decode::decode(&mut TrailingZeroInput::new(entropy.as_ref())) + .expect("infinite length input; no invalid inputs for type; qed") } } diff --git a/frame/utility/src/tests.rs b/frame/utility/src/tests.rs index bbfbb417e23d..44b07f70db14 100644 --- a/frame/utility/src/tests.rs +++ b/frame/utility/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,7 +26,7 @@ use frame_support::{ assert_err_ignore_postinfo, assert_noop, assert_ok, dispatch::{DispatchError, DispatchErrorWithPostInfo, Dispatchable}, parameter_types, storage, - traits::Contains, + traits::{ConstU32, ConstU64, Contains}, weights::{Pays, Weight}, }; use sp_core::H256; @@ -38,7 +38,6 @@ use sp_runtime::{ // example module to test behaviors. #[frame_support::pallet] pub mod example { - use super::*; use frame_support::{dispatch::WithPostDispatchInfo, pallet_prelude::*}; use frame_system::pallet_prelude::*; @@ -99,7 +98,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(Weight::max_value()); } @@ -118,7 +116,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type Header = Header; type Event = Event; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; @@ -127,10 +125,9 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } -parameter_types! { - pub const ExistentialDeposit: u64 = 1; -} + impl pallet_balances::Config for Test { type MaxLocks = (); type MaxReserves = (); @@ -138,7 +135,7 @@ impl pallet_balances::Config for Test { type Balance = u64; type DustRemoval = (); type Event = Event; - type ExistentialDeposit = ExistentialDeposit; + type ExistentialDeposit = ConstU64<1>; type AccountStore = System; type WeightInfo = (); } @@ -168,6 +165,7 @@ impl Contains for TestBaseCallFilter { impl Config for Test { type Event = Event; type Call = Call; + type PalletsOrigin = OriginCaller; type WeightInfo = (); } @@ -288,7 +286,7 @@ fn as_derivative_filters() { value: 1 })), ), - DispatchError::BadOrigin + DispatchError::from(frame_system::Error::::CallFiltered), ); }); } @@ -338,7 +336,11 @@ fn batch_with_signed_filters() { vec![Call::Balances(pallet_balances::Call::transfer_keep_alive { dest: 2, value: 1 })] ),); System::assert_last_event( - utility::Event::BatchInterrupted(0, DispatchError::BadOrigin).into(), + utility::Event::BatchInterrupted { + index: 0, + error: frame_system::Error::::CallFiltered.into(), + } + .into(), ); }); } @@ -409,7 +411,7 @@ fn batch_handles_weight_refund() { let result = call.dispatch(Origin::signed(1)); assert_ok!(result); System::assert_last_event( - utility::Event::BatchInterrupted(1, DispatchError::Other("")).into(), + utility::Event::BatchInterrupted { index: 1, error: DispatchError::Other("") }.into(), ); // No weight is refunded assert_eq!(extract_actual_weight(&result, &info), info.weight); @@ -424,7 +426,7 @@ fn batch_handles_weight_refund() { let result = call.dispatch(Origin::signed(1)); assert_ok!(result); System::assert_last_event( - utility::Event::BatchInterrupted(1, DispatchError::Other("")).into(), + utility::Event::BatchInterrupted { index: 1, error: DispatchError::Other("") }.into(), ); assert_eq!(extract_actual_weight(&result, &info), info.weight - diff * batch_len); @@ -437,7 +439,7 @@ fn batch_handles_weight_refund() { let result = call.dispatch(Origin::signed(1)); assert_ok!(result); System::assert_last_event( - utility::Event::BatchInterrupted(1, DispatchError::Other("")).into(), + utility::Event::BatchInterrupted { index: 1, error: DispatchError::Other("") }.into(), ); assert_eq!( extract_actual_weight(&result, &info), @@ -573,7 +575,7 @@ fn batch_all_does_not_nest() { actual_weight: Some(::WeightInfo::batch_all(1) + info.weight), pays_fee: Pays::Yes }, - error: DispatchError::BadOrigin, + error: frame_system::Error::::CallFiltered.into(), } ); @@ -585,7 +587,11 @@ fn batch_all_does_not_nest() { // and balances. assert_ok!(Utility::batch_all(Origin::signed(1), vec![batch_nested])); System::assert_has_event( - utility::Event::BatchInterrupted(0, DispatchError::BadOrigin).into(), + utility::Event::BatchInterrupted { + index: 0, + error: frame_system::Error::::CallFiltered.into(), + } + .into(), ); assert_eq!(Balances::free_balance(1), 10); assert_eq!(Balances::free_balance(2), 10); diff --git a/frame/utility/src/weights.rs b/frame/utility/src/weights.rs index 6ac23419e3ef..e5f3cb0f58fd 100644 --- a/frame/utility/src/weights.rs +++ b/frame/utility/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_utility //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-07, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/utility/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -48,39 +49,46 @@ pub trait WeightInfo { fn batch(c: u32, ) -> Weight; fn as_derivative() -> Weight; fn batch_all(c: u32, ) -> Weight; + fn dispatch_as() -> Weight; } /// Weights for pallet_utility using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { fn batch(c: u32, ) -> Weight { - (30_319_000 as Weight) - // Standard Error: 3_000 - .saturating_add((6_759_000 as Weight).saturating_mul(c as Weight)) + (18_598_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_374_000 as Weight).saturating_mul(c as Weight)) } fn as_derivative() -> Weight { - (4_030_000 as Weight) + (1_650_000 as Weight) } fn batch_all(c: u32, ) -> Weight { - (26_621_000 as Weight) - // Standard Error: 3_000 - .saturating_add((7_251_000 as Weight).saturating_mul(c as Weight)) + (13_988_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_481_000 as Weight).saturating_mul(c as Weight)) + } + fn dispatch_as() -> Weight { + (8_463_000 as Weight) } } // For backwards compatibility and tests impl WeightInfo for () { fn batch(c: u32, ) -> Weight { - (30_319_000 as Weight) - // Standard Error: 3_000 - .saturating_add((6_759_000 as Weight).saturating_mul(c as Weight)) + (18_598_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_374_000 as Weight).saturating_mul(c as Weight)) } fn as_derivative() -> Weight { - (4_030_000 as Weight) + (1_650_000 as Weight) } fn batch_all(c: u32, ) -> Weight { - (26_621_000 as Weight) - // Standard Error: 3_000 - .saturating_add((7_251_000 as Weight).saturating_mul(c as Weight)) + (13_988_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_481_000 as Weight).saturating_mul(c as Weight)) + } + fn dispatch_as() -> Weight { + (8_463_000 as Weight) } } diff --git a/frame/vesting/Cargo.toml b/frame/vesting/Cargo.toml index 806e0e603686..83f897847f79 100644 --- a/frame/vesting/Cargo.toml +++ b/frame/vesting/Cargo.toml @@ -2,9 +2,9 @@ name = "pallet-vesting" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME pallet for manage vesting" readme = "README.md" @@ -13,20 +13,20 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } log = { version = "0.4.0", default-features = false } [dev-dependencies] -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } pallet-balances = { version = "4.0.0-dev", path = "../balances" } [features] @@ -39,5 +39,5 @@ std = [ "frame-support/std", "frame-system/std", ] -runtime-benchmarks = ["frame-benchmarking"] +runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/vesting/src/benchmarking.rs b/frame/vesting/src/benchmarking.rs index 5cdc14c8fdac..1693fdd3f1cb 100644 --- a/frame/vesting/src/benchmarking.rs +++ b/frame/vesting/src/benchmarking.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,7 @@ #![cfg(feature = "runtime-benchmarks")] -use frame_benchmarking::{account, benchmarks, impl_benchmark_test_suite, whitelisted_caller}; +use frame_benchmarking::{account, benchmarks, whitelisted_caller}; use frame_support::assert_ok; use frame_system::{Pallet as System, RawOrigin}; use sp_runtime::traits::{Bounded, CheckedDiv, CheckedMul}; @@ -374,10 +374,10 @@ benchmarks! { T::Currency::transfer(&caller, &test_dest, expected_balance, ExistenceRequirement::AllowDeath) ); } -} -impl_benchmark_test_suite!( - Vesting, - crate::mock::ExtBuilder::default().existential_deposit(256).build(), - crate::mock::Test, -); + impl_benchmark_test_suite!( + Vesting, + crate::mock::ExtBuilder::default().existential_deposit(256).build(), + crate::mock::Test, + ); +} diff --git a/frame/vesting/src/lib.rs b/frame/vesting/src/lib.rs index 27862a5ca4b7..13841a0443ce 100644 --- a/frame/vesting/src/lib.rs +++ b/frame/vesting/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -45,13 +45,14 @@ #![cfg_attr(not(feature = "std"), no_std)] mod benchmarking; -mod migrations; + #[cfg(test)] mod mock; #[cfg(test)] mod tests; mod vesting_info; +pub mod migrations; pub mod weights; use codec::{Decode, Encode, MaxEncodedLen}; @@ -73,7 +74,7 @@ use sp_runtime::{ }, RuntimeDebug, }; -use sp_std::{convert::TryInto, fmt::Debug, prelude::*}; +use sp_std::{fmt::Debug, prelude::*}; pub use vesting_info::*; pub use weights::WeightInfo; @@ -104,9 +105,9 @@ enum VestingAction { /// Do not actively remove any schedules. Passive, /// Remove the schedule specified by the index. - Remove(usize), + Remove { index: usize }, /// Remove the two schedules, specified by index, so they can be merged. - Merge(usize, usize), + Merge { index1: usize, index2: usize }, } impl VestingAction { @@ -114,8 +115,8 @@ impl VestingAction { fn should_remove(&self, index: usize) -> bool { match self { Self::Passive => false, - Self::Remove(index1) => *index1 == index, - Self::Merge(index1, index2) => *index1 == index || *index2 == index, + Self::Remove { index: index1 } => *index1 == index, + Self::Merge { index1, index2 } => *index1 == index || *index2 == index, } } @@ -170,34 +171,14 @@ pub mod pallet { #[pallet::extra_constants] impl Pallet { - // TODO: rename to snake case after https://github.com/paritytech/substrate/issues/8826 fixed. - #[allow(non_snake_case)] - fn MaxVestingSchedules() -> u32 { + #[pallet::constant_name(MaxVestingSchedules)] + fn max_vesting_schedules() -> u32 { T::MAX_VESTING_SCHEDULES } } #[pallet::hooks] impl Hooks> for Pallet { - #[cfg(feature = "try-runtime")] - fn pre_upgrade() -> Result<(), &'static str> { - migrations::v1::pre_migrate::() - } - - fn on_runtime_upgrade() -> Weight { - if StorageVersion::::get() == Releases::V0 { - StorageVersion::::put(Releases::V1); - migrations::v1::migrate::().saturating_add(T::DbWeight::get().reads_writes(1, 1)) - } else { - T::DbWeight::get().reads(1) - } - } - - #[cfg(feature = "try-runtime")] - fn post_upgrade() -> Result<(), &'static str> { - migrations::v1::post_migrate::() - } - fn integrity_test() { assert!(T::MAX_VESTING_SCHEDULES > 0, "`MaxVestingSchedules` must ge greater than 0"); } @@ -221,7 +202,6 @@ pub mod pallet { #[pallet::pallet] #[pallet::generate_store(pub(super) trait Store)] - #[pallet::generate_storage_info] pub struct Pallet(_); #[pallet::genesis_config] @@ -275,10 +255,9 @@ pub mod pallet { pub enum Event { /// The amount vested has been updated. This could indicate a change in funds available. /// The balance given is the amount which is left unvested (and thus locked). - /// \[account, unvested\] - VestingUpdated(T::AccountId, BalanceOf), + VestingUpdated { account: T::AccountId, unvested: BalanceOf }, /// An \[account\] has become fully vested. - VestingCompleted(T::AccountId), + VestingCompleted { account: T::AccountId }, } /// Error for the vesting pallet. @@ -446,7 +425,8 @@ pub mod pallet { let schedule2_index = schedule2_index as usize; let schedules = Self::vesting(&who).ok_or(Error::::NotVesting)?; - let merge_action = VestingAction::Merge(schedule1_index, schedule2_index); + let merge_action = + VestingAction::Merge { index1: schedule1_index, index2: schedule2_index }; let (schedules, locked_now) = Self::exec_action(schedules.to_vec(), merge_action)?; @@ -568,14 +548,13 @@ impl Pallet { let mut total_locked_now: BalanceOf = Zero::zero(); let filtered_schedules = action .pick_schedules::(schedules) - .filter_map(|schedule| { + .filter(|schedule| { let locked_now = schedule.locked_at::(now); - if locked_now.is_zero() { - None - } else { + let keep = !locked_now.is_zero(); + if keep { total_locked_now = total_locked_now.saturating_add(locked_now); - Some(schedule) } + keep }) .collect::>(); @@ -586,11 +565,14 @@ impl Pallet { fn write_lock(who: &T::AccountId, total_locked_now: BalanceOf) { if total_locked_now.is_zero() { T::Currency::remove_lock(VESTING_ID, who); - Self::deposit_event(Event::::VestingCompleted(who.clone())); + Self::deposit_event(Event::::VestingCompleted { account: who.clone() }); } else { let reasons = WithdrawReasons::TRANSFER | WithdrawReasons::RESERVE; T::Currency::set_lock(VESTING_ID, who, total_locked_now, reasons); - Self::deposit_event(Event::::VestingUpdated(who.clone(), total_locked_now)); + Self::deposit_event(Event::::VestingUpdated { + account: who.clone(), + unvested: total_locked_now, + }); }; } @@ -633,7 +615,7 @@ impl Pallet { action: VestingAction, ) -> Result<(Vec, T::BlockNumber>>, BalanceOf), DispatchError> { let (schedules, locked_now) = match action { - VestingAction::Merge(idx1, idx2) => { + VestingAction::Merge { index1: idx1, index2: idx2 } => { // The schedule index is based off of the schedule ordering prior to filtering out // any schedules that may be ending at this block. let schedule1 = *schedules.get(idx1).ok_or(Error::::ScheduleIndexOutOfBounds)?; @@ -758,7 +740,7 @@ where /// Remove a vesting schedule for a given account. fn remove_vesting_schedule(who: &T::AccountId, schedule_index: u32) -> DispatchResult { let schedules = Self::vesting(who).ok_or(Error::::NotVesting)?; - let remove_action = VestingAction::Remove(schedule_index as usize); + let remove_action = VestingAction::Remove { index: schedule_index as usize }; let (schedules, locked_now) = Self::exec_action(schedules.to_vec(), remove_action)?; diff --git a/frame/vesting/src/migrations.rs b/frame/vesting/src/migrations.rs index 086257d285ea..15668425b4b2 100644 --- a/frame/vesting/src/migrations.rs +++ b/frame/vesting/src/migrations.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,11 +20,11 @@ use super::*; // Migration from single schedule to multiple schedules. -pub(crate) mod v1 { +pub mod v1 { use super::*; #[cfg(feature = "try-runtime")] - pub(crate) fn pre_migrate() -> Result<(), &'static str> { + pub fn pre_migrate() -> Result<(), &'static str> { assert!(StorageVersion::::get() == Releases::V0, "Storage version too high."); log::debug!( @@ -37,7 +37,7 @@ pub(crate) mod v1 { /// Migrate from single schedule to multi schedule storage. /// WARNING: This migration will delete schedules if `MaxVestingSchedules < 1`. - pub(crate) fn migrate() -> Weight { + pub fn migrate() -> Weight { let mut reads_writes = 0; Vesting::::translate::, T::BlockNumber>, _>( @@ -65,12 +65,12 @@ pub(crate) mod v1 { } #[cfg(feature = "try-runtime")] - pub(crate) fn post_migrate() -> Result<(), &'static str> { + pub fn post_migrate() -> Result<(), &'static str> { assert_eq!(StorageVersion::::get(), Releases::V1); for (_key, schedules) in Vesting::::iter() { assert!( - schedules.len() == 1, + schedules.len() >= 1, "A bounded vec with incorrect count of items was created." ); diff --git a/frame/vesting/src/mock.rs b/frame/vesting/src/mock.rs index cb8961150003..8a830d72b26b 100644 --- a/frame/vesting/src/mock.rs +++ b/frame/vesting/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +15,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use frame_support::parameter_types; +use frame_support::{ + parameter_types, + traits::{ConstU32, ConstU64}, +}; use sp_core::H256; use sp_runtime::{ testing::Header, @@ -41,7 +44,6 @@ frame_support::construct_runtime!( ); parameter_types! { - pub const BlockHashCount: u64 = 250; pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights::simple_max(1024); } @@ -49,7 +51,7 @@ impl frame_system::Config for Test { type AccountData = pallet_balances::AccountData; type AccountId = u64; type BaseCallFilter = frame_support::traits::Everything; - type BlockHashCount = BlockHashCount; + type BlockHashCount = ConstU64<250>; type BlockLength = (); type BlockNumber = u64; type BlockWeights = (); @@ -64,22 +66,21 @@ impl frame_system::Config for Test { type OnKilledAccount = (); type OnNewAccount = (); type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; type Origin = Origin; type PalletInfo = PalletInfo; type SS58Prefix = (); type SystemWeightInfo = (); type Version = (); } -parameter_types! { - pub const MaxLocks: u32 = 10; -} + impl pallet_balances::Config for Test { type AccountStore = System; type Balance = u64; type DustRemoval = (); type Event = Event; type ExistentialDeposit = ExistentialDeposit; - type MaxLocks = MaxLocks; + type MaxLocks = ConstU32<10>; type MaxReserves = (); type ReserveIdentifier = [u8; 8]; type WeightInfo = (); diff --git a/frame/vesting/src/tests.rs b/frame/vesting/src/tests.rs index 2a6dd0520c3b..cbc2e09c8319 100644 --- a/frame/vesting/src/tests.rs +++ b/frame/vesting/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/vesting/src/vesting_info.rs b/frame/vesting/src/vesting_info.rs index 81bffa199fd7..9069b6948276 100644 --- a/frame/vesting/src/vesting_info.rs +++ b/frame/vesting/src/vesting_info.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/frame/vesting/src/weights.rs b/frame/vesting/src/weights.rs index 3ccc1a5bda36..140c1889d6e5 100644 --- a/frame/vesting/src/weights.rs +++ b/frame/vesting/src/weights.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,11 +18,11 @@ //! Autogenerated weights for pallet_vesting //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2021-08-10, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 +//! DATE: 2022-01-31, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 // Executed Command: -// target/release/substrate +// ./target/production/substrate // benchmark // --chain=dev // --steps=50 @@ -33,8 +33,9 @@ // --wasm-execution=compiled // --heap-pages=4096 // --output=./frame/vesting/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs - +// --template=.maintain/frame-weight-template.hbs +// --header=HEADER-APACHE2 +// --raw #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -61,22 +62,22 @@ impl WeightInfo for SubstrateWeight { // Storage: Vesting Vesting (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vest_locked(l: u32, s: u32, ) -> Weight { - (50_642_000 as Weight) + (27_037_000 as Weight) // Standard Error: 1_000 - .saturating_add((144_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 3_000 - .saturating_add((177_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((88_000 as Weight).saturating_mul(l as Weight)) + // Standard Error: 2_000 + .saturating_add((69_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } // Storage: Vesting Vesting (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vest_unlocked(l: u32, s: u32, ) -> Weight { - (50_830_000 as Weight) + (26_627_000 as Weight) // Standard Error: 1_000 - .saturating_add((115_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 3_000 - .saturating_add((112_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((82_000 as Weight).saturating_mul(l as Weight)) + // Standard Error: 2_000 + .saturating_add((53_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } @@ -84,11 +85,11 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn vest_other_locked(l: u32, s: u32, ) -> Weight { - (52_151_000 as Weight) + (26_672_000 as Weight) // Standard Error: 1_000 - .saturating_add((130_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 3_000 - .saturating_add((162_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((85_000 as Weight).saturating_mul(l as Weight)) + // Standard Error: 2_000 + .saturating_add((77_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -96,11 +97,11 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn vest_other_unlocked(l: u32, s: u32, ) -> Weight { - (51_009_000 as Weight) - // Standard Error: 4_000 - .saturating_add((123_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 9_000 - .saturating_add((118_000 as Weight).saturating_mul(s as Weight)) + (26_682_000 as Weight) + // Standard Error: 1_000 + .saturating_add((74_000 as Weight).saturating_mul(l as Weight)) + // Standard Error: 2_000 + .saturating_add((51_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -108,11 +109,11 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vested_transfer(l: u32, s: u32, ) -> Weight { - (89_517_000 as Weight) - // Standard Error: 5_000 - .saturating_add((114_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 10_000 - .saturating_add((23_000 as Weight).saturating_mul(s as Weight)) + (42_066_000 as Weight) + // Standard Error: 1_000 + .saturating_add((83_000 as Weight).saturating_mul(l as Weight)) + // Standard Error: 3_000 + .saturating_add((43_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -120,11 +121,11 @@ impl WeightInfo for SubstrateWeight { // Storage: System Account (r:2 w:2) // Storage: Balances Locks (r:1 w:1) fn force_vested_transfer(l: u32, s: u32, ) -> Weight { - (87_903_000 as Weight) - // Standard Error: 6_000 - .saturating_add((121_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 12_000 - .saturating_add((56_000 as Weight).saturating_mul(s as Weight)) + (41_672_000 as Weight) + // Standard Error: 1_000 + .saturating_add((84_000 as Weight).saturating_mul(l as Weight)) + // Standard Error: 3_000 + .saturating_add((46_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } @@ -132,11 +133,11 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn not_unlocking_merge_schedules(l: u32, s: u32, ) -> Weight { - (54_463_000 as Weight) + (27_627_000 as Weight) + // Standard Error: 0 + .saturating_add((86_000 as Weight).saturating_mul(l as Weight)) // Standard Error: 2_000 - .saturating_add((123_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 5_000 - .saturating_add((149_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((68_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -144,11 +145,11 @@ impl WeightInfo for SubstrateWeight { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn unlocking_merge_schedules(l: u32, s: u32, ) -> Weight { - (53_674_000 as Weight) + (27_143_000 as Weight) + // Standard Error: 0 + .saturating_add((88_000 as Weight).saturating_mul(l as Weight)) // Standard Error: 1_000 - .saturating_add((137_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 4_000 - .saturating_add((152_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((72_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } @@ -159,22 +160,22 @@ impl WeightInfo for () { // Storage: Vesting Vesting (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vest_locked(l: u32, s: u32, ) -> Weight { - (50_642_000 as Weight) + (27_037_000 as Weight) // Standard Error: 1_000 - .saturating_add((144_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 3_000 - .saturating_add((177_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((88_000 as Weight).saturating_mul(l as Weight)) + // Standard Error: 2_000 + .saturating_add((69_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } // Storage: Vesting Vesting (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vest_unlocked(l: u32, s: u32, ) -> Weight { - (50_830_000 as Weight) + (26_627_000 as Weight) // Standard Error: 1_000 - .saturating_add((115_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 3_000 - .saturating_add((112_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((82_000 as Weight).saturating_mul(l as Weight)) + // Standard Error: 2_000 + .saturating_add((53_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } @@ -182,11 +183,11 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn vest_other_locked(l: u32, s: u32, ) -> Weight { - (52_151_000 as Weight) + (26_672_000 as Weight) // Standard Error: 1_000 - .saturating_add((130_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 3_000 - .saturating_add((162_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((85_000 as Weight).saturating_mul(l as Weight)) + // Standard Error: 2_000 + .saturating_add((77_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -194,11 +195,11 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn vest_other_unlocked(l: u32, s: u32, ) -> Weight { - (51_009_000 as Weight) - // Standard Error: 4_000 - .saturating_add((123_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 9_000 - .saturating_add((118_000 as Weight).saturating_mul(s as Weight)) + (26_682_000 as Weight) + // Standard Error: 1_000 + .saturating_add((74_000 as Weight).saturating_mul(l as Weight)) + // Standard Error: 2_000 + .saturating_add((51_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -206,11 +207,11 @@ impl WeightInfo for () { // Storage: System Account (r:1 w:1) // Storage: Balances Locks (r:1 w:1) fn vested_transfer(l: u32, s: u32, ) -> Weight { - (89_517_000 as Weight) - // Standard Error: 5_000 - .saturating_add((114_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 10_000 - .saturating_add((23_000 as Weight).saturating_mul(s as Weight)) + (42_066_000 as Weight) + // Standard Error: 1_000 + .saturating_add((83_000 as Weight).saturating_mul(l as Weight)) + // Standard Error: 3_000 + .saturating_add((43_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -218,11 +219,11 @@ impl WeightInfo for () { // Storage: System Account (r:2 w:2) // Storage: Balances Locks (r:1 w:1) fn force_vested_transfer(l: u32, s: u32, ) -> Weight { - (87_903_000 as Weight) - // Standard Error: 6_000 - .saturating_add((121_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 12_000 - .saturating_add((56_000 as Weight).saturating_mul(s as Weight)) + (41_672_000 as Weight) + // Standard Error: 1_000 + .saturating_add((84_000 as Weight).saturating_mul(l as Weight)) + // Standard Error: 3_000 + .saturating_add((46_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } @@ -230,11 +231,11 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn not_unlocking_merge_schedules(l: u32, s: u32, ) -> Weight { - (54_463_000 as Weight) + (27_627_000 as Weight) + // Standard Error: 0 + .saturating_add((86_000 as Weight).saturating_mul(l as Weight)) // Standard Error: 2_000 - .saturating_add((123_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 5_000 - .saturating_add((149_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((68_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } @@ -242,11 +243,11 @@ impl WeightInfo for () { // Storage: Balances Locks (r:1 w:1) // Storage: System Account (r:1 w:1) fn unlocking_merge_schedules(l: u32, s: u32, ) -> Weight { - (53_674_000 as Weight) + (27_143_000 as Weight) + // Standard Error: 0 + .saturating_add((88_000 as Weight).saturating_mul(l as Weight)) // Standard Error: 1_000 - .saturating_add((137_000 as Weight).saturating_mul(l as Weight)) - // Standard Error: 4_000 - .saturating_add((152_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((72_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } diff --git a/frame/whitelist/Cargo.toml b/frame/whitelist/Cargo.toml new file mode 100644 index 000000000000..5f414e5d3203 --- /dev/null +++ b/frame/whitelist/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "pallet-whitelist" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet for whitelisting call, and dispatch from specific origin" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } +scale-info = { version = "2.0", default-features = false, features = ["derive"] } +sp-api = { version = "4.0.0-dev", default-features = false, path = "../../primitives/api" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } + +frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" } +frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true } + +[dev-dependencies] +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../primitives/io" } +pallet-balances = { version = "4.0.0-dev", path = "../balances" } +pallet-preimage = { version = "4.0.0-dev", path = "../preimage" } + +[features] +default = ["std"] +std = [ + "codec/std", + "scale-info/std", + "sp-api/std", + "sp-runtime/std", + "sp-std/std", + "frame-support/std", + "frame-system/std", +] +runtime-benchmarks = [ + "frame-benchmarking", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", +] +try-runtime = ["frame-support/try-runtime"] diff --git a/frame/whitelist/src/benchmarking.rs b/frame/whitelist/src/benchmarking.rs new file mode 100644 index 000000000000..50809ddef7ec --- /dev/null +++ b/frame/whitelist/src/benchmarking.rs @@ -0,0 +1,121 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Whitelist pallet benchmarking. + +#![cfg(feature = "runtime-benchmarks")] + +use super::*; +use frame_benchmarking::benchmarks; +use frame_support::{ + ensure, + traits::{EnsureOrigin, Get, PreimageRecipient}, +}; + +#[cfg(test)] +use crate::Pallet as Whitelist; + +benchmarks! { + whitelist_call { + let origin = T::WhitelistOrigin::successful_origin(); + let call_hash = Default::default(); + }: _(origin, call_hash) + verify { + ensure!( + WhitelistedCall::::contains_key(call_hash), + "call not whitelisted" + ); + ensure!( + T::PreimageProvider::preimage_requested(&call_hash), + "preimage not requested" + ); + } + + remove_whitelisted_call { + let origin = T::WhitelistOrigin::successful_origin(); + let call_hash = Default::default(); + Pallet::::whitelist_call(origin.clone(), call_hash) + .expect("whitelisting call must be successful"); + }: _(origin, call_hash) + verify { + ensure!( + !WhitelistedCall::::contains_key(call_hash), + "whitelist not removed" + ); + ensure!( + !T::PreimageProvider::preimage_requested(&call_hash), + "preimage still requested" + ); + } + + // We benchmark with the maximum possible size for a call. + // If the resulting weight is too big, maybe it worth having a weight which depends + // on the size of the call, with a new witness in parameter. + dispatch_whitelisted_call { + let origin = T::DispatchWhitelistedOrigin::successful_origin(); + // NOTE: we remove `10` because we need some bytes to encode the variants and vec length + let remark_len = >::MaxSize::get() - 10; + let remark = sp_std::vec![1_8; remark_len as usize]; + + let call: ::Call = frame_system::Call::remark { remark }.into(); + let call_weight = call.get_dispatch_info().weight; + let encoded_call = call.encode(); + let call_hash = T::Hashing::hash(&encoded_call[..]); + + Pallet::::whitelist_call(origin.clone(), call_hash) + .expect("whitelisting call must be successful"); + + let encoded_call = encoded_call.try_into().expect("encoded_call must be small enough"); + T::PreimageProvider::note_preimage(encoded_call); + + }: _(origin, call_hash, call_weight) + verify { + ensure!( + !WhitelistedCall::::contains_key(call_hash), + "whitelist not removed" + ); + ensure!( + !T::PreimageProvider::preimage_requested(&call_hash), + "preimage still requested" + ); + } + + dispatch_whitelisted_call_with_preimage { + let n in 1 .. 10_000; + + let origin = T::DispatchWhitelistedOrigin::successful_origin(); + let remark = sp_std::vec![1u8; n as usize]; + + let call: ::Call = frame_system::Call::remark { remark }.into(); + let call_hash = T::Hashing::hash_of(&call); + + Pallet::::whitelist_call(origin.clone(), call_hash) + .expect("whitelisting call must be successful"); + }: _(origin, Box::new(call)) + verify { + ensure!( + !WhitelistedCall::::contains_key(call_hash), + "whitelist not removed" + ); + ensure!( + !T::PreimageProvider::preimage_requested(&call_hash), + "preimage still requested" + ); + } + + impl_benchmark_test_suite!(Whitelist, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/frame/whitelist/src/lib.rs b/frame/whitelist/src/lib.rs new file mode 100644 index 000000000000..239f0fd28016 --- /dev/null +++ b/frame/whitelist/src/lib.rs @@ -0,0 +1,235 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Whitelist Pallet +//! +//! - [`Config`] +//! - [`Call`] +//! +//! ## Overview +//! +//! Allow some configurable origin: [`Config::WhitelistOrigin`] to whitelist some hash of a call, +//! and allow another configurable origin: [`Config::DispatchWhitelistedOrigin`] to dispatch them +//! with the root origin. +//! +//! In the meantime the call corresponding to the hash must have been submitted to the to the +//! pre-image handler [`PreimageProvider`]. + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "runtime-benchmarks")] +mod benchmarking; +#[cfg(test)] +mod mock; +#[cfg(test)] +mod tests; +pub mod weights; + +use codec::{DecodeLimit, Encode, FullCodec}; +use frame_support::{ + ensure, + traits::{PreimageProvider, PreimageRecipient}, + weights::{GetDispatchInfo, PostDispatchInfo, Weight}, +}; +use scale_info::TypeInfo; +use sp_runtime::traits::{Dispatchable, Hash}; +use sp_std::prelude::*; + +pub use pallet::*; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use crate::weights::WeightInfo; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::config] + pub trait Config: frame_system::Config { + /// The overarching event type. + type Event: From> + IsType<::Event>; + + /// The overarching call type. + type Call: IsType<::Call> + + Dispatchable + + GetDispatchInfo + + FullCodec + + TypeInfo + + From> + + Parameter; + + /// Required origin for whitelisting a call. + type WhitelistOrigin: EnsureOrigin; + + /// Required origin for dispatching whitelisted call with root origin. + type DispatchWhitelistedOrigin: EnsureOrigin; + + /// The handler of pre-images. + // NOTE: recipient is only needed for benchmarks. + type PreimageProvider: PreimageProvider + PreimageRecipient; + + /// The weight information for this pallet. + type WeightInfo: WeightInfo; + } + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(_); + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + CallWhitelisted { call_hash: T::Hash }, + WhitelistedCallRemoved { call_hash: T::Hash }, + WhitelistedCallDispatched { call_hash: T::Hash, result: DispatchResultWithPostInfo }, + } + + #[pallet::error] + pub enum Error { + /// The preimage of the call hash could not be loaded. + UnavailablePreImage, + /// The call could not be decoded. + UndecodableCall, + /// The weight of the decoded call was higher than the witness. + InvalidCallWeightWitness, + /// The call was not whitelisted. + CallIsNotWhitelisted, + /// The call was already whitelisted; No-Op. + CallAlreadyWhitelisted, + } + + #[pallet::storage] + pub type WhitelistedCall = StorageMap<_, Twox64Concat, T::Hash, (), OptionQuery>; + + #[pallet::call] + impl Pallet { + #[pallet::weight(T::WeightInfo::whitelist_call())] + pub fn whitelist_call(origin: OriginFor, call_hash: T::Hash) -> DispatchResult { + T::WhitelistOrigin::ensure_origin(origin)?; + + ensure!( + !WhitelistedCall::::contains_key(call_hash), + Error::::CallAlreadyWhitelisted, + ); + + WhitelistedCall::::insert(call_hash, ()); + T::PreimageProvider::request_preimage(&call_hash); + + Self::deposit_event(Event::::CallWhitelisted { call_hash }); + + Ok(()) + } + + #[pallet::weight(T::WeightInfo::remove_whitelisted_call())] + pub fn remove_whitelisted_call(origin: OriginFor, call_hash: T::Hash) -> DispatchResult { + T::WhitelistOrigin::ensure_origin(origin)?; + + WhitelistedCall::::take(call_hash).ok_or(Error::::CallIsNotWhitelisted)?; + + T::PreimageProvider::unrequest_preimage(&call_hash); + + Self::deposit_event(Event::::WhitelistedCallRemoved { call_hash }); + + Ok(()) + } + + #[pallet::weight( + T::WeightInfo::dispatch_whitelisted_call().saturating_add(*call_weight_witness) + )] + pub fn dispatch_whitelisted_call( + origin: OriginFor, + call_hash: T::Hash, + call_weight_witness: Weight, + ) -> DispatchResultWithPostInfo { + T::DispatchWhitelistedOrigin::ensure_origin(origin)?; + + ensure!( + WhitelistedCall::::contains_key(call_hash), + Error::::CallIsNotWhitelisted, + ); + + let call = T::PreimageProvider::get_preimage(&call_hash) + .ok_or(Error::::UnavailablePreImage)?; + + let call = ::Call::decode_all_with_depth_limit( + sp_api::MAX_EXTRINSIC_DEPTH, + &mut &call[..], + ) + .map_err(|_| Error::::UndecodableCall)?; + + ensure!( + call.get_dispatch_info().weight <= call_weight_witness, + Error::::InvalidCallWeightWitness + ); + + let actual_weight = Self::clean_and_dispatch(call_hash, call) + .map(|w| w.saturating_add(T::WeightInfo::dispatch_whitelisted_call())); + + Ok(actual_weight.into()) + } + + #[pallet::weight({ + let call_weight = call.get_dispatch_info().weight; + let call_len = call.encoded_size() as u32; + + T::WeightInfo::dispatch_whitelisted_call_with_preimage(call_len) + .saturating_add(call_weight) + })] + pub fn dispatch_whitelisted_call_with_preimage( + origin: OriginFor, + call: Box<::Call>, + ) -> DispatchResultWithPostInfo { + T::DispatchWhitelistedOrigin::ensure_origin(origin)?; + + let call_hash = ::Hashing::hash_of(&call); + + ensure!( + WhitelistedCall::::contains_key(call_hash), + Error::::CallIsNotWhitelisted, + ); + + let call_len = call.encoded_size() as u32; + let actual_weight = Self::clean_and_dispatch(call_hash, *call).map(|w| { + w.saturating_add(T::WeightInfo::dispatch_whitelisted_call_with_preimage(call_len)) + }); + + Ok(actual_weight.into()) + } + } +} + +impl Pallet { + /// Clean whitelisting/preimage and dispatch call. + /// + /// Return the call actual weight of the dispatched call if there is some. + fn clean_and_dispatch(call_hash: T::Hash, call: ::Call) -> Option { + WhitelistedCall::::remove(call_hash); + + T::PreimageProvider::unrequest_preimage(&call_hash); + + let result = call.dispatch(frame_system::Origin::::Root.into()); + + let call_actual_weight = match result { + Ok(call_post_info) => call_post_info.actual_weight, + Err(call_err) => call_err.post_info.actual_weight, + }; + + Self::deposit_event(Event::::WhitelistedCallDispatched { call_hash, result }); + + call_actual_weight + } +} diff --git a/frame/whitelist/src/mock.rs b/frame/whitelist/src/mock.rs new file mode 100644 index 000000000000..3009a6f6b5d5 --- /dev/null +++ b/frame/whitelist/src/mock.rs @@ -0,0 +1,124 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Mock for Whitelist Pallet + +#![cfg(test)] + +use crate as pallet_whitelist; + +use frame_support::{ + parameter_types, + traits::{ConstU32, ConstU64, Nothing}, +}; +use frame_system::EnsureRoot; +use sp_core::H256; +use sp_runtime::{ + testing::Header, + traits::{BlakeTwo256, IdentityLookup}, + BuildStorage, +}; + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; + +frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system, + Balances: pallet_balances, + Whitelist: pallet_whitelist, + Preimage: pallet_preimage, + } +); + +frame_support::parameter_types! { + pub BlockWeights: frame_system::limits::BlockWeights = + frame_system::limits::BlockWeights::simple_max(1024); +} +impl frame_system::Config for Test { + type BaseCallFilter = Nothing; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type Origin = Origin; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Call = Call; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = ConstU64<250>; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = ConstU32<16>; +} + +impl pallet_balances::Config for Test { + type MaxLocks = (); + type MaxReserves = (); + type ReserveIdentifier = [u8; 8]; + type Balance = u64; + type Event = Event; + type DustRemoval = (); + type ExistentialDeposit = ConstU64<1>; + type AccountStore = System; + type WeightInfo = (); +} + +parameter_types! { + // Taken from Polkadot as reference. + pub const PreimageMaxSize: u32 = 4096 * 1024; +} + +impl pallet_preimage::Config for Test { + type Event = Event; + type Currency = Balances; + type ManagerOrigin = EnsureRoot; + type MaxSize = PreimageMaxSize; + type BaseDeposit = ConstU64<1>; + type ByteDeposit = ConstU64<1>; + type WeightInfo = (); +} + +impl pallet_whitelist::Config for Test { + type Event = Event; + type Call = Call; + type WhitelistOrigin = EnsureRoot; + type DispatchWhitelistedOrigin = EnsureRoot; + type PreimageProvider = Preimage; + type WeightInfo = (); +} + +pub fn new_test_ext() -> sp_io::TestExternalities { + let t = GenesisConfig::default().build_storage().unwrap(); + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext +} diff --git a/frame/whitelist/src/tests.rs b/frame/whitelist/src/tests.rs new file mode 100644 index 000000000000..67bccaeaeebe --- /dev/null +++ b/frame/whitelist/src/tests.rs @@ -0,0 +1,175 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for Whitelist Pallet + +use crate::mock::*; +use codec::Encode; +use frame_support::{assert_noop, assert_ok, dispatch::GetDispatchInfo, traits::PreimageProvider}; +use sp_runtime::{traits::Hash, DispatchError}; + +#[test] +fn test_whitelist_call_and_remove() { + new_test_ext().execute_with(|| { + let call = Call::System(frame_system::Call::remark { remark: vec![] }); + let encoded_call = call.encode(); + let call_hash = ::Hashing::hash(&encoded_call[..]); + + assert_noop!( + Whitelist::remove_whitelisted_call(Origin::root(), call_hash), + crate::Error::::CallIsNotWhitelisted, + ); + + assert_noop!( + Whitelist::whitelist_call(Origin::signed(1), call_hash), + DispatchError::BadOrigin, + ); + + assert_ok!(Whitelist::whitelist_call(Origin::root(), call_hash)); + + assert!(Preimage::preimage_requested(&call_hash)); + + assert_noop!( + Whitelist::whitelist_call(Origin::root(), call_hash), + crate::Error::::CallAlreadyWhitelisted, + ); + + assert_noop!( + Whitelist::remove_whitelisted_call(Origin::signed(1), call_hash), + DispatchError::BadOrigin, + ); + + assert_ok!(Whitelist::remove_whitelisted_call(Origin::root(), call_hash)); + + assert!(!Preimage::preimage_requested(&call_hash)); + + assert_noop!( + Whitelist::remove_whitelisted_call(Origin::root(), call_hash), + crate::Error::::CallIsNotWhitelisted, + ); + }); +} + +#[test] +fn test_whitelist_call_and_execute() { + new_test_ext().execute_with(|| { + let call = Call::System(frame_system::Call::remark_with_event { remark: vec![1] }); + let call_weight = call.get_dispatch_info().weight; + let encoded_call = call.encode(); + let call_hash = ::Hashing::hash(&encoded_call[..]); + + assert_noop!( + Whitelist::dispatch_whitelisted_call(Origin::root(), call_hash, call_weight), + crate::Error::::CallIsNotWhitelisted, + ); + + assert_ok!(Whitelist::whitelist_call(Origin::root(), call_hash)); + + assert_noop!( + Whitelist::dispatch_whitelisted_call(Origin::signed(1), call_hash, call_weight), + DispatchError::BadOrigin, + ); + + assert_noop!( + Whitelist::dispatch_whitelisted_call(Origin::root(), call_hash, call_weight), + crate::Error::::UnavailablePreImage, + ); + + assert_ok!(Preimage::note_preimage(Origin::root(), encoded_call)); + + assert!(Preimage::preimage_requested(&call_hash)); + + assert_noop!( + Whitelist::dispatch_whitelisted_call(Origin::root(), call_hash, call_weight - 1), + crate::Error::::InvalidCallWeightWitness, + ); + + assert_ok!(Whitelist::dispatch_whitelisted_call(Origin::root(), call_hash, call_weight)); + + assert!(!Preimage::preimage_requested(&call_hash)); + + assert_noop!( + Whitelist::dispatch_whitelisted_call(Origin::root(), call_hash, call_weight), + crate::Error::::CallIsNotWhitelisted, + ); + }); +} + +#[test] +fn test_whitelist_call_and_execute_failing_call() { + new_test_ext().execute_with(|| { + let call = Call::Whitelist(crate::Call::dispatch_whitelisted_call { + call_hash: Default::default(), + call_weight_witness: 0, + }); + let call_weight = call.get_dispatch_info().weight; + let encoded_call = call.encode(); + let call_hash = ::Hashing::hash(&encoded_call[..]); + + assert_ok!(Whitelist::whitelist_call(Origin::root(), call_hash)); + assert_ok!(Preimage::note_preimage(Origin::root(), encoded_call)); + assert!(Preimage::preimage_requested(&call_hash)); + assert_ok!(Whitelist::dispatch_whitelisted_call(Origin::root(), call_hash, call_weight)); + assert!(!Preimage::preimage_requested(&call_hash)); + }); +} + +#[test] +fn test_whitelist_call_and_execute_without_note_preimage() { + new_test_ext().execute_with(|| { + let call = + Box::new(Call::System(frame_system::Call::remark_with_event { remark: vec![1] })); + let call_hash = ::Hashing::hash_of(&call); + + assert_ok!(Whitelist::whitelist_call(Origin::root(), call_hash)); + assert!(Preimage::preimage_requested(&call_hash)); + + assert_ok!(Whitelist::dispatch_whitelisted_call_with_preimage( + Origin::root(), + call.clone() + )); + + assert!(!Preimage::preimage_requested(&call_hash)); + + assert_noop!( + Whitelist::dispatch_whitelisted_call_with_preimage(Origin::root(), call), + crate::Error::::CallIsNotWhitelisted, + ); + }); +} + +#[test] +fn test_whitelist_call_and_execute_decode_consumes_all() { + new_test_ext().execute_with(|| { + let call = Call::System(frame_system::Call::remark_with_event { remark: vec![1] }); + let call_weight = call.get_dispatch_info().weight; + let mut call = call.encode(); + // Appending something does not make the encoded call invalid. + // This tests that the decode function consumes all data. + call.extend(call.clone()); + + let call_hash = ::Hashing::hash(&call[..]); + + assert_ok!(Preimage::note_preimage(Origin::root(), call)); + assert_ok!(Whitelist::whitelist_call(Origin::root(), call_hash)); + + assert_noop!( + Whitelist::dispatch_whitelisted_call(Origin::root(), call_hash, call_weight), + crate::Error::::UndecodableCall, + ); + }); +} diff --git a/frame/whitelist/src/weights.rs b/frame/whitelist/src/weights.rs new file mode 100644 index 000000000000..fdba734db64b --- /dev/null +++ b/frame/whitelist/src/weights.rs @@ -0,0 +1,126 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for pallet_whitelist +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2022-02-25, STEPS: `50`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 + +// Executed Command: +// target/production/substrate +// benchmark +// --chain=dev +// --steps=50 +// --repeat=20 +// --pallet=pallet_whitelist +// --extrinsic=* +// --execution=wasm +// --wasm-execution=compiled +// --heap-pages=4096 +// --output=./frame/whitelist/src/weights.rs +// --template=./.maintain/frame-weight-template.hbs + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use sp_std::marker::PhantomData; + +/// Weight functions needed for pallet_whitelist. +pub trait WeightInfo { + fn whitelist_call() -> Weight; + fn remove_whitelisted_call() -> Weight; + fn dispatch_whitelisted_call() -> Weight; + fn dispatch_whitelisted_call_with_preimage(n: u32, ) -> Weight; +} + +/// Weights for pallet_whitelist using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + // Storage: Whitelist WhitelistedCall (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + fn whitelist_call() -> Weight { + (16_254_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + } + // Storage: Whitelist WhitelistedCall (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Preimage PreimageFor (r:0 w:1) + fn remove_whitelisted_call() -> Weight { + (18_348_000 as Weight) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Whitelist WhitelistedCall (r:1 w:1) + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + fn dispatch_whitelisted_call() -> Weight { + (6_618_241_000 as Weight) + .saturating_add(T::DbWeight::get().reads(3 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } + // Storage: Whitelist WhitelistedCall (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Preimage PreimageFor (r:0 w:1) + fn dispatch_whitelisted_call_with_preimage(n: u32, ) -> Weight { + (20_619_000 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(3 as Weight)) + } +} + +// For backwards compatibility and tests +impl WeightInfo for () { + // Storage: Whitelist WhitelistedCall (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + fn whitelist_call() -> Weight { + (16_254_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + } + // Storage: Whitelist WhitelistedCall (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Preimage PreimageFor (r:0 w:1) + fn remove_whitelisted_call() -> Weight { + (18_348_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Whitelist WhitelistedCall (r:1 w:1) + // Storage: Preimage PreimageFor (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + fn dispatch_whitelisted_call() -> Weight { + (6_618_241_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } + // Storage: Whitelist WhitelistedCall (r:1 w:1) + // Storage: Preimage StatusFor (r:1 w:1) + // Storage: Preimage PreimageFor (r:0 w:1) + fn dispatch_whitelisted_call_with_preimage(n: u32, ) -> Weight { + (20_619_000 as Weight) + // Standard Error: 0 + .saturating_add((2_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(3 as Weight)) + } +} diff --git a/primitives/api/Cargo.toml b/primitives/api/Cargo.toml index 7e751232acb5..a0dd8eeb31e7 100644 --- a/primitives/api/Cargo.toml +++ b/primitives/api/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-api" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate runtime api primitives" readme = "README.md" @@ -13,15 +13,15 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } sp-api-proc-macro = { version = "4.0.0-dev", path = "proc-macro" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../runtime" } -sp-version = { version = "4.0.0-dev", default-features = false, path = "../version" } -sp-state-machine = { version = "0.10.0-dev", optional = true, path = "../state-machine" } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } +sp-version = { version = "5.0.0", default-features = false, path = "../version" } +sp-state-machine = { version = "0.12.0", optional = true, path = "../state-machine" } hash-db = { version = "0.15.2", optional = true } -thiserror = { version = "1.0.21", optional = true } +thiserror = { version = "1.0.30", optional = true } log = { version = "0.4.14", default-features = false } diff --git a/primitives/api/proc-macro/Cargo.toml b/primitives/api/proc-macro/Cargo.toml index d5909967ac5a..dc5deb2efa66 100644 --- a/primitives/api/proc-macro/Cargo.toml +++ b/primitives/api/proc-macro/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-api-proc-macro" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Macros for declaring and implementing runtime apis." documentation = "https://docs.rs/sp-api-proc-macro" @@ -16,11 +16,11 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -quote = "1.0.3" -syn = { version = "1.0.58", features = ["full", "fold", "extra-traits", "visit"] } -proc-macro2 = "1.0.29" -blake2-rfc = { version = "0.2.18", default-features = false } -proc-macro-crate = "1.0.0" +quote = "1.0.10" +syn = { version = "1.0.82", features = ["full", "fold", "extra-traits", "visit"] } +proc-macro2 = "1.0.36" +blake2 = { version = "0.10.2", default-features = false } +proc-macro-crate = "1.1.3" # Required for the doc tests [features] diff --git a/primitives/api/proc-macro/src/decl_runtime_apis.rs b/primitives/api/proc-macro/src/decl_runtime_apis.rs index 510a2eeaa530..2301f531590e 100644 --- a/primitives/api/proc-macro/src/decl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/decl_runtime_apis.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -39,8 +39,6 @@ use syn::{ use std::collections::HashMap; -use blake2_rfc; - /// The ident used for the block generic parameter. const BLOCK_GENERIC_IDENT: &str = "Block"; @@ -183,7 +181,7 @@ fn generate_native_call_generators(decl: &ItemTrait) -> Result { { ::decode_with_depth_limit( #crate_::MAX_EXTRINSIC_DEPTH, - &#crate_::Encode::encode(input)[..], + &mut &#crate_::Encode::encode(input)[..], ).map_err(map_error) } )); @@ -235,9 +233,7 @@ fn generate_native_call_generators(decl: &ItemTrait) -> Result { // compatible. To ensure that we forward it by ref/value, we use the value given by the // the user. Otherwise if it is not using the block, we don't need to add anything. let input_borrows = - params - .iter() - .map(|v| if type_is_using_block(&v.1) { v.2.clone() } else { None }); + params.iter().map(|v| if type_is_using_block(&v.1) { v.2 } else { None }); // Replace all `Block` with `NodeBlock`, add `'a` lifetime to references and collect // all the function inputs. @@ -382,21 +378,21 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { #[cfg(any(feature = "std", test))] #[allow(clippy::too_many_arguments)] pub fn #fn_name< - R: #crate_::Encode + #crate_::Decode + PartialEq, + R: #crate_::Encode + #crate_::Decode + std::cmp::PartialEq, NC: FnOnce() -> std::result::Result + std::panic::UnwindSafe, Block: #crate_::BlockT, T: #crate_::CallApiAt, >( call_runtime_at: &T, at: &#crate_::BlockId, - args: Vec, + args: std::vec::Vec, changes: &std::cell::RefCell<#crate_::OverlayedChanges>, storage_transaction_cache: &std::cell::RefCell< #crate_::StorageTransactionCache >, - native_call: Option, + native_call: std::option::Option, context: #crate_::ExecutionContext, - recorder: &Option<#crate_::ProofRecorder>, + recorder: &std::option::Option<#crate_::ProofRecorder>, ) -> std::result::Result<#crate_::NativeOrEncoded, #crate_::ApiError> { let version = call_runtime_at.runtime_version_at(at)?; @@ -416,7 +412,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { recorder, }; - let ret = call_runtime_at.call_api_at(params)?; + let ret = #crate_::CallApiAt::::call_api_at(call_runtime_at, params)?; return Ok(ret) } @@ -433,7 +429,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { recorder, }; - call_runtime_at.call_api_at(params) + #crate_::CallApiAt::::call_api_at(call_runtime_at, params) } )); } @@ -681,13 +677,13 @@ impl<'a> ToClientSideDecl<'a> { #native_handling }, #crate_::NativeOrEncoded::Encoded(r) => { - <#ret_type as #crate_::Decode>::decode(&mut &r[..]) - .map_err(|err| - #crate_::ApiError::FailedToDecodeReturnValue { - function: #function_name, - error: err, - } - ) + std::result::Result::map_err( + <#ret_type as #crate_::Decode>::decode(&mut &r[..]), + |err| #crate_::ApiError::FailedToDecodeReturnValue { + function: #function_name, + error: err, + } + ) } } ) @@ -752,8 +748,10 @@ fn parse_runtime_api_version(version: &Attribute) -> Result { /// Generates the identifier as const variable for the given `trait_name` /// by hashing the `trait_name`. fn generate_runtime_api_id(trait_name: &str) -> TokenStream { + use blake2::digest::{consts::U8, Digest}; + let mut res = [0; 8]; - res.copy_from_slice(blake2_rfc::blake2b::blake2b(8, &[], trait_name.as_bytes()).as_bytes()); + res.copy_from_slice(blake2::Blake2b::::digest(trait_name).as_slice()); quote!( const ID: [u8; 8] = [ #( #res ),* ]; ) } @@ -786,7 +784,7 @@ fn generate_runtime_info_impl(trait_: &ItemTrait, version: u64) -> TokenStream { quote!( #[cfg(any(feature = "std", test))] impl < #( #impl_generics, )* > #crate_::RuntimeApiInfo - for #trait_name < #( #ty_generics, )* > + for dyn #trait_name < #( #ty_generics, )* > { #id #version diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index bc0f027e1efa..f594a743fcf9 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -86,7 +86,7 @@ fn generate_impl_call( let (#( #pnames ),*) : ( #( #ptypes ),* ) = match #c::DecodeLimit::decode_all_with_depth_limit( #c::MAX_EXTRINSIC_DEPTH, - &#input, + &mut #input, ) { Ok(res) => res, Err(e) => panic!("Bad input data provided to {}: {}", #fn_name_str, e), @@ -207,7 +207,7 @@ fn generate_runtime_api_base_structures() -> Result { storage_transaction_cache: std::cell::RefCell< #crate_::StorageTransactionCache >, - recorder: Option<#crate_::ProofRecorder>, + recorder: std::option::Option<#crate_::ProofRecorder>, } // `RuntimeApi` itself is not threadsafe. However, an instance is only available in a @@ -233,12 +233,12 @@ fn generate_runtime_api_base_structures() -> Result { &self, call: F, ) -> R where Self: Sized { - self.changes.borrow_mut().start_transaction(); - *self.commit_on_success.borrow_mut() = false; + #crate_::OverlayedChanges::start_transaction(&mut std::cell::RefCell::borrow_mut(&self.changes)); + *std::cell::RefCell::borrow_mut(&self.commit_on_success) = false; let res = call(self); - *self.commit_on_success.borrow_mut() = true; + *std::cell::RefCell::borrow_mut(&self.commit_on_success) = true; - self.commit_or_rollback(matches!(res, #crate_::TransactionOutcome::Commit(_))); + self.commit_or_rollback(std::matches!(res, #crate_::TransactionOutcome::Commit(_))); res.into_inner() } @@ -247,9 +247,8 @@ fn generate_runtime_api_base_structures() -> Result { &self, at: &#crate_::BlockId, ) -> std::result::Result where Self: Sized { - self.call - .runtime_version_at(at) - .map(|v| v.has_api_with(&A::ID, |v| v == A::VERSION)) + #crate_::CallApiAt::::runtime_version_at(self.call, at) + .map(|v| #crate_::RuntimeVersion::has_api_with(&v, &A::ID, |v| v == A::VERSION)) } fn has_api_with bool>( @@ -257,51 +256,49 @@ fn generate_runtime_api_base_structures() -> Result { at: &#crate_::BlockId, pred: P, ) -> std::result::Result where Self: Sized { - self.call - .runtime_version_at(at) - .map(|v| v.has_api_with(&A::ID, pred)) + #crate_::CallApiAt::::runtime_version_at(self.call, at) + .map(|v| #crate_::RuntimeVersion::has_api_with(&v, &A::ID, pred)) } fn api_version( &self, at: &#crate_::BlockId, ) -> std::result::Result, #crate_::ApiError> where Self: Sized { - self.call - .runtime_version_at(at) - .map(|v| v.api_version(&A::ID)) + #crate_::CallApiAt::::runtime_version_at(self.call, at) + .map(|v| #crate_::RuntimeVersion::api_version(&v, &A::ID)) } fn record_proof(&mut self) { - self.recorder = Some(Default::default()); + self.recorder = std::option::Option::Some(std::default::Default::default()); } - fn proof_recorder(&self) -> Option<#crate_::ProofRecorder> { - self.recorder.clone() + fn proof_recorder(&self) -> std::option::Option<#crate_::ProofRecorder> { + std::clone::Clone::clone(&self.recorder) } - fn extract_proof(&mut self) -> Option<#crate_::StorageProof> { - self.recorder - .take() - .map(|recorder| recorder.to_storage_proof()) + fn extract_proof(&mut self) -> std::option::Option<#crate_::StorageProof> { + std::option::Option::take(&mut self.recorder) + .map(|recorder| #crate_::ProofRecorder::::to_storage_proof(&recorder)) } fn into_storage_changes( &self, backend: &Self::StateBackend, - changes_trie_state: Option<&#crate_::ChangesTrieState< - #crate_::HashFor, - #crate_::NumberFor, - >>, parent_hash: Block::Hash, - ) -> std::result::Result< + ) -> core::result::Result< #crate_::StorageChanges, String > where Self: Sized { - self.changes.replace(Default::default()).into_storage_changes( + let at = #crate_::BlockId::Hash(std::clone::Clone::clone(&parent_hash)); + let state_version = #crate_::CallApiAt::::runtime_version_at(self.call, &at) + .map(|v| #crate_::RuntimeVersion::state_version(&v)) + .map_err(|e| format!("Failed to get state version: {}", e))?; + + #crate_::OverlayedChanges::into_storage_changes( + std::cell::RefCell::take(&self.changes), backend, - changes_trie_state, - parent_hash, - self.storage_transaction_cache.replace(Default::default()), + core::cell::RefCell::take(&self.storage_transaction_cache), + state_version, ) } } @@ -320,9 +317,9 @@ fn generate_runtime_api_base_structures() -> Result { RuntimeApiImpl { call: unsafe { std::mem::transmute(call) }, commit_on_success: true.into(), - changes: Default::default(), - recorder: Default::default(), - storage_transaction_cache: Default::default(), + changes: std::default::Default::default(), + recorder: std::default::Default::default(), + storage_transaction_cache: std::default::Default::default(), }.into() } } @@ -330,20 +327,22 @@ fn generate_runtime_api_base_structures() -> Result { #[cfg(any(feature = "std", test))] impl> RuntimeApiImpl { fn call_api_at< - R: #crate_::Encode + #crate_::Decode + PartialEq, + R: #crate_::Encode + #crate_::Decode + std::cmp::PartialEq, F: FnOnce( &C, &std::cell::RefCell<#crate_::OverlayedChanges>, &std::cell::RefCell<#crate_::StorageTransactionCache>, - &Option<#crate_::ProofRecorder>, + &std::option::Option<#crate_::ProofRecorder>, ) -> std::result::Result<#crate_::NativeOrEncoded, E>, E, >( &self, call_api_at: F, ) -> std::result::Result<#crate_::NativeOrEncoded, E> { - if *self.commit_on_success.borrow() { - self.changes.borrow_mut().start_transaction(); + if *std::cell::RefCell::borrow(&self.commit_on_success) { + #crate_::OverlayedChanges::start_transaction( + &mut std::cell::RefCell::borrow_mut(&self.changes) + ); } let res = call_api_at( &self.call, @@ -352,7 +351,7 @@ fn generate_runtime_api_base_structures() -> Result { &self.recorder, ); - self.commit_or_rollback(res.is_ok()); + self.commit_or_rollback(std::result::Result::is_ok(&res)); res } @@ -361,13 +360,19 @@ fn generate_runtime_api_base_structures() -> Result { We only close a transaction when we opened one ourself. Other parts of the runtime that make use of transactions (state-machine) also balance their transactions. The runtime cannot close client initiated - transactions. qed"; - if *self.commit_on_success.borrow() { - if commit { - self.changes.borrow_mut().commit_transaction().expect(proof); + transactions; qed"; + if *std::cell::RefCell::borrow(&self.commit_on_success) { + let res = if commit { + #crate_::OverlayedChanges::commit_transaction( + &mut std::cell::RefCell::borrow_mut(&self.changes) + ) } else { - self.changes.borrow_mut().rollback_transaction().expect(proof); - } + #crate_::OverlayedChanges::rollback_transaction( + &mut std::cell::RefCell::borrow_mut(&self.changes) + ) + }; + + std::result::Result::expect(res, proof); } } } diff --git a/primitives/api/proc-macro/src/lib.rs b/primitives/api/proc-macro/src/lib.rs index b8731d70ca3c..20a2f76f2c83 100644 --- a/primitives/api/proc-macro/src/lib.rs +++ b/primitives/api/proc-macro/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs b/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs index 77f8a07f85c4..ffc158ac94d2 100644 --- a/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -116,10 +116,6 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result, - #crate_::NumberFor<#block_type>, - >>, _: <#block_type as #crate_::BlockT>::Hash, ) -> std::result::Result< #crate_::StorageChanges, diff --git a/primitives/api/proc-macro/src/utils.rs b/primitives/api/proc-macro/src/utils.rs index a3f21638751e..2aa6a657aa9c 100644 --- a/primitives/api/proc-macro/src/utils.rs +++ b/primitives/api/proc-macro/src/utils.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index 82954d193e60..964ef15ce5f5 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -84,6 +84,8 @@ pub use sp_core::NativeOrEncoded; use sp_core::OpaqueMetadata; #[doc(hidden)] pub use sp_core::{offchain, ExecutionContext}; +#[cfg(feature = "std")] +pub use sp_runtime::StateVersion; #[doc(hidden)] pub use sp_runtime::{ generic::BlockId, @@ -97,7 +99,7 @@ pub use sp_runtime::{ #[doc(hidden)] #[cfg(feature = "std")] pub use sp_state_machine::{ - Backend as StateBackend, ChangesTrieState, InMemoryBackend, OverlayedChanges, StorageProof, + Backend as StateBackend, InMemoryBackend, OverlayedChanges, StorageProof, }; #[cfg(feature = "std")] use sp_std::result; @@ -269,6 +271,7 @@ pub use sp_api_proc_macro::decl_runtime_apis; /// // Here we are exposing the runtime api versions. /// apis: RUNTIME_API_VERSIONS, /// transaction_version: 1, +/// state_version: 1, /// }; /// /// # fn main() {} @@ -394,14 +397,12 @@ pub type ProofRecorder = sp_state_machine::ProofRecorder<::Hash> pub type StorageTransactionCache = sp_state_machine::StorageTransactionCache< >>::Transaction, HashFor, - NumberFor, >; #[cfg(feature = "std")] pub type StorageChanges = sp_state_machine::StorageChanges< >>::Transaction, HashFor, - NumberFor, >; /// Extract the state backend type for a type that implements `ProvideRuntimeApi`. @@ -514,7 +515,6 @@ pub trait ApiExt { fn into_storage_changes( &self, backend: &Self::StateBackend, - changes_trie_state: Option<&ChangesTrieState, NumberFor>>, parent_hash: Block::Hash, ) -> Result, String> where @@ -644,8 +644,6 @@ pub const fn serialize_runtime_api_info(id: [u8; 8], version: u32) -> [u8; RUNTI /// Deserialize the runtime API info serialized by [`serialize_runtime_api_info`]. pub fn deserialize_runtime_api_info(bytes: [u8; RUNTIME_API_INFO_SIZE]) -> ([u8; 8], u32) { - use sp_std::convert::TryInto; - let id: [u8; 8] = bytes[0..8] .try_into() .expect("the source slice size is equal to the dest array length; qed"); @@ -659,53 +657,13 @@ pub fn deserialize_runtime_api_info(bytes: [u8; RUNTIME_API_INFO_SIZE]) -> ([u8; (id, version) } -#[derive(codec::Encode, codec::Decode)] -pub struct OldRuntimeVersion { - pub spec_name: RuntimeString, - pub impl_name: RuntimeString, - pub authoring_version: u32, - pub spec_version: u32, - pub impl_version: u32, - pub apis: ApisVec, -} - -impl From for RuntimeVersion { - fn from(x: OldRuntimeVersion) -> Self { - Self { - spec_name: x.spec_name, - impl_name: x.impl_name, - authoring_version: x.authoring_version, - spec_version: x.spec_version, - impl_version: x.impl_version, - apis: x.apis, - transaction_version: 1, - } - } -} - -impl From for OldRuntimeVersion { - fn from(x: RuntimeVersion) -> Self { - Self { - spec_name: x.spec_name, - impl_name: x.impl_name, - authoring_version: x.authoring_version, - spec_version: x.spec_version, - impl_version: x.impl_version, - apis: x.apis, - } - } -} - decl_runtime_apis! { /// The `Core` runtime api that every Substrate runtime needs to implement. #[core_trait] - #[api_version(3)] + #[api_version(4)] pub trait Core { /// Returns the version of the runtime. fn version() -> RuntimeVersion; - /// Returns the version of the runtime. - #[changed_in(3)] - fn version() -> OldRuntimeVersion; /// Execute the given block. fn execute_block(block: Block); /// Initialize a block with the given header. diff --git a/primitives/api/test/Cargo.toml b/primitives/api/test/Cargo.toml index b78c9abb80dc..9f9f399234db 100644 --- a/primitives/api/test/Cargo.toml +++ b/primitives/api/test/Cargo.toml @@ -2,10 +2,10 @@ name = "sp-api-test" version = "2.0.1" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" publish = false -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] @@ -14,21 +14,21 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sp-api = { version = "4.0.0-dev", path = "../" } substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } -sp-version = { version = "4.0.0-dev", path = "../../version" } -sp-tracing = { version = "4.0.0-dev", path = "../../tracing" } -sp-runtime = { version = "4.0.0-dev", path = "../../runtime" } +sp-version = { version = "5.0.0", path = "../../version" } +sp-tracing = { version = "5.0.0", path = "../../tracing" } +sp-runtime = { version = "6.0.0", path = "../../runtime" } sp-consensus = { version = "0.10.0-dev", path = "../../consensus/common" } sc-block-builder = { version = "0.10.0-dev", path = "../../../client/block-builder" } -codec = { package = "parity-scale-codec", version = "2.0.0" } -sp-state-machine = { version = "0.10.0-dev", path = "../../state-machine" } -trybuild = "1.0.43" -rustversion = "1.0.0" +codec = { package = "parity-scale-codec", version = "3.0.0" } +sp-state-machine = { version = "0.12.0", path = "../../state-machine" } +trybuild = "1.0.53" +rustversion = "1.0.6" [dev-dependencies] criterion = "0.3.0" -futures = "0.3.9" +futures = "0.3.21" log = "0.4.14" -sp-core = { version = "4.0.0-dev", path = "../../core" } +sp-core = { version = "6.0.0", path = "../../core" } [[bench]] name = "bench" diff --git a/primitives/api/test/benches/bench.rs b/primitives/api/test/benches/bench.rs index b3d96a2db6a5..2682c91f9410 100644 --- a/primitives/api/test/benches/bench.rs +++ b/primitives/api/test/benches/bench.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/api/test/tests/decl_and_impl.rs b/primitives/api/test/tests/decl_and_impl.rs index 8d1b04a37a9f..1db416a1d3db 100644 --- a/primitives/api/test/tests/decl_and_impl.rs +++ b/primitives/api/test/tests/decl_and_impl.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/api/test/tests/runtime_calls.rs b/primitives/api/test/tests/runtime_calls.rs index 101f92fd6c7d..ba42b342377c 100644 --- a/primitives/api/test/tests/runtime_calls.rs +++ b/primitives/api/test/tests/runtime_calls.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -59,7 +59,7 @@ fn calling_native_runtime_function_with_non_decodable_parameter() { .build(); let runtime_api = client.runtime_api(); let block_id = BlockId::Number(client.chain_info().best_number); - runtime_api.fail_convert_parameter(&block_id, DecodeFails::new()).unwrap(); + runtime_api.fail_convert_parameter(&block_id, DecodeFails::default()).unwrap(); } #[test] @@ -187,7 +187,7 @@ fn record_proof_works() { amount: 1000, nonce: 0, from: AccountKeyring::Alice.into(), - to: Default::default(), + to: AccountKeyring::Bob.into(), } .into_signed_tx(); @@ -210,8 +210,9 @@ fn record_proof_works() { WasmExecutionMethod::Interpreted, None, 8, + 2, ); - execution_proof_check_on_trie_backend::<_, u64, _, _>( + execution_proof_check_on_trie_backend( &backend, &mut overlay, &executor, diff --git a/primitives/api/test/tests/trybuild.rs b/primitives/api/test/tests/trybuild.rs index 5a6025f463af..9e3af145dc56 100644 --- a/primitives/api/test/tests/trybuild.rs +++ b/primitives/api/test/tests/trybuild.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr b/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr index 063cbff60f81..d11aebbf149b 100644 --- a/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr +++ b/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.stderr @@ -58,7 +58,10 @@ error[E0308]: mismatched types --> $DIR/type_reference_in_impl_runtime_apis_call.rs:19:11 | 19 | fn test(data: &u64) { - | ^^^^^^^ - | | - | expected `u64`, found `&u64` - | help: consider removing the borrow: `data` + | ^^^^^^^ expected `u64`, found `&u64` + | +help: consider removing the borrow + | +19 - fn test(data: &u64) { +19 + fn test(data: &u64) { + | diff --git a/primitives/application-crypto/Cargo.toml b/primitives/application-crypto/Cargo.toml index 6849dc25f856..a8181ca5380c 100644 --- a/primitives/application-crypto/Cargo.toml +++ b/primitives/application-crypto/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "sp-application-crypto" -version = "4.0.0-dev" +version = "6.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" description = "Provides facilities for generating application specific crypto wrapper types." license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sp-application-crypto" readme = "README.md" @@ -15,12 +15,12 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.126", optional = true, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../io" } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +serde = { version = "1.0.136", optional = true, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +sp-io = { version = "6.0.0", default-features = false, path = "../io" } [features] default = [ "std" ] diff --git a/primitives/application-crypto/src/ecdsa.rs b/primitives/application-crypto/src/ecdsa.rs index 915e16ba3b1a..6a0eb7ab2f84 100644 --- a/primitives/application-crypto/src/ecdsa.rs +++ b/primitives/application-crypto/src/ecdsa.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -57,6 +57,6 @@ impl RuntimePublic for Public { } fn to_raw_vec(&self) -> Vec { - sp_core::crypto::Public::to_raw_vec(self) + sp_core::crypto::ByteArray::to_raw_vec(self) } } diff --git a/primitives/application-crypto/src/ed25519.rs b/primitives/application-crypto/src/ed25519.rs index 09ce48fcb274..f5ec40233ca9 100644 --- a/primitives/application-crypto/src/ed25519.rs +++ b/primitives/application-crypto/src/ed25519.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -57,6 +57,6 @@ impl RuntimePublic for Public { } fn to_raw_vec(&self) -> Vec { - sp_core::crypto::Public::to_raw_vec(self) + sp_core::crypto::ByteArray::to_raw_vec(self) } } diff --git a/primitives/application-crypto/src/lib.rs b/primitives/application-crypto/src/lib.rs index baa656066705..05f89c40ef99 100644 --- a/primitives/application-crypto/src/lib.rs +++ b/primitives/application-crypto/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,7 +27,10 @@ pub use sp_core::crypto::{DeriveJunction, Pair, SecretStringError, Ss58Codec}; #[doc(hidden)] pub use sp_core::{ self, - crypto::{CryptoType, CryptoTypePublicPair, Derive, IsWrappedBy, Public, Wraps}, + crypto::{ + ByteArray, CryptoType, CryptoTypePublicPair, Derive, IsWrappedBy, Public, UncheckedFrom, + Wraps, + }, RuntimeDebug, }; @@ -39,7 +42,7 @@ pub use scale_info; #[cfg(feature = "std")] pub use serde; #[doc(hidden)] -pub use sp_std::{convert::TryFrom, ops::Deref, vec::Vec}; +pub use sp_std::{ops::Deref, vec::Vec}; pub mod ecdsa; pub mod ed25519; @@ -221,7 +224,7 @@ macro_rules! app_crypto_public_full_crypto { $crate::wrap! { /// A generic `AppPublic` wrapper type over $public crypto; this has no specific App. #[derive( - Clone, Default, Eq, Hash, PartialEq, PartialOrd, Ord, + Clone, Eq, Hash, PartialEq, PartialOrd, Ord, $crate::codec::Encode, $crate::codec::Decode, $crate::RuntimeDebug, @@ -258,7 +261,7 @@ macro_rules! app_crypto_public_not_full_crypto { $crate::wrap! { /// A generic `AppPublic` wrapper type over $public crypto; this has no specific App. #[derive( - Clone, Default, Eq, PartialEq, Ord, PartialOrd, + Clone, Eq, PartialEq, Ord, PartialOrd, $crate::codec::Encode, $crate::codec::Decode, $crate::RuntimeDebug, @@ -301,13 +304,12 @@ macro_rules! app_crypto_public_common { } } + impl $crate::ByteArray for Public { + const LEN: usize = <$public>::LEN; + } impl $crate::Public for Public { - fn from_slice(x: &[u8]) -> Self { - Self(<$public>::from_slice(x)) - } - fn to_public_crypto_pair(&self) -> $crate::CryptoTypePublicPair { - $crate::CryptoTypePublicPair($crypto_type, self.to_raw_vec()) + $crate::CryptoTypePublicPair($crypto_type, $crate::ByteArray::to_raw_vec(self)) } } @@ -357,11 +359,11 @@ macro_rules! app_crypto_public_common { impl From<&Public> for $crate::CryptoTypePublicPair { fn from(key: &Public) -> Self { - $crate::CryptoTypePublicPair($crypto_type, $crate::Public::to_raw_vec(key)) + $crate::CryptoTypePublicPair($crypto_type, $crate::ByteArray::to_raw_vec(key)) } } - impl<'a> $crate::TryFrom<&'a [u8]> for Public { + impl<'a> TryFrom<&'a [u8]> for Public { type Error = (); fn try_from(data: &'a [u8]) -> Result { @@ -435,7 +437,7 @@ macro_rules! app_crypto_signature_full_crypto { ($sig:ty, $key_type:expr, $crypto_type:expr) => { $crate::wrap! { /// A generic `AppPublic` wrapper type over $public crypto; this has no specific App. - #[derive(Clone, Default, Eq, PartialEq, + #[derive(Clone, Eq, PartialEq, $crate::codec::Encode, $crate::codec::Decode, $crate::RuntimeDebug, @@ -470,7 +472,7 @@ macro_rules! app_crypto_signature_not_full_crypto { ($sig:ty, $key_type:expr, $crypto_type:expr) => { $crate::wrap! { /// A generic `AppPublic` wrapper type over $public crypto; this has no specific App. - #[derive(Clone, Default, Eq, PartialEq, + #[derive(Clone, Eq, PartialEq, $crate::codec::Encode, $crate::codec::Decode, $crate::scale_info::TypeInfo, @@ -516,11 +518,19 @@ macro_rules! app_crypto_signature_common { type Generic = $sig; } - impl $crate::TryFrom<$crate::Vec> for Signature { + impl<'a> TryFrom<&'a [u8]> for Signature { + type Error = (); + + fn try_from(data: &'a [u8]) -> Result { + <$sig>::try_from(data).map(Into::into) + } + } + + impl TryFrom<$crate::Vec> for Signature { type Error = (); fn try_from(data: $crate::Vec) -> Result { - Ok(<$sig>::try_from(data.as_slice())?.into()) + Self::try_from(&data[..]) } } }; diff --git a/primitives/application-crypto/src/sr25519.rs b/primitives/application-crypto/src/sr25519.rs index f51236f2ab38..81c5320efd71 100644 --- a/primitives/application-crypto/src/sr25519.rs +++ b/primitives/application-crypto/src/sr25519.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -57,6 +57,6 @@ impl RuntimePublic for Public { } fn to_raw_vec(&self) -> Vec { - sp_core::crypto::Public::to_raw_vec(self) + sp_core::crypto::ByteArray::to_raw_vec(self) } } diff --git a/primitives/application-crypto/src/traits.rs b/primitives/application-crypto/src/traits.rs index 376d12f0c7a3..7a99c144b69f 100644 --- a/primitives/application-crypto/src/traits.rs +++ b/primitives/application-crypto/src/traits.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/application-crypto/test/Cargo.toml b/primitives/application-crypto/test/Cargo.toml index 468bfee3cc01..9e93e78f69ff 100644 --- a/primitives/application-crypto/test/Cargo.toml +++ b/primitives/application-crypto/test/Cargo.toml @@ -2,20 +2,20 @@ name = "sp-application-crypto-test" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" description = "Integration tests for application-crypto" license = "Apache-2.0" publish = false -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../core" } -sp-keystore = { version = "0.10.0-dev", path = "../../keystore", default-features = false } +sp-core = { version = "6.0.0", default-features = false, path = "../../core" } +sp-keystore = { version = "0.12.0", path = "../../keystore", default-features = false } substrate-test-runtime-client = { version = "2.0.0", path = "../../../test-utils/runtime/client" } -sp-runtime = { version = "4.0.0-dev", path = "../../runtime" } +sp-runtime = { version = "6.0.0", path = "../../runtime" } sp-api = { version = "4.0.0-dev", path = "../../api" } -sp-application-crypto = { version = "4.0.0-dev", path = "../" } +sp-application-crypto = { version = "6.0.0", path = "../" } diff --git a/primitives/application-crypto/test/src/ecdsa.rs b/primitives/application-crypto/test/src/ecdsa.rs index c4aa6a2afbd6..e45a3d5f8f86 100644 --- a/primitives/application-crypto/test/src/ecdsa.rs +++ b/primitives/application-crypto/test/src/ecdsa.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/application-crypto/test/src/ed25519.rs b/primitives/application-crypto/test/src/ed25519.rs index 7cfd801388c7..ef2df9fe9196 100644 --- a/primitives/application-crypto/test/src/ed25519.rs +++ b/primitives/application-crypto/test/src/ed25519.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/application-crypto/test/src/lib.rs b/primitives/application-crypto/test/src/lib.rs index 6b7734764e79..7cc3f8b0780e 100644 --- a/primitives/application-crypto/test/src/lib.rs +++ b/primitives/application-crypto/test/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/application-crypto/test/src/sr25519.rs b/primitives/application-crypto/test/src/sr25519.rs index 12dfbc609fb0..e15ffe82a1c3 100644 --- a/primitives/application-crypto/test/src/sr25519.rs +++ b/primitives/application-crypto/test/src/sr25519.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/arithmetic/Cargo.toml b/primitives/arithmetic/Cargo.toml index abdbd4e60d04..26ee7677363c 100644 --- a/primitives/arithmetic/Cargo.toml +++ b/primitives/arithmetic/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-arithmetic" -version = "4.0.0-dev" +version = "5.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Minimal fixed point arithmetic primitives and types for runtime." documentation = "https://docs.rs/sp-arithmetic" @@ -15,21 +15,22 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", + "max-encoded-len", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } integer-sqrt = "0.1.2" static_assertions = "1.1.0" num-traits = { version = "0.2.8", default-features = false } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -serde = { version = "1.0.126", optional = true, features = ["derive"] } -sp-debug-derive = { version = "3.0.0", default-features = false, path = "../debug-derive" } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +serde = { version = "1.0.136", optional = true, features = ["derive"] } +sp-debug-derive = { version = "4.0.0", default-features = false, path = "../debug-derive" } [dev-dependencies] rand = "0.7.2" criterion = "0.3" -primitive-types = "0.10.1" +primitive-types = "0.11.1" [features] default = ["std"] diff --git a/primitives/arithmetic/benches/bench.rs b/primitives/arithmetic/benches/bench.rs index 02db00aa0bf8..3e4fafe2a4a9 100644 --- a/primitives/arithmetic/benches/bench.rs +++ b/primitives/arithmetic/benches/bench.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/arithmetic/fuzzer/Cargo.toml b/primitives/arithmetic/fuzzer/Cargo.toml index d10eccfc7c74..e51dd4e415a6 100644 --- a/primitives/arithmetic/fuzzer/Cargo.toml +++ b/primitives/arithmetic/fuzzer/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-arithmetic-fuzzer" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Fuzzer for fixed point arithmetic primitives." documentation = "https://docs.rs/sp-arithmetic-fuzzer" @@ -14,9 +14,9 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-arithmetic = { version = "4.0.0-dev", path = ".." } +sp-arithmetic = { version = "5.0.0", path = ".." } honggfuzz = "0.5.49" -primitive-types = "0.10.1" +primitive-types = "0.11.1" num-bigint = "0.2" [[bin]] diff --git a/primitives/arithmetic/fuzzer/src/biguint.rs b/primitives/arithmetic/fuzzer/src/biguint.rs index ca5b8379afff..f49743a4b8a6 100644 --- a/primitives/arithmetic/fuzzer/src/biguint.rs +++ b/primitives/arithmetic/fuzzer/src/biguint.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,7 +29,6 @@ use honggfuzz::fuzz; use sp_arithmetic::biguint::{BigUint, Single}; -use std::convert::TryFrom; fn main() { loop { diff --git a/primitives/arithmetic/fuzzer/src/fixed_point.rs b/primitives/arithmetic/fuzzer/src/fixed_point.rs index d8f058ae51e2..c1b93f8c63a1 100644 --- a/primitives/arithmetic/fuzzer/src/fixed_point.rs +++ b/primitives/arithmetic/fuzzer/src/fixed_point.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/arithmetic/fuzzer/src/multiply_by_rational.rs b/primitives/arithmetic/fuzzer/src/multiply_by_rational.rs index 3089d4b09218..019cf0ec39b7 100644 --- a/primitives/arithmetic/fuzzer/src/multiply_by_rational.rs +++ b/primitives/arithmetic/fuzzer/src/multiply_by_rational.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/arithmetic/fuzzer/src/normalize.rs b/primitives/arithmetic/fuzzer/src/normalize.rs index 2662565106e6..dd717115a5c9 100644 --- a/primitives/arithmetic/fuzzer/src/normalize.rs +++ b/primitives/arithmetic/fuzzer/src/normalize.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,7 +25,6 @@ use honggfuzz::fuzz; use sp_arithmetic::Normalizable; -use std::convert::TryInto; type Ty = u64; diff --git a/primitives/arithmetic/fuzzer/src/per_thing_rational.rs b/primitives/arithmetic/fuzzer/src/per_thing_rational.rs index 7b90faa94069..7021c54c0ba0 100644 --- a/primitives/arithmetic/fuzzer/src/per_thing_rational.rs +++ b/primitives/arithmetic/fuzzer/src/per_thing_rational.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/arithmetic/src/biguint.rs b/primitives/arithmetic/src/biguint.rs index 17ed323dc0ce..33f0960ee378 100644 --- a/primitives/arithmetic/src/biguint.rs +++ b/primitives/arithmetic/src/biguint.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,7 @@ use codec::{Decode, Encode}; use num_traits::{One, Zero}; -use sp_std::{cell::RefCell, cmp::Ordering, convert::TryFrom, ops, prelude::*, vec}; +use sp_std::{cell::RefCell, cmp::Ordering, ops, prelude::*, vec}; // A sensible value for this would be half of the dword size of the host machine. Since the // runtime is compiled to 32bit webassembly, using 32 and 64 for single and double respectively @@ -664,7 +664,6 @@ pub mod tests { #[test] fn can_try_build_numbers_from_types() { - use sp_std::convert::TryFrom; assert_eq!(u64::try_from(with_limbs(1)).unwrap(), 1); assert_eq!(u64::try_from(with_limbs(2)).unwrap(), u32::MAX as u64 + 2); assert_eq!(u64::try_from(with_limbs(3)).unwrap_err(), "cannot fit a number into u64"); diff --git a/primitives/arithmetic/src/fixed_point.rs b/primitives/arithmetic/src/fixed_point.rs index 7a81f222c492..7ce17bb72611 100644 --- a/primitives/arithmetic/src/fixed_point.rs +++ b/primitives/arithmetic/src/fixed_point.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,7 +27,6 @@ use crate::{ }; use codec::{CompactAs, Decode, Encode}; use sp_std::{ - convert::{TryFrom, TryInto}, fmt::Debug, ops::{self, Add, Div, Mul, Sub}, prelude::*, @@ -122,7 +121,8 @@ pub trait FixedPointNumber: /// Creates `self` from an integer number `int`. /// /// Returns `None` if `int` exceeds accuracy. - fn checked_from_integer(int: Self::Inner) -> Option { + fn checked_from_integer>(int: N) -> Option { + let int: Self::Inner = int.into(); int.checked_mul(&Self::DIV).map(Self::from_inner) } @@ -369,6 +369,7 @@ macro_rules! implement_fixed { Default, Copy, Clone, + codec::MaxEncodedLen, PartialEq, Eq, PartialOrd, @@ -667,6 +668,15 @@ macro_rules! implement_fixed { assert!($name::DIV > 0); } + #[test] + fn has_max_encoded_len() { + struct AsMaxEncodedLen { + _data: T, + } + + let _ = AsMaxEncodedLen { _data: $name::min_value() }; + } + #[test] fn from_i129_works() { let a = I129 { value: 1, negative: true }; @@ -889,31 +899,32 @@ macro_rules! implement_fixed { let accuracy = $name::accuracy(); // Case where integer fits. - let a = $name::checked_from_integer(42).expect("42 * accuracy <= inner_max; qed"); + let a = $name::checked_from_integer::<$inner_type>(42) + .expect("42 * accuracy <= inner_max; qed"); assert_eq!(a.into_inner(), 42 * accuracy); // Max integer that fit. - let a = $name::checked_from_integer(inner_max / accuracy) + let a = $name::checked_from_integer::<$inner_type>(inner_max / accuracy) .expect("(inner_max / accuracy) * accuracy <= inner_max; qed"); assert_eq!(a.into_inner(), (inner_max / accuracy) * accuracy); // Case where integer doesn't fit, so it returns `None`. - let a = $name::checked_from_integer(inner_max / accuracy + 1); + let a = $name::checked_from_integer::<$inner_type>(inner_max / accuracy + 1); assert_eq!(a, None); if $name::SIGNED { // Case where integer fits. - let a = $name::checked_from_integer(0.saturating_sub(42)) + let a = $name::checked_from_integer::<$inner_type>(0.saturating_sub(42)) .expect("-42 * accuracy >= inner_min; qed"); assert_eq!(a.into_inner(), 0 - 42 * accuracy); // Min integer that fit. - let a = $name::checked_from_integer(inner_min / accuracy) + let a = $name::checked_from_integer::<$inner_type>(inner_min / accuracy) .expect("(inner_min / accuracy) * accuracy <= inner_min; qed"); assert_eq!(a.into_inner(), (inner_min / accuracy) * accuracy); // Case where integer doesn't fit, so it returns `None`. - let a = $name::checked_from_integer(inner_min / accuracy - 1); + let a = $name::checked_from_integer::<$inner_type>(inner_min / accuracy - 1); assert_eq!(a, None); } } diff --git a/primitives/arithmetic/src/helpers_128bit.rs b/primitives/arithmetic/src/helpers_128bit.rs index bbf69ea359fe..735b11287cbe 100644 --- a/primitives/arithmetic/src/helpers_128bit.rs +++ b/primitives/arithmetic/src/helpers_128bit.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,7 +24,6 @@ use crate::biguint; use num_traits::Zero; use sp_std::{ cmp::{max, min}, - convert::TryInto, mem, }; diff --git a/primitives/arithmetic/src/lib.rs b/primitives/arithmetic/src/lib.rs index 8671ceb0396e..729da123757c 100644 --- a/primitives/arithmetic/src/lib.rs +++ b/primitives/arithmetic/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -44,7 +44,7 @@ pub use fixed_point::{FixedI128, FixedI64, FixedPointNumber, FixedPointOperand, pub use per_things::{InnerOf, PerThing, PerU16, Perbill, Percent, Permill, Perquintill, UpperOf}; pub use rational::{Rational128, RationalInfinite}; -use sp_std::{cmp::Ordering, convert::TryInto, fmt::Debug, prelude::*}; +use sp_std::{cmp::Ordering, fmt::Debug, prelude::*}; use traits::{BaseArithmetic, One, SaturatedConversion, Unsigned, Zero}; /// Trait for comparing two numbers with an threshold. @@ -55,7 +55,7 @@ use traits::{BaseArithmetic, One, SaturatedConversion, Unsigned, Zero}; /// - `Ordering::Equal` otherwise. pub trait ThresholdOrd { /// Compare if `self` is `threshold` greater or less than `other`. - fn tcmp(&self, other: &T, epsilon: T) -> Ordering; + fn tcmp(&self, other: &T, threshold: T) -> Ordering; } impl ThresholdOrd for T diff --git a/primitives/arithmetic/src/per_things.rs b/primitives/arithmetic/src/per_things.rs index f388c19de6b4..1b9e6d91a2cd 100644 --- a/primitives/arithmetic/src/per_things.rs +++ b/primitives/arithmetic/src/per_things.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,9 +24,7 @@ use crate::traits::{ }; use codec::{CompactAs, Encode}; use num_traits::{Pow, SaturatingAdd, SaturatingSub}; -use sp_debug_derive::RuntimeDebug; use sp_std::{ - convert::{TryFrom, TryInto}, fmt, ops, ops::{Add, Sub}, prelude::*, @@ -425,7 +423,7 @@ macro_rules! implement_per_thing { /// #[doc = $title] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] - #[derive(Encode, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug, scale_info::TypeInfo)] + #[derive(Encode, Copy, Clone, PartialEq, Eq, codec::MaxEncodedLen, PartialOrd, Ord, sp_std::fmt::Debug, scale_info::TypeInfo)] pub struct $name($type); /// Implementation makes any compact encoding of `PerThing::Inner` valid, @@ -833,11 +831,21 @@ macro_rules! implement_per_thing { } } + impl $crate::traits::Zero for $name { + fn zero() -> Self { + Self::zero() + } + + fn is_zero(&self) -> bool { + self == &Self::zero() + } + } + #[cfg(test)] mod $test_mod { use codec::{Encode, Decode}; - use super::{$name, Saturating, RuntimeDebug, PerThing}; + use super::{$name, Saturating, PerThing}; use crate::traits::Zero; #[test] @@ -861,7 +869,7 @@ macro_rules! implement_per_thing { assert!(<$upper_type>::from($max) * <$upper_type>::from($max) < <$upper_type>::max_value()); } - #[derive(Encode, Decode, PartialEq, Eq, RuntimeDebug)] + #[derive(Encode, Decode, PartialEq, Eq, Debug)] struct WithCompact { data: T, } @@ -895,6 +903,15 @@ macro_rules! implement_per_thing { } } + #[test] + fn has_max_encoded_len() { + struct AsMaxEncodedLen { + _data: T, + } + + let _ = AsMaxEncodedLen { _data: $name(1) }; + } + #[test] fn fail_on_invalid_encoded_value() { let value = <$upper_type>::from($max) * 2; diff --git a/primitives/arithmetic/src/rational.rs b/primitives/arithmetic/src/rational.rs index 225e1d952182..63ae6e65bc9e 100644 --- a/primitives/arithmetic/src/rational.rs +++ b/primitives/arithmetic/src/rational.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/arithmetic/src/traits.rs b/primitives/arithmetic/src/traits.rs index 53341117b1fe..748aaed2a7cf 100644 --- a/primitives/arithmetic/src/traits.rs +++ b/primitives/arithmetic/src/traits.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,12 +23,8 @@ pub use num_traits::{ checked_pow, Bounded, CheckedAdd, CheckedDiv, CheckedMul, CheckedNeg, CheckedRem, CheckedShl, CheckedShr, CheckedSub, One, Signed, Unsigned, Zero, }; -use sp_std::{ - self, - convert::{TryFrom, TryInto}, - ops::{ - Add, AddAssign, Div, DivAssign, Mul, MulAssign, Rem, RemAssign, Shl, Shr, Sub, SubAssign, - }, +use sp_std::ops::{ + Add, AddAssign, Div, DivAssign, Mul, MulAssign, Rem, RemAssign, Shl, Shr, Sub, SubAssign, }; /// A meta trait for arithmetic type operations, regardless of any limitation on size. diff --git a/primitives/authority-discovery/Cargo.toml b/primitives/authority-discovery/Cargo.toml index 6638e478b4cd..c452aaa89202 100644 --- a/primitives/authority-discovery/Cargo.toml +++ b/primitives/authority-discovery/Cargo.toml @@ -3,9 +3,9 @@ name = "sp-authority-discovery" version = "4.0.0-dev" authors = ["Parity Technologies "] description = "Authority discovery primitives" -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,12 +13,12 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../application-crypto" } -codec = { package = "parity-scale-codec", default-features = false, version = "2.0.0" } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../application-crypto" } +codec = { package = "parity-scale-codec", default-features = false, version = "3.0.0" } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } sp-api = { version = "4.0.0-dev", default-features = false, path = "../api" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } [features] default = ["std"] diff --git a/primitives/authority-discovery/src/lib.rs b/primitives/authority-discovery/src/lib.rs index 871a35e6bf48..95bb458b1be8 100644 --- a/primitives/authority-discovery/src/lib.rs +++ b/primitives/authority-discovery/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/authorship/Cargo.toml b/primitives/authorship/Cargo.toml index 15e4dc57ff5a..75e94b895f12 100644 --- a/primitives/authorship/Cargo.toml +++ b/primitives/authorship/Cargo.toml @@ -3,9 +3,9 @@ name = "sp-authorship" version = "4.0.0-dev" authors = ["Parity Technologies "] description = "Authorship primitives" -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -14,9 +14,9 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../inherents" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } async-trait = { version = "0.1.50", optional = true } [features] diff --git a/primitives/authorship/src/lib.rs b/primitives/authorship/src/lib.rs index ac4b5fd315dc..7ea19d9ea5ff 100644 --- a/primitives/authorship/src/lib.rs +++ b/primitives/authorship/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -88,13 +88,13 @@ impl sp_inherents::InherentDataProvider for InherentDataProvider async fn try_handle_error( &self, identifier: &InherentIdentifier, - error: &[u8], + mut error: &[u8], ) -> Option> { if *identifier != INHERENT_IDENTIFIER { return None } - let error = InherentError::decode(&mut &error[..]).ok()?; + let error = InherentError::decode(&mut error).ok()?; Some(Err(Error::Application(Box::from(format!("{:?}", error))))) } diff --git a/primitives/beefy/Cargo.toml b/primitives/beefy/Cargo.toml index 633ac0e8fbcd..cf901f4a34fc 100644 --- a/primitives/beefy/Cargo.toml +++ b/primitives/beefy/Cargo.toml @@ -2,23 +2,30 @@ name = "beefy-primitives" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate" +description = "Primitives for BEEFY protocol." +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { version = "2.2.0", package = "parity-scale-codec", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { version = "3.0.0", package = "parity-scale-codec", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } sp-api = { version = "4.0.0-dev", path = "../api", default-features = false } -sp-application-crypto = { version = "4.0.0-dev", path = "../application-crypto", default-features = false } -sp-core = { version = "4.0.0-dev", path = "../core", default-features = false } -sp-runtime = { version = "4.0.0-dev", path = "../runtime", default-features = false } -sp-std = { version = "4.0.0-dev", path = "../std", default-features = false } +sp-application-crypto = { version = "6.0.0", path = "../application-crypto", default-features = false } +sp-core = { version = "6.0.0", path = "../core", default-features = false } +sp-runtime = { version = "6.0.0", path = "../runtime", default-features = false } +sp-std = { version = "4.0.0", path = "../std", default-features = false } [dev-dependencies] +hex = "0.4.3" hex-literal = "0.3" - -sp-keystore = { version = "0.10.0-dev", path = "../keystore" } +sp-keystore = { version = "0.12.0", path = "../keystore" } [features] default = ["std"] diff --git a/primitives/beefy/src/commitment.rs b/primitives/beefy/src/commitment.rs index 7aab93bbcb97..ed392139de13 100644 --- a/primitives/beefy/src/commitment.rs +++ b/primitives/beefy/src/commitment.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,26 +15,83 @@ // See the License for the specific language governing permissions and // limitations under the License. +use codec::{Decode, Encode, Error, Input}; use sp_std::{cmp, prelude::*}; -use crate::{crypto::Signature, ValidatorSetId}; +use crate::ValidatorSetId; + +/// Id of different payloads in the [`Commitment`] data +pub type BeefyPayloadId = [u8; 2]; + +/// Registry of all known [`BeefyPayloadId`]. +pub mod known_payload_ids { + use crate::BeefyPayloadId; + + /// A [`Payload`](super::Payload) identifier for Merkle Mountain Range root hash. + /// + /// Encoded value should contain a [`crate::MmrRootHash`] type (i.e. 32-bytes hash). + pub const MMR_ROOT_ID: BeefyPayloadId = *b"mh"; +} + +/// A BEEFY payload type allowing for future extensibility of adding additional kinds of payloads. +/// +/// The idea is to store a vector of SCALE-encoded values with an extra identifier. +/// Identifiers MUST be sorted by the [`BeefyPayloadId`] to allow efficient lookup of expected +/// value. Duplicated identifiers are disallowed. It's okay for different implementations to only +/// support a subset of possible values. +#[derive(Decode, Encode, Debug, PartialEq, Eq, Clone, Ord, PartialOrd, Hash)] +pub struct Payload(Vec<(BeefyPayloadId, Vec)>); + +impl Payload { + /// Construct a new payload given an initial vallue + pub fn new(id: BeefyPayloadId, value: Vec) -> Self { + Self(vec![(id, value)]) + } + + /// Returns a raw payload under given `id`. + /// + /// If the [`BeefyPayloadId`] is not found in the payload `None` is returned. + pub fn get_raw(&self, id: &BeefyPayloadId) -> Option<&Vec> { + let index = self.0.binary_search_by(|probe| probe.0.cmp(id)).ok()?; + Some(&self.0[index].1) + } + + /// Returns a decoded payload value under given `id`. + /// + /// In case the value is not there or it cannot be decoded does not match `None` is returned. + pub fn get_decoded(&self, id: &BeefyPayloadId) -> Option { + self.get_raw(id).and_then(|raw| T::decode(&mut &raw[..]).ok()) + } + + /// Push a `Vec` with a given id into the payload vec. + /// This method will internally sort the payload vec after every push. + /// + /// Returns self to allow for daisy chaining. + pub fn push_raw(mut self, id: BeefyPayloadId, value: Vec) -> Self { + self.0.push((id, value)); + self.0.sort_by_key(|(id, _)| *id); + self + } +} /// A commitment signed by GRANDPA validators as part of BEEFY protocol. /// -/// The commitment contains a [payload] extracted from the finalized block at height [block_number]. +/// The commitment contains a [payload](Commitment::payload) extracted from the finalized block at +/// height [block_number](Commitment::block_number). /// GRANDPA validators collect signatures on commitments and a stream of such signed commitments /// (see [SignedCommitment]) forms the BEEFY protocol. -#[derive(Clone, Debug, PartialEq, Eq, codec::Encode, codec::Decode)] -pub struct Commitment { - /// The payload being signed. +#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode)] +pub struct Commitment { + /// A collection of payloads to be signed, see [`Payload`] for details. /// - /// This should be some form of cumulative representation of the chain (think MMR root hash). - /// The payload should also contain some details that allow the light client to verify next - /// validator set. The protocol does not enforce any particular format of this data, - /// nor how often it should be present in commitments, however the light client has to be - /// provided with full validator set whenever it performs the transition (i.e. importing first - /// block with [validator_set_id] incremented). - pub payload: TPayload, + /// One of the payloads should be some form of cumulative representation of the chain (think + /// MMR root hash). Additionally one of the payloads should also contain some details that + /// allow the light client to verify next validator set. The protocol does not enforce any + /// particular format of this data, nor how often it should be present in commitments, however + /// the light client has to be provided with full validator set whenever it performs the + /// transition (i.e. importing first block with + /// [validator_set_id](Commitment::validator_set_id) incremented). + pub payload: Payload, /// Finalized block number this commitment is for. /// @@ -51,25 +108,23 @@ pub struct Commitment { /// /// Validator set is changing once per epoch. The Light Client must be provided by details /// about the validator set whenever it's importing first commitment with a new - /// `validator_set_id`. Validator set data MUST be verifiable, for instance using [payload] - /// information. + /// `validator_set_id`. Validator set data MUST be verifiable, for instance using + /// [payload](Commitment::payload) information. pub validator_set_id: ValidatorSetId, } -impl cmp::PartialOrd for Commitment +impl cmp::PartialOrd for Commitment where TBlockNumber: cmp::Ord, - TPayload: cmp::Eq, { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } -impl cmp::Ord for Commitment +impl cmp::Ord for Commitment where TBlockNumber: cmp::Ord, - TPayload: cmp::Eq, { fn cmp(&self, other: &Self) -> cmp::Ordering { self.validator_set_id @@ -79,32 +134,163 @@ where } /// A commitment with matching GRANDPA validators' signatures. -#[derive(Clone, Debug, PartialEq, Eq, codec::Encode, codec::Decode)] -pub struct SignedCommitment { +/// +/// Note that SCALE-encoding of the structure is optimized for size efficiency over the wire, +/// please take a look at custom [`Encode`] and [`Decode`] implementations and +/// `CompactSignedCommitment` struct. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SignedCommitment { /// The commitment signatures are collected for. - pub commitment: Commitment, + pub commitment: Commitment, /// GRANDPA validators' signatures for the commitment. /// /// The length of this `Vec` must match number of validators in the current set (see /// [Commitment::validator_set_id]). - pub signatures: Vec>, + pub signatures: Vec>, } -impl SignedCommitment { +impl SignedCommitment { /// Return the number of collected signatures. pub fn no_of_signatures(&self) -> usize { self.signatures.iter().filter(|x| x.is_some()).count() } } -/// A [SignedCommitment] with a version number. This variant will be appended -/// to the block justifications for the block for which the signed commitment -/// has been generated. +/// Type to be used to denote placement of signatures +type BitField = Vec; +/// Compress 8 bit values into a single u8 Byte +const CONTAINER_BIT_SIZE: usize = 8; + +/// Compressed representation of [`SignedCommitment`], used for encoding efficiency. +#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode)] +struct CompactSignedCommitment { + /// The commitment, unchanged compared to regular [`SignedCommitment`]. + commitment: Commitment, + /// A bitfield representing presence of a signature coming from a validator at some index. + /// + /// The bit at index `0` is set to `1` in case we have a signature coming from a validator at + /// index `0` in in the original validator set. In case the [`SignedCommitment`] does not + /// contain that signature the `bit` will be set to `0`. Bits are packed into `Vec` + signatures_from: BitField, + /// Number of validators in the Validator Set and hence number of significant bits in the + /// [`signatures_from`] collection. + /// + /// Note this might be smaller than the size of `signatures_compact` in case some signatures + /// are missing. + validator_set_len: u32, + /// A `Vec` containing all `Signature`s present in the original [`SignedCommitment`]. + /// + /// Note that in order to associate a `Signature` from this `Vec` with a validator, one needs + /// to look at the `signatures_from` bitfield, since some validators might have not produced a + /// signature. + signatures_compact: Vec, +} + +impl<'a, TBlockNumber: Clone, TSignature> CompactSignedCommitment { + /// Packs a `SignedCommitment` into the compressed `CompactSignedCommitment` format for + /// efficient network transport. + fn pack(signed_commitment: &'a SignedCommitment) -> Self { + let SignedCommitment { commitment, signatures } = signed_commitment; + let validator_set_len = signatures.len() as u32; + + let signatures_compact: Vec<&'a TSignature> = + signatures.iter().filter_map(|x| x.as_ref()).collect(); + let bits = { + let mut bits: Vec = + signatures.iter().map(|x| if x.is_some() { 1 } else { 0 }).collect(); + // Resize with excess bits for placement purposes + let excess_bits_len = + CONTAINER_BIT_SIZE - (validator_set_len as usize % CONTAINER_BIT_SIZE); + bits.resize(bits.len() + excess_bits_len, 0); + bits + }; + + let mut signatures_from: BitField = vec![]; + let chunks = bits.chunks(CONTAINER_BIT_SIZE); + for chunk in chunks { + let mut iter = chunk.iter().copied(); + let mut v = iter.next().unwrap() as u8; + + for bit in iter { + v <<= 1; + v |= bit as u8; + } + + signatures_from.push(v); + } + + Self { + commitment: commitment.clone(), + signatures_from, + validator_set_len, + signatures_compact, + } + } + + /// Unpacks a `CompactSignedCommitment` into the uncompressed `SignedCommitment` form. + fn unpack( + temporary_signatures: CompactSignedCommitment, + ) -> SignedCommitment { + let CompactSignedCommitment { + commitment, + signatures_from, + validator_set_len, + signatures_compact, + } = temporary_signatures; + let mut bits: Vec = vec![]; + + for block in signatures_from { + for bit in 0..CONTAINER_BIT_SIZE { + bits.push((block >> (CONTAINER_BIT_SIZE - bit - 1)) & 1); + } + } + + bits.truncate(validator_set_len as usize); + + let mut next_signature = signatures_compact.into_iter(); + let signatures: Vec> = bits + .iter() + .map(|&x| if x == 1 { next_signature.next() } else { None }) + .collect(); + + SignedCommitment { commitment, signatures } + } +} + +impl Encode for SignedCommitment +where + TBlockNumber: Encode + Clone, + TSignature: Encode, +{ + fn using_encoded R>(&self, f: F) -> R { + let temp = CompactSignedCommitment::pack(self); + temp.using_encoded(f) + } +} + +impl Decode for SignedCommitment +where + TBlockNumber: Decode + Clone, + TSignature: Decode, +{ + fn decode(input: &mut I) -> Result { + let temp = CompactSignedCommitment::decode(input)?; + Ok(CompactSignedCommitment::unpack(temp)) + } +} + +/// A [SignedCommitment] with a version number. +/// +/// This variant will be appended to the block justifications for the block +/// for which the signed commitment has been generated. +/// +/// Note that this enum is subject to change in the future with introduction +/// of additional cryptographic primitives to BEEFY. #[derive(Clone, Debug, PartialEq, codec::Encode, codec::Decode)] -pub enum VersionedCommitment { +pub enum VersionedFinalityProof { #[codec(index = 1)] /// Current active version - V1(SignedCommitment), + V1(SignedCommitment), } #[cfg(test)] @@ -118,9 +304,9 @@ mod tests { use crate::{crypto, KEY_TYPE}; - type TestCommitment = Commitment; - type TestSignedCommitment = SignedCommitment; - type TestVersionedCommitment = VersionedCommitment; + type TestCommitment = Commitment; + type TestSignedCommitment = SignedCommitment; + type TestVersionedFinalityProof = VersionedFinalityProof; // The mock signatures are equivalent to the ones produced by the BEEFY keystore fn mock_signatures() -> (crypto::Signature, crypto::Signature) { @@ -147,8 +333,9 @@ mod tests { #[test] fn commitment_encode_decode() { // given + let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode()); let commitment: TestCommitment = - Commitment { payload: "Hello World!".into(), block_number: 5, validator_set_id: 0 }; + Commitment { payload, block_number: 5, validator_set_id: 0 }; // when let encoded = codec::Encode::encode(&commitment); @@ -159,7 +346,7 @@ mod tests { assert_eq!( encoded, hex_literal::hex!( - "3048656c6c6f20576f726c6421050000000000000000000000000000000000000000000000" + "046d68343048656c6c6f20576f726c6421050000000000000000000000000000000000000000000000" ) ); } @@ -167,8 +354,9 @@ mod tests { #[test] fn signed_commitment_encode_decode() { // given + let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode()); let commitment: TestCommitment = - Commitment { payload: "Hello World!".into(), block_number: 5, validator_set_id: 0 }; + Commitment { payload, block_number: 5, validator_set_id: 0 }; let sigs = mock_signatures(); @@ -186,10 +374,11 @@ mod tests { assert_eq!( encoded, hex_literal::hex!( - "3048656c6c6f20576f726c64210500000000000000000000000000000000000000000000001000 - 0001558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d - 10dd3cd68ce3dc0c33c86e99bcb7816f9ba01012d6e1f8105c337a86cdd9aaacdc496577f3db8c55ef9e6fd48f2c5c05a - 2274707491635d8ba3df64f324575b7b2a34487bca2324b6a0046395a71681be3d0c2a00" + "046d68343048656c6c6f20576f726c6421050000000000000000000000000000000000000000000000 + 04300400000008558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746 + cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba012d6e1f8105c337a86cdd9a + aacdc496577f3db8c55ef9e6fd48f2c5c05a2274707491635d8ba3df64f324575b7b2a34487bca2324b + 6a0046395a71681be3d0c2a00" ) ); } @@ -197,8 +386,9 @@ mod tests { #[test] fn signed_commitment_count_signatures() { // given + let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode()); let commitment: TestCommitment = - Commitment { payload: "Hello World!".into(), block_number: 5, validator_set_id: 0 }; + Commitment { payload, block_number: 5, validator_set_id: 0 }; let sigs = mock_signatures(); @@ -221,7 +411,8 @@ mod tests { block_number: u128, validator_set_id: crate::ValidatorSetId, ) -> TestCommitment { - Commitment { payload: "Hello World!".into(), block_number, validator_set_id } + let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode()); + Commitment { payload, block_number, validator_set_id } } // given @@ -240,8 +431,9 @@ mod tests { #[test] fn versioned_commitment_encode_decode() { + let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode()); let commitment: TestCommitment = - Commitment { payload: "Hello World!".into(), block_number: 5, validator_set_id: 0 }; + Commitment { payload, block_number: 5, validator_set_id: 0 }; let sigs = mock_signatures(); @@ -250,15 +442,89 @@ mod tests { signatures: vec![None, None, Some(sigs.0), Some(sigs.1)], }; - let versioned = TestVersionedCommitment::V1(signed.clone()); + let versioned = TestVersionedFinalityProof::V1(signed.clone()); let encoded = codec::Encode::encode(&versioned); assert_eq!(1, encoded[0]); assert_eq!(encoded[1..], codec::Encode::encode(&signed)); - let decoded = TestVersionedCommitment::decode(&mut &*encoded); + let decoded = TestVersionedFinalityProof::decode(&mut &*encoded); assert_eq!(decoded, Ok(versioned)); } + + #[test] + fn large_signed_commitment_encode_decode() { + // given + let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode()); + let commitment: TestCommitment = + Commitment { payload, block_number: 5, validator_set_id: 0 }; + + let sigs = mock_signatures(); + + let signatures: Vec> = (0..1024) + .into_iter() + .map(|x| if x < 340 { None } else { Some(sigs.0.clone()) }) + .collect(); + let signed = SignedCommitment { commitment, signatures }; + + // when + let encoded = codec::Encode::encode(&signed); + let decoded = TestSignedCommitment::decode(&mut &*encoded); + + // then + assert_eq!(decoded, Ok(signed)); + assert_eq!( + encoded, + hex_literal::hex!( + "046d68343048656c6c6f20576f726c6421050000000000000000000000000000000000000000000000 + 05020000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff + fffffffffff0000040000b10a558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed + 4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad812 + 79df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d1 + 0dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2 + ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01 + 558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e9 + 9a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72 + d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bc + b7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc3 + 21f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc9855 + 80e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0 + c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da + 8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279d + f0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd + 3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac8 + 0a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558 + 455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a8 + 30e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d94 + 8d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb78 + 16f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f + 2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e + 4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33 + c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da848 + 0c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df07 + 95cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd + 68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a0 + 9abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455 + ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e + 314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1 + 107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f + 9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f231 + 9a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb + 75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86 + e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c7 + 46cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795c + c985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68c + e3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09ab + ed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad8 + 1279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314 + d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107 + b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba + 01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01" + ) + ); + } } diff --git a/primitives/beefy/src/lib.rs b/primitives/beefy/src/lib.rs index 790b915ab98d..8dbdd66f3559 100644 --- a/primitives/beefy/src/lib.rs +++ b/primitives/beefy/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -35,7 +35,10 @@ mod commitment; pub mod mmr; pub mod witness; -pub use commitment::{Commitment, SignedCommitment, VersionedCommitment}; +pub use commitment::{ + known_payload_ids, BeefyPayloadId, Commitment, Payload, SignedCommitment, + VersionedFinalityProof, +}; use codec::{Codec, Decode, Encode}; use scale_info::TypeInfo; @@ -81,15 +84,39 @@ pub type ValidatorSetId = u64; #[derive(Decode, Encode, Debug, PartialEq, Clone, TypeInfo)] pub struct ValidatorSet { /// Public keys of the validator set elements - pub validators: Vec, + validators: Vec, /// Identifier of the validator set - pub id: ValidatorSetId, + id: ValidatorSetId, } impl ValidatorSet { - /// Return an empty validator set with id of 0. - pub fn empty() -> Self { - Self { validators: Default::default(), id: Default::default() } + /// Return a validator set with the given validators and set id. + pub fn new(validators: I, id: ValidatorSetId) -> Option + where + I: IntoIterator, + { + let validators: Vec = validators.into_iter().collect(); + if validators.is_empty() { + // No validators; the set would be empty. + None + } else { + Some(Self { validators, id }) + } + } + + /// Return a reference to the vec of validators. + pub fn validators(&self) -> &[AuthorityId] { + &self.validators + } + + /// Return the validator set id. + pub fn id(&self) -> ValidatorSetId { + self.id + } + + /// Return the number of validators in the set. + pub fn len(&self) -> usize { + self.validators.len() } } @@ -118,9 +145,9 @@ pub enum ConsensusLog { /// A vote message is a direct vote created by a BEEFY node on every voting round /// and is gossiped to its peers. #[derive(Debug, Decode, Encode, TypeInfo)] -pub struct VoteMessage { +pub struct VoteMessage { /// Commit to information extracted from a finalized block - pub commitment: Commitment, + pub commitment: Commitment, /// Node authority id pub id: Id, /// Node signature @@ -132,6 +159,26 @@ sp_api::decl_runtime_apis! { pub trait BeefyApi { /// Return the current active BEEFY validator set - fn validator_set() -> ValidatorSet; + fn validator_set() -> Option>; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use sp_application_crypto::ecdsa::{self, Public}; + use sp_core::Pair; + + #[test] + fn validator_set() { + // Empty set not allowed. + assert_eq!(ValidatorSet::::new(vec![], 0), None); + + let alice = ecdsa::Pair::from_string("//Alice", None).unwrap(); + let set_id = 0; + let validators = ValidatorSet::::new(vec![alice.public()], set_id).unwrap(); + + assert_eq!(validators.id(), set_id); + assert_eq!(validators.validators(), &vec![alice.public()]); } } diff --git a/primitives/beefy/src/mmr.rs b/primitives/beefy/src/mmr.rs index e428c0ea0121..426a1ba5ff80 100644 --- a/primitives/beefy/src/mmr.rs +++ b/primitives/beefy/src/mmr.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,12 +26,26 @@ //! but we imagine they will be useful for other chains that either want to bridge with Polkadot //! or are completely standalone, but heavily inspired by Polkadot. -use codec::{Decode, Encode}; +use crate::Vec; +use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; +/// A provider for extra data that gets added to the Mmr leaf +pub trait BeefyDataProvider { + /// Return a vector of bytes, ideally should be a merkle root hash + fn extra_data() -> ExtraData; +} + +/// A default implementation for runtimes. +impl BeefyDataProvider> for () { + fn extra_data() -> Vec { + Vec::new() + } +} + /// A standard leaf that gets added every block to the MMR constructed by Substrate's `pallet_mmr`. #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] -pub struct MmrLeaf { +pub struct MmrLeaf { /// Version of the leaf format. /// /// Can be used to enable future format migrations and compatibility. @@ -41,8 +55,9 @@ pub struct MmrLeaf { pub parent_number_and_hash: (BlockNumber, Hash), /// A merkle root of the next BEEFY authority set. pub beefy_next_authority_set: BeefyNextAuthoritySet, - /// A merkle root of all registered parachain heads. - pub parachain_heads: MerkleRoot, + /// Arbitrary extra leaf data to be used by downstream pallets to include custom data in the + /// [`MmrLeaf`] + pub leaf_extra: ExtraData, } /// A MMR leaf versioning scheme. @@ -81,7 +96,7 @@ impl MmrLeafVersion { } /// Details of the next BEEFY authority set. -#[derive(Debug, Default, PartialEq, Eq, Clone, Encode, Decode, TypeInfo)] +#[derive(Debug, Default, PartialEq, Eq, Clone, Encode, Decode, TypeInfo, MaxEncodedLen)] pub struct BeefyNextAuthoritySet { /// Id of the next set. /// diff --git a/primitives/beefy/src/witness.rs b/primitives/beefy/src/witness.rs index c28a464e72df..aae060815053 100644 --- a/primitives/beefy/src/witness.rs +++ b/primitives/beefy/src/witness.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,10 +25,7 @@ use sp_std::prelude::*; -use crate::{ - commitment::{Commitment, SignedCommitment}, - crypto::Signature, -}; +use crate::commitment::{Commitment, SignedCommitment}; /// A light form of [SignedCommitment]. /// @@ -40,9 +37,9 @@ use crate::{ /// Ethereum Mainnet), in a commit-reveal like scheme, where first we submit only the signed /// commitment witness and later on, the client picks only some signatures to verify at random. #[derive(Debug, PartialEq, Eq, codec::Encode, codec::Decode)] -pub struct SignedCommitmentWitness { +pub struct SignedCommitmentWitness { /// The full content of the commitment. - pub commitment: Commitment, + pub commitment: Commitment, /// The bit vector of validators who signed the commitment. pub signed_by: Vec, // TODO [ToDr] Consider replacing with bitvec crate @@ -51,9 +48,7 @@ pub struct SignedCommitmentWitness { pub signatures_merkle_root: TMerkleRoot, } -impl - SignedCommitmentWitness -{ +impl SignedCommitmentWitness { /// Convert [SignedCommitment] into [SignedCommitmentWitness]. /// /// This takes a [SignedCommitment], which contains full signatures @@ -62,12 +57,12 @@ impl /// and a merkle root of all signatures. /// /// Returns the full list of signatures along with the witness. - pub fn from_signed( - signed: SignedCommitment, + pub fn from_signed( + signed: SignedCommitment, merkelize: TMerkelize, - ) -> (Self, Vec>) + ) -> (Self, Vec>) where - TMerkelize: FnOnce(&[Option]) -> TMerkleRoot, + TMerkelize: FnOnce(&[Option]) -> TMerkleRoot, { let SignedCommitment { commitment, signatures } = signed; let signed_by = signatures.iter().map(|s| s.is_some()).collect(); @@ -86,12 +81,12 @@ mod tests { use super::*; use codec::Decode; - use crate::{crypto, KEY_TYPE}; + use crate::{crypto, known_payload_ids, Payload, KEY_TYPE}; - type TestCommitment = Commitment; - type TestSignedCommitment = SignedCommitment; + type TestCommitment = Commitment; + type TestSignedCommitment = SignedCommitment; type TestSignedCommitmentWitness = - SignedCommitmentWitness>>; + SignedCommitmentWitness>>; // The mock signatures are equivalent to the ones produced by the BEEFY keystore fn mock_signatures() -> (crypto::Signature, crypto::Signature) { @@ -116,8 +111,10 @@ mod tests { } fn signed_commitment() -> TestSignedCommitment { + let payload = + Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".as_bytes().to_vec()); let commitment: TestCommitment = - Commitment { payload: "Hello World!".into(), block_number: 5, validator_set_id: 0 }; + Commitment { payload, block_number: 5, validator_set_id: 0 }; let sigs = mock_signatures(); @@ -152,10 +149,11 @@ mod tests { assert_eq!( encoded, hex_literal::hex!( - "3048656c6c6f20576f726c64210500000000000000000000000000000000000000000000001000 - 00010110000001558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e9 - 9a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01012d6e1f8105c337a86cdd9aaacdc496577f3db8c55ef9e6fd - 48f2c5c05a2274707491635d8ba3df64f324575b7b2a34487bca2324b6a0046395a71681be3d0c2a00" + "046d683048656c6c6f20576f726c642105000000000000000000000000000000000000000000000010 + 0000010110000001558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c + 746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01012d6e1f8105c337a86 + cdd9aaacdc496577f3db8c55ef9e6fd48f2c5c05a2274707491635d8ba3df64f324575b7b2a34487bc + a2324b6a0046395a71681be3d0c2a00" ) ); } diff --git a/primitives/block-builder/Cargo.toml b/primitives/block-builder/Cargo.toml index d7fa0f2ef85c..6d7a0a2789c2 100644 --- a/primitives/block-builder/Cargo.toml +++ b/primitives/block-builder/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-block-builder" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "The block builder runtime api." readme = "README.md" @@ -13,10 +13,10 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } sp-api = { version = "4.0.0-dev", default-features = false, path = "../api" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../inherents" } [features] diff --git a/primitives/block-builder/src/lib.rs b/primitives/block-builder/src/lib.rs index 3741b1920064..1b74c27b7ae4 100644 --- a/primitives/block-builder/src/lib.rs +++ b/primitives/block-builder/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,11 +20,14 @@ #![cfg_attr(not(feature = "std"), no_std)] use sp_inherents::{CheckInherentsResult, InherentData}; -use sp_runtime::{traits::Block as BlockT, ApplyExtrinsicResult}; +use sp_runtime::{ + legacy::byte_sized_error::ApplyExtrinsicResult as ApplyExtrinsicResultBeforeV6, + traits::Block as BlockT, ApplyExtrinsicResult, +}; sp_api::decl_runtime_apis! { /// The `BlockBuilder` api trait that provides the required functionality for building a block. - #[api_version(5)] + #[api_version(6)] pub trait BlockBuilder { /// Apply the given extrinsic. /// @@ -32,6 +35,9 @@ sp_api::decl_runtime_apis! { /// this block or not. fn apply_extrinsic(extrinsic: ::Extrinsic) -> ApplyExtrinsicResult; + #[changed_in(6)] + fn apply_extrinsic(extrinsic: ::Extrinsic) -> ApplyExtrinsicResultBeforeV6; + /// Finish the current block. #[renamed("finalise_block", 3)] fn finalize_block() -> ::Header; diff --git a/primitives/blockchain/Cargo.toml b/primitives/blockchain/Cargo.toml index 66d9152c230d..c6fcdd3cb1c0 100644 --- a/primitives/blockchain/Cargo.toml +++ b/primitives/blockchain/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-blockchain" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate blockchain traits and primitives." documentation = "https://docs.rs/sp-blockchain" @@ -15,13 +15,13 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] log = "0.4.11" -lru = "0.6.6" -parking_lot = "0.11.1" -thiserror = "1.0.21" -futures = "0.3.9" -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +lru = "0.7.5" +parking_lot = "0.12.0" +thiserror = "1.0.30" +futures = "0.3.21" +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } sp-consensus = { version = "0.10.0-dev", path = "../consensus/common" } -sp-runtime = { version = "4.0.0-dev", path = "../runtime" } -sp-state-machine = { version = "0.10.0-dev", path = "../state-machine" } +sp-runtime = { version = "6.0.0", path = "../runtime" } +sp-state-machine = { version = "0.12.0", path = "../state-machine" } sp-database = { version = "4.0.0-dev", path = "../database" } sp-api = { version = "4.0.0-dev", path = "../api" } diff --git a/primitives/blockchain/src/backend.rs b/primitives/blockchain/src/backend.rs index bb34a0449b5f..3c6419648388 100644 --- a/primitives/blockchain/src/backend.rs +++ b/primitives/blockchain/src/backend.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,8 +17,6 @@ //! Substrate blockchain trait -use std::sync::Arc; - use log::warn; use parking_lot::RwLock; use sp_runtime::{ @@ -96,8 +94,6 @@ pub trait Backend: fn justifications(&self, id: BlockId) -> Result>; /// Get last finalized block hash. fn last_finalized(&self) -> Result; - /// Returns data cache reference, if it is enabled on this backend. - fn cache(&self) -> Option>>; /// Returns hashes of all blocks that are leaves of the block tree. /// in other words, that have no children, are chain heads. @@ -237,33 +233,6 @@ pub trait Backend: fn block_indexed_body(&self, id: BlockId) -> Result>>>; } -/// Provides access to the optional cache. -pub trait ProvideCache { - /// Returns data cache reference, if it is enabled on this backend. - fn cache(&self) -> Option>>; -} - -/// Blockchain optional data cache. -pub trait Cache: Send + Sync { - /// Initialize genesis value for the given cache. - /// - /// The operation should be performed once before anything else is inserted in the cache. - /// Otherwise cache may end up in inconsistent state. - fn initialize(&self, key: &well_known_cache_keys::Id, value_at_genesis: Vec) -> Result<()>; - /// Returns cached value by the given key. - /// - /// Returned tuple is the range where value has been active and the value itself. - /// Fails if read from cache storage fails or if the value for block is discarded - /// (i.e. if block is earlier that best finalized, but it is not in canonical chain). - fn get_at( - &self, - key: &well_known_cache_keys::Id, - block: &BlockId, - ) -> Result< - Option<((NumberFor, Block::Hash), Option<(NumberFor, Block::Hash)>, Vec)>, - >; -} - /// Blockchain info #[derive(Debug, Eq, PartialEq)] pub struct Info { @@ -281,6 +250,8 @@ pub struct Info { pub finalized_state: Option<(Block::Hash, <::Header as HeaderT>::Number)>, /// Number of concurrent leave forks. pub number_leaves: usize, + /// Missing blocks after warp sync. (start, end). + pub block_gap: Option<(NumberFor, NumberFor)>, } /// Block status. diff --git a/primitives/blockchain/src/error.rs b/primitives/blockchain/src/error.rs index ef3afa5bce94..c82fb9bebf4e 100644 --- a/primitives/blockchain/src/error.rs +++ b/primitives/blockchain/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -69,7 +69,7 @@ pub enum Error { ExtrinsicRootInvalid { received: String, expected: String }, // `inner` cannot be made member, since it lacks `std::error::Error` trait bounds. - #[error("Execution failed: {0:?}")] + #[error("Execution failed: {0}")] Execution(Box), #[error("Blockchain")] diff --git a/primitives/blockchain/src/header_metadata.rs b/primitives/blockchain/src/header_metadata.rs index 928409963bcd..c21c82b9fbc2 100644 --- a/primitives/blockchain/src/header_metadata.rs +++ b/primitives/blockchain/src/header_metadata.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,7 @@ use lru::LruCache; use parking_lot::RwLock; -use sp_runtime::traits::{Block as BlockT, Header, NumberFor}; +use sp_runtime::traits::{Block as BlockT, Header, NumberFor, One}; /// Set to the expected max difference between `best` and `finalized` blocks at sync. const LRU_CACHE_SIZE: usize = 5_000; @@ -37,7 +37,14 @@ pub fn lowest_common_ancestor + ?Sized>( id_two: Block::Hash, ) -> Result, T::Error> { let mut header_one = backend.header_metadata(id_one)?; + if header_one.parent == id_two { + return Ok(HashAndNumber { hash: id_two, number: header_one.number - One::one() }) + } + let mut header_two = backend.header_metadata(id_two)?; + if header_two.parent == id_one { + return Ok(HashAndNumber { hash: id_one, number: header_one.number }) + } let mut orig_header_one = header_one.clone(); let mut orig_header_two = header_two.clone(); @@ -199,7 +206,7 @@ impl TreeRoute { /// Handles header metadata: hash, number, parent hash, etc. pub trait HeaderMetadata { /// Error used in case the header metadata is not found. - type Error; + type Error: std::error::Error; fn header_metadata( &self, diff --git a/primitives/blockchain/src/lib.rs b/primitives/blockchain/src/lib.rs index cd36cabe1551..2fdef6cba9e5 100644 --- a/primitives/blockchain/src/lib.rs +++ b/primitives/blockchain/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/consensus/aura/Cargo.toml b/primitives/consensus/aura/Cargo.toml index c228b88fd657..2c4a2f0a0de9 100644 --- a/primitives/consensus/aura/Cargo.toml +++ b/primitives/consensus/aura/Cargo.toml @@ -3,9 +3,9 @@ name = "sp-consensus-aura" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "Primitives for Aura consensus" -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,12 +13,12 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../application-crypto" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../std" } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../../application-crypto" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../std" } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../api" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../runtime" } sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../inherents" } sp-timestamp = { version = "4.0.0-dev", default-features = false, path = "../../timestamp" } sp-consensus-slots = { version = "0.10.0-dev", default-features = false, path = "../slots" } diff --git a/primitives/consensus/aura/src/digests.rs b/primitives/consensus/aura/src/digests.rs index eaa29036d98a..b71930b6c6b8 100644 --- a/primitives/consensus/aura/src/digests.rs +++ b/primitives/consensus/aura/src/digests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -25,7 +25,6 @@ use crate::AURA_ENGINE_ID; use codec::{Codec, Encode}; use sp_consensus_slots::Slot; use sp_runtime::generic::DigestItem; -use sp_std::fmt::Debug; /// A digest item which is usable with aura consensus. pub trait CompatibleDigestItem: Sized { @@ -42,10 +41,9 @@ pub trait CompatibleDigestItem: Sized { fn as_aura_pre_digest(&self) -> Option; } -impl CompatibleDigestItem for DigestItem +impl CompatibleDigestItem for DigestItem where Signature: Codec, - Hash: Debug + Send + Sync + Eq + Clone + Codec + 'static, { fn aura_seal(signature: Signature) -> Self { DigestItem::Seal(AURA_ENGINE_ID, signature.encode()) diff --git a/primitives/consensus/aura/src/inherents.rs b/primitives/consensus/aura/src/inherents.rs index 2a797b5d3f39..ce3d832c78ee 100644 --- a/primitives/consensus/aura/src/inherents.rs +++ b/primitives/consensus/aura/src/inherents.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -58,12 +58,11 @@ impl InherentDataProvider { /// Creates the inherent data provider by calculating the slot from the given /// `timestamp` and `duration`. - pub fn from_timestamp_and_duration( + pub fn from_timestamp_and_slot_duration( timestamp: sp_timestamp::Timestamp, - duration: std::time::Duration, + slot_duration: sp_consensus_slots::SlotDuration, ) -> Self { - let slot = - InherentType::from((timestamp.as_duration().as_millis() / duration.as_millis()) as u64); + let slot = InherentType::from_timestamp(timestamp, slot_duration); Self { slot } } diff --git a/primitives/consensus/aura/src/lib.rs b/primitives/consensus/aura/src/lib.rs index e6a319c1d159..3e47adf0bf92 100644 --- a/primitives/consensus/aura/src/lib.rs +++ b/primitives/consensus/aura/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -62,7 +62,7 @@ pub mod ed25519 { pub type AuthorityId = app_ed25519::Public; } -pub use sp_consensus_slots::Slot; +pub use sp_consensus_slots::{Slot, SlotDuration}; /// The `ConsensusEngineId` of AuRa. pub const AURA_ENGINE_ID: ConsensusEngineId = [b'a', b'u', b'r', b'a']; @@ -93,30 +93,3 @@ sp_api::decl_runtime_apis! { fn authorities() -> Vec; } } - -/// Aura slot duration. -/// -/// Internally stored as milliseconds. -#[derive(sp_runtime::RuntimeDebug, Encode, Decode, PartialEq, Clone, Copy)] -pub struct SlotDuration(u64); - -impl SlotDuration { - /// Initialize from the given milliseconds. - pub fn from_millis(val: u64) -> Self { - Self(val) - } - - /// Returns the slot duration in milli seconds. - pub fn get(&self) -> u64 { - self.0 - } -} - -#[cfg(feature = "std")] -impl sp_consensus::SlotData for SlotDuration { - fn slot_duration(&self) -> std::time::Duration { - std::time::Duration::from_millis(self.0) - } - - const SLOT_KEY: &'static [u8] = b"aura_slot_duration"; -} diff --git a/primitives/consensus/babe/Cargo.toml b/primitives/consensus/babe/Cargo.toml index 5f6bfec21973..189dc5b2e80f 100644 --- a/primitives/consensus/babe/Cargo.toml +++ b/primitives/consensus/babe/Cargo.toml @@ -3,9 +3,9 @@ name = "sp-consensus-babe" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "Primitives for BABE consensus" -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,21 +13,21 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../application-crypto" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../../application-crypto" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } merlin = { version = "2.0", default-features = false } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../std" } +sp-std = { version = "4.0.0", default-features = false, path = "../../std" } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../api" } sp-consensus = { version = "0.10.0-dev", optional = true, path = "../common" } sp-consensus-slots = { version = "0.10.0-dev", default-features = false, path = "../slots" } sp-consensus-vrf = { version = "0.10.0-dev", path = "../vrf", default-features = false } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../core" } +sp-core = { version = "6.0.0", default-features = false, path = "../../core" } sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../inherents" } -sp-keystore = { version = "0.10.0-dev", default-features = false, path = "../../keystore", optional = true } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../runtime" } +sp-keystore = { version = "0.12.0", default-features = false, path = "../../keystore", optional = true } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../runtime" } sp-timestamp = { version = "4.0.0-dev", path = "../../timestamp", optional = true } -serde = { version = "1.0.126", features = ["derive"], optional = true } +serde = { version = "1.0.136", features = ["derive"], optional = true } async-trait = { version = "0.1.50", optional = true } [features] diff --git a/primitives/consensus/babe/src/digests.rs b/primitives/consensus/babe/src/digests.rs index 1c908fe61fc0..0f21c913ac57 100644 --- a/primitives/consensus/babe/src/digests.rs +++ b/primitives/consensus/babe/src/digests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use super::{ AllowedSlots, AuthorityId, AuthorityIndex, AuthoritySignature, BabeAuthorityWeight, BabeEpochConfiguration, Slot, BABE_ENGINE_ID, }; -use codec::{Codec, Decode, Encode, MaxEncodedLen}; +use codec::{Decode, Encode, MaxEncodedLen}; use sp_runtime::{DigestItem, RuntimeDebug}; use sp_std::vec::Vec; @@ -177,10 +177,7 @@ pub trait CompatibleDigestItem: Sized { fn as_next_config_descriptor(&self) -> Option; } -impl CompatibleDigestItem for DigestItem -where - Hash: Send + Sync + Eq + Clone + Codec + 'static, -{ +impl CompatibleDigestItem for DigestItem { fn babe_pre_digest(digest: PreDigest) -> Self { DigestItem::PreRuntime(BABE_ENGINE_ID, digest.encode()) } diff --git a/primitives/consensus/babe/src/inherents.rs b/primitives/consensus/babe/src/inherents.rs index cecd61998a4d..c26dc514ae15 100644 --- a/primitives/consensus/babe/src/inherents.rs +++ b/primitives/consensus/babe/src/inherents.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,6 @@ //! Inherents for BABE use sp_inherents::{Error, InherentData, InherentIdentifier}; - use sp_std::result::Result; /// The BABE inherent identifier. @@ -60,12 +59,11 @@ impl InherentDataProvider { /// Creates the inherent data provider by calculating the slot from the given /// `timestamp` and `duration`. - pub fn from_timestamp_and_duration( + pub fn from_timestamp_and_slot_duration( timestamp: sp_timestamp::Timestamp, - duration: std::time::Duration, + slot_duration: sp_consensus_slots::SlotDuration, ) -> Self { - let slot = - InherentType::from((timestamp.as_duration().as_millis() / duration.as_millis()) as u64); + let slot = InherentType::from_timestamp(timestamp, slot_duration); Self { slot } } diff --git a/primitives/consensus/babe/src/lib.rs b/primitives/consensus/babe/src/lib.rs index 560866cfb2ab..492d1a9a7238 100644 --- a/primitives/consensus/babe/src/lib.rs +++ b/primitives/consensus/babe/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -79,7 +79,7 @@ pub const MEDIAN_ALGORITHM_CARDINALITY: usize = 1200; // arbitrary suggestion by /// The index of an authority. pub type AuthorityIndex = u32; -pub use sp_consensus_slots::Slot; +pub use sp_consensus_slots::{Slot, SlotDuration}; /// An equivocation proof for multiple block authorships on the same slot (i.e. double vote). pub type EquivocationProof = sp_consensus_slots::EquivocationProof; @@ -237,15 +237,6 @@ impl AllowedSlots { } } -#[cfg(feature = "std")] -impl sp_consensus::SlotData for BabeGenesisConfiguration { - fn slot_duration(&self) -> std::time::Duration { - std::time::Duration::from_millis(self.slot_duration) - } - - const SLOT_KEY: &'static [u8] = b"babe_configuration"; -} - /// Configuration data used by the BABE consensus engine. #[derive(Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug, MaxEncodedLen, TypeInfo)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] diff --git a/primitives/consensus/common/Cargo.toml b/primitives/consensus/common/Cargo.toml index ecfc1c1b3182..35bd7f33fbf4 100644 --- a/primitives/consensus/common/Cargo.toml +++ b/primitives/consensus/common/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-consensus" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Common utilities for building and using consensus engines in substrate." documentation = "https://docs.rs/sp-consensus/" @@ -15,22 +15,22 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] async-trait = "0.1.42" -codec = { package = "parity-scale-codec", version = "2.0.0", features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", features = [ "derive", ] } -futures = { version = "0.3.1", features = ["thread-pool"] } +futures = { version = "0.3.21", features = ["thread-pool"] } log = "0.4.8" -sp-core = { path = "../../core", version = "4.0.0-dev" } +sp-core = { path = "../../core", version = "6.0.0"} sp-inherents = { version = "4.0.0-dev", path = "../../inherents" } -sp-state-machine = { version = "0.10.0-dev", path = "../../state-machine" } +sp-state-machine = { version = "0.12.0", path = "../../state-machine" } futures-timer = "3.0.1" -sp-std = { version = "4.0.0-dev", path = "../../std" } -sp-version = { version = "4.0.0-dev", path = "../../version" } -sp-runtime = { version = "4.0.0-dev", path = "../../runtime" } -thiserror = "1.0.21" +sp-std = { version = "4.0.0", path = "../../std" } +sp-version = { version = "5.0.0", path = "../../version" } +sp-runtime = { version = "6.0.0", path = "../../runtime" } +thiserror = "1.0.30" [dev-dependencies] -futures = "0.3.9" +futures = "0.3.21" sp-test-primitives = { version = "2.0.0", path = "../../test-primitives" } [features] diff --git a/primitives/consensus/common/src/block_validation.rs b/primitives/consensus/common/src/block_validation.rs index 54a70a402b06..71f3a80b27a6 100644 --- a/primitives/consensus/common/src/block_validation.rs +++ b/primitives/consensus/common/src/block_validation.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/consensus/common/src/error.rs b/primitives/consensus/common/src/error.rs index 546f30d3e820..0656b5761fb3 100644 --- a/primitives/consensus/common/src/error.rs +++ b/primitives/consensus/common/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -86,13 +86,13 @@ pub enum Error { CannotSign(Vec, String), } -impl core::convert::From for Error { +impl From for Error { fn from(p: Public) -> Self { Self::InvalidAuthority(p) } } -impl core::convert::From for Error { +impl From for Error { fn from(s: String) -> Self { Self::StateUnavailable(s) } diff --git a/primitives/consensus/common/src/evaluation.rs b/primitives/consensus/common/src/evaluation.rs index 19be5e552634..d0ddbb6fab81 100644 --- a/primitives/consensus/common/src/evaluation.rs +++ b/primitives/consensus/common/src/evaluation.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/consensus/common/src/lib.rs b/primitives/consensus/common/src/lib.rs index d7979baf47c1..59bbf7618dfc 100644 --- a/primitives/consensus/common/src/lib.rs +++ b/primitives/consensus/common/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,7 +26,8 @@ use std::{sync::Arc, time::Duration}; use futures::prelude::*; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, DigestFor, HashFor, NumberFor}, + traits::{Block as BlockT, HashFor}, + Digest, }; use sp_state_machine::StorageProof; @@ -97,7 +98,7 @@ pub trait Environment { + Unpin + 'static; /// Error which can occur upon creation. - type Error: From + std::fmt::Debug + 'static; + type Error: From + std::error::Error + 'static; /// Initialize the proposal logic on top of a specific header. Provide /// the authorities at that header. @@ -111,8 +112,7 @@ pub struct Proposal { /// Proof that was recorded while building the block. pub proof: Proof, /// The storage changes while building this block. - pub storage_changes: - sp_state_machine::StorageChanges, NumberFor>, + pub storage_changes: sp_state_machine::StorageChanges>, } /// Error that is returned when [`ProofRecording`] requested to record a proof, @@ -191,7 +191,7 @@ mod private { /// Proposers are generic over bits of "consensus data" which are engine-specific. pub trait Proposer { /// Error type which can occur when proposing or evaluating. - type Error: From + std::fmt::Debug + 'static; + type Error: From + std::error::Error + 'static; /// The transaction type used by the backend. type Transaction: Default + Send + 'static; /// Future that resolves to a committed proposal with an optional proof. @@ -224,7 +224,7 @@ pub trait Proposer { fn propose( self, inherent_data: InherentData, - inherent_digests: DigestFor, + inherent_digests: Digest, max_duration: Duration, block_size_limit: Option, ) -> Self::Proposal; @@ -327,12 +327,3 @@ impl CanAuthorWith for NeverCanAuthor { Err("Authoring is always disabled.".to_string()) } } - -/// A type from which a slot duration can be obtained. -pub trait SlotData { - /// Gets the slot duration. - fn slot_duration(&self) -> sp_std::time::Duration; - - /// The static slot key - const SLOT_KEY: &'static [u8]; -} diff --git a/primitives/consensus/common/src/select_chain.rs b/primitives/consensus/common/src/select_chain.rs index fd8b06ecf8ab..f366cd34c51e 100644 --- a/primitives/consensus/common/src/select_chain.rs +++ b/primitives/consensus/common/src/select_chain.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/consensus/pow/Cargo.toml b/primitives/consensus/pow/Cargo.toml index f93eeca2fb24..eb2db085c482 100644 --- a/primitives/consensus/pow/Cargo.toml +++ b/primitives/consensus/pow/Cargo.toml @@ -3,9 +3,9 @@ name = "sp-consensus-pow" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "Primitives for Aura consensus" -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -14,10 +14,10 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sp-api = { version = "4.0.0-dev", default-features = false, path = "../../api" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../runtime" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../core" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../../std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../runtime" } +sp-core = { version = "6.0.0", default-features = false, path = "../../core" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } [features] default = ["std"] diff --git a/primitives/consensus/pow/src/lib.rs b/primitives/consensus/pow/src/lib.rs index ac8bc589c136..fe10ee808db9 100644 --- a/primitives/consensus/pow/src/lib.rs +++ b/primitives/consensus/pow/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/consensus/slots/Cargo.toml b/primitives/consensus/slots/Cargo.toml index 3ad204f97396..a334b10d6586 100644 --- a/primitives/consensus/slots/Cargo.toml +++ b/primitives/consensus/slots/Cargo.toml @@ -3,9 +3,9 @@ name = "sp-consensus-slots" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "Primitives for slots-based consensus" -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,16 +13,22 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.2.0", default-features = false, features = ["derive", "max-encoded-len"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../runtime" } -sp-arithmetic = { version = "4.0.0-dev", default-features = false, path = "../../arithmetic" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } +scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } +serde = { version = "1.0", features = ["derive"], optional = true } +sp-arithmetic = { version = "5.0.0", default-features = false, path = "../../arithmetic" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../std" } +sp-timestamp = { version = "4.0.0-dev", default-features = false, path = "../../timestamp" } [features] default = ["std"] std = [ "codec/std", "scale-info/std", - "sp-runtime/std", + "serde", "sp-arithmetic/std", + "sp-runtime/std", + "sp-std/std", + "sp-timestamp/std", ] diff --git a/primitives/consensus/slots/src/lib.rs b/primitives/consensus/slots/src/lib.rs index 89b57dca8308..21b3cad1e716 100644 --- a/primitives/consensus/slots/src/lib.rs +++ b/primitives/consensus/slots/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,9 +21,11 @@ use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; +use sp_timestamp::Timestamp; /// Unit type wrapper that represents a slot. #[derive(Debug, Encode, MaxEncodedLen, Decode, Eq, Clone, Copy, Default, Ord, TypeInfo)] +#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] pub struct Slot(u64); impl core::ops::Deref for Slot { @@ -63,6 +65,11 @@ impl + Copy> core::cmp::PartialOrd for Slot { } impl Slot { + /// Create a new slot by calculating it from the given timestamp and slot duration. + pub const fn from_timestamp(timestamp: Timestamp, slot_duration: SlotDuration) -> Self { + Slot(timestamp.as_millis() / slot_duration.as_millis()) + } + /// Saturating addition. pub fn saturating_add>(self, rhs: T) -> Self { Self(self.0.saturating_add(rhs.into())) @@ -93,6 +100,32 @@ impl From for u64 { } } +/// A slot duration defined in milliseconds. +#[derive(Clone, Copy, Debug, Encode, Decode, Hash, PartialOrd, Ord, PartialEq, Eq, TypeInfo)] +pub struct SlotDuration(u64); + +impl SlotDuration { + /// Initialize from the given milliseconds. + pub const fn from_millis(millis: u64) -> Self { + Self(millis) + } +} + +impl SlotDuration { + /// Returns `self` as a `u64` representing the duration in milliseconds. + pub const fn as_millis(&self) -> u64 { + self.0 + } +} + +#[cfg(feature = "std")] +impl SlotDuration { + /// Returns `self` as [`sp_std::time::Duration`]. + pub const fn as_duration(&self) -> sp_std::time::Duration { + sp_std::time::Duration::from_millis(self.0) + } +} + /// Represents an equivocation proof. An equivocation happens when a validator /// produces more than one block on the same slot. The proof of equivocation /// are the given distinct headers that were signed by the validator and which diff --git a/primitives/consensus/vrf/Cargo.toml b/primitives/consensus/vrf/Cargo.toml index 124cbf423f06..80d2d1ddb09d 100644 --- a/primitives/consensus/vrf/Cargo.toml +++ b/primitives/consensus/vrf/Cargo.toml @@ -3,21 +3,21 @@ name = "sp-consensus-vrf" version = "0.10.0-dev" authors = ["Parity Technologies "] description = "Primitives for VRF based consensus" -edition = "2018" +edition = "2021" license = "Apache-2.0" repository = "https://github.com/paritytech/substrate/" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" readme = "README.md" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { version = "2.0.0", package = "parity-scale-codec", default-features = false } +codec = { version = "3.0.0", package = "parity-scale-codec", default-features = false } schnorrkel = { version = "0.9.1", features = ["preaudit_deprecated", "u64_backend"], default-features = false } -sp-std = { version = "4.0.0-dev", path = "../../std", default-features = false } -sp-core = { version = "4.0.0-dev", path = "../../core", default-features = false } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../runtime" } +sp-std = { version = "4.0.0", path = "../../std", default-features = false } +sp-core = { version = "6.0.0", path = "../../core", default-features = false } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../runtime" } [features] default = ["std"] diff --git a/primitives/consensus/vrf/src/lib.rs b/primitives/consensus/vrf/src/lib.rs index 19391c6c1c84..07e3f2c31970 100644 --- a/primitives/consensus/vrf/src/lib.rs +++ b/primitives/consensus/vrf/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/consensus/vrf/src/schnorrkel.rs b/primitives/consensus/vrf/src/schnorrkel.rs index 687e0bd23182..094a398893ff 100644 --- a/primitives/consensus/vrf/src/schnorrkel.rs +++ b/primitives/consensus/vrf/src/schnorrkel.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,6 @@ use codec::{Decode, Encode, EncodeLike}; use schnorrkel::errors::MultiSignatureStage; use sp_core::U512; use sp_std::{ - convert::TryFrom, ops::{Deref, DerefMut}, prelude::*, }; diff --git a/primitives/core/Cargo.toml b/primitives/core/Cargo.toml index 73c3d454ed58..fded98c6c315 100644 --- a/primitives/core/Cargo.toml +++ b/primitives/core/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-core" -version = "4.0.0-dev" +version = "6.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Shareable Substrate types." documentation = "https://docs.rs/sp-core" @@ -13,68 +13,62 @@ documentation = "https://docs.rs/sp-core" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -codec = { package = "parity-scale-codec", version = "2.2.0", default-features = false, features = [ +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive", "max-encoded-len", ] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } log = { version = "0.4.11", default-features = false } -serde = { version = "1.0.126", optional = true, features = ["derive"] } +serde = { version = "1.0.136", optional = true, features = ["derive"] } byteorder = { version = "1.3.2", default-features = false } -primitive-types = { version = "0.10.1", default-features = false, features = [ - "codec", - "scale-info" -] } +primitive-types = { version = "0.11.1", default-features = false, features = ["codec", "scale-info"] } impl-serde = { version = "0.3.0", optional = true } -wasmi = { version = "0.9.0", optional = true } +wasmi = { version = "0.9.1", optional = true } hash-db = { version = "0.15.2", default-features = false } hash256-std-hasher = { version = "0.15.2", default-features = false } -base58 = { version = "0.1.0", optional = true } +base58 = { version = "0.2.0", optional = true } rand = { version = "0.7.3", optional = true, features = ["small_rng"] } -substrate-bip39 = { version = "0.4.2", optional = true } -tiny-bip39 = { version = "0.8", optional = true } -regex = { version = "1.4.2", optional = true } +substrate-bip39 = { version = "0.4.4", optional = true } +tiny-bip39 = { version = "0.8.2", optional = true } +regex = { version = "1.5.4", optional = true } num-traits = { version = "0.2.8", default-features = false } -zeroize = { version = "1.4.1", default-features = false } -secrecy = { version = "0.7.0", default-features = false } +# zeroize = { version = "1.5.4", default-features = false } +zeroize = { version = "1.4.3", default-features = false } +secrecy = { version = "0.8.0", default-features = false } lazy_static = { version = "1.4.0", default-features = false, optional = true } -parking_lot = { version = "0.11.1", optional = true } -sp-debug-derive = { version = "3.0.0", path = "../debug-derive" } -sp-externalities = { version = "0.10.0-dev", optional = true, path = "../externalities" } -sp-storage = { version = "4.0.0-dev", default-features = false, path = "../storage" } -parity-util-mem = { version = "0.10.0", default-features = false, features = [ - "primitive-types", -] } -futures = { version = "0.3.1", optional = true } +parking_lot = { version = "0.12.0", optional = true } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +sp-debug-derive = { version = "4.0.0", default-features = false, path = "../debug-derive" } +sp-storage = { version = "6.0.0", default-features = false, path = "../storage" } +sp-externalities = { version = "0.12.0", optional = true, path = "../externalities" } +parity-util-mem = { version = "0.11.0", default-features = false, features = ["primitive-types"] } +futures = { version = "0.3.21", optional = true } dyn-clonable = { version = "0.9.0", optional = true } -thiserror = { version = "1.0.21", optional = true } +thiserror = { version = "1.0.30", optional = true } +bitflags = "1.3" # full crypto -ed25519-dalek = { version = "1.0.1", default-features = false, features = [ - "u64_backend", - "alloc", -], optional = true } +ed25519-dalek = { version = "1.0.1", default-features = false, features = ["u64_backend", "alloc"], optional = true } blake2-rfc = { version = "0.2.18", default-features = false, optional = true } -tiny-keccak = { version = "2.0.1", features = ["keccak"], optional = true } schnorrkel = { version = "0.9.1", features = [ "preaudit_deprecated", "u64_backend", ], default-features = false, optional = true } -sha2 = { version = "0.9.2", default-features = false, optional = true } hex = { version = "0.4", default-features = false, optional = true } -twox-hash = { version = "1.5.0", default-features = false, optional = true } -libsecp256k1 = { version = "0.6", default-features = false, features = ["hmac", "static-context"], optional = true } +libsecp256k1 = { version = "0.7", default-features = false, features = ["static-context"], optional = true } merlin = { version = "2.0", default-features = false, optional = true } - -sp-runtime-interface = { version = "4.0.0-dev", default-features = false, path = "../runtime-interface" } +secp256k1 = { version = "0.21.2", default-features = false, features = ["recovery", "alloc"], optional = true } +ss58-registry = { version = "1.11.0", default-features = false } +sp-core-hashing = { version = "4.0.0", path = "./hashing", default-features = false, optional = true } +sp-runtime-interface = { version = "6.0.0", default-features = false, path = "../runtime-interface" } [dev-dependencies] -sp-serializer = { version = "3.0.0", path = "../serializer" } -hex-literal = "0.3.1" +sp-serializer = { version = "4.0.0-dev", path = "../serializer" } +hex-literal = "0.3.4" rand = "0.7.2" criterion = "0.3.3" serde_json = "1.0" +sp-core-hashing-proc-macro = { version = "5.0.0", path = "./hashing/proc-macro" } [[bench]] name = "bench" @@ -103,7 +97,6 @@ std = [ "hash-db/std", "sp-std/std", "serde", - "twox-hash/std", "blake2-rfc/std", "ed25519-dalek/std", "hex/std", @@ -112,15 +105,17 @@ std = [ "tiny-bip39", "byteorder/std", "rand", - "sha2/std", "schnorrkel/std", "regex", "num-traits/std", - "tiny-keccak", + "secp256k1/std", + "secp256k1/global-context", + "sp-core-hashing/std", "sp-debug-derive/std", "sp-externalities", "sp-storage/std", "sp-runtime-interface/std", + "ss58-registry/std", "zeroize/alloc", "secrecy/alloc", "futures", @@ -135,12 +130,11 @@ std = [ full_crypto = [ "ed25519-dalek", "blake2-rfc", - "tiny-keccak", "schnorrkel", "hex", - "sha2", - "twox-hash", "libsecp256k1", + "secp256k1", + "sp-core-hashing", "sp-runtime-interface/disable_target_static_assertions", "merlin", ] diff --git a/primitives/core/hashing/Cargo.toml b/primitives/core/hashing/Cargo.toml new file mode 100644 index 000000000000..978cb8906d5d --- /dev/null +++ b/primitives/core/hashing/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "sp-core-hashing" +version = "4.0.0" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Primitive core crate hashing implementation." +documentation = "https://docs.rs/sp-core-hashing" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +sp-std = { version = "4.0.0", default-features = false, path = "../../std" } +byteorder = { version = "1.3.2", default-features = false } + +digest = { version = "0.10.3", default-features = false } +blake2 = { version = "0.10.2", default-features = false } +sha2 = { version = "0.10.1", default-features = false } +sha3 = { version = "0.10.0", default-features = false } +twox-hash = { version = "1.6.2", default-features = false, features = ["digest_0_10"] } + +[features] +default = ["std"] +std = [ + "sp-std/std", + "blake2/std", + "sha2/std", + "sha3/std", + "twox-hash/std", +] diff --git a/primitives/core/hashing/proc-macro/Cargo.toml b/primitives/core/hashing/proc-macro/Cargo.toml new file mode 100644 index 000000000000..b3dc155cd8bf --- /dev/null +++ b/primitives/core/hashing/proc-macro/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "sp-core-hashing-proc-macro" +version = "5.0.0" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "This crate provides procedural macros for calculating static hash." +documentation = "https://docs.rs/sp-core-hashing-proc-macro" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0.82", features = ["full", "parsing"] } +quote = "1.0.6" +proc-macro2 = "1.0.36" +sp-core-hashing = { version = "4.0.0", path = "../", default-features = false } diff --git a/primitives/core/hashing/proc-macro/src/impls.rs b/primitives/core/hashing/proc-macro/src/impls.rs new file mode 100644 index 000000000000..ff9593ea1844 --- /dev/null +++ b/primitives/core/hashing/proc-macro/src/impls.rs @@ -0,0 +1,124 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use quote::quote; +use syn::parse::{Parse, ParseStream}; + +use proc_macro::TokenStream; + +pub(super) struct InputBytes(pub Vec); + +pub(super) struct MultipleInputBytes(pub Vec>); + +impl MultipleInputBytes { + pub(super) fn concatenated(mut self) -> Vec { + if self.0.len() == 0 { + Vec::new() + } else { + let mut result = core::mem::take(&mut self.0[0]); + for other in self.0[1..].iter_mut() { + result.append(other); + } + result + } + } +} + +impl Parse for InputBytes { + fn parse(input: ParseStream) -> syn::Result { + match syn::ExprArray::parse(input) { + Ok(array) => { + let mut bytes = Vec::::new(); + for expr in array.elems.iter() { + match expr { + syn::Expr::Lit(lit) => match &lit.lit { + syn::Lit::Int(b) => bytes.push(b.base10_parse()?), + syn::Lit::Byte(b) => bytes.push(b.value()), + _ => + return Err(syn::Error::new( + input.span(), + "Expected array of u8 elements.".to_string(), + )), + }, + _ => + return Err(syn::Error::new( + input.span(), + "Expected array of u8 elements.".to_string(), + )), + } + } + return Ok(InputBytes(bytes)) + }, + Err(_e) => (), + } + // use rust names as a vec of their utf8 bytecode. + match syn::Ident::parse(input) { + Ok(ident) => return Ok(InputBytes(ident.to_string().as_bytes().to_vec())), + Err(_e) => (), + } + Ok(InputBytes(syn::LitByteStr::parse(input)?.value())) + } +} + +impl Parse for MultipleInputBytes { + fn parse(input: ParseStream) -> syn::Result { + let elts = + syn::punctuated::Punctuated::::parse_terminated(input)?; + Ok(MultipleInputBytes(elts.into_iter().map(|elt| elt.0).collect())) + } +} + +pub(super) fn twox_64(bytes: Vec) -> TokenStream { + bytes_to_array(sp_core_hashing::twox_64(bytes.as_slice())) +} + +pub(super) fn twox_128(bytes: Vec) -> TokenStream { + bytes_to_array(sp_core_hashing::twox_128(bytes.as_slice())) +} + +pub(super) fn blake2b_512(bytes: Vec) -> TokenStream { + bytes_to_array(sp_core_hashing::blake2_512(bytes.as_slice())) +} + +pub(super) fn blake2b_256(bytes: Vec) -> TokenStream { + bytes_to_array(sp_core_hashing::blake2_256(bytes.as_slice())) +} + +pub(super) fn blake2b_64(bytes: Vec) -> TokenStream { + bytes_to_array(sp_core_hashing::blake2_64(bytes.as_slice())) +} + +pub(super) fn keccak_256(bytes: Vec) -> TokenStream { + bytes_to_array(sp_core_hashing::keccak_256(bytes.as_slice())) +} + +pub(super) fn keccak_512(bytes: Vec) -> TokenStream { + bytes_to_array(sp_core_hashing::keccak_512(bytes.as_slice())) +} + +pub(super) fn sha2_256(bytes: Vec) -> TokenStream { + bytes_to_array(sp_core_hashing::sha2_256(bytes.as_slice())) +} + +fn bytes_to_array(bytes: impl IntoIterator) -> TokenStream { + let bytes = bytes.into_iter(); + + quote!( + [ #( #bytes ),* ] + ) + .into() +} diff --git a/primitives/core/hashing/proc-macro/src/lib.rs b/primitives/core/hashing/proc-macro/src/lib.rs new file mode 100644 index 000000000000..2db292d8dc0c --- /dev/null +++ b/primitives/core/hashing/proc-macro/src/lib.rs @@ -0,0 +1,129 @@ +// This file is part of Substrate. + +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Macros to calculate constant hash bytes result. +//! +//! Macros from this crate does apply a specific hash function on input. +//! Input can be literal byte array as `b"content"` or array of bytes +//! as `[1, 2, 3]`. +//! Rust identifier can also be use, in this case we use their utf8 string +//! byte representation, for instance if the ident is `MyStruct`, then +//! `b"MyStruct"` will be hashed. +//! If multiple arguments comma separated are passed, they are concatenated +//! then hashed. +//! +//! Examples: +//! +//! ```rust +//! assert_eq!( +//! sp_core_hashing_proc_macro::blake2b_256!(b"test"), +//! sp_core_hashing::blake2_256(b"test"), +//! ); +//! assert_eq!( +//! sp_core_hashing_proc_macro::blake2b_256!([1u8]), +//! sp_core_hashing::blake2_256(&[1u8]), +//! ); +//! assert_eq!( +//! sp_core_hashing_proc_macro::blake2b_256!([1, 2, 3]), +//! sp_core_hashing::blake2_256(&[1, 2, 3]), +//! ); +//! assert_eq!( +//! sp_core_hashing_proc_macro::blake2b_256!(identifier), +//! sp_core_hashing::blake2_256(b"identifier"), +//! ); +//! assert_eq!( +//! sp_core_hashing_proc_macro::blake2b_256!(identifier, b"/string"), +//! sp_core_hashing::blake2_256(b"identifier/string"), +//! ); +//! ``` + +mod impls; + +use impls::MultipleInputBytes; +use proc_macro::TokenStream; + +/// Process a Blake2 64-bit hash of bytes parameter outputs a `[u8; 8]`. +/// Multiple inputs are concatenated before hashing. +/// Input can be identifier (name of identifier as bytes is used), byte string or +/// array of bytes. +#[proc_macro] +pub fn blake2b_64(input: TokenStream) -> TokenStream { + impls::blake2b_64(syn::parse_macro_input!(input as MultipleInputBytes).concatenated()) +} + +/// Apply a Blake2 256-bit hash of bytes parameter, outputs a `[u8; 32]`. +/// Multiple inputs are concatenated before hashing. +/// Input can be identifier (name of identifier as bytes is used), byte string or +/// array of bytes. +#[proc_macro] +pub fn blake2b_256(input: TokenStream) -> TokenStream { + impls::blake2b_256(syn::parse_macro_input!(input as MultipleInputBytes).concatenated()) +} + +/// Apply a Blake2 512-bit hash of bytes parameter, outputs a `[u8; 64]`. +/// Multiple inputs are concatenated before hashing. +/// Input can be identifier (name of identifier as bytes is used), byte string or +/// array of bytes. +#[proc_macro] +pub fn blake2b_512(input: TokenStream) -> TokenStream { + impls::blake2b_512(syn::parse_macro_input!(input as MultipleInputBytes).concatenated()) +} + +/// Apply a XX 64-bit hash on its bytes parameter, outputs a `[u8; 8]`. +/// Multiple inputs are concatenated before hashing. +/// Input can be identifier (name of identifier as bytes is used), byte string or +/// array of bytes. +#[proc_macro] +pub fn twox_64(input: TokenStream) -> TokenStream { + impls::twox_64(syn::parse_macro_input!(input as MultipleInputBytes).concatenated()) +} + +/// Apply a XX 128-bit hash on its bytes parameter, outputs a `[u8; 16]`. +/// Multiple inputs are concatenated before hashing. +/// Input can be identifier (name of identifier as bytes is used), byte string or +/// array of bytes. +#[proc_macro] +pub fn twox_128(input: TokenStream) -> TokenStream { + impls::twox_128(syn::parse_macro_input!(input as MultipleInputBytes).concatenated()) +} + +/// Apply a keccak 256-bit hash on its bytes parameter, outputs a `[u8; 32]`. +/// Multiple inputs are concatenated before hashing. +/// Input can be identifier (name of identifier as bytes is used), byte string or +/// array of bytes. +#[proc_macro] +pub fn keccak_256(input: TokenStream) -> TokenStream { + impls::keccak_256(syn::parse_macro_input!(input as MultipleInputBytes).concatenated()) +} + +/// Apply a keccak 512-bit hash on its bytes parameter, outputs a `[u8; 64]`. +/// Multiple inputs are concatenated before hashing. +/// Input can be identifier (name of identifier as bytes is used), byte string or +/// array of bytes. +#[proc_macro] +pub fn keccak_512(input: TokenStream) -> TokenStream { + impls::keccak_512(syn::parse_macro_input!(input as MultipleInputBytes).concatenated()) +} + +/// Apply a sha2 256-bit hash on its bytes parameter, outputs a `[u8; 32]`. +/// Multiple inputs are concatenated before hashing. +/// Input can be identifier (name of identifier as bytes is used), byte string or +/// array of bytes. +#[proc_macro] +pub fn sha2_256(input: TokenStream) -> TokenStream { + impls::sha2_256(syn::parse_macro_input!(input as MultipleInputBytes).concatenated()) +} diff --git a/primitives/core/hashing/src/lib.rs b/primitives/core/hashing/src/lib.rs new file mode 100644 index 000000000000..e6ccd5aaa8fb --- /dev/null +++ b/primitives/core/hashing/src/lib.rs @@ -0,0 +1,148 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Hashing Functions. + +#![warn(missing_docs)] +#![cfg_attr(not(feature = "std"), no_std)] + +use core::hash::Hasher; + +use byteorder::{ByteOrder, LittleEndian}; +use digest::{ + consts::{U16, U32, U8}, + Digest, +}; + +/// Do a Blake2 512-bit hash and place result in `dest`. +pub fn blake2_512_into(data: &[u8], dest: &mut [u8; 64]) { + dest.copy_from_slice(blake2::Blake2b512::digest(data).as_slice()); +} + +/// Do a Blake2 512-bit hash and return result. +pub fn blake2_512(data: &[u8]) -> [u8; 64] { + let mut r = [0; 64]; + blake2_512_into(data, &mut r); + r +} + +/// Do a Blake2 256-bit hash and place result in `dest`. +pub fn blake2_256_into(data: &[u8], dest: &mut [u8; 32]) { + type Blake2b256 = blake2::Blake2b; + dest.copy_from_slice(Blake2b256::digest(data).as_slice()); +} + +/// Do a Blake2 256-bit hash and return result. +pub fn blake2_256(data: &[u8]) -> [u8; 32] { + let mut r = [0; 32]; + blake2_256_into(data, &mut r); + r +} + +/// Do a Blake2 128-bit hash and place result in `dest`. +pub fn blake2_128_into(data: &[u8], dest: &mut [u8; 16]) { + type Blake2b128 = blake2::Blake2b; + dest.copy_from_slice(Blake2b128::digest(data).as_slice()); +} + +/// Do a Blake2 128-bit hash and return result. +pub fn blake2_128(data: &[u8]) -> [u8; 16] { + let mut r = [0; 16]; + blake2_128_into(data, &mut r); + r +} + +/// Do a Blake2 64-bit hash and place result in `dest`. +pub fn blake2_64_into(data: &[u8], dest: &mut [u8; 8]) { + type Blake2b64 = blake2::Blake2b; + dest.copy_from_slice(Blake2b64::digest(data).as_slice()); +} + +/// Do a Blake2 64-bit hash and return result. +pub fn blake2_64(data: &[u8]) -> [u8; 8] { + let mut r = [0; 8]; + blake2_64_into(data, &mut r); + r +} + +/// Do a XX 64-bit hash and place result in `dest`. +pub fn twox_64_into(data: &[u8], dest: &mut [u8; 8]) { + let r0 = twox_hash::XxHash::with_seed(0).chain_update(data).finish(); + LittleEndian::write_u64(&mut dest[0..8], r0); +} + +/// Do a XX 64-bit hash and return result. +pub fn twox_64(data: &[u8]) -> [u8; 8] { + let mut r: [u8; 8] = [0; 8]; + twox_64_into(data, &mut r); + r +} + +/// Do a XX 128-bit hash and place result in `dest`. +pub fn twox_128_into(data: &[u8], dest: &mut [u8; 16]) { + let r0 = twox_hash::XxHash::with_seed(0).chain_update(data).finish(); + let r1 = twox_hash::XxHash::with_seed(1).chain_update(data).finish(); + LittleEndian::write_u64(&mut dest[0..8], r0); + LittleEndian::write_u64(&mut dest[8..16], r1); +} + +/// Do a XX 128-bit hash and return result. +pub fn twox_128(data: &[u8]) -> [u8; 16] { + let mut r: [u8; 16] = [0; 16]; + twox_128_into(data, &mut r); + r +} + +/// Do a XX 256-bit hash and place result in `dest`. +pub fn twox_256_into(data: &[u8], dest: &mut [u8; 32]) { + let r0 = twox_hash::XxHash::with_seed(0).chain_update(data).finish(); + let r1 = twox_hash::XxHash::with_seed(1).chain_update(data).finish(); + let r2 = twox_hash::XxHash::with_seed(2).chain_update(data).finish(); + let r3 = twox_hash::XxHash::with_seed(3).chain_update(data).finish(); + LittleEndian::write_u64(&mut dest[0..8], r0); + LittleEndian::write_u64(&mut dest[8..16], r1); + LittleEndian::write_u64(&mut dest[16..24], r2); + LittleEndian::write_u64(&mut dest[24..32], r3); +} + +/// Do a XX 256-bit hash and return result. +pub fn twox_256(data: &[u8]) -> [u8; 32] { + let mut r: [u8; 32] = [0; 32]; + twox_256_into(data, &mut r); + r +} + +/// Do a keccak 256-bit hash and return result. +pub fn keccak_256(data: &[u8]) -> [u8; 32] { + let mut output = [0u8; 32]; + output.copy_from_slice(sha3::Keccak256::digest(data).as_slice()); + output +} + +/// Do a keccak 512-bit hash and return result. +pub fn keccak_512(data: &[u8]) -> [u8; 64] { + let mut output = [0u8; 64]; + output.copy_from_slice(sha3::Keccak512::digest(data).as_slice()); + output +} + +/// Do a sha2 256-bit hash and return result. +pub fn sha2_256(data: &[u8]) -> [u8; 32] { + let mut output = [0u8; 32]; + output.copy_from_slice(sha2::Sha256::digest(data).as_slice()); + output +} diff --git a/primitives/core/src/changes_trie.rs b/primitives/core/src/changes_trie.rs deleted file mode 100644 index f4ce83dc2c87..000000000000 --- a/primitives/core/src/changes_trie.rs +++ /dev/null @@ -1,321 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Substrate changes trie configuration. - -use codec::{Decode, Encode}; -use num_traits::Zero; -#[cfg(any(feature = "std", test))] -use serde::{Deserialize, Serialize}; - -/// Substrate changes trie configuration. -#[cfg_attr( - any(feature = "std", test), - derive(Serialize, Deserialize, parity_util_mem::MallocSizeOf) -)] -#[derive(Debug, Clone, PartialEq, Eq, Default, Encode, Decode, scale_info::TypeInfo)] -pub struct ChangesTrieConfiguration { - /// Interval (in blocks) at which level1-digests are created. Digests are not - /// created when this is less or equal to 1. - pub digest_interval: u32, - /// Maximal number of digest levels in hierarchy. 0 means that digests are not - /// created at all (even level1 digests). 1 means only level1-digests are created. - /// 2 means that every digest_interval^2 there will be a level2-digest, and so on. - /// Please ensure that maximum digest interval (i.e. digest_interval^digest_levels) - /// is within `u32` limits. Otherwise you'll never see digests covering such intervals - /// && maximal digests interval will be truncated to the last interval that fits - /// `u32` limits. - pub digest_levels: u32, -} - -/// Substrate changes trie configuration range. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ChangesTrieConfigurationRange { - /// Zero block of configuration. - pub zero: (Number, Hash), - /// Last block of configuration (if configuration has been deactivated at some point). - pub end: Option<(Number, Hash)>, - /// The configuration itself. None if changes tries were disabled in this range. - pub config: Option, -} - -impl ChangesTrieConfiguration { - /// Create new configuration given digest interval and levels. - pub fn new(digest_interval: u32, digest_levels: u32) -> Self { - Self { digest_interval, digest_levels } - } - - /// Is digest build enabled? - pub fn is_digest_build_enabled(&self) -> bool { - self.digest_interval > 1 && self.digest_levels > 0 - } - - /// Do we need to build digest at given block? - pub fn is_digest_build_required_at_block(&self, zero: Number, block: Number) -> bool - where - Number: From - + PartialEq - + ::sp_std::ops::Rem - + ::sp_std::ops::Sub - + ::sp_std::cmp::PartialOrd - + Zero, - { - block > zero && - self.is_digest_build_enabled() && - ((block - zero) % self.digest_interval.into()).is_zero() - } - - /// Returns max digest interval. One if digests are not created at all. - pub fn max_digest_interval(&self) -> u32 { - if !self.is_digest_build_enabled() { - return 1 - } - - // we'll get >1 loop iteration only when bad configuration parameters are selected - let mut current_level = self.digest_levels; - loop { - if let Some(max_digest_interval) = self.digest_interval.checked_pow(current_level) { - return max_digest_interval - } - - current_level -= 1; - } - } - - /// Returns max level digest block number that has been created at block <= passed block number. - /// - /// Returns None if digests are not created at all. - pub fn prev_max_level_digest_block(&self, zero: Number, block: Number) -> Option - where - Number: Clone - + From - + PartialOrd - + PartialEq - + ::sp_std::ops::Add - + ::sp_std::ops::Sub - + ::sp_std::ops::Div - + ::sp_std::ops::Mul - + Zero, - { - if block <= zero { - return None - } - - let (next_begin, next_end) = - self.next_max_level_digest_range(zero.clone(), block.clone())?; - - // if 'next' digest includes our block, then it is a also a previous digest - if next_end == block { - return Some(block) - } - - // if previous digest ends at zero block, then there are no previous digest - let prev_end = next_begin - 1.into(); - if prev_end == zero { - None - } else { - Some(prev_end) - } - } - - /// Returns max level digest blocks range (inclusive) which includes passed block. - /// - /// Returns None if digests are not created at all. - /// It will return the first max-level digest if block is <= zero. - pub fn next_max_level_digest_range( - &self, - zero: Number, - mut block: Number, - ) -> Option<(Number, Number)> - where - Number: Clone - + From - + PartialOrd - + PartialEq - + ::sp_std::ops::Add - + ::sp_std::ops::Sub - + ::sp_std::ops::Div - + ::sp_std::ops::Mul, - { - if !self.is_digest_build_enabled() { - return None - } - - if block <= zero { - block = zero.clone() + 1.into(); - } - - let max_digest_interval: Number = self.max_digest_interval().into(); - let max_digests_since_zero = (block.clone() - zero.clone()) / max_digest_interval.clone(); - if max_digests_since_zero == 0.into() { - return Some((zero.clone() + 1.into(), zero + max_digest_interval)) - } - let last_max_digest_block = zero + max_digests_since_zero * max_digest_interval.clone(); - Some(if block == last_max_digest_block { - (block.clone() - max_digest_interval + 1.into(), block) - } else { - (last_max_digest_block.clone() + 1.into(), last_max_digest_block + max_digest_interval) - }) - } - - /// Returns Some if digest must be built at given block number. - /// The tuple is: - /// ( - /// digest level - /// digest interval (in blocks) - /// step between blocks we're interested in when digest is built - /// ) - pub fn digest_level_at_block( - &self, - zero: Number, - block: Number, - ) -> Option<(u32, u32, u32)> - where - Number: Clone - + From - + PartialEq - + ::sp_std::ops::Rem - + ::sp_std::ops::Sub - + ::sp_std::cmp::PartialOrd - + Zero, - { - if !self.is_digest_build_required_at_block(zero.clone(), block.clone()) { - return None - } - - let relative_block = block - zero; - let mut digest_interval = self.digest_interval; - let mut current_level = 1u32; - let mut digest_step = 1u32; - while current_level < self.digest_levels { - let new_digest_interval = match digest_interval.checked_mul(self.digest_interval) { - Some(new_digest_interval) - if (relative_block.clone() % new_digest_interval.into()).is_zero() => - new_digest_interval, - _ => break, - }; - - digest_step = digest_interval; - digest_interval = new_digest_interval; - current_level += 1; - } - - Some((current_level, digest_interval, digest_step)) - } -} - -#[cfg(test)] -mod tests { - use super::ChangesTrieConfiguration; - - fn config(interval: u32, levels: u32) -> ChangesTrieConfiguration { - ChangesTrieConfiguration { digest_interval: interval, digest_levels: levels } - } - - #[test] - fn is_digest_build_enabled_works() { - assert!(!config(0, 100).is_digest_build_enabled()); - assert!(!config(1, 100).is_digest_build_enabled()); - assert!(config(2, 100).is_digest_build_enabled()); - assert!(!config(100, 0).is_digest_build_enabled()); - assert!(config(100, 1).is_digest_build_enabled()); - } - - #[test] - fn is_digest_build_required_at_block_works() { - fn test_with_zero(zero: u64) { - assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero)); - assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 1u64)); - assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 2u64)); - assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 4u64)); - assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 8u64)); - assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 9u64)); - assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 64u64)); - assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 64u64)); - assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 512u64)); - assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 4096u64)); - assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 4103u64)); - assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 4104u64)); - assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 4108u64)); - } - - test_with_zero(0); - test_with_zero(8); - test_with_zero(17); - } - - #[test] - fn digest_level_at_block_works() { - fn test_with_zero(zero: u64) { - assert_eq!(config(8, 4).digest_level_at_block(zero, zero), None); - assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 7u64), None); - assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 63u64), None); - assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 8u64), Some((1, 8, 1))); - assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 64u64), Some((2, 64, 8))); - assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 512u64), Some((3, 512, 64))); - assert_eq!( - config(8, 4).digest_level_at_block(zero, zero + 4096u64), - Some((4, 4096, 512)) - ); - assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 4112u64), Some((1, 8, 1))); - } - - test_with_zero(0); - test_with_zero(8); - test_with_zero(17); - } - - #[test] - fn max_digest_interval_works() { - assert_eq!(config(0, 0).max_digest_interval(), 1); - assert_eq!(config(2, 2).max_digest_interval(), 4); - assert_eq!(config(8, 4).max_digest_interval(), 4096); - assert_eq!(config(::std::u32::MAX, 1024).max_digest_interval(), ::std::u32::MAX); - } - - #[test] - fn next_max_level_digest_range_works() { - assert_eq!(config(0, 0).next_max_level_digest_range(0u64, 16), None); - assert_eq!(config(1, 1).next_max_level_digest_range(0u64, 16), None); - assert_eq!(config(2, 1).next_max_level_digest_range(0u64, 16), Some((15, 16))); - assert_eq!(config(4, 1).next_max_level_digest_range(0u64, 16), Some((13, 16))); - assert_eq!(config(32, 1).next_max_level_digest_range(0u64, 16), Some((1, 32))); - assert_eq!(config(2, 3).next_max_level_digest_range(0u64, 10), Some((9, 16))); - assert_eq!(config(2, 3).next_max_level_digest_range(0u64, 8), Some((1, 8))); - assert_eq!(config(2, 1).next_max_level_digest_range(1u64, 1), Some((2, 3))); - assert_eq!(config(2, 2).next_max_level_digest_range(7u64, 9), Some((8, 11))); - - assert_eq!(config(2, 2).next_max_level_digest_range(7u64, 5), Some((8, 11))); - } - - #[test] - fn prev_max_level_digest_block_works() { - assert_eq!(config(0, 0).prev_max_level_digest_block(0u64, 16), None); - assert_eq!(config(1, 1).prev_max_level_digest_block(0u64, 16), None); - assert_eq!(config(2, 1).prev_max_level_digest_block(0u64, 16), Some(16)); - assert_eq!(config(4, 1).prev_max_level_digest_block(0u64, 16), Some(16)); - assert_eq!(config(4, 2).prev_max_level_digest_block(0u64, 16), Some(16)); - assert_eq!(config(4, 2).prev_max_level_digest_block(0u64, 17), Some(16)); - assert_eq!(config(4, 2).prev_max_level_digest_block(0u64, 33), Some(32)); - assert_eq!(config(32, 1).prev_max_level_digest_block(0u64, 16), None); - assert_eq!(config(2, 3).prev_max_level_digest_block(0u64, 10), Some(8)); - assert_eq!(config(2, 3).prev_max_level_digest_block(0u64, 8), Some(8)); - assert_eq!(config(2, 2).prev_max_level_digest_block(7u64, 8), None); - - assert_eq!(config(2, 2).prev_max_level_digest_block(7u64, 5), None); - } -} diff --git a/primitives/core/src/crypto.rs b/primitives/core/src/crypto.rs index cf7be5f2166e..f994da151535 100644 --- a/primitives/core/src/crypto.rs +++ b/primitives/core/src/crypto.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,8 +26,6 @@ use crate::{ed25519, sr25519}; use base58::{FromBase58, ToBase58}; use codec::{Decode, Encode, MaxEncodedLen}; #[cfg(feature = "std")] -use parking_lot::Mutex; -#[cfg(feature = "std")] use rand::{rngs::OsRng, RngCore}; #[cfg(feature = "std")] use regex::Regex; @@ -38,14 +36,15 @@ pub use secrecy::ExposeSecret; #[cfg(feature = "std")] pub use secrecy::SecretString; use sp_runtime_interface::pass_by::PassByInner; -#[cfg(feature = "std")] -use sp_std::convert::TryInto; #[doc(hidden)] pub use sp_std::ops::Deref; -use sp_std::{convert::TryFrom, hash::Hash, str, vec::Vec}; +use sp_std::{hash::Hash, str, vec::Vec}; /// Trait to zeroize a memory buffer. pub use zeroize::Zeroize; +#[cfg(feature = "full_crypto")] +pub use ss58_registry::{from_known_address_format, Ss58AddressFormat, Ss58AddressFormatRegistry}; + /// The root phrase for our publicly known keys. pub const DEV_PHRASE: &str = "bottom drive obey lake curtain smoke basket hold race lonely fit walk"; @@ -199,35 +198,55 @@ impl> From for DeriveJunction { } /// An error type for SS58 decoding. +#[cfg_attr(feature = "std", derive(thiserror::Error))] +#[cfg_attr(not(feature = "std"), derive(Debug))] +#[derive(Clone, Copy, Eq, PartialEq)] +#[allow(missing_docs)] #[cfg(feature = "full_crypto")] -#[derive(Clone, Copy, Eq, PartialEq, Debug)] pub enum PublicError { - /// Bad alphabet. + #[cfg_attr(feature = "std", error("Base 58 requirement is violated"))] BadBase58, - /// Bad length. + #[cfg_attr(feature = "std", error("Length is bad"))] BadLength, - /// Unknown identifier for the encoding. - UnknownVersion, - /// Invalid checksum. + #[cfg_attr( + feature = "std", + error( + "Unknown SS58 address format `{}`. ` \ + `To support this address format, you need to call `set_default_ss58_version` at node start up.", + _0 + ) + )] + UnknownSs58AddressFormat(Ss58AddressFormat), + #[cfg_attr(feature = "std", error("Invalid checksum"))] InvalidChecksum, - /// Invalid format. + #[cfg_attr(feature = "std", error("Invalid SS58 prefix byte."))] + InvalidPrefix, + #[cfg_attr(feature = "std", error("Invalid SS58 format."))] InvalidFormat, - /// Invalid derivation path. + #[cfg_attr(feature = "std", error("Invalid derivation path."))] InvalidPath, - /// Disallowed SS58 Address Format for this datatype. + #[cfg_attr(feature = "std", error("Disallowed SS58 Address Format for this datatype."))] FormatNotAllowed, } +#[cfg(feature = "std")] +impl sp_std::fmt::Debug for PublicError { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { + // Just use the `Display` implementation + write!(f, "{}", self) + } +} + /// Key that can be encoded to/from SS58. /// -/// See +/// See /// for information on the codec. #[cfg(feature = "full_crypto")] -pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + Default { +pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + ByteArray { /// A format filterer, can be used to ensure that `from_ss58check` family only decode for /// allowed identifiers. By default just refuses the two reserved identifiers. fn format_is_allowed(f: Ss58AddressFormat) -> bool { - !matches!(f, Ss58AddressFormat::Reserved46 | Ss58AddressFormat::Reserved47) + !f.is_reserved() } /// Some if the string is a properly encoded SS58Check address. @@ -235,8 +254,8 @@ pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + Default { fn from_ss58check(s: &str) -> Result { Self::from_ss58check_with_version(s).and_then(|(r, v)| match v { v if !v.is_custom() => Ok(r), - v if v == *DEFAULT_VERSION.lock() => Ok(r), - _ => Err(PublicError::UnknownVersion), + v if v == default_ss58_version() => Ok(r), + v => Err(PublicError::UnknownSs58AddressFormat(v)), }) } @@ -244,10 +263,7 @@ pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + Default { #[cfg(feature = "std")] fn from_ss58check_with_version(s: &str) -> Result<(Self, Ss58AddressFormat), PublicError> { const CHECKSUM_LEN: usize = 2; - let mut res = Self::default(); - - // Must decode to our type. - let body_len = res.as_mut().len(); + let body_len = Self::LEN; let data = s.from_base58().map_err(|_| PublicError::BadBase58)?; if data.len() < 2 { @@ -265,12 +281,12 @@ pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + Default { let upper = data[1] & 0b00111111; (2, (lower as u16) | ((upper as u16) << 8)) }, - _ => return Err(PublicError::UnknownVersion), + _ => return Err(PublicError::InvalidPrefix), }; if data.len() != prefix_len + body_len + CHECKSUM_LEN { return Err(PublicError::BadLength) } - let format = ident.try_into().map_err(|_: ()| PublicError::UnknownVersion)?; + let format = ident.into(); if !Self::format_is_allowed(format) { return Err(PublicError::FormatNotAllowed) } @@ -281,8 +297,10 @@ pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + Default { // Invalid checksum. return Err(PublicError::InvalidChecksum) } - res.as_mut().copy_from_slice(&data[prefix_len..body_len + prefix_len]); - Ok((res, format)) + + let result = Self::from_slice(&data[prefix_len..body_len + prefix_len]) + .map_err(|()| PublicError::BadLength)?; + Ok((result, format)) } /// Some if the string is a properly encoded SS58Check address, optionally with @@ -291,8 +309,8 @@ pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + Default { fn from_string(s: &str) -> Result { Self::from_string_with_version(s).and_then(|(r, v)| match v { v if !v.is_custom() => Ok(r), - v if v == *DEFAULT_VERSION.lock() => Ok(r), - _ => Err(PublicError::UnknownVersion), + v if v == default_ss58_version() => Ok(r), + v => Err(PublicError::UnknownSs58AddressFormat(v)), }) } @@ -322,7 +340,7 @@ pub trait Ss58Codec: Sized + AsMut<[u8]> + AsRef<[u8]> + Default { /// Return the ss58-check string for this key. #[cfg(feature = "std")] fn to_ss58check(&self) -> String { - self.to_ss58check_with_version(*DEFAULT_VERSION.lock()) + self.to_ss58check_with_version(default_ss58_version()) } /// Some if the string is a properly encoded SS58Check address, optionally with @@ -355,282 +373,36 @@ fn ss58hash(data: &[u8]) -> blake2_rfc::blake2b::Blake2bResult { context.finalize() } +/// Default prefix number #[cfg(feature = "std")] -lazy_static::lazy_static! { - static ref DEFAULT_VERSION: Mutex - = Mutex::new(Ss58AddressFormat::SubstrateAccount); -} - -#[cfg(feature = "full_crypto")] -macro_rules! ss58_address_format { - ( $( $identifier:tt => ($number:expr, $name:expr, $desc:tt) )* ) => ( - /// A known address (sub)format/network ID for SS58. - #[derive(Copy, Clone, PartialEq, Eq, crate::RuntimeDebug)] - pub enum Ss58AddressFormat { - $(#[doc = $desc] $identifier),*, - /// Use a manually provided numeric value as a standard identifier - Custom(u16), - } - - #[cfg(feature = "std")] - impl std::fmt::Display for Ss58AddressFormat { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - $( - Ss58AddressFormat::$identifier => write!(f, "{}", $name), - )* - Ss58AddressFormat::Custom(x) => write!(f, "{}", x), - } - - } - } - - static ALL_SS58_ADDRESS_FORMATS: [Ss58AddressFormat; 0 $(+ { let _ = $number; 1})*] = [ - $(Ss58AddressFormat::$identifier),*, - ]; - - impl Ss58AddressFormat { - /// names of all address formats - pub fn all_names() -> &'static [&'static str] { - &[ - $($name),*, - ] - } - /// All known address formats. - pub fn all() -> &'static [Ss58AddressFormat] { - &ALL_SS58_ADDRESS_FORMATS - } - - /// Whether the address is custom. - pub fn is_custom(&self) -> bool { - matches!(self, Self::Custom(_)) - } - } - - impl TryFrom for Ss58AddressFormat { - type Error = (); - - fn try_from(x: u8) -> Result { - Ss58AddressFormat::try_from(x as u16) - } - } - - impl From for u16 { - fn from(x: Ss58AddressFormat) -> u16 { - match x { - $(Ss58AddressFormat::$identifier => $number),*, - Ss58AddressFormat::Custom(n) => n, - } - } - } - - impl TryFrom for Ss58AddressFormat { - type Error = (); - - fn try_from(x: u16) -> Result { - match x { - $($number => Ok(Ss58AddressFormat::$identifier)),*, - _ => Ok(Ss58AddressFormat::Custom(x)), - } - } - } - - /// Error encountered while parsing `Ss58AddressFormat` from &'_ str - /// unit struct for now. - #[derive(Copy, Clone, PartialEq, Eq, crate::RuntimeDebug)] - pub struct ParseError; - - impl<'a> TryFrom<&'a str> for Ss58AddressFormat { - type Error = ParseError; - - fn try_from(x: &'a str) -> Result { - match x { - $($name => Ok(Ss58AddressFormat::$identifier)),*, - a => a.parse::().map(Ss58AddressFormat::Custom).map_err(|_| ParseError), - } - } - } - - #[cfg(feature = "std")] - impl std::str::FromStr for Ss58AddressFormat { - type Err = ParseError; - - fn from_str(data: &str) -> Result { - Self::try_from(data) - } - } - - #[cfg(feature = "std")] - impl std::fmt::Display for ParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "failed to parse network value as u8") - } - } - - #[cfg(feature = "std")] - impl Default for Ss58AddressFormat { - fn default() -> Self { - *DEFAULT_VERSION.lock() - } - } +static DEFAULT_VERSION: core::sync::atomic::AtomicU16 = std::sync::atomic::AtomicU16::new( + from_known_address_format(Ss58AddressFormatRegistry::SubstrateAccount), +); - #[cfg(feature = "std")] - impl From for String { - fn from(x: Ss58AddressFormat) -> String { - x.to_string() - } - } - ) +/// Returns default SS58 format used by the current active process. +#[cfg(feature = "std")] +pub fn default_ss58_version() -> Ss58AddressFormat { + DEFAULT_VERSION.load(std::sync::atomic::Ordering::Relaxed).into() } -#[cfg(feature = "full_crypto")] -ss58_address_format!( - PolkadotAccount => - (0, "polkadot", "Polkadot Relay-chain, standard account (*25519).") - BareSr25519 => - (1, "sr25519", "Bare 32-bit Schnorr/Ristretto 25519 (S/R 25519) key.") - KusamaAccount => - (2, "kusama", "Kusama Relay-chain, standard account (*25519).") - BareEd25519 => - (3, "ed25519", "Bare 32-bit Edwards Ed25519 key.") - KatalChainAccount => - (4, "katalchain", "Katal Chain, standard account (*25519).") - PlasmAccount => - (5, "plasm", "Plasm Network, standard account (*25519).") - BifrostAccount => - (6, "bifrost", "Bifrost mainnet, direct checksum, standard account (*25519).") - EdgewareAccount => - (7, "edgeware", "Edgeware mainnet, standard account (*25519).") - KaruraAccount => - (8, "karura", "Acala Karura canary network, standard account (*25519).") - ReynoldsAccount => - (9, "reynolds", "Laminar Reynolds canary network, standard account (*25519).") - AcalaAccount => - (10, "acala", "Acala mainnet, standard account (*25519).") - LaminarAccount => - (11, "laminar", "Laminar mainnet, standard account (*25519).") - PolymathAccount => - (12, "polymath", "Polymath network, standard account (*25519).") - SubstraTeeAccount => - (13, "substratee", "Any SubstraTEE off-chain network private account (*25519).") - TotemAccount => - (14, "totem", "Any Totem Live Accounting network standard account (*25519).") - SynesthesiaAccount => - (15, "synesthesia", "Synesthesia mainnet, standard account (*25519).") - KulupuAccount => - (16, "kulupu", "Kulupu mainnet, standard account (*25519).") - DarkAccount => - (17, "dark", "Dark mainnet, standard account (*25519).") - DarwiniaAccount => - (18, "darwinia", "Darwinia Chain mainnet, standard account (*25519).") - GeekAccount => - (19, "geek", "GeekCash mainnet, standard account (*25519).") - StafiAccount => - (20, "stafi", "Stafi mainnet, standard account (*25519).") - DockTestAccount => - (21, "dock-testnet", "Dock testnet, standard account (*25519).") - DockMainAccount => - (22, "dock-mainnet", "Dock mainnet, standard account (*25519).") - ShiftNrg => - (23, "shift", "ShiftNrg mainnet, standard account (*25519).") - ZeroAccount => - (24, "zero", "ZERO mainnet, standard account (*25519).") - AlphavilleAccount => - (25, "alphaville", "ZERO testnet, standard account (*25519).") - JupiterAccount => - (26, "jupiter", "Jupiter testnet, standard account (*25519).") - SubsocialAccount => - (28, "subsocial", "Subsocial network, standard account (*25519).") - DhiwayAccount => - (29, "cord", "Dhiway CORD network, standard account (*25519).") - PhalaAccount => - (30, "phala", "Phala Network, standard account (*25519).") - LitentryAccount => - (31, "litentry", "Litentry Network, standard account (*25519).") - RobonomicsAccount => - (32, "robonomics", "Any Robonomics network standard account (*25519).") - DataHighwayAccount => - (33, "datahighway", "DataHighway mainnet, standard account (*25519).") - AresAccount => - (34, "ares", "Ares Protocol, standard account (*25519).") - ValiuAccount => - (35, "vln", "Valiu Liquidity Network mainnet, standard account (*25519).") - CentrifugeAccount => - (36, "centrifuge", "Centrifuge Chain mainnet, standard account (*25519).") - NodleAccount => - (37, "nodle", "Nodle Chain mainnet, standard account (*25519).") - KiltAccount => - (38, "kilt", "KILT Chain mainnet, standard account (*25519).") - PolimecAccount => - (41, "poli", "Polimec Chain mainnet, standard account (*25519).") - SubstrateAccount => - (42, "substrate", "Any Substrate network, standard account (*25519).") - BareSecp256k1 => - (43, "secp256k1", "Bare ECDSA SECP256k1 key.") - ChainXAccount => - (44, "chainx", "ChainX mainnet, standard account (*25519).") - UniartsAccount => - (45, "uniarts", "UniArts Chain mainnet, standard account (*25519).") - Reserved46 => - (46, "reserved46", "Reserved for future use (46).") - Reserved47 => - (47, "reserved47", "Reserved for future use (47).") - NeatcoinAccount => - (48, "neatcoin", "Neatcoin mainnet, standard account (*25519).") - PicassoAccount => - (49, "picasso", "Composable Canary Network, standard account (*25519).") - ComposableAccount => - (50, "composable", "Composable mainnet, standard account (*25519).") - HydraDXAccount => - (63, "hydradx", "HydraDX standard account (*25519).") - AventusAccount => - (65, "aventus", "Aventus Chain mainnet, standard account (*25519).") - CrustAccount => - (66, "crust", "Crust Network, standard account (*25519).") - EquilibriumAccount => - (67, "equilibrium", "Equilibrium Network, standard account (*25519).") - SoraAccount => - (69, "sora", "SORA Network, standard account (*25519).") - ZeitgeistAccount => - (73, "zeitgeist", "Zeitgeist network, standard account (*25519).") - MantaAccount => - (77, "manta", "Manta Network, standard account (*25519).") - CalamariAccount => - (78, "calamari", "Manta Canary Network, standard account (*25519).") - Polkadex => - (88, "polkadex", "Polkadex Mainnet, standard account (*25519).") - PolkaSmith => - (98, "polkasmith", "PolkaSmith Canary Network, standard account (*25519).") - PolkaFoundry => - (99, "polkafoundry", "PolkaFoundry Network, standard account (*25519).") - OriginTrailAccount => - (101, "origintrail-parachain", "OriginTrail Parachain, ethereumm account (ECDSA).") - HeikoAccount => - (110, "heiko", "Heiko, session key (*25519).") - CloverAccount => - (128, "clover", "Clover Finance, standard account (*25519).") - ParallelAccount => - (172, "parallel", "Parallel, session key (*25519).") - SocialAccount => - (252, "social-network", "Social Network, standard account (*25519).") - Moonbeam => - (1284, "moonbeam", "Moonbeam, session key (*25519).") - Moonriver => - (1285, "moonriver", "Moonriver, session key (*25519).") - BasiliskAccount => - (10041, "basilisk", "Basilisk standard account (*25519).") - - // Note: 16384 and above are reserved. -); +/// Returns either the input address format or the default. +#[cfg(feature = "std")] +pub fn unwrap_or_default_ss58_version(network: Option) -> Ss58AddressFormat { + network.unwrap_or_else(default_ss58_version) +} -/// Set the default "version" (actually, this is a bit of a misnomer and the version byte is -/// typically used not just to encode format/version but also network identity) that is used for -/// encoding and decoding SS58 addresses. If an unknown version is provided then it fails. +/// Set the default SS58 "version". +/// +/// This SS58 version/format will be used when encoding/decoding SS58 addresses. +/// +/// If you want to support a custom SS58 prefix (that isn't yet registered in the `ss58-registry`), +/// you are required to call this function with your desired prefix [`Ss58AddressFormat::custom`]. +/// This will enable the node to decode ss58 addresses with this prefix. /// -/// See `ss58_address_format!` for all current known "versions". +/// This SS58 version/format is also only used by the node and not by the runtime. #[cfg(feature = "std")] -pub fn set_default_ss58_version(version: Ss58AddressFormat) { - *DEFAULT_VERSION.lock() = version +pub fn set_default_ss58_version(new_default: Ss58AddressFormat) { + DEFAULT_VERSION.store(new_default.into(), std::sync::atomic::Ordering::Relaxed); } #[cfg(feature = "std")] @@ -644,19 +416,13 @@ lazy_static::lazy_static! { } #[cfg(feature = "std")] -impl + AsRef<[u8]> + Default + Derive> Ss58Codec for T { +impl + AsRef<[u8]> + Public + Derive> Ss58Codec for T { fn from_string(s: &str) -> Result { let cap = SS58_REGEX.captures(s).ok_or(PublicError::InvalidFormat)?; let s = cap.name("ss58").map(|r| r.as_str()).unwrap_or(DEV_ADDRESS); let addr = if let Some(stripped) = s.strip_prefix("0x") { let d = hex::decode(stripped).map_err(|_| PublicError::InvalidFormat)?; - let mut r = Self::default(); - if d.len() == r.as_ref().len() { - r.as_mut().copy_from_slice(&d); - r - } else { - return Err(PublicError::BadLength) - } + Self::from_slice(&d).map_err(|()| PublicError::BadLength)? } else { Self::from_ss58check(s)? }; @@ -684,25 +450,15 @@ impl + AsRef<[u8]> + Default + Derive> Ss58Codec for T { } } -/// Trait suitable for typical cryptographic PKI key public type. -pub trait Public: - AsRef<[u8]> - + AsMut<[u8]> - + Default - + Derive - + CryptoType - + PartialEq - + Eq - + Clone - + Send - + Sync - + for<'a> TryFrom<&'a [u8]> -{ - /// A new instance from the given slice. - /// - /// NOTE: No checking goes on to ensure this is a real public key. Only use it if - /// you are certain that the array actually is a pubkey. GIGO! - fn from_slice(data: &[u8]) -> Self; +/// Trait used for types that are really just a fixed-length array. +pub trait ByteArray: AsRef<[u8]> + AsMut<[u8]> + for<'a> TryFrom<&'a [u8], Error = ()> { + /// The "length" of the values of this type, which is always the same. + const LEN: usize; + + /// A new instance from the given slice that should be `Self::LEN` bytes long. + fn from_slice(data: &[u8]) -> Result { + Self::try_from(data) + } /// Return a `Vec` filled with raw data. fn to_raw_vec(&self) -> Vec { @@ -713,14 +469,16 @@ pub trait Public: fn as_slice(&self) -> &[u8] { self.as_ref() } +} + +/// Trait suitable for typical cryptographic PKI key public type. +pub trait Public: ByteArray + Derive + CryptoType + PartialEq + Eq + Clone + Send + Sync { /// Return `CryptoTypePublicPair` from public key. fn to_public_crypto_pair(&self) -> CryptoTypePublicPair; } /// An opaque 32-byte cryptographic identifier. -#[derive( - Clone, Eq, PartialEq, Ord, PartialOrd, Default, Encode, Decode, MaxEncodedLen, TypeInfo, -)] +#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Encode, Decode, MaxEncodedLen, TypeInfo)] #[cfg_attr(feature = "std", derive(Hash))] pub struct AccountId32([u8; 32]); @@ -740,6 +498,10 @@ impl UncheckedFrom for AccountId32 { } } +impl ByteArray for AccountId32 { + const LEN: usize = 32; +} + #[cfg(feature = "std")] impl Ss58Codec for AccountId32 {} @@ -773,13 +535,13 @@ impl From<[u8; 32]> for AccountId32 { } } -impl<'a> sp_std::convert::TryFrom<&'a [u8]> for AccountId32 { +impl<'a> TryFrom<&'a [u8]> for AccountId32 { type Error = (); fn try_from(x: &'a [u8]) -> Result { if x.len() == 32 { - let mut r = AccountId32::default(); - r.0.copy_from_slice(x); - Ok(r) + let mut data = [0; 32]; + data.copy_from_slice(x); + Ok(AccountId32(data)) } else { Err(()) } @@ -902,9 +664,10 @@ mod dummy { impl Derive for Dummy {} - impl Public for Dummy { - fn from_slice(_: &[u8]) -> Self { - Self + impl ByteArray for Dummy { + const LEN: usize = 0; + fn from_slice(_: &[u8]) -> Result { + Ok(Self) } #[cfg(feature = "std")] fn to_raw_vec(&self) -> Vec { @@ -913,8 +676,10 @@ mod dummy { fn as_slice(&self) -> &[u8] { b"" } + } + impl Public for Dummy { fn to_public_crypto_pair(&self) -> CryptoTypePublicPair { - CryptoTypePublicPair(CryptoTypeId(*b"dumm"), Public::to_raw_vec(self)) + CryptoTypePublicPair(CryptoTypeId(*b"dumm"), ::to_raw_vec(self)) } } @@ -962,6 +727,104 @@ mod dummy { } } +/// A secret uri (`SURI`) that can be used to generate a key pair. +/// +/// The `SURI` can be parsed from a string. The string is interpreted in the following way: +/// +/// - If `string` is a possibly `0x` prefixed 64-digit hex string, then it will be interpreted +/// directly as a `MiniSecretKey` (aka "seed" in `subkey`). +/// - If `string` is a valid BIP-39 key phrase of 12, 15, 18, 21 or 24 words, then the key will +/// be derived from it. In this case: +/// - the phrase may be followed by one or more items delimited by `/` characters. +/// - the path may be followed by `///`, in which case everything after the `///` is treated +/// as a password. +/// - If `string` begins with a `/` character it is prefixed with the Substrate public `DEV_PHRASE` +/// and interpreted as above. +/// +/// In this case they are interpreted as HDKD junctions; purely numeric items are interpreted as +/// integers, non-numeric items as strings. Junctions prefixed with `/` are interpreted as soft +/// junctions, and with `//` as hard junctions. +/// +/// There is no correspondence mapping between `SURI` strings and the keys they represent. +/// Two different non-identical strings can actually lead to the same secret being derived. +/// Notably, integer junction indices may be legally prefixed with arbitrary number of zeros. +/// Similarly an empty password (ending the `SURI` with `///`) is perfectly valid and will +/// generally be equivalent to no password at all. +/// +/// # Example +/// +/// Parse [`DEV_PHRASE`] secret uri with junction: +/// +/// ``` +/// # use sp_core::crypto::{SecretUri, DeriveJunction, DEV_PHRASE, ExposeSecret}; +/// # use std::str::FromStr; +/// let suri = SecretUri::from_str("//Alice").expect("Parse SURI"); +/// +/// assert_eq!(vec![DeriveJunction::from("Alice").harden()], suri.junctions); +/// assert_eq!(DEV_PHRASE, suri.phrase.expose_secret()); +/// assert!(suri.password.is_none()); +/// ``` +/// +/// Parse [`DEV_PHRASE`] secret ui with junction and password: +/// +/// ``` +/// # use sp_core::crypto::{SecretUri, DeriveJunction, DEV_PHRASE, ExposeSecret}; +/// # use std::str::FromStr; +/// let suri = SecretUri::from_str("//Alice///SECRET_PASSWORD").expect("Parse SURI"); +/// +/// assert_eq!(vec![DeriveJunction::from("Alice").harden()], suri.junctions); +/// assert_eq!(DEV_PHRASE, suri.phrase.expose_secret()); +/// assert_eq!("SECRET_PASSWORD", suri.password.unwrap().expose_secret()); +/// ``` +/// +/// Parse [`DEV_PHRASE`] secret ui with hex phrase and junction: +/// +/// ``` +/// # use sp_core::crypto::{SecretUri, DeriveJunction, DEV_PHRASE, ExposeSecret}; +/// # use std::str::FromStr; +/// let suri = SecretUri::from_str("0xe5be9a5092b81bca64be81d212e7f2f9eba183bb7a90954f7b76361f6edb5c0a//Alice").expect("Parse SURI"); +/// +/// assert_eq!(vec![DeriveJunction::from("Alice").harden()], suri.junctions); +/// assert_eq!("0xe5be9a5092b81bca64be81d212e7f2f9eba183bb7a90954f7b76361f6edb5c0a", suri.phrase.expose_secret()); +/// assert!(suri.password.is_none()); +/// ``` +#[cfg(feature = "std")] +pub struct SecretUri { + /// The phrase to derive the private key. + /// + /// This can either be a 64-bit hex string or a BIP-39 key phrase. + pub phrase: SecretString, + /// Optional password as given as part of the uri. + pub password: Option, + /// The junctions as part of the uri. + pub junctions: Vec, +} + +#[cfg(feature = "std")] +impl sp_std::str::FromStr for SecretUri { + type Err = SecretStringError; + + fn from_str(s: &str) -> Result { + let cap = SECRET_PHRASE_REGEX.captures(s).ok_or(SecretStringError::InvalidFormat)?; + + let junctions = JUNCTION_REGEX + .captures_iter(&cap["path"]) + .map(|f| DeriveJunction::from(&f[1])) + .collect::>(); + + let phrase = cap.name("phrase").map(|r| r.as_str()).unwrap_or(DEV_PHRASE); + let password = cap.name("password"); + + Ok(Self { + phrase: SecretString::from_str(phrase).expect("Returns infallible error; qed"), + password: password.map(|v| { + SecretString::from_str(v.as_str()).expect("Returns infallible error; qed") + }), + junctions, + }) + } +} + /// Trait suitable for typical cryptographic PKI key pair type. /// /// For now it just specifies how to create a key from a phrase and derivation path. @@ -1074,14 +937,12 @@ pub trait Pair: CryptoType + Sized + Clone + Send + Sync + 'static { s: &str, password_override: Option<&str>, ) -> Result<(Self, Option), SecretStringError> { - let cap = SECRET_PHRASE_REGEX.captures(s).ok_or(SecretStringError::InvalidFormat)?; - - let path = JUNCTION_REGEX.captures_iter(&cap["path"]).map(|f| DeriveJunction::from(&f[1])); - - let phrase = cap.name("phrase").map(|r| r.as_str()).unwrap_or(DEV_PHRASE); - let password = password_override.or_else(|| cap.name("password").map(|m| m.as_str())); + use sp_std::str::FromStr; + let SecretUri { junctions, phrase, password } = SecretUri::from_str(s)?; + let password = + password_override.or_else(|| password.as_ref().map(|p| p.expose_secret().as_str())); - let (root, seed) = if let Some(stripped) = phrase.strip_prefix("0x") { + let (root, seed) = if let Some(stripped) = phrase.expose_secret().strip_prefix("0x") { hex::decode(stripped) .ok() .and_then(|seed_vec| { @@ -1095,9 +956,11 @@ pub trait Pair: CryptoType + Sized + Clone + Send + Sync + 'static { }) .ok_or(SecretStringError::InvalidSeed)? } else { - Self::from_phrase(phrase, password).map_err(|_| SecretStringError::InvalidPhrase)? + Self::from_phrase(phrase.expose_secret().as_str(), password) + .map_err(|_| SecretStringError::InvalidPhrase)? }; - root.derive(path, Some(seed)).map_err(|_| SecretStringError::InvalidPath) + root.derive(junctions.into_iter(), Some(seed)) + .map_err(|_| SecretStringError::InvalidPath) } /// Interprets the string `s` in order to generate a key pair. @@ -1256,8 +1119,6 @@ pub mod key_types { pub const AUTHORITY_DISCOVERY: KeyTypeId = KeyTypeId(*b"audi"); /// Key type for staking, built-in. Identified as `stak`. pub const STAKING: KeyTypeId = KeyTypeId(*b"stak"); - /// Key type for equivocation reporting, built-in. Identified as `fish`. - pub const REPORTING: KeyTypeId = KeyTypeId(*b"fish"); /// A key type ID useful for tests. pub const DUMMY: KeyTypeId = KeyTypeId(*b"dumy"); } @@ -1300,17 +1161,22 @@ mod tests { impl<'a> TryFrom<&'a [u8]> for TestPublic { type Error = (); - fn try_from(_: &'a [u8]) -> Result { - Ok(Self) + fn try_from(data: &'a [u8]) -> Result { + Self::from_slice(data) } } impl CryptoType for TestPublic { type Pair = TestPair; } impl Derive for TestPublic {} - impl Public for TestPublic { - fn from_slice(_bytes: &[u8]) -> Self { - Self + impl ByteArray for TestPublic { + const LEN: usize = 0; + fn from_slice(bytes: &[u8]) -> Result { + if bytes.len() == 0 { + Ok(Self) + } else { + Err(()) + } } fn as_slice(&self) -> &[u8] { &[] @@ -1318,6 +1184,8 @@ mod tests { fn to_raw_vec(&self) -> Vec { vec![] } + } + impl Public for TestPublic { fn to_public_crypto_pair(&self) -> CryptoTypePublicPair { CryptoTypePublicPair(CryptoTypeId(*b"dumm"), self.to_raw_vec()) } diff --git a/primitives/core/src/ecdsa.rs b/primitives/core/src/ecdsa.rs index 11e9b9d71d80..6343e3f4dfd0 100644 --- a/primitives/core/src/ecdsa.rs +++ b/primitives/core/src/ecdsa.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,19 +15,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -// tag::description[] -//! Simple ECDSA API. -// end::description[] +//! Simple ECDSA secp256k1 API. use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; use sp_runtime_interface::pass_by::PassByInner; -use sp_std::cmp::Ordering; #[cfg(feature = "std")] use crate::crypto::Ss58Codec; use crate::crypto::{ - CryptoType, CryptoTypeId, CryptoTypePublicPair, Derive, Public as TraitPublic, UncheckedFrom, + ByteArray, CryptoType, CryptoTypeId, CryptoTypePublicPair, Derive, Public as TraitPublic, + UncheckedFrom, }; #[cfg(feature = "full_crypto")] use crate::{ @@ -36,10 +34,15 @@ use crate::{ }; #[cfg(feature = "std")] use bip39::{Language, Mnemonic, MnemonicType}; +#[cfg(all(feature = "full_crypto", not(feature = "std")))] +use secp256k1::Secp256k1; +#[cfg(feature = "std")] +use secp256k1::SECP256K1; #[cfg(feature = "full_crypto")] -use core::convert::{TryFrom, TryInto}; -#[cfg(feature = "full_crypto")] -use libsecp256k1::{PublicKey, SecretKey}; +use secp256k1::{ + ecdsa::{RecoverableSignature, RecoveryId}, + Message, PublicKey, SecretKey, +}; #[cfg(feature = "std")] use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; #[cfg(feature = "full_crypto")] @@ -55,43 +58,22 @@ pub const CRYPTO_ID: CryptoTypeId = CryptoTypeId(*b"ecds"); type Seed = [u8; 32]; /// The ECDSA compressed public key. -#[derive(Clone, Encode, Decode, PassByInner, MaxEncodedLen, TypeInfo)] +#[cfg_attr(feature = "full_crypto", derive(Hash))] +#[derive( + Clone, + Copy, + Encode, + Decode, + PassByInner, + MaxEncodedLen, + TypeInfo, + Eq, + PartialEq, + PartialOrd, + Ord, +)] pub struct Public(pub [u8; 33]); -impl PartialOrd for Public { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Public { - fn cmp(&self, other: &Self) -> Ordering { - self.as_ref().cmp(&other.as_ref()) - } -} - -impl PartialEq for Public { - fn eq(&self, other: &Self) -> bool { - self.as_ref() == other.as_ref() - } -} - -impl Eq for Public {} - -/// An error type for SS58 decoding. -#[cfg(feature = "std")] -#[derive(Clone, Copy, Eq, PartialEq, Debug)] -pub enum PublicError { - /// Bad alphabet. - BadBase58, - /// Bad length. - BadLength, - /// Unknown version. - UnknownVersion, - /// Invalid checksum. - InvalidChecksum, -} - impl Public { /// A new instance from the given 33-byte `data`. /// @@ -106,24 +88,24 @@ impl Public { /// This will convert the full public key into the compressed format. #[cfg(feature = "std")] pub fn from_full(full: &[u8]) -> Result { - libsecp256k1::PublicKey::parse_slice(full, None) - .map(|k| k.serialize_compressed()) - .map(Self) - .map_err(|_| ()) + let pubkey = if full.len() == 64 { + // Tag it as uncompressed public key. + let mut tagged_full = [0u8; 65]; + tagged_full[0] = 0x04; + tagged_full[1..].copy_from_slice(full); + secp256k1::PublicKey::from_slice(&tagged_full) + } else { + secp256k1::PublicKey::from_slice(full) + }; + pubkey.map(|k| Self(k.serialize())).map_err(|_| ()) } } -impl TraitPublic for Public { - /// A new instance from the given slice that should be 33 bytes long. - /// - /// NOTE: No checking goes on to ensure this is a real public key. Only use it if - /// you are certain that the array actually is a pubkey. GIGO! - fn from_slice(data: &[u8]) -> Self { - let mut r = [0u8; 33]; - r.copy_from_slice(data); - Self(r) - } +impl ByteArray for Public { + const LEN: usize = 33; +} +impl TraitPublic for Public { fn to_public_crypto_pair(&self) -> CryptoTypePublicPair { CryptoTypePublicPair(CRYPTO_ID, self.to_raw_vec()) } @@ -143,12 +125,6 @@ impl From<&Public> for CryptoTypePublicPair { impl Derive for Public {} -impl Default for Public { - fn default() -> Self { - Public([0u8; 33]) - } -} - impl AsRef<[u8]> for Public { fn as_ref(&self) -> &[u8] { &self.0[..] @@ -161,15 +137,16 @@ impl AsMut<[u8]> for Public { } } -impl sp_std::convert::TryFrom<&[u8]> for Public { +impl TryFrom<&[u8]> for Public { type Error = (); fn try_from(data: &[u8]) -> Result { - if data.len() == 33 { - Ok(Self::from_slice(data)) - } else { - Err(()) + if data.len() != Self::LEN { + return Err(()) } + let mut r = [0u8; Self::LEN]; + r.copy_from_slice(data); + Ok(Self::unchecked_from(r)) } } @@ -227,18 +204,12 @@ impl<'de> Deserialize<'de> for Public { } } -#[cfg(feature = "full_crypto")] -impl sp_std::hash::Hash for Public { - fn hash(&self, state: &mut H) { - self.as_ref().hash(state); - } -} - /// A signature (a 512-bit value, plus 8 bits for recovery ID). -#[derive(Encode, Decode, PassByInner, TypeInfo)] +#[cfg_attr(feature = "full_crypto", derive(Hash))] +#[derive(Encode, Decode, MaxEncodedLen, PassByInner, TypeInfo, PartialEq, Eq)] pub struct Signature(pub [u8; 65]); -impl sp_std::convert::TryFrom<&[u8]> for Signature { +impl TryFrom<&[u8]> for Signature { type Error = (); fn try_from(data: &[u8]) -> Result { @@ -289,14 +260,6 @@ impl Default for Signature { } } -impl PartialEq for Signature { - fn eq(&self, b: &Self) -> bool { - self.0[..] == b.0[..] - } -} - -impl Eq for Signature {} - impl From for [u8; 65] { fn from(v: Signature) -> [u8; 65] { v.0 @@ -333,10 +296,9 @@ impl sp_std::fmt::Debug for Signature { } } -#[cfg(feature = "full_crypto")] -impl sp_std::hash::Hash for Signature { - fn hash(&self, state: &mut H) { - sp_std::hash::Hash::hash(&self.0[..], state); +impl UncheckedFrom<[u8; 65]> for Signature { + fn unchecked_from(data: [u8; 65]) -> Signature { + Signature(data) } } @@ -353,63 +315,56 @@ impl Signature { /// /// NOTE: No checking goes on to ensure this is a real signature. Only use it if /// you are certain that the array actually is a signature. GIGO! - pub fn from_slice(data: &[u8]) -> Self { + pub fn from_slice(data: &[u8]) -> Option { + if data.len() != 65 { + return None + } let mut r = [0u8; 65]; r.copy_from_slice(data); - Signature(r) + Some(Signature(r)) } /// Recover the public key from this signature and a message. #[cfg(feature = "full_crypto")] pub fn recover>(&self, message: M) -> Option { - let message = libsecp256k1::Message::parse(&blake2_256(message.as_ref())); - let sig: (_, _) = self.try_into().ok()?; - libsecp256k1::recover(&message, &sig.0, &sig.1) - .ok() - .map(|recovered| Public(recovered.serialize_compressed())) + self.recover_prehashed(&blake2_256(message.as_ref())) } /// Recover the public key from this signature and a pre-hashed message. #[cfg(feature = "full_crypto")] pub fn recover_prehashed(&self, message: &[u8; 32]) -> Option { - let message = libsecp256k1::Message::parse(message); + let rid = RecoveryId::from_i32(self.0[64] as i32).ok()?; + let sig = RecoverableSignature::from_compact(&self.0[..64], rid).ok()?; + let message = Message::from_slice(message).expect("Message is 32 bytes; qed"); - let sig: (_, _) = self.try_into().ok()?; + #[cfg(feature = "std")] + let context = SECP256K1; + #[cfg(not(feature = "std"))] + let context = Secp256k1::verification_only(); - libsecp256k1::recover(&message, &sig.0, &sig.1) + context + .recover_ecdsa(&message, &sig) .ok() - .map(|key| Public(key.serialize_compressed())) + .map(|pubkey| Public(pubkey.serialize())) } } #[cfg(feature = "full_crypto")] -impl From<(libsecp256k1::Signature, libsecp256k1::RecoveryId)> for Signature { - fn from(x: (libsecp256k1::Signature, libsecp256k1::RecoveryId)) -> Signature { +impl From for Signature { + fn from(recsig: RecoverableSignature) -> Signature { let mut r = Self::default(); - r.0[0..64].copy_from_slice(&x.0.serialize()[..]); - r.0[64] = x.1.serialize(); + let (recid, sig) = recsig.serialize_compact(); + r.0[..64].copy_from_slice(&sig); + // This is safe due to the limited range of possible valid ids. + r.0[64] = recid.to_i32() as u8; r } } -#[cfg(feature = "full_crypto")] -impl<'a> TryFrom<&'a Signature> for (libsecp256k1::Signature, libsecp256k1::RecoveryId) { - type Error = (); - fn try_from( - x: &'a Signature, - ) -> Result<(libsecp256k1::Signature, libsecp256k1::RecoveryId), Self::Error> { - parse_signature_standard(&x.0).map_err(|_| ()) - } -} - /// Derive a single hard junction. #[cfg(feature = "full_crypto")] fn derive_hard_junction(secret_seed: &Seed, cc: &[u8; 32]) -> Seed { - ("Secp256k1HDKD", secret_seed, cc).using_encoded(|data| { - let mut res = [0u8; 32]; - res.copy_from_slice(blake2_rfc::blake2b::blake2b(32, &[], data).as_bytes()); - res - }) + ("Secp256k1HDKD", secret_seed, cc).using_encoded(|data| sp_core_hashing::blake2_256(data)) } /// An error when deriving a key. @@ -423,7 +378,7 @@ pub enum DeriveError { #[cfg(feature = "full_crypto")] #[derive(Clone)] pub struct Pair { - public: PublicKey, + public: Public, secret: SecretKey, } @@ -477,8 +432,15 @@ impl TraitPair for Pair { /// You should never need to use this; generate(), generate_with_phrase fn from_seed_slice(seed_slice: &[u8]) -> Result { let secret = - SecretKey::parse_slice(seed_slice).map_err(|_| SecretStringError::InvalidSeedLength)?; - let public = PublicKey::from_secret_key(&secret); + SecretKey::from_slice(seed_slice).map_err(|_| SecretStringError::InvalidSeedLength)?; + + #[cfg(feature = "std")] + let context = SECP256K1; + #[cfg(not(feature = "std"))] + let context = Secp256k1::signing_only(); + + let public = PublicKey::from_secret_key(&context, &secret); + let public = Public(public.serialize()); Ok(Pair { public, secret }) } @@ -488,7 +450,7 @@ impl TraitPair for Pair { path: Iter, _seed: Option, ) -> Result<(Pair, Option), DeriveError> { - let mut acc = self.secret.serialize(); + let mut acc = self.seed(); for j in path { match j { DeriveJunction::Soft(_cc) => return Err(DeriveError::SoftKeyInPath), @@ -500,25 +462,19 @@ impl TraitPair for Pair { /// Get the public key. fn public(&self) -> Public { - Public(self.public.serialize_compressed()) + self.public } /// Sign a message. fn sign(&self, message: &[u8]) -> Signature { - let message = libsecp256k1::Message::parse(&blake2_256(message)); - libsecp256k1::sign(&message, &self.secret).into() + self.sign_prehashed(&blake2_256(message)) } /// Verify a signature on a message. Returns true if the signature is good. fn verify>(sig: &Self::Signature, message: M, pubkey: &Self::Public) -> bool { - let message = libsecp256k1::Message::parse(&blake2_256(message.as_ref())); - let sig: (_, _) = match sig.try_into() { - Ok(x) => x, - _ => return false, - }; - match libsecp256k1::recover(&message, &sig.0, &sig.1) { - Ok(actual) => pubkey.0[..] == actual.serialize_compressed()[..], - _ => false, + match sig.recover(message) { + Some(actual) => actual == *pubkey, + None => false, } } @@ -527,17 +483,9 @@ impl TraitPair for Pair { /// This doesn't use the type system to ensure that `sig` and `pubkey` are the correct /// size. Use it only if you're coming from byte buffers and need the speed. fn verify_weak, M: AsRef<[u8]>>(sig: &[u8], message: M, pubkey: P) -> bool { - let message = libsecp256k1::Message::parse(&blake2_256(message.as_ref())); - if sig.len() != 65 { - return false - } - let (sig, ri) = match parse_signature_standard(&sig) { - Ok(sigri) => sigri, - _ => return false, - }; - match libsecp256k1::recover(&message, &sig, &ri) { - Ok(actual) => pubkey.as_ref() == &actual.serialize()[1..], - _ => false, + match Signature::from_slice(sig).and_then(|sig| sig.recover(message)) { + Some(actual) => actual.as_ref() == pubkey.as_ref(), + None => false, } } @@ -551,7 +499,7 @@ impl TraitPair for Pair { impl Pair { /// Get the seed for this key. pub fn seed(&self) -> Seed { - self.secret.serialize() + self.secret.serialize_secret() } /// Exactly as `from_string` except that if no matches are found then, the the first 32 @@ -568,57 +516,63 @@ impl Pair { /// Sign a pre-hashed message pub fn sign_prehashed(&self, message: &[u8; 32]) -> Signature { - let message = libsecp256k1::Message::parse(message); - libsecp256k1::sign(&message, &self.secret).into() + let message = Message::from_slice(message).expect("Message is 32 bytes; qed"); + + #[cfg(feature = "std")] + let context = SECP256K1; + #[cfg(not(feature = "std"))] + let context = Secp256k1::signing_only(); + + context.sign_ecdsa_recoverable(&message, &self.secret).into() } /// Verify a signature on a pre-hashed message. Return `true` if the signature is valid /// and thus matches the given `public` key. pub fn verify_prehashed(sig: &Signature, message: &[u8; 32], public: &Public) -> bool { - let message = libsecp256k1::Message::parse(message); - - let sig: (_, _) = match sig.try_into() { - Ok(x) => x, - _ => return false, - }; - - match libsecp256k1::recover(&message, &sig.0, &sig.1) { - Ok(actual) => public.0[..] == actual.serialize_compressed()[..], - _ => false, + match sig.recover_prehashed(message) { + Some(actual) => actual == *public, + None => false, } } /// Verify a signature on a message. Returns true if the signature is good. - /// Parses Signature using parse_overflowing_slice + /// Parses Signature using parse_overflowing_slice. + #[deprecated(note = "please use `verify` instead")] pub fn verify_deprecated>(sig: &Signature, message: M, pubkey: &Public) -> bool { let message = libsecp256k1::Message::parse(&blake2_256(message.as_ref())); - let (sig, ri) = match parse_signature_overflowing(&sig.0) { - Ok(sigri) => sigri, + + let parse_signature_overflowing = |x: [u8; 65]| { + let sig = libsecp256k1::Signature::parse_overflowing_slice(&x[..64]).ok()?; + let rid = libsecp256k1::RecoveryId::parse(x[64]).ok()?; + Some((sig, rid)) + }; + + let (sig, rid) = match parse_signature_overflowing(sig.0) { + Some(sigri) => sigri, _ => return false, }; - match libsecp256k1::recover(&message, &sig, &ri) { - Ok(actual) => pubkey.0[..] == actual.serialize_compressed()[..], + match libsecp256k1::recover(&message, &sig, &rid) { + Ok(actual) => pubkey.0 == actual.serialize_compressed(), _ => false, } } } +// The `secp256k1` backend doesn't implement cleanup for their private keys. +// Currently we should take care of wiping the secret from memory. +// NOTE: this solution is not effective when `Pair` is moved around memory. +// The very same problem affects other cryptographic backends that are just using +// `zeroize`for their secrets. #[cfg(feature = "full_crypto")] -fn parse_signature_standard( - x: &[u8], -) -> Result<(libsecp256k1::Signature, libsecp256k1::RecoveryId), libsecp256k1::Error> { - let sig = libsecp256k1::Signature::parse_standard_slice(&x[..64])?; - let ri = libsecp256k1::RecoveryId::parse(x[64])?; - Ok((sig, ri)) -} - -#[cfg(feature = "full_crypto")] -fn parse_signature_overflowing( - x: &[u8], -) -> Result<(libsecp256k1::Signature, libsecp256k1::RecoveryId), libsecp256k1::Error> { - let sig = libsecp256k1::Signature::parse_overflowing_slice(&x[..64])?; - let ri = libsecp256k1::RecoveryId::parse(x[64])?; - Ok((sig, ri)) +impl Drop for Pair { + fn drop(&mut self) { + let ptr = self.secret.as_mut_ptr(); + for off in 0..self.secret.len() { + unsafe { + core::ptr::write_volatile(ptr.add(off), 0); + } + } + } } impl CryptoType for Public { @@ -639,9 +593,9 @@ impl CryptoType for Pair { #[cfg(test)] mod test { use super::*; - use crate::{ - crypto::{set_default_ss58_version, PublicError, DEV_PHRASE}, - keccak_256, + use crate::crypto::{ + set_default_ss58_version, PublicError, Ss58AddressFormat, Ss58AddressFormatRegistry, + DEV_PHRASE, }; use hex_literal::hex; use serde_json; @@ -772,26 +726,24 @@ mod test { #[test] fn ss58check_format_check_works() { - use crate::crypto::Ss58AddressFormat; let pair = Pair::from_seed(b"12345678901234567890123456789012"); let public = pair.public(); - let format = Ss58AddressFormat::Reserved46; + let format = Ss58AddressFormatRegistry::Reserved46Account.into(); let s = public.to_ss58check_with_version(format); assert_eq!(Public::from_ss58check_with_version(&s), Err(PublicError::FormatNotAllowed)); } #[test] fn ss58check_full_roundtrip_works() { - use crate::crypto::Ss58AddressFormat; let pair = Pair::from_seed(b"12345678901234567890123456789012"); let public = pair.public(); - let format = Ss58AddressFormat::PolkadotAccount; + let format = Ss58AddressFormatRegistry::PolkadotAccount.into(); let s = public.to_ss58check_with_version(format); let (k, f) = Public::from_ss58check_with_version(&s).unwrap(); assert_eq!(k, public); assert_eq!(f, format); - let format = Ss58AddressFormat::Custom(64); + let format = Ss58AddressFormat::custom(64); let s = public.to_ss58check_with_version(format); let (k, f) = Public::from_ss58check_with_version(&s).unwrap(); assert_eq!(k, public); @@ -805,10 +757,10 @@ mod test { if std::env::var("RUN_CUSTOM_FORMAT_TEST") == Ok("1".into()) { use crate::crypto::Ss58AddressFormat; // temp save default format version - let default_format = Ss58AddressFormat::default(); + let default_format = crate::crypto::default_ss58_version(); // set current ss58 version is custom "200" `Ss58AddressFormat::Custom(200)` - set_default_ss58_version(Ss58AddressFormat::Custom(200)); + set_default_ss58_version(Ss58AddressFormat::custom(200)); // custom addr encoded by version 200 let addr = "4pbsSkWcBaYoFHrKJZp5fDVUKbqSYD9dhZZGvpp3vQ5ysVs5ybV"; Public::from_ss58check(&addr).unwrap(); @@ -862,22 +814,20 @@ mod test { // `msg` shouldn't be mangled let msg = [0u8; 32]; let sig1 = pair.sign_prehashed(&msg); - let sig2: Signature = - libsecp256k1::sign(&libsecp256k1::Message::parse(&msg), &pair.secret).into(); - + let sig2: Signature = { + let message = Message::from_slice(&msg).unwrap(); + SECP256K1.sign_ecdsa_recoverable(&message, &pair.secret).into() + }; assert_eq!(sig1, sig2); // signature is actually different let sig2 = pair.sign(&msg); - assert_ne!(sig1, sig2); // using pre-hashed `msg` works - let msg = keccak_256(b"this should be hashed"); - let sig1 = pair.sign_prehashed(&msg); - let sig2: Signature = - libsecp256k1::sign(&libsecp256k1::Message::parse(&msg), &pair.secret).into(); - + let msg = b"this should be hashed"; + let sig1 = pair.sign_prehashed(&blake2_256(msg)); + let sig2 = pair.sign(msg); assert_eq!(sig1, sig2); } @@ -886,12 +836,12 @@ mod test { let (pair, _, _) = Pair::generate_with_phrase(Some("password")); // `msg` and `sig` match - let msg = keccak_256(b"this should be hashed"); + let msg = blake2_256(b"this should be hashed"); let sig = pair.sign_prehashed(&msg); assert!(Pair::verify_prehashed(&sig, &msg, &pair.public())); // `msg` and `sig` don't match - let msg = keccak_256(b"this is a different message"); + let msg = blake2_256(b"this is a different message"); assert!(!Pair::verify_prehashed(&sig, &msg, &pair.public())); } @@ -900,7 +850,7 @@ mod test { let (pair, _, _) = Pair::generate_with_phrase(Some("password")); // recovered key matches signing key - let msg = keccak_256(b"this should be hashed"); + let msg = blake2_256(b"this should be hashed"); let sig = pair.sign_prehashed(&msg); let key = sig.recover_prehashed(&msg).unwrap(); assert_eq!(pair.public(), key); @@ -909,7 +859,7 @@ mod test { assert!(Pair::verify_prehashed(&sig, &msg, &key)); // recovered key and signing key don't match - let msg = keccak_256(b"this is a different message"); + let msg = blake2_256(b"this is a different message"); let key = sig.recover_prehashed(&msg).unwrap(); assert_ne!(pair.public(), key); } diff --git a/primitives/core/src/ed25519.rs b/primitives/core/src/ed25519.rs index d786ee9d255f..0bde9e2e5303 100644 --- a/primitives/core/src/ed25519.rs +++ b/primitives/core/src/ed25519.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,10 @@ #[cfg(feature = "full_crypto")] use sp_std::vec::Vec; -use crate::hash::{H256, H512}; +use crate::{ + crypto::ByteArray, + hash::{H256, H512}, +}; use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; @@ -36,8 +39,6 @@ use crate::crypto::{DeriveJunction, Pair as TraitPair, SecretStringError}; #[cfg(feature = "std")] use bip39::{Language, Mnemonic, MnemonicType}; #[cfg(feature = "full_crypto")] -use core::convert::TryFrom; -#[cfg(feature = "full_crypto")] use ed25519_dalek::{Signer as _, Verifier as _}; #[cfg(feature = "std")] use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; @@ -66,7 +67,6 @@ type Seed = [u8; 32]; Copy, Encode, Decode, - Default, PassByInner, MaxEncodedLen, TypeInfo, @@ -114,17 +114,16 @@ impl Deref for Public { } } -impl sp_std::convert::TryFrom<&[u8]> for Public { +impl TryFrom<&[u8]> for Public { type Error = (); fn try_from(data: &[u8]) -> Result { - if data.len() == 32 { - let mut inner = [0u8; 32]; - inner.copy_from_slice(data); - Ok(Public(inner)) - } else { - Err(()) + if data.len() != Self::LEN { + return Err(()) } + let mut r = [0u8; Self::LEN]; + r.copy_from_slice(data); + Ok(Self::unchecked_from(r)) } } @@ -210,10 +209,11 @@ impl<'de> Deserialize<'de> for Public { } /// A signature (a 512-bit value). -#[derive(Encode, Decode, PassByInner, TypeInfo)] +#[cfg_attr(feature = "full_crypto", derive(Hash))] +#[derive(Encode, Decode, MaxEncodedLen, PassByInner, TypeInfo, PartialEq, Eq)] pub struct Signature(pub [u8; 64]); -impl sp_std::convert::TryFrom<&[u8]> for Signature { +impl TryFrom<&[u8]> for Signature { type Error = (); fn try_from(data: &[u8]) -> Result { @@ -258,20 +258,6 @@ impl Clone for Signature { } } -impl Default for Signature { - fn default() -> Self { - Signature([0u8; 64]) - } -} - -impl PartialEq for Signature { - fn eq(&self, b: &Self) -> bool { - self.0[..] == b.0[..] - } -} - -impl Eq for Signature {} - impl From for H512 { fn from(v: Signature) -> H512 { H512::from(v.0) @@ -314,10 +300,9 @@ impl sp_std::fmt::Debug for Signature { } } -#[cfg(feature = "full_crypto")] -impl sp_std::hash::Hash for Signature { - fn hash(&self, state: &mut H) { - sp_std::hash::Hash::hash(&self.0[..], state); +impl UncheckedFrom<[u8; 64]> for Signature { + fn unchecked_from(data: [u8; 64]) -> Signature { + Signature(data) } } @@ -334,10 +319,13 @@ impl Signature { /// /// NOTE: No checking goes on to ensure this is a real signature. Only use it if /// you are certain that the array actually is a signature. GIGO! - pub fn from_slice(data: &[u8]) -> Self { + pub fn from_slice(data: &[u8]) -> Option { + if data.len() != 64 { + return None + } let mut r = [0u8; 64]; r.copy_from_slice(data); - Signature(r) + Some(Signature(r)) } /// A new instance from an H512. @@ -359,24 +347,6 @@ pub struct LocalizedSignature { pub signature: Signature, } -/// An error type for SS58 decoding. -#[cfg(feature = "std")] -#[derive(Clone, Copy, Eq, PartialEq, Debug, thiserror::Error)] -pub enum PublicError { - /// Bad alphabet. - #[error("Base 58 requirement is violated")] - BadBase58, - /// Bad length. - #[error("Length is bad")] - BadLength, - /// Unknown version. - #[error("Unknown version")] - UnknownVersion, - /// Invalid checksum. - #[error("Invalid checksum")] - InvalidChecksum, -} - impl Public { /// A new instance from the given 32-byte `data`. /// @@ -400,17 +370,11 @@ impl Public { } } -impl TraitPublic for Public { - /// A new instance from the given slice that should be 32 bytes long. - /// - /// NOTE: No checking goes on to ensure this is a real public key. Only use it if - /// you are certain that the array actually is a pubkey. GIGO! - fn from_slice(data: &[u8]) -> Self { - let mut r = [0u8; 32]; - r.copy_from_slice(data); - Public(r) - } +impl ByteArray for Public { + const LEN: usize = 32; +} +impl TraitPublic for Public { fn to_public_crypto_pair(&self) -> CryptoTypePublicPair { CryptoTypePublicPair(CRYPTO_ID, self.to_raw_vec()) } @@ -433,11 +397,7 @@ impl From<&Public> for CryptoTypePublicPair { /// Derive a single hard junction. #[cfg(feature = "full_crypto")] fn derive_hard_junction(secret_seed: &Seed, cc: &[u8; 32]) -> Seed { - ("Ed25519HDKD", secret_seed, cc).using_encoded(|data| { - let mut res = [0u8; 32]; - res.copy_from_slice(blake2_rfc::blake2b::blake2b(32, &[], data).as_bytes()); - res - }) + ("Ed25519HDKD", secret_seed, cc).using_encoded(|data| sp_core_hashing::blake2_256(data)) } /// An error when deriving a key. diff --git a/primitives/core/src/hash.rs b/primitives/core/src/hash.rs index 55a9664c9dad..f2974e9372ad 100644 --- a/primitives/core/src/hash.rs +++ b/primitives/core/src/hash.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/core/src/hasher.rs b/primitives/core/src/hasher.rs index 01680de08376..173bd560ad1e 100644 --- a/primitives/core/src/hasher.rs +++ b/primitives/core/src/hasher.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/core/src/hashing.rs b/primitives/core/src/hashing.rs index 4c719f7c6983..1c439355a33c 100644 --- a/primitives/core/src/hashing.rs +++ b/primitives/core/src/hashing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,147 +22,45 @@ //! unless you know what you're doing. Using `sp_io` will be more performant, since instead of //! computing the hash in WASM it delegates that computation to the host client. -use sha2::{Digest, Sha256}; -use tiny_keccak::{Hasher, Keccak}; - -/// Do a Blake2 512-bit hash and place result in `dest`. -pub fn blake2_512_into(data: &[u8], dest: &mut [u8; 64]) { - dest.copy_from_slice(blake2_rfc::blake2b::blake2b(64, &[], data).as_bytes()); -} - -/// Do a Blake2 512-bit hash and return result. -pub fn blake2_512(data: &[u8]) -> [u8; 64] { - let mut r = [0; 64]; - blake2_512_into(data, &mut r); - r -} - -/// Do a Blake2 256-bit hash and place result in `dest`. -pub fn blake2_256_into(data: &[u8], dest: &mut [u8; 32]) { - dest.copy_from_slice(blake2_rfc::blake2b::blake2b(32, &[], data).as_bytes()); -} - -/// Do a Blake2 256-bit hash and return result. -pub fn blake2_256(data: &[u8]) -> [u8; 32] { - let mut r = [0; 32]; - blake2_256_into(data, &mut r); - r -} - -/// Do a Blake2 128-bit hash and place result in `dest`. -pub fn blake2_128_into(data: &[u8], dest: &mut [u8; 16]) { - dest.copy_from_slice(blake2_rfc::blake2b::blake2b(16, &[], data).as_bytes()); -} - -/// Do a Blake2 128-bit hash and return result. -pub fn blake2_128(data: &[u8]) -> [u8; 16] { - let mut r = [0; 16]; - blake2_128_into(data, &mut r); - r -} - -/// Do a Blake2 64-bit hash and place result in `dest`. -pub fn blake2_64_into(data: &[u8], dest: &mut [u8; 8]) { - dest.copy_from_slice(blake2_rfc::blake2b::blake2b(8, &[], data).as_bytes()); -} - -/// Do a Blake2 64-bit hash and return result. -pub fn blake2_64(data: &[u8]) -> [u8; 8] { - let mut r = [0; 8]; - blake2_64_into(data, &mut r); - r -} - -/// Do a XX 64-bit hash and place result in `dest`. -pub fn twox_64_into(data: &[u8], dest: &mut [u8; 8]) { - use core::hash::Hasher; - let mut h0 = twox_hash::XxHash::with_seed(0); - h0.write(data); - let r0 = h0.finish(); - use byteorder::{ByteOrder, LittleEndian}; - LittleEndian::write_u64(&mut dest[0..8], r0); -} - -/// Do a XX 64-bit hash and return result. -pub fn twox_64(data: &[u8]) -> [u8; 8] { - let mut r: [u8; 8] = [0; 8]; - twox_64_into(data, &mut r); - r -} - -/// Do a XX 128-bit hash and place result in `dest`. -pub fn twox_128_into(data: &[u8], dest: &mut [u8; 16]) { - use core::hash::Hasher; - let mut h0 = twox_hash::XxHash::with_seed(0); - let mut h1 = twox_hash::XxHash::with_seed(1); - h0.write(data); - h1.write(data); - let r0 = h0.finish(); - let r1 = h1.finish(); - use byteorder::{ByteOrder, LittleEndian}; - LittleEndian::write_u64(&mut dest[0..8], r0); - LittleEndian::write_u64(&mut dest[8..16], r1); -} - -/// Do a XX 128-bit hash and return result. -pub fn twox_128(data: &[u8]) -> [u8; 16] { - let mut r: [u8; 16] = [0; 16]; - twox_128_into(data, &mut r); - r -} - -/// Do a XX 256-bit hash and place result in `dest`. -pub fn twox_256_into(data: &[u8], dest: &mut [u8; 32]) { - use ::core::hash::Hasher; - use byteorder::{ByteOrder, LittleEndian}; - let mut h0 = twox_hash::XxHash::with_seed(0); - let mut h1 = twox_hash::XxHash::with_seed(1); - let mut h2 = twox_hash::XxHash::with_seed(2); - let mut h3 = twox_hash::XxHash::with_seed(3); - h0.write(data); - h1.write(data); - h2.write(data); - h3.write(data); - let r0 = h0.finish(); - let r1 = h1.finish(); - let r2 = h2.finish(); - let r3 = h3.finish(); - LittleEndian::write_u64(&mut dest[0..8], r0); - LittleEndian::write_u64(&mut dest[8..16], r1); - LittleEndian::write_u64(&mut dest[16..24], r2); - LittleEndian::write_u64(&mut dest[24..32], r3); -} - -/// Do a XX 256-bit hash and return result. -pub fn twox_256(data: &[u8]) -> [u8; 32] { - let mut r: [u8; 32] = [0; 32]; - twox_256_into(data, &mut r); - r -} - -/// Do a keccak 256-bit hash and return result. -pub fn keccak_256(data: &[u8]) -> [u8; 32] { - let mut keccak = Keccak::v256(); - keccak.update(data); - let mut output = [0u8; 32]; - keccak.finalize(&mut output); - output -} - -/// Do a keccak 512-bit hash and return result. -pub fn keccak_512(data: &[u8]) -> [u8; 64] { - let mut keccak = Keccak::v512(); - keccak.update(data); - let mut output = [0u8; 64]; - keccak.finalize(&mut output); - output -} - -/// Do a sha2 256-bit hash and return result. -pub fn sha2_256(data: &[u8]) -> [u8; 32] { - let mut hasher = Sha256::new(); - hasher.update(data); - let mut output = [0u8; 32]; - output.copy_from_slice(&hasher.finalize()); - output +pub use sp_core_hashing::*; + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn blake2b() { + assert_eq!(sp_core_hashing_proc_macro::blake2b_64!(b""), blake2_64(b"")[..]); + assert_eq!(sp_core_hashing_proc_macro::blake2b_256!(b"test"), blake2_256(b"test")[..]); + assert_eq!(sp_core_hashing_proc_macro::blake2b_512!(b""), blake2_512(b"")[..]); + } + + #[test] + fn keccak() { + assert_eq!(sp_core_hashing_proc_macro::keccak_256!(b"test"), keccak_256(b"test")[..]); + assert_eq!(sp_core_hashing_proc_macro::keccak_512!(b"test"), keccak_512(b"test")[..]); + } + + #[test] + fn sha2() { + assert_eq!(sp_core_hashing_proc_macro::sha2_256!(b"test"), sha2_256(b"test")[..]); + } + + #[test] + fn twox() { + assert_eq!(sp_core_hashing_proc_macro::twox_128!(b"test"), twox_128(b"test")[..]); + assert_eq!(sp_core_hashing_proc_macro::twox_64!(b""), twox_64(b"")[..]); + } + + #[test] + fn twox_concats() { + assert_eq!( + sp_core_hashing_proc_macro::twox_128!(b"test", b"123", b"45", b"", b"67890"), + super::twox_128(&b"test1234567890"[..]), + ); + assert_eq!( + sp_core_hashing_proc_macro::twox_128!(b"test", test, b"45", b"", b"67890"), + super::twox_128(&b"testtest4567890"[..]), + ); + } } diff --git a/primitives/core/src/hexdisplay.rs b/primitives/core/src/hexdisplay.rs index 4d91db156792..e5262ba8f657 100644 --- a/primitives/core/src/hexdisplay.rs +++ b/primitives/core/src/hexdisplay.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/core/src/lib.rs b/primitives/core/src/lib.rs index a6229fe43a1a..b7c8b69e8a0a 100644 --- a/primitives/core/src/lib.rs +++ b/primitives/core/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -56,9 +56,6 @@ pub use hashing::{blake2_128, blake2_256, keccak_256, twox_128, twox_256, twox_6 pub mod crypto; pub mod hexdisplay; -pub mod u32_trait; - -mod changes_trie; pub mod ecdsa; pub mod ed25519; pub mod hash; @@ -76,9 +73,8 @@ pub use self::{ hash::{convert_hash, H160, H256, H512}, uint::{U256, U512}, }; -pub use changes_trie::{ChangesTrieConfiguration, ChangesTrieConfigurationRange}; #[cfg(feature = "full_crypto")] -pub use crypto::{DeriveJunction, Pair, Public}; +pub use crypto::{ByteArray, DeriveJunction, Pair, Public}; #[cfg(feature = "std")] pub use self::hasher::blake2::Blake2Hasher; @@ -100,7 +96,7 @@ pub enum ExecutionContext { /// We distinguish between major sync and import so that validators who are running /// their initial sync (or catching up after some time offline) can use the faster /// native runtime (since we can reasonably assume the network as a whole has already - /// come to a broad conensus on the block and it probably hasn't been crafted + /// come to a broad consensus on the block and it probably hasn't been crafted /// specifically to attack this node), but when importing blocks at the head of the /// chain in normal operation they can use the safer Wasm version. Syncing, @@ -118,15 +114,13 @@ impl ExecutionContext { use ExecutionContext::*; match self { - Importing | Syncing | BlockConstruction => offchain::Capabilities::none(), + Importing | Syncing | BlockConstruction => offchain::Capabilities::empty(), // Enable keystore, transaction pool and Offchain DB reads by default for offchain // calls. - OffchainCall(None) => [ - offchain::Capability::Keystore, - offchain::Capability::OffchainDbRead, - offchain::Capability::TransactionPool, - ][..] - .into(), + OffchainCall(None) => + offchain::Capabilities::KEYSTORE | + offchain::Capabilities::OFFCHAIN_DB_READ | + offchain::Capabilities::TRANSACTION_POOL, OffchainCall(Some((_, capabilities))) => *capabilities, } } diff --git a/primitives/core/src/offchain/mod.rs b/primitives/core/src/offchain/mod.rs index 895787011383..92c8e4e53c8d 100644 --- a/primitives/core/src/offchain/mod.rs +++ b/primitives/core/src/offchain/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,10 +21,7 @@ use crate::{OpaquePeerId, RuntimeDebug}; use codec::{Decode, Encode}; use scale_info::TypeInfo; use sp_runtime_interface::pass_by::{PassByCodec, PassByEnum, PassByInner}; -use sp_std::{ - convert::TryFrom, - prelude::{Box, Vec}, -}; +use sp_std::prelude::{Box, Vec}; pub use crate::crypto::KeyTypeId; @@ -258,6 +255,30 @@ impl Timestamp { } } +bitflags::bitflags! { + /// Execution context extra capabilities. + pub struct Capabilities: u32 { + /// Access to transaction pool. + const TRANSACTION_POOL = 0b0000_0000_0001; + /// External http calls. + const HTTP = 0b0000_0000_0010; + /// Keystore access. + const KEYSTORE = 0b0000_0000_0100; + /// Randomness source. + const RANDOMNESS = 0b0000_0000_1000; + /// Access to opaque network state. + const NETWORK_STATE = 0b0000_0001_0000; + /// Access to offchain worker DB (read only). + const OFFCHAIN_DB_READ = 0b0000_0010_0000; + /// Access to offchain worker DB (writes). + const OFFCHAIN_DB_WRITE = 0b0000_0100_0000; + /// Manage the authorized nodes + const NODE_AUTHORIZATION = 0b0000_1000_0000; + /// Access time related functionality + const TIME = 0b0001_0000_0000; + const Ipfs = 255; + } +} /// A request that can be handled by an IPFS node. #[derive(Clone, PartialEq, Eq, RuntimeDebug, Encode, Decode, PassByCodec)] pub enum IpfsRequest { @@ -399,67 +420,13 @@ pub enum IpfsRequestStatus { Finished(IpfsResponse), } -/// Execution context extra capabilities. -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -#[repr(u8)] -pub enum Capability { - /// Access to transaction pool. - TransactionPool = 1, - /// External http calls. - Http = 2, - /// Keystore access. - Keystore = 4, - /// Randomness source. - Randomness = 8, - /// Access to opaque network state. - NetworkState = 16, - /// Access to offchain worker DB (read only). - OffchainDbRead = 32, - /// Access to offchain worker DB (writes). - OffchainDbWrite = 64, - /// Manage the authorized nodes - NodeAuthorization = 128, - /// Access to an ipfs node - Ipfs = 255, -} - -/// A set of capabilities -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub struct Capabilities(u8); - impl Capabilities { - /// Return an object representing an empty set of capabilities. - pub fn none() -> Self { - Self(0) - } - - /// Return an object representing all capabilities enabled. - pub fn all() -> Self { - Self(u8::MAX) - } - /// Return capabilities for rich offchain calls. /// /// Those calls should be allowed to sign and submit transactions /// and access offchain workers database (but read only!). pub fn rich_offchain_call() -> Self { - [Capability::TransactionPool, Capability::Keystore, Capability::OffchainDbRead][..].into() - } - - /// Check if particular capability is enabled. - pub fn has(&self, capability: Capability) -> bool { - self.0 & capability as u8 != 0 - } - - /// Check if this capability object represents all capabilities. - pub fn has_all(&self) -> bool { - self == &Capabilities::all() - } -} - -impl<'a> From<&'a [Capability]> for Capabilities { - fn from(list: &'a [Capability]) -> Self { - Capabilities(list.iter().fold(0_u8, |a, b| a | *b as u8)) + Capabilities::TRANSACTION_POOL | Capabilities::KEYSTORE | Capabilities::OFFCHAIN_DB_READ } } @@ -717,8 +684,8 @@ impl LimitedExternalities { /// Check if given capability is allowed. /// /// Panics in case it is not. - fn check(&self, capability: Capability, name: &'static str) { - if !self.capabilities.has(capability) { + fn check(&self, capability: Capabilities, name: &'static str) { + if !self.capabilities.contains(capability) { panic!("Accessing a forbidden API: {}. No: {:?} capability.", name, capability); } } @@ -726,27 +693,27 @@ impl LimitedExternalities { impl Externalities for LimitedExternalities { fn is_validator(&self) -> bool { - self.check(Capability::Keystore, "is_validator"); + self.check(Capabilities::KEYSTORE, "is_validator"); self.externalities.is_validator() } fn network_state(&self) -> Result { - self.check(Capability::NetworkState, "network_state"); + self.check(Capabilities::NETWORK_STATE, "network_state"); self.externalities.network_state() } fn timestamp(&mut self) -> Timestamp { - self.check(Capability::Http, "timestamp"); + self.check(Capabilities::TIME, "timestamp"); self.externalities.timestamp() } fn sleep_until(&mut self, deadline: Timestamp) { - self.check(Capability::Http, "sleep_until"); + self.check(Capabilities::TIME, "sleep_until"); self.externalities.sleep_until(deadline) } fn random_seed(&mut self) -> [u8; 32] { - self.check(Capability::Randomness, "random_seed"); + self.check(Capabilities::RANDOMNESS, "random_seed"); self.externalities.random_seed() } @@ -756,7 +723,7 @@ impl Externalities for LimitedExternalities { uri: &str, meta: &[u8], ) -> Result { - self.check(Capability::Http, "http_request_start"); + self.check(Capabilities::HTTP, "http_request_start"); self.externalities.http_request_start(method, uri, meta) } @@ -766,7 +733,7 @@ impl Externalities for LimitedExternalities { name: &str, value: &str, ) -> Result<(), ()> { - self.check(Capability::Http, "http_request_add_header"); + self.check(Capabilities::HTTP, "http_request_add_header"); self.externalities.http_request_add_header(request_id, name, value) } @@ -776,7 +743,7 @@ impl Externalities for LimitedExternalities { chunk: &[u8], deadline: Option, ) -> Result<(), HttpError> { - self.check(Capability::Http, "http_request_write_body"); + self.check(Capabilities::HTTP, "http_request_write_body"); self.externalities.http_request_write_body(request_id, chunk, deadline) } @@ -785,12 +752,12 @@ impl Externalities for LimitedExternalities { ids: &[HttpRequestId], deadline: Option, ) -> Vec { - self.check(Capability::Http, "http_response_wait"); + self.check(Capabilities::HTTP, "http_response_wait"); self.externalities.http_response_wait(ids, deadline) } fn http_response_headers(&mut self, request_id: HttpRequestId) -> Vec<(Vec, Vec)> { - self.check(Capability::Http, "http_response_headers"); + self.check(Capabilities::HTTP, "http_response_headers"); self.externalities.http_response_headers(request_id) } @@ -800,22 +767,22 @@ impl Externalities for LimitedExternalities { buffer: &mut [u8], deadline: Option, ) -> Result { - self.check(Capability::Http, "http_response_read_body"); + self.check(Capabilities::HTTP, "http_response_read_body"); self.externalities.http_response_read_body(request_id, buffer, deadline) } fn ipfs_request_start(&mut self, request: IpfsRequest) -> Result { - self.check(Capability::Ipfs, "ipfs_request_start"); + self.check(Capabilities::Ipfs, "ipfs_request_start"); self.externalities.ipfs_request_start(request) } fn ipfs_response_wait(&mut self, ids: &[IpfsRequestId], deadline: Option) -> Vec { - self.check(Capability::Ipfs, "ipfs_response_wait"); + self.check(Capabilities::Ipfs, "ipfs_response_wait"); self.externalities.ipfs_response_wait(ids, deadline) } fn set_authorized_nodes(&mut self, nodes: Vec, authorized_only: bool) { - self.check(Capability::NodeAuthorization, "set_authorized_nodes"); + self.check(Capabilities::NODE_AUTHORIZATION, "set_authorized_nodes"); self.externalities.set_authorized_nodes(nodes, authorized_only) } } @@ -899,12 +866,12 @@ impl DbExternalities for Box { impl DbExternalities for LimitedExternalities { fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { - self.check(Capability::OffchainDbWrite, "local_storage_set"); + self.check(Capabilities::OFFCHAIN_DB_WRITE, "local_storage_set"); self.externalities.local_storage_set(kind, key, value) } fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { - self.check(Capability::OffchainDbWrite, "local_storage_clear"); + self.check(Capabilities::OFFCHAIN_DB_WRITE, "local_storage_clear"); self.externalities.local_storage_clear(kind, key) } @@ -915,13 +882,13 @@ impl DbExternalities for LimitedExternalities { old_value: Option<&[u8]>, new_value: &[u8], ) -> bool { - self.check(Capability::OffchainDbWrite, "local_storage_compare_and_set"); + self.check(Capabilities::OFFCHAIN_DB_WRITE, "local_storage_compare_and_set"); self.externalities .local_storage_compare_and_set(kind, key, old_value, new_value) } fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { - self.check(Capability::OffchainDbRead, "local_storage_get"); + self.check(Capabilities::OFFCHAIN_DB_READ, "local_storage_get"); self.externalities.local_storage_get(kind, key) } } @@ -990,15 +957,15 @@ mod tests { #[test] fn capabilities() { - let none = Capabilities::none(); + let none = Capabilities::empty(); let all = Capabilities::all(); - let some = Capabilities::from(&[Capability::Keystore, Capability::Randomness][..]); - - assert!(!none.has(Capability::Keystore)); - assert!(all.has(Capability::Keystore)); - assert!(some.has(Capability::Keystore)); - assert!(!none.has(Capability::TransactionPool)); - assert!(all.has(Capability::TransactionPool)); - assert!(!some.has(Capability::TransactionPool)); + let some = Capabilities::KEYSTORE | Capabilities::RANDOMNESS; + + assert!(!none.contains(Capabilities::KEYSTORE)); + assert!(all.contains(Capabilities::KEYSTORE)); + assert!(some.contains(Capabilities::KEYSTORE)); + assert!(!none.contains(Capabilities::TRANSACTION_POOL)); + assert!(all.contains(Capabilities::TRANSACTION_POOL)); + assert!(!some.contains(Capabilities::TRANSACTION_POOL)); } } diff --git a/primitives/core/src/offchain/storage.rs b/primitives/core/src/offchain/storage.rs index ff72006cffd6..cf2c93641f24 100644 --- a/primitives/core/src/offchain/storage.rs +++ b/primitives/core/src/offchain/storage.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/core/src/offchain/testing.rs b/primitives/core/src/offchain/testing.rs index 35aa2f3fe629..39e776ca1572 100644 --- a/primitives/core/src/offchain/testing.rs +++ b/primitives/core/src/offchain/testing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -401,7 +401,7 @@ impl offchain::Externalities for TestOffchainExt { Ok(0) } else { let read = std::cmp::min(buffer.len(), response[req.read..].len()); - buffer[0..read].copy_from_slice(&response[req.read..read]); + buffer[0..read].copy_from_slice(&response[req.read..req.read + read]); req.read += read; Ok(read) } diff --git a/primitives/core/src/sandbox.rs b/primitives/core/src/sandbox.rs index acc3fda5e9b1..1f408a3b8cc0 100644 --- a/primitives/core/src/sandbox.rs +++ b/primitives/core/src/sandbox.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/core/src/sr25519.rs b/primitives/core/src/sr25519.rs index 4787c2d9d13e..ef033c2099b5 100644 --- a/primitives/core/src/sr25519.rs +++ b/primitives/core/src/sr25519.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -35,13 +35,11 @@ use schnorrkel::{ #[cfg(feature = "full_crypto")] use sp_std::vec::Vec; #[cfg(feature = "std")] -use std::convert::TryFrom; -#[cfg(feature = "std")] use substrate_bip39::mini_secret_from_entropy; use crate::{ crypto::{ - CryptoType, CryptoTypeId, CryptoTypePublicPair, Derive, Public as TraitPublic, + ByteArray, CryptoType, CryptoTypeId, CryptoTypePublicPair, Derive, Public as TraitPublic, UncheckedFrom, }, hash::{H256, H512}, @@ -74,7 +72,6 @@ pub const CRYPTO_ID: CryptoTypeId = CryptoTypeId(*b"sr25"); Copy, Encode, Decode, - Default, PassByInner, MaxEncodedLen, TypeInfo, @@ -143,17 +140,16 @@ impl std::str::FromStr for Public { } } -impl sp_std::convert::TryFrom<&[u8]> for Public { +impl TryFrom<&[u8]> for Public { type Error = (); fn try_from(data: &[u8]) -> Result { - if data.len() == 32 { - let mut inner = [0u8; 32]; - inner.copy_from_slice(data); - Ok(Public(inner)) - } else { - Err(()) + if data.len() != Self::LEN { + return Err(()) } + let mut r = [0u8; 32]; + r.copy_from_slice(data); + Ok(Self::unchecked_from(r)) } } @@ -213,10 +209,11 @@ impl<'de> Deserialize<'de> for Public { /// An Schnorrkel/Ristretto x25519 ("sr25519") signature. /// /// Instead of importing it for the local module, alias it to be available as a public type -#[derive(Encode, Decode, PassByInner, TypeInfo)] +#[cfg_attr(feature = "full_crypto", derive(Hash))] +#[derive(Encode, Decode, MaxEncodedLen, PassByInner, TypeInfo, PartialEq, Eq)] pub struct Signature(pub [u8; 64]); -impl sp_std::convert::TryFrom<&[u8]> for Signature { +impl TryFrom<&[u8]> for Signature { type Error = (); fn try_from(data: &[u8]) -> Result { @@ -261,20 +258,6 @@ impl Clone for Signature { } } -impl Default for Signature { - fn default() -> Self { - Signature([0u8; 64]) - } -} - -impl PartialEq for Signature { - fn eq(&self, b: &Self) -> bool { - self.0[..] == b.0[..] - } -} - -impl Eq for Signature {} - impl From for [u8; 64] { fn from(v: Signature) -> [u8; 64] { v.0 @@ -324,13 +307,6 @@ impl sp_std::fmt::Debug for Signature { } } -#[cfg(feature = "full_crypto")] -impl sp_std::hash::Hash for Signature { - fn hash(&self, state: &mut H) { - sp_std::hash::Hash::hash(&self.0[..], state); - } -} - /// A localized signature also contains sender information. /// NOTE: Encode and Decode traits are supported in ed25519 but not possible for now here. #[cfg(feature = "std")] @@ -342,6 +318,12 @@ pub struct LocalizedSignature { pub signature: Signature, } +impl UncheckedFrom<[u8; 64]> for Signature { + fn unchecked_from(data: [u8; 64]) -> Signature { + Signature(data) + } +} + impl Signature { /// A new instance from the given 64-byte `data`. /// @@ -357,10 +339,13 @@ impl Signature { /// /// NOTE: No checking goes on to ensure this is a real signature. Only use it if /// you are certain that the array actually is a signature. GIGO! - pub fn from_slice(data: &[u8]) -> Self { + pub fn from_slice(data: &[u8]) -> Option { + if data.len() != 64 { + return None + } let mut r = [0u8; 64]; r.copy_from_slice(data); - Signature(r) + Some(Signature(r)) } /// A new instance from an H512. @@ -412,17 +397,11 @@ impl Public { } } -impl TraitPublic for Public { - /// A new instance from the given slice that should be 32 bytes long. - /// - /// NOTE: No checking goes on to ensure this is a real public key. Only use it if - /// you are certain that the array actually is a pubkey. GIGO! - fn from_slice(data: &[u8]) -> Self { - let mut r = [0u8; 32]; - r.copy_from_slice(data); - Public(r) - } +impl ByteArray for Public { + const LEN: usize = 32; +} +impl TraitPublic for Public { fn to_public_crypto_pair(&self) -> CryptoTypePublicPair { CryptoTypePublicPair(CRYPTO_ID, self.to_raw_vec()) } diff --git a/primitives/core/src/testing.rs b/primitives/core/src/testing.rs index a7fff0def83f..d5ca1dc45fa0 100644 --- a/primitives/core/src/testing.rs +++ b/primitives/core/src/testing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -152,10 +152,20 @@ impl Default for TaskExecutor { #[cfg(feature = "std")] impl crate::traits::SpawnNamed for TaskExecutor { - fn spawn_blocking(&self, _: &'static str, future: futures::future::BoxFuture<'static, ()>) { + fn spawn_blocking( + &self, + _name: &'static str, + _group: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ) { self.0.spawn_ok(future); } - fn spawn(&self, _: &'static str, future: futures::future::BoxFuture<'static, ()>) { + fn spawn( + &self, + _name: &'static str, + _group: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ) { self.0.spawn_ok(future); } } @@ -165,11 +175,17 @@ impl crate::traits::SpawnEssentialNamed for TaskExecutor { fn spawn_essential_blocking( &self, _: &'static str, + _: Option<&'static str>, future: futures::future::BoxFuture<'static, ()>, ) { self.0.spawn_ok(future); } - fn spawn_essential(&self, _: &'static str, future: futures::future::BoxFuture<'static, ()>) { + fn spawn_essential( + &self, + _: &'static str, + _: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ) { self.0.spawn_ok(future); } } diff --git a/primitives/core/src/traits.rs b/primitives/core/src/traits.rs index 47639f9d87ba..80e8963a2909 100644 --- a/primitives/core/src/traits.rs +++ b/primitives/core/src/traits.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -190,58 +190,91 @@ sp_externalities::decl_extension! { pub struct RuntimeSpawnExt(Box); } -/// Something that can spawn tasks (blocking and non-blocking) with an assigned name. +/// Something that can spawn tasks (blocking and non-blocking) with an assigned name +/// and optional group. #[dyn_clonable::clonable] pub trait SpawnNamed: Clone + Send + Sync { /// Spawn the given blocking future. /// - /// The given `name` is used to identify the future in tracing. - fn spawn_blocking(&self, name: &'static str, future: futures::future::BoxFuture<'static, ()>); + /// The given `group` and `name` is used to identify the future in tracing. + fn spawn_blocking( + &self, + name: &'static str, + group: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ); /// Spawn the given non-blocking future. /// - /// The given `name` is used to identify the future in tracing. - fn spawn(&self, name: &'static str, future: futures::future::BoxFuture<'static, ()>); + /// The given `group` and `name` is used to identify the future in tracing. + fn spawn( + &self, + name: &'static str, + group: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ); } impl SpawnNamed for Box { - fn spawn_blocking(&self, name: &'static str, future: futures::future::BoxFuture<'static, ()>) { - (**self).spawn_blocking(name, future) + fn spawn_blocking( + &self, + name: &'static str, + group: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ) { + (**self).spawn_blocking(name, group, future) } - - fn spawn(&self, name: &'static str, future: futures::future::BoxFuture<'static, ()>) { - (**self).spawn(name, future) + fn spawn( + &self, + name: &'static str, + group: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ) { + (**self).spawn(name, group, future) } } -/// Something that can spawn essential tasks (blocking and non-blocking) with an assigned name. +/// Something that can spawn essential tasks (blocking and non-blocking) with an assigned name +/// and optional group. /// /// Essential tasks are special tasks that should take down the node when they end. #[dyn_clonable::clonable] pub trait SpawnEssentialNamed: Clone + Send + Sync { /// Spawn the given blocking future. /// - /// The given `name` is used to identify the future in tracing. + /// The given `group` and `name` is used to identify the future in tracing. fn spawn_essential_blocking( &self, name: &'static str, + group: Option<&'static str>, future: futures::future::BoxFuture<'static, ()>, ); /// Spawn the given non-blocking future. /// - /// The given `name` is used to identify the future in tracing. - fn spawn_essential(&self, name: &'static str, future: futures::future::BoxFuture<'static, ()>); + /// The given `group` and `name` is used to identify the future in tracing. + fn spawn_essential( + &self, + name: &'static str, + group: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ); } impl SpawnEssentialNamed for Box { fn spawn_essential_blocking( &self, name: &'static str, + group: Option<&'static str>, future: futures::future::BoxFuture<'static, ()>, ) { - (**self).spawn_essential_blocking(name, future) + (**self).spawn_essential_blocking(name, group, future) } - fn spawn_essential(&self, name: &'static str, future: futures::future::BoxFuture<'static, ()>) { - (**self).spawn_essential(name, future) + fn spawn_essential( + &self, + name: &'static str, + group: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ) { + (**self).spawn_essential(name, group, future) } } diff --git a/primitives/core/src/u32_trait.rs b/primitives/core/src/u32_trait.rs deleted file mode 100644 index 37837e7c0548..000000000000 --- a/primitives/core/src/u32_trait.rs +++ /dev/null @@ -1,570 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! An u32 trait with "values" as impl'd types. - -/// A u32 value, wrapped in a trait because we don't yet have const generics. -pub trait Value { - /// The actual value represented by the impl'ing type. - const VALUE: u32; -} - -/// Type representing the value 0 for the `Value` trait. -pub struct _0; -impl Value for _0 { - const VALUE: u32 = 0; -} -/// Type representing the value 1 for the `Value` trait. -pub struct _1; -impl Value for _1 { - const VALUE: u32 = 1; -} -/// Type representing the value 2 for the `Value` trait. -pub struct _2; -impl Value for _2 { - const VALUE: u32 = 2; -} -/// Type representing the value 3 for the `Value` trait. -pub struct _3; -impl Value for _3 { - const VALUE: u32 = 3; -} -/// Type representing the value 4 for the `Value` trait. -pub struct _4; -impl Value for _4 { - const VALUE: u32 = 4; -} -/// Type representing the value 5 for the `Value` trait. -pub struct _5; -impl Value for _5 { - const VALUE: u32 = 5; -} -/// Type representing the value 6 for the `Value` trait. -pub struct _6; -impl Value for _6 { - const VALUE: u32 = 6; -} -/// Type representing the value 7 for the `Value` trait. -pub struct _7; -impl Value for _7 { - const VALUE: u32 = 7; -} -/// Type representing the value 8 for the `Value` trait. -pub struct _8; -impl Value for _8 { - const VALUE: u32 = 8; -} -/// Type representing the value 9 for the `Value` trait. -pub struct _9; -impl Value for _9 { - const VALUE: u32 = 9; -} -/// Type representing the value 10 for the `Value` trait. -pub struct _10; -impl Value for _10 { - const VALUE: u32 = 10; -} -/// Type representing the value 11 for the `Value` trait. -pub struct _11; -impl Value for _11 { - const VALUE: u32 = 11; -} -/// Type representing the value 12 for the `Value` trait. -pub struct _12; -impl Value for _12 { - const VALUE: u32 = 12; -} -/// Type representing the value 13 for the `Value` trait. -pub struct _13; -impl Value for _13 { - const VALUE: u32 = 13; -} -/// Type representing the value 14 for the `Value` trait. -pub struct _14; -impl Value for _14 { - const VALUE: u32 = 14; -} -/// Type representing the value 15 for the `Value` trait. -pub struct _15; -impl Value for _15 { - const VALUE: u32 = 15; -} -/// Type representing the value 16 for the `Value` trait. -pub struct _16; -impl Value for _16 { - const VALUE: u32 = 16; -} -/// Type representing the value 17 for the `Value` trait. -pub struct _17; -impl Value for _17 { - const VALUE: u32 = 17; -} -/// Type representing the value 18 for the `Value` trait. -pub struct _18; -impl Value for _18 { - const VALUE: u32 = 18; -} -/// Type representing the value 19 for the `Value` trait. -pub struct _19; -impl Value for _19 { - const VALUE: u32 = 19; -} -/// Type representing the value 20 for the `Value` trait. -pub struct _20; -impl Value for _20 { - const VALUE: u32 = 20; -} -/// Type representing the value 21 for the `Value` trait. -pub struct _21; -impl Value for _21 { - const VALUE: u32 = 21; -} -/// Type representing the value 22 for the `Value` trait. -pub struct _22; -impl Value for _22 { - const VALUE: u32 = 22; -} -/// Type representing the value 23 for the `Value` trait. -pub struct _23; -impl Value for _23 { - const VALUE: u32 = 23; -} -/// Type representing the value 24 for the `Value` trait. -pub struct _24; -impl Value for _24 { - const VALUE: u32 = 24; -} -/// Type representing the value 25 for the `Value` trait. -pub struct _25; -impl Value for _25 { - const VALUE: u32 = 25; -} -/// Type representing the value 26 for the `Value` trait. -pub struct _26; -impl Value for _26 { - const VALUE: u32 = 26; -} -/// Type representing the value 27 for the `Value` trait. -pub struct _27; -impl Value for _27 { - const VALUE: u32 = 27; -} -/// Type representing the value 28 for the `Value` trait. -pub struct _28; -impl Value for _28 { - const VALUE: u32 = 28; -} -/// Type representing the value 29 for the `Value` trait. -pub struct _29; -impl Value for _29 { - const VALUE: u32 = 29; -} -/// Type representing the value 30 for the `Value` trait. -pub struct _30; -impl Value for _30 { - const VALUE: u32 = 30; -} -/// Type representing the value 31 for the `Value` trait. -pub struct _31; -impl Value for _31 { - const VALUE: u32 = 31; -} -/// Type representing the value 32 for the `Value` trait. -pub struct _32; -impl Value for _32 { - const VALUE: u32 = 32; -} -/// Type representing the value 33 for the `Value` trait. -pub struct _33; -impl Value for _33 { - const VALUE: u32 = 33; -} -/// Type representing the value 34 for the `Value` trait. -pub struct _34; -impl Value for _34 { - const VALUE: u32 = 34; -} -/// Type representing the value 35 for the `Value` trait. -pub struct _35; -impl Value for _35 { - const VALUE: u32 = 35; -} -/// Type representing the value 36 for the `Value` trait. -pub struct _36; -impl Value for _36 { - const VALUE: u32 = 36; -} -/// Type representing the value 37 for the `Value` trait. -pub struct _37; -impl Value for _37 { - const VALUE: u32 = 37; -} -/// Type representing the value 38 for the `Value` trait. -pub struct _38; -impl Value for _38 { - const VALUE: u32 = 38; -} -/// Type representing the value 39 for the `Value` trait. -pub struct _39; -impl Value for _39 { - const VALUE: u32 = 39; -} -/// Type representing the value 40 for the `Value` trait. -pub struct _40; -impl Value for _40 { - const VALUE: u32 = 40; -} -/// Type representing the value 41 for the `Value` trait. -pub struct _41; -impl Value for _41 { - const VALUE: u32 = 41; -} -/// Type representing the value 42 for the `Value` trait. -pub struct _42; -impl Value for _42 { - const VALUE: u32 = 42; -} -/// Type representing the value 43 for the `Value` trait. -pub struct _43; -impl Value for _43 { - const VALUE: u32 = 43; -} -/// Type representing the value 44 for the `Value` trait. -pub struct _44; -impl Value for _44 { - const VALUE: u32 = 44; -} -/// Type representing the value 45 for the `Value` trait. -pub struct _45; -impl Value for _45 { - const VALUE: u32 = 45; -} -/// Type representing the value 46 for the `Value` trait. -pub struct _46; -impl Value for _46 { - const VALUE: u32 = 46; -} -/// Type representing the value 47 for the `Value` trait. -pub struct _47; -impl Value for _47 { - const VALUE: u32 = 47; -} -/// Type representing the value 48 for the `Value` trait. -pub struct _48; -impl Value for _48 { - const VALUE: u32 = 48; -} -/// Type representing the value 49 for the `Value` trait. -pub struct _49; -impl Value for _49 { - const VALUE: u32 = 49; -} -/// Type representing the value 50 for the `Value` trait. -pub struct _50; -impl Value for _50 { - const VALUE: u32 = 50; -} -/// Type representing the value 51 for the `Value` trait. -pub struct _51; -impl Value for _51 { - const VALUE: u32 = 51; -} -/// Type representing the value 52 for the `Value` trait. -pub struct _52; -impl Value for _52 { - const VALUE: u32 = 52; -} -/// Type representing the value 53 for the `Value` trait. -pub struct _53; -impl Value for _53 { - const VALUE: u32 = 53; -} -/// Type representing the value 54 for the `Value` trait. -pub struct _54; -impl Value for _54 { - const VALUE: u32 = 54; -} -/// Type representing the value 55 for the `Value` trait. -pub struct _55; -impl Value for _55 { - const VALUE: u32 = 55; -} -/// Type representing the value 56 for the `Value` trait. -pub struct _56; -impl Value for _56 { - const VALUE: u32 = 56; -} -/// Type representing the value 57 for the `Value` trait. -pub struct _57; -impl Value for _57 { - const VALUE: u32 = 57; -} -/// Type representing the value 58 for the `Value` trait. -pub struct _58; -impl Value for _58 { - const VALUE: u32 = 58; -} -/// Type representing the value 59 for the `Value` trait. -pub struct _59; -impl Value for _59 { - const VALUE: u32 = 59; -} -/// Type representing the value 60 for the `Value` trait. -pub struct _60; -impl Value for _60 { - const VALUE: u32 = 60; -} -/// Type representing the value 61 for the `Value` trait. -pub struct _61; -impl Value for _61 { - const VALUE: u32 = 61; -} -/// Type representing the value 62 for the `Value` trait. -pub struct _62; -impl Value for _62 { - const VALUE: u32 = 62; -} -/// Type representing the value 63 for the `Value` trait. -pub struct _63; -impl Value for _63 { - const VALUE: u32 = 63; -} -/// Type representing the value 64 for the `Value` trait. -pub struct _64; -impl Value for _64 { - const VALUE: u32 = 64; -} -/// Type representing the value 65 for the `Value` trait. -pub struct _65; -impl Value for _65 { - const VALUE: u32 = 65; -} -/// Type representing the value 66 for the `Value` trait. -pub struct _66; -impl Value for _66 { - const VALUE: u32 = 66; -} -/// Type representing the value 67 for the `Value` trait. -pub struct _67; -impl Value for _67 { - const VALUE: u32 = 67; -} -/// Type representing the value 68 for the `Value` trait. -pub struct _68; -impl Value for _68 { - const VALUE: u32 = 68; -} -/// Type representing the value 69 for the `Value` trait. -pub struct _69; -impl Value for _69 { - const VALUE: u32 = 69; -} -/// Type representing the value 70 for the `Value` trait. -pub struct _70; -impl Value for _70 { - const VALUE: u32 = 70; -} -/// Type representing the value 71 for the `Value` trait. -pub struct _71; -impl Value for _71 { - const VALUE: u32 = 71; -} -/// Type representing the value 72 for the `Value` trait. -pub struct _72; -impl Value for _72 { - const VALUE: u32 = 72; -} -/// Type representing the value 73 for the `Value` trait. -pub struct _73; -impl Value for _73 { - const VALUE: u32 = 73; -} -/// Type representing the value 74 for the `Value` trait. -pub struct _74; -impl Value for _74 { - const VALUE: u32 = 74; -} -/// Type representing the value 75 for the `Value` trait. -pub struct _75; -impl Value for _75 { - const VALUE: u32 = 75; -} -/// Type representing the value 76 for the `Value` trait. -pub struct _76; -impl Value for _76 { - const VALUE: u32 = 76; -} -/// Type representing the value 77 for the `Value` trait. -pub struct _77; -impl Value for _77 { - const VALUE: u32 = 77; -} -/// Type representing the value 78 for the `Value` trait. -pub struct _78; -impl Value for _78 { - const VALUE: u32 = 78; -} -/// Type representing the value 79 for the `Value` trait. -pub struct _79; -impl Value for _79 { - const VALUE: u32 = 79; -} -/// Type representing the value 80 for the `Value` trait. -pub struct _80; -impl Value for _80 { - const VALUE: u32 = 80; -} -/// Type representing the value 81 for the `Value` trait. -pub struct _81; -impl Value for _81 { - const VALUE: u32 = 81; -} -/// Type representing the value 82 for the `Value` trait. -pub struct _82; -impl Value for _82 { - const VALUE: u32 = 82; -} -/// Type representing the value 83 for the `Value` trait. -pub struct _83; -impl Value for _83 { - const VALUE: u32 = 83; -} -/// Type representing the value 84 for the `Value` trait. -pub struct _84; -impl Value for _84 { - const VALUE: u32 = 84; -} -/// Type representing the value 85 for the `Value` trait. -pub struct _85; -impl Value for _85 { - const VALUE: u32 = 85; -} -/// Type representing the value 86 for the `Value` trait. -pub struct _86; -impl Value for _86 { - const VALUE: u32 = 86; -} -/// Type representing the value 87 for the `Value` trait. -pub struct _87; -impl Value for _87 { - const VALUE: u32 = 87; -} -/// Type representing the value 88 for the `Value` trait. -pub struct _88; -impl Value for _88 { - const VALUE: u32 = 88; -} -/// Type representing the value 89 for the `Value` trait. -pub struct _89; -impl Value for _89 { - const VALUE: u32 = 89; -} -/// Type representing the value 90 for the `Value` trait. -pub struct _90; -impl Value for _90 { - const VALUE: u32 = 90; -} -/// Type representing the value 91 for the `Value` trait. -pub struct _91; -impl Value for _91 { - const VALUE: u32 = 91; -} -/// Type representing the value 92 for the `Value` trait. -pub struct _92; -impl Value for _92 { - const VALUE: u32 = 92; -} -/// Type representing the value 93 for the `Value` trait. -pub struct _93; -impl Value for _93 { - const VALUE: u32 = 93; -} -/// Type representing the value 94 for the `Value` trait. -pub struct _94; -impl Value for _94 { - const VALUE: u32 = 94; -} -/// Type representing the value 95 for the `Value` trait. -pub struct _95; -impl Value for _95 { - const VALUE: u32 = 95; -} -/// Type representing the value 96 for the `Value` trait. -pub struct _96; -impl Value for _96 { - const VALUE: u32 = 96; -} -/// Type representing the value 97 for the `Value` trait. -pub struct _97; -impl Value for _97 { - const VALUE: u32 = 97; -} -/// Type representing the value 98 for the `Value` trait. -pub struct _98; -impl Value for _98 { - const VALUE: u32 = 98; -} -/// Type representing the value 99 for the `Value` trait. -pub struct _99; -impl Value for _99 { - const VALUE: u32 = 99; -} -/// Type representing the value 100 for the `Value` trait. -pub struct _100; -impl Value for _100 { - const VALUE: u32 = 100; -} -/// Type representing the value 112 for the `Value` trait. -pub struct _112; -impl Value for _112 { - const VALUE: u32 = 112; -} -/// Type representing the value 128 for the `Value` trait. -pub struct _128; -impl Value for _128 { - const VALUE: u32 = 128; -} -/// Type representing the value 160 for the `Value` trait. -pub struct _160; -impl Value for _160 { - const VALUE: u32 = 160; -} -/// Type representing the value 192 for the `Value` trait. -pub struct _192; -impl Value for _192 { - const VALUE: u32 = 192; -} -/// Type representing the value 224 for the `Value` trait. -pub struct _224; -impl Value for _224 { - const VALUE: u32 = 224; -} -/// Type representing the value 256 for the `Value` trait. -pub struct _256; -impl Value for _256 { - const VALUE: u32 = 256; -} -/// Type representing the value 384 for the `Value` trait. -pub struct _384; -impl Value for _384 { - const VALUE: u32 = 384; -} -/// Type representing the value 512 for the `Value` trait. -pub struct _512; -impl Value for _512 { - const VALUE: u32 = 512; -} diff --git a/primitives/core/src/uint.rs b/primitives/core/src/uint.rs index a74980332ad2..f4eb3a19ac36 100644 --- a/primitives/core/src/uint.rs +++ b/primitives/core/src/uint.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/database/Cargo.toml b/primitives/database/Cargo.toml index c99651d4ef04..198f44510209 100644 --- a/primitives/database/Cargo.toml +++ b/primitives/database/Cargo.toml @@ -2,15 +2,15 @@ name = "sp-database" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate database trait." documentation = "https://docs.rs/sp-database" readme = "README.md" [dependencies] -parking_lot = "0.11.1" -kvdb = "0.10.0" +parking_lot = "0.12.0" +kvdb = "0.11.0" diff --git a/primitives/database/src/error.rs b/primitives/database/src/error.rs index 4bf5a20aff40..78646427a2b5 100644 --- a/primitives/database/src/error.rs +++ b/primitives/database/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/database/src/kvdb.rs b/primitives/database/src/kvdb.rs index 1a2b0513dc28..5fe5fda307a1 100644 --- a/primitives/database/src/kvdb.rs +++ b/primitives/database/src/kvdb.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/database/src/lib.rs b/primitives/database/src/lib.rs index d30c7eb3323e..3ec62a2b7814 100644 --- a/primitives/database/src/lib.rs +++ b/primitives/database/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -110,6 +110,11 @@ pub trait Database>: Send + Sync { fn supports_ref_counting(&self) -> bool { false } + + /// Remove a possible path-prefix from the key. + /// + /// Not all database implementations use a prefix for keys, so this function may be a noop. + fn sanitize_key(&self, _key: &mut Vec) {} } impl std::fmt::Debug for dyn Database { diff --git a/primitives/database/src/mem.rs b/primitives/database/src/mem.rs index d1b1861e98fd..6ecd44c6005b 100644 --- a/primitives/database/src/mem.rs +++ b/primitives/database/src/mem.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/debug-derive/Cargo.toml b/primitives/debug-derive/Cargo.toml index 0d3ba805100c..5852bd428d3e 100644 --- a/primitives/debug-derive/Cargo.toml +++ b/primitives/debug-derive/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-debug-derive" -version = "3.0.0" +version = "4.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Macros to derive runtime debug implementation." documentation = "https://docs.rs/sp-debug-derive" @@ -17,11 +17,17 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -quote = "1.0.3" -syn = "1.0.58" +quote = "1.0.10" +syn = "1.0.82" proc-macro2 = "1.0" [features] +default = [ "std" ] std = [] +# By default `RuntimeDebug` implements `Debug` that outputs `` when `std` is +# disabled. However, sometimes downstream users need to have the real `Debug` implementation for +# debugging purposes. If this is required, a user only needs to add this crate as a dependency of +# their runtime and enable the `force-debug` feature. +force-debug = [] [dev-dependencies] diff --git a/primitives/debug-derive/src/impls.rs b/primitives/debug-derive/src/impls.rs index 4d79ee988016..060997fe9782 100644 --- a/primitives/debug-derive/src/impls.rs +++ b/primitives/debug-derive/src/impls.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -43,7 +43,7 @@ pub fn debug_derive(ast: DeriveInput) -> proc_macro::TokenStream { gen.into() } -#[cfg(not(feature = "std"))] +#[cfg(all(not(feature = "std"), not(feature = "force-debug")))] mod implementation { use super::*; @@ -58,7 +58,7 @@ mod implementation { } } -#[cfg(feature = "std")] +#[cfg(any(feature = "std", feature = "force-debug"))] mod implementation { use super::*; use proc_macro2::Span; @@ -101,7 +101,7 @@ mod implementation { } } - fn derive_fields<'a>(name_str: &str, fields: Fields) -> TokenStream { + fn derive_fields(name_str: &str, fields: Fields) -> TokenStream { match fields { Fields::Named { names, this } => { let names_str: Vec<_> = names.iter().map(|x| x.to_string()).collect(); diff --git a/primitives/debug-derive/src/lib.rs b/primitives/debug-derive/src/lib.rs index 7eaa3a0020e9..c98610ce4780 100644 --- a/primitives/debug-derive/src/lib.rs +++ b/primitives/debug-derive/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/debug-derive/tests/tests.rs b/primitives/debug-derive/tests/tests.rs index 4f4c7f4caabc..39414da86bf4 100644 --- a/primitives/debug-derive/tests/tests.rs +++ b/primitives/debug-derive/tests/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/externalities/Cargo.toml b/primitives/externalities/Cargo.toml index 52a6300688cd..4fc8619a9ba5 100644 --- a/primitives/externalities/Cargo.toml +++ b/primitives/externalities/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-externalities" -version = "0.10.0-dev" +version = "0.12.0" license = "Apache-2.0" authors = ["Parity Technologies "] -edition = "2018" -homepage = "https://substrate.dev" +edition = "2021" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate externalities abstraction" documentation = "https://docs.rs/sp-externalities" @@ -14,10 +14,10 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-storage = { version = "4.0.0-dev", path = "../storage", default-features = false } -sp-std = { version = "4.0.0-dev", path = "../std", default-features = false } +sp-storage = { version = "6.0.0", path = "../storage", default-features = false } +sp-std = { version = "4.0.0", path = "../std", default-features = false } environmental = { version = "1.1.3", default-features = false } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } [features] default = ["std"] diff --git a/primitives/externalities/src/extensions.rs b/primitives/externalities/src/extensions.rs index 37086a707b64..d58edb749e9e 100644 --- a/primitives/externalities/src/extensions.rs +++ b/primitives/externalities/src/extensions.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index e6a8f8caa8d3..49b190b4ae26 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,7 +31,7 @@ use sp_std::{ vec::Vec, }; -use sp_storage::{ChildInfo, TrackedStorageKey}; +use sp_storage::{ChildInfo, StateVersion, TrackedStorageKey}; pub use extensions::{Extension, ExtensionStore, Extensions}; pub use scope_limited::{set_and_run_with_externalities, with_externalities}; @@ -157,7 +157,7 @@ pub trait Externalities: ExtensionStore { /// This will also update all child storage keys in the top-level storage map. /// /// The returned hash is defined by the `Block` and is SCALE encoded. - fn storage_root(&mut self) -> Vec; + fn storage_root(&mut self, state_version: StateVersion) -> Vec; /// Get the trie root of a child storage map. /// @@ -165,7 +165,11 @@ pub trait Externalities: ExtensionStore { /// /// If the storage root equals the default hash as defined by the trie, the key in the top-level /// storage map will be removed. - fn child_storage_root(&mut self, child_info: &ChildInfo) -> Vec; + fn child_storage_root( + &mut self, + child_info: &ChildInfo, + state_version: StateVersion, + ) -> Vec; /// Append storage item. /// @@ -173,13 +177,6 @@ pub trait Externalities: ExtensionStore { /// operation. fn storage_append(&mut self, key: Vec, value: Vec); - /// Get the changes trie root of the current storage overlay at a block with given `parent`. - /// - /// `parent` expects a SCALE encoded hash. - /// - /// The returned hash is defined by the `Block` and is SCALE encoded. - fn storage_changes_root(&mut self, parent: &[u8]) -> Result>, ()>; - /// Start a new nested transaction. /// /// This allows to either commit or roll back all changes made after this call to the diff --git a/primitives/externalities/src/scope_limited.rs b/primitives/externalities/src/scope_limited.rs index 15a670a9abee..1d2e6863c9e5 100644 --- a/primitives/externalities/src/scope_limited.rs +++ b/primitives/externalities/src/scope_limited.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/finality-grandpa/Cargo.toml b/primitives/finality-grandpa/Cargo.toml index c0c2a654270f..ebc067e630c1 100644 --- a/primitives/finality-grandpa/Cargo.toml +++ b/primitives/finality-grandpa/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-finality-grandpa" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Primitives for GRANDPA integration, suitable for WASM compilation." documentation = "https://docs.rs/sp-finality-grandpa" @@ -15,17 +15,17 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -grandpa = { package = "finality-grandpa", version = "0.14.1", default-features = false, features = ["derive-codec"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +grandpa = { package = "finality-grandpa", version = "0.15.0", default-features = false, features = ["derive-codec"] } log = { version = "0.4.8", optional = true } -serde = { version = "1.0.126", optional = true, features = ["derive"] } +serde = { version = "1.0.136", optional = true, features = ["derive"] } sp-api = { version = "4.0.0-dev", default-features = false, path = "../api" } -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../application-crypto" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } -sp-keystore = { version = "0.10.0-dev", default-features = false, path = "../keystore", optional = true } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../application-crypto" } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +sp-keystore = { version = "0.12.0", default-features = false, path = "../keystore", optional = true } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } [features] default = ["std"] diff --git a/primitives/finality-grandpa/src/lib.rs b/primitives/finality-grandpa/src/lib.rs index d99a4c188222..dd5cef85a2ba 100644 --- a/primitives/finality-grandpa/src/lib.rs +++ b/primitives/finality-grandpa/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -389,7 +389,6 @@ where { use sp_application_crypto::AppKey; use sp_core::crypto::Public; - use sp_std::convert::TryInto; let encoded = localized_payload(round, set_id, &message); let signature = SyncCryptoStore::sign_with( diff --git a/primitives/inherents/Cargo.toml b/primitives/inherents/Cargo.toml index 23558750b5cf..0e701a397d7d 100644 --- a/primitives/inherents/Cargo.toml +++ b/primitives/inherents/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-inherents" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Provides types and traits for creating and checking inherents." documentation = "https://docs.rs/sp-inherents" @@ -15,16 +15,16 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } -sp-runtime = { version = "4.0.0-dev", path = "../runtime", optional = true } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -thiserror = { version = "1.0.21", optional = true } -impl-trait-for-tuples = "0.2.0" +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +sp-runtime = { version = "6.0.0", path = "../runtime", optional = true } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +thiserror = { version = "1.0.30", optional = true } +impl-trait-for-tuples = "0.2.2" async-trait = { version = "0.1.50", optional = true } [dev-dependencies] -futures = "0.3.9" +futures = "0.3.21" [features] default = [ "std" ] diff --git a/primitives/inherents/src/client_side.rs b/primitives/inherents/src/client_side.rs index 18877cae5f34..42068e24e029 100644 --- a/primitives/inherents/src/client_side.rs +++ b/primitives/inherents/src/client_side.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/inherents/src/lib.rs b/primitives/inherents/src/lib.rs index 90f4e455a42d..a3ef963c47b3 100644 --- a/primitives/inherents/src/lib.rs +++ b/primitives/inherents/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,14 +15,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Substrate inherent extrinsics +//! Substrate Inherent Extrinsics //! //! Inherent extrinsics are extrinsics that are inherently added to each block. However, it is up to -//! runtime implementation to require an inherent for each block or to make it optional. Inherents -//! are mainly used to pass data from the block producer to the runtime. So, inherents require some -//! part that is running on the client side and some part that is running on the runtime side. Any -//! data that is required by an inherent is passed as [`InherentData`] from the client to the -//! runtime when the inherents are constructed. +//! the runtime implementation to require an inherent for each block or to make it optional. +//! Inherents are mainly used to pass data from the block producer to the runtime. So, inherents +//! require some part that is running on the client side and some part that is running on the +//! runtime side. Any data that is required by an inherent is passed as [`InherentData`] from the +//! client to the runtime when the inherents are constructed. //! //! The process of constructing and applying inherents is the following: //! diff --git a/primitives/io/Cargo.toml b/primitives/io/Cargo.toml index 5a8c1c4af4f9..bdae6174a87c 100644 --- a/primitives/io/Cargo.toml +++ b/primitives/io/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-io" -version = "4.0.0-dev" +version = "6.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "I/O for Substrate runtimes" documentation = "https://docs.rs/sp-io" @@ -15,22 +15,23 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } hash-db = { version = "0.15.2", default-features = false } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } -sp-keystore = { version = "0.10.0-dev", default-features = false, optional = true, path = "../keystore" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -libsecp256k1 = { version = "0.6", optional = true } -sp-state-machine = { version = "0.10.0-dev", optional = true, path = "../state-machine" } -sp-wasm-interface = { version = "4.0.0-dev", path = "../wasm-interface", default-features = false } -sp-runtime-interface = { version = "4.0.0-dev", default-features = false, path = "../runtime-interface" } -sp-trie = { version = "4.0.0-dev", optional = true, path = "../trie" } -sp-externalities = { version = "0.10.0-dev", optional = true, path = "../externalities" } -sp-tracing = { version = "4.0.0-dev", default-features = false, path = "../tracing" } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +sp-keystore = { version = "0.12.0", default-features = false, optional = true, path = "../keystore" } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +libsecp256k1 = { version = "0.7", optional = true } +sp-state-machine = { version = "0.12.0", optional = true, path = "../state-machine" } +sp-wasm-interface = { version = "6.0.0", path = "../wasm-interface", default-features = false } +sp-runtime-interface = { version = "6.0.0", default-features = false, path = "../runtime-interface" } +sp-trie = { version = "6.0.0", optional = true, path = "../trie" } +sp-externalities = { version = "0.12.0", optional = true, path = "../externalities" } +sp-tracing = { version = "5.0.0", default-features = false, path = "../tracing" } log = { version = "0.4.8", optional = true } -futures = { version = "0.3.1", features = ["thread-pool"], optional = true } -parking_lot = { version = "0.11.1", optional = true } -tracing = { version = "0.1.25", default-features = false } +futures = { version = "0.3.21", features = ["thread-pool"], optional = true } +parking_lot = { version = "0.12.0", optional = true } +secp256k1 = { version = "0.21.2", features = ["recovery", "global-context"], optional = true } +tracing = { version = "0.1.29", default-features = false } tracing-core = { version = "0.1.17", default-features = false} [features] @@ -44,6 +45,7 @@ std = [ "sp-trie", "sp-state-machine", "libsecp256k1", + "secp256k1", "sp-runtime-interface/std", "sp-externalities", "sp-wasm-interface/std", @@ -66,3 +68,22 @@ with-tracing = [ disable_panic_handler = [] disable_oom = [] disable_allocator = [] + +# This feature flag controls the runtime's behavior when encountering +# a panic or when it runs out of memory, improving the diagnostics. +# +# When enabled the runtime will marshal the relevant error message +# to the host through the `PanicHandler::abort_on_panic` runtime interface. +# This gives the caller direct programmatic access to the error message. +# +# When disabled the error message will only be printed out in the +# logs, with the caller receving a generic "wasm `unreachable` instruction executed" +# error message. +# +# This has no effect if both `disable_panic_handler` and `disable_oom` +# are enabled. +# +# WARNING: Enabling this feature flag requires the `PanicHandler::abort_on_panic` +# host function to be supported by the host. Do *not* enable it for your +# runtime without first upgrading your host client! +improved_panic_error_reporting = [] diff --git a/primitives/io/src/batch_verifier.rs b/primitives/io/src/batch_verifier.rs index b6da1d85907b..501d986758bf 100644 --- a/primitives/io/src/batch_verifier.rs +++ b/primitives/io/src/batch_verifier.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -74,6 +74,7 @@ impl BatchVerifier { self.scheduler.spawn( name, + None, async move { if !f() { invalid_clone.store(true, AtomicOrdering::Relaxed); @@ -177,7 +178,8 @@ impl BatchVerifier { if pending.len() > 0 { let (sender, receiver) = std::sync::mpsc::channel(); self.scheduler.spawn( - "substrate_batch_verify_join", + "substrate-batch-verify-join", + None, async move { futures::future::join_all(pending).await; sender.send(()).expect( diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 919b0ceaeac7..be28c28dd5b2 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -52,11 +52,13 @@ use sp_core::{ HttpError, HttpRequestId, HttpRequestStatus, OpaqueNetworkState, StorageKind, Timestamp, IpfsRequest, IpfsRequestId, IpfsRequestStatus, }, - sr25519, LogLevel, LogLevelFilter, OpaquePeerId, H256, + sr25519, + storage::StateVersion, + LogLevel, LogLevelFilter, OpaquePeerId, H256, }; #[cfg(feature = "std")] -use sp_trie::{trie_types::Layout, TrieConfiguration}; +use sp_trie::{LayoutV0, LayoutV1, TrieConfiguration}; use sp_runtime_interface::{ pass_by::{PassBy, PassByCodec}, @@ -65,6 +67,12 @@ use sp_runtime_interface::{ use codec::{Decode, Encode}; +#[cfg(feature = "std")] +use secp256k1::{ + ecdsa::{RecoverableSignature, RecoveryId}, + Message, SECP256K1, +}; + #[cfg(feature = "std")] use sp_externalities::{Externalities, ExternalitiesExt}; @@ -89,12 +97,12 @@ pub enum EcdsaVerifyError { } /// The outcome of calling `storage_kill`. Returned value is the number of storage items -/// removed from the trie from making the `storage_kill` call. +/// removed from the backend from making the `storage_kill` call. #[derive(PassByCodec, Encode, Decode)] pub enum KillStorageResult { - /// No key remains in the child trie. + /// All key to remove were removed, return number of key removed from backend. AllRemoved(u32), - /// At least one key still resides in the child trie due to the supplied limit. + /// Not all key to remove were removed, return number of key removed from backend. SomeRemaining(u32), } @@ -151,9 +159,7 @@ pub trait Storage { /// The limit can be used to partially delete a prefix storage in case it is too large /// to delete in one go (block). /// - /// It returns a boolean false iff some keys are remaining in - /// the prefix after the functions returns. Also returns a `u32` with - /// the number of keys removed from the process. + /// Returns [`KillStorageResult`] to inform about the result. /// /// # Note /// @@ -164,8 +170,10 @@ pub trait Storage { /// /// Calling this function multiple times per block for the same `prefix` does /// not make much sense because it is not cumulative when called inside the same block. - /// Use this function to distribute the deletion of a single child trie across multiple - /// blocks. + /// The deletion would always start from `prefix` resulting in the same keys being deleted + /// every time this function is called with the exact same arguments per block. This happens + /// because the keys in the overlay are not taken into account when deleting keys in the + /// backend. #[version(2)] fn clear_prefix(&mut self, prefix: &[u8], limit: Option) -> KillStorageResult { let (all_removed, num_removed) = Externalities::clear_prefix(*self, prefix, limit); @@ -193,19 +201,22 @@ pub trait Storage { /// /// Returns a `Vec` that holds the SCALE encoded hash. fn root(&mut self) -> Vec { - self.storage_root() + self.storage_root(StateVersion::V0) } - /// "Commit" all existing operations and get the resulting storage change root. - /// `parent_hash` is a SCALE encoded hash. + /// "Commit" all existing operations and compute the resulting storage root. /// /// The hashing algorithm is defined by the `Block`. /// - /// Returns `Some(Vec)` which holds the SCALE encoded hash or `None` when - /// changes trie is disabled. - fn changes_root(&mut self, parent_hash: &[u8]) -> Option> { - self.storage_changes_root(parent_hash) - .expect("Invalid `parent_hash` given to `changes_root`.") + /// Returns a `Vec` that holds the SCALE encoded hash. + #[version(2)] + fn root(&mut self, version: StateVersion) -> Vec { + self.storage_root(version) + } + + /// Always returns `None`. This function exists for compatibility reasons. + fn changes_root(&mut self, _parent_hash: &[u8]) -> Option> { + None } /// Get the next key in storage after the given one in lexicographic order. @@ -381,7 +392,19 @@ pub trait DefaultChildStorage { /// Returns a `Vec` that holds the SCALE encoded hash. fn root(&mut self, storage_key: &[u8]) -> Vec { let child_info = ChildInfo::new_default(storage_key); - self.child_storage_root(&child_info) + self.child_storage_root(&child_info, StateVersion::V0) + } + + /// Default child root calculation. + /// + /// "Commit" all existing operations and compute the resulting child storage root. + /// The hashing algorithm is defined by the `Block`. + /// + /// Returns a `Vec` that holds the SCALE encoded hash. + #[version(2)] + fn root(&mut self, storage_key: &[u8], version: StateVersion) -> Vec { + let child_info = ChildInfo::new_default(storage_key); + self.child_storage_root(&child_info, version) } /// Child storage key iteration. @@ -398,27 +421,63 @@ pub trait DefaultChildStorage { pub trait Trie { /// A trie root formed from the iterated items. fn blake2_256_root(input: Vec<(Vec, Vec)>) -> H256 { - Layout::::trie_root(input) + LayoutV0::::trie_root(input) + } + + /// A trie root formed from the iterated items. + #[version(2)] + fn blake2_256_root(input: Vec<(Vec, Vec)>, version: StateVersion) -> H256 { + match version { + StateVersion::V0 => LayoutV0::::trie_root(input), + StateVersion::V1 => LayoutV1::::trie_root(input), + } } /// A trie root formed from the enumerated items. fn blake2_256_ordered_root(input: Vec>) -> H256 { - Layout::::ordered_trie_root(input) + LayoutV0::::ordered_trie_root(input) + } + + /// A trie root formed from the enumerated items. + #[version(2)] + fn blake2_256_ordered_root(input: Vec>, version: StateVersion) -> H256 { + match version { + StateVersion::V0 => LayoutV0::::ordered_trie_root(input), + StateVersion::V1 => LayoutV1::::ordered_trie_root(input), + } } /// A trie root formed from the iterated items. fn keccak_256_root(input: Vec<(Vec, Vec)>) -> H256 { - Layout::::trie_root(input) + LayoutV0::::trie_root(input) + } + + /// A trie root formed from the iterated items. + #[version(2)] + fn keccak_256_root(input: Vec<(Vec, Vec)>, version: StateVersion) -> H256 { + match version { + StateVersion::V0 => LayoutV0::::trie_root(input), + StateVersion::V1 => LayoutV1::::trie_root(input), + } } /// A trie root formed from the enumerated items. fn keccak_256_ordered_root(input: Vec>) -> H256 { - Layout::::ordered_trie_root(input) + LayoutV0::::ordered_trie_root(input) + } + + /// A trie root formed from the enumerated items. + #[version(2)] + fn keccak_256_ordered_root(input: Vec>, version: StateVersion) -> H256 { + match version { + StateVersion::V0 => LayoutV0::::ordered_trie_root(input), + StateVersion::V1 => LayoutV1::::ordered_trie_root(input), + } } /// Verify trie proof fn blake2_256_verify_proof(root: H256, proof: &[Vec], key: &[u8], value: &[u8]) -> bool { - sp_trie::verify_trie_proof::, _, _, _>( + sp_trie::verify_trie_proof::, _, _, _>( &root, proof, &[(key, Some(value))], @@ -426,15 +485,69 @@ pub trait Trie { .is_ok() } + /// Verify trie proof + #[version(2)] + fn blake2_256_verify_proof( + root: H256, + proof: &[Vec], + key: &[u8], + value: &[u8], + version: StateVersion, + ) -> bool { + match version { + StateVersion::V0 => sp_trie::verify_trie_proof::< + LayoutV0, + _, + _, + _, + >(&root, proof, &[(key, Some(value))]) + .is_ok(), + StateVersion::V1 => sp_trie::verify_trie_proof::< + LayoutV1, + _, + _, + _, + >(&root, proof, &[(key, Some(value))]) + .is_ok(), + } + } + /// Verify trie proof fn keccak_256_verify_proof(root: H256, proof: &[Vec], key: &[u8], value: &[u8]) -> bool { - sp_trie::verify_trie_proof::, _, _, _>( + sp_trie::verify_trie_proof::, _, _, _>( &root, proof, &[(key, Some(value))], ) .is_ok() } + + /// Verify trie proof + #[version(2)] + fn keccak_256_verify_proof( + root: H256, + proof: &[Vec], + key: &[u8], + value: &[u8], + version: StateVersion, + ) -> bool { + match version { + StateVersion::V0 => sp_trie::verify_trie_proof::< + LayoutV0, + _, + _, + _, + >(&root, proof, &[(key, Some(value))]) + .is_ok(), + StateVersion::V1 => sp_trie::verify_trie_proof::< + LayoutV1, + _, + _, + _, + >(&root, proof, &[(key, Some(value))]) + .is_ok(), + } + } } /// Interface that provides miscellaneous functions for communicating between the runtime and the @@ -541,7 +654,7 @@ pub trait Crypto { SyncCryptoStore::sign_with(keystore, id, &pub_key.into(), msg) .ok() .flatten() - .map(|sig| ed25519::Signature::from_slice(sig.as_slice())) + .and_then(|sig| ed25519::Signature::from_slice(&sig)) } /// Verify `ed25519` signature. @@ -665,7 +778,7 @@ pub trait Crypto { SyncCryptoStore::sign_with(keystore, id, &pub_key.into(), msg) .ok() .flatten() - .map(|sig| sr25519::Signature::from_slice(sig.as_slice())) + .and_then(|sig| sr25519::Signature::from_slice(&sig)) } /// Verify an `sr25519` signature. @@ -714,13 +827,31 @@ pub trait Crypto { SyncCryptoStore::sign_with(keystore, id, &pub_key.into(), msg) .ok() .flatten() - .map(|sig| ecdsa::Signature::from_slice(sig.as_slice())) + .and_then(|sig| ecdsa::Signature::from_slice(&sig)) + } + + /// Sign the given a pre-hashed `msg` with the `ecdsa` key that corresponds to the given public + /// key and key type in the keystore. + /// + /// Returns the signature. + fn ecdsa_sign_prehashed( + &mut self, + id: KeyTypeId, + pub_key: &ecdsa::Public, + msg: &[u8; 32], + ) -> Option { + let keystore = &***self + .extension::() + .expect("No `keystore` associated for the current context!"); + SyncCryptoStore::ecdsa_sign_prehashed(keystore, id, pub_key, msg).ok().flatten() } /// Verify `ecdsa` signature. /// /// Returns `true` when the verification was successful. + /// This version is able to handle, non-standard, overflowing signatures. fn ecdsa_verify(sig: &ecdsa::Signature, msg: &[u8], pub_key: &ecdsa::Public) -> bool { + #[allow(deprecated)] ecdsa::Pair::verify_deprecated(sig, msg, pub_key) } @@ -732,6 +863,17 @@ pub trait Crypto { ecdsa::Pair::verify(sig, msg, pub_key) } + /// Verify `ecdsa` signature with pre-hashed `msg`. + /// + /// Returns `true` when the verification was successful. + fn ecdsa_verify_prehashed( + sig: &ecdsa::Signature, + msg: &[u8; 32], + pub_key: &ecdsa::Public, + ) -> bool { + ecdsa::Pair::verify_prehashed(sig, msg, pub_key) + } + /// Register a `ecdsa` signature for batch verification. /// /// Batch verification must be enabled by calling [`start_batch_verify`]. @@ -758,18 +900,20 @@ pub trait Crypto { /// /// Returns `Err` if the signature is bad, otherwise the 64-byte pubkey /// (doesn't include the 0x04 prefix). + /// This version is able to handle, non-standard, overflowing signatures. fn secp256k1_ecdsa_recover( sig: &[u8; 65], msg: &[u8; 32], ) -> Result<[u8; 64], EcdsaVerifyError> { - let rs = libsecp256k1::Signature::parse_overflowing_slice(&sig[0..64]) - .map_err(|_| EcdsaVerifyError::BadRS)?; - let v = libsecp256k1::RecoveryId::parse( - if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as u8 + let rid = libsecp256k1::RecoveryId::parse( + if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as u8, ) .map_err(|_| EcdsaVerifyError::BadV)?; - let pubkey = libsecp256k1::recover(&libsecp256k1::Message::parse(msg), &rs, &v) - .map_err(|_| EcdsaVerifyError::BadSignature)?; + let sig = libsecp256k1::Signature::parse_overflowing_slice(&sig[..64]) + .map_err(|_| EcdsaVerifyError::BadRS)?; + let msg = libsecp256k1::Message::parse(msg); + let pubkey = + libsecp256k1::recover(&msg, &sig, &rid).map_err(|_| EcdsaVerifyError::BadSignature)?; let mut res = [0u8; 64]; res.copy_from_slice(&pubkey.serialize()[1..65]); Ok(res) @@ -787,16 +931,16 @@ pub trait Crypto { sig: &[u8; 65], msg: &[u8; 32], ) -> Result<[u8; 64], EcdsaVerifyError> { - let rs = libsecp256k1::Signature::parse_standard_slice(&sig[0..64]) + let rid = RecoveryId::from_i32(if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as i32) + .map_err(|_| EcdsaVerifyError::BadV)?; + let sig = RecoverableSignature::from_compact(&sig[..64], rid) .map_err(|_| EcdsaVerifyError::BadRS)?; - let v = libsecp256k1::RecoveryId::parse( - if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as u8 - ) - .map_err(|_| EcdsaVerifyError::BadV)?; - let pubkey = libsecp256k1::recover(&libsecp256k1::Message::parse(msg), &rs, &v) + let msg = Message::from_slice(msg).expect("Message is 32 bytes; qed"); + let pubkey = SECP256K1 + .recover_ecdsa(&msg, &sig) .map_err(|_| EcdsaVerifyError::BadSignature)?; let mut res = [0u8; 64]; - res.copy_from_slice(&pubkey.serialize()[1..65]); + res.copy_from_slice(&pubkey.serialize_uncompressed()[1..]); Ok(res) } @@ -810,14 +954,15 @@ pub trait Crypto { sig: &[u8; 65], msg: &[u8; 32], ) -> Result<[u8; 33], EcdsaVerifyError> { - let rs = libsecp256k1::Signature::parse_overflowing_slice(&sig[0..64]) - .map_err(|_| EcdsaVerifyError::BadRS)?; - let v = libsecp256k1::RecoveryId::parse( - if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as u8 + let rid = libsecp256k1::RecoveryId::parse( + if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as u8, ) .map_err(|_| EcdsaVerifyError::BadV)?; - let pubkey = libsecp256k1::recover(&libsecp256k1::Message::parse(msg), &rs, &v) - .map_err(|_| EcdsaVerifyError::BadSignature)?; + let sig = libsecp256k1::Signature::parse_overflowing_slice(&sig[0..64]) + .map_err(|_| EcdsaVerifyError::BadRS)?; + let msg = libsecp256k1::Message::parse(msg); + let pubkey = + libsecp256k1::recover(&msg, &sig, &rid).map_err(|_| EcdsaVerifyError::BadSignature)?; Ok(pubkey.serialize_compressed()) } @@ -832,15 +977,15 @@ pub trait Crypto { sig: &[u8; 65], msg: &[u8; 32], ) -> Result<[u8; 33], EcdsaVerifyError> { - let rs = libsecp256k1::Signature::parse_standard_slice(&sig[0..64]) + let rid = RecoveryId::from_i32(if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as i32) + .map_err(|_| EcdsaVerifyError::BadV)?; + let sig = RecoverableSignature::from_compact(&sig[..64], rid) .map_err(|_| EcdsaVerifyError::BadRS)?; - let v = libsecp256k1::RecoveryId::parse( - if sig[64] > 26 { sig[64] - 27 } else { sig[64] } as u8 - ) - .map_err(|_| EcdsaVerifyError::BadV)?; - let pubkey = libsecp256k1::recover(&libsecp256k1::Message::parse(msg), &rs, &v) + let msg = Message::from_slice(msg).expect("Message is 32 bytes; qed"); + let pubkey = SECP256K1 + .recover_ecdsa(&msg, &sig) .map_err(|_| EcdsaVerifyError::BadSignature)?; - Ok(pubkey.serialize_compressed()) + Ok(pubkey.serialize()) } } @@ -1175,6 +1320,17 @@ pub trait Allocator { } } +/// WASM-only interface which allows for aborting the execution in case +/// of an unrecoverable error. +#[runtime_interface(wasm_only)] +pub trait PanicHandler { + /// Aborts the current execution with the given error message. + #[trap_on_return] + fn abort_on_panic(&mut self, message: &str) { + self.register_panic_error_message(message); + } +} + /// Interface that provides functions for logging from within the runtime. #[runtime_interface] pub trait Logging { @@ -1473,14 +1629,14 @@ pub trait RuntimeTasks { } /// Allocator used by Substrate when executing the Wasm runtime. -#[cfg(not(feature = "std"))] +#[cfg(all(target_arch = "wasm32", not(feature = "std")))] struct WasmAllocator; -#[cfg(all(not(feature = "disable_allocator"), not(feature = "std")))] +#[cfg(all(target_arch = "wasm32", not(feature = "disable_allocator"), not(feature = "std")))] #[global_allocator] static ALLOCATOR: WasmAllocator = WasmAllocator; -#[cfg(not(feature = "std"))] +#[cfg(all(target_arch = "wasm32", not(feature = "std")))] mod allocator_impl { use super::*; use core::alloc::{GlobalAlloc, Layout}; @@ -1502,21 +1658,35 @@ mod allocator_impl { #[no_mangle] pub fn panic(info: &core::panic::PanicInfo) -> ! { let message = sp_std::alloc::format!("{}", info); - logging::log(LogLevel::Error, "runtime", message.as_bytes()); - core::arch::wasm32::unreachable(); + #[cfg(feature = "improved_panic_error_reporting")] + { + panic_handler::abort_on_panic(&message); + } + #[cfg(not(feature = "improved_panic_error_reporting"))] + { + logging::log(LogLevel::Error, "runtime", message.as_bytes()); + core::arch::wasm32::unreachable(); + } } /// A default OOM handler for WASM environment. #[cfg(all(not(feature = "disable_oom"), not(feature = "std")))] #[alloc_error_handler] pub fn oom(_: core::alloc::Layout) -> ! { - logging::log(LogLevel::Error, "runtime", b"Runtime memory exhausted. Aborting"); - core::arch::wasm32::unreachable(); + #[cfg(feature = "improved_panic_error_reporting")] + { + panic_handler::abort_on_panic("Runtime memory exhausted."); + } + #[cfg(not(feature = "improved_panic_error_reporting"))] + { + logging::log(LogLevel::Error, "runtime", b"Runtime memory exhausted. Aborting"); + core::arch::wasm32::unreachable(); + } } /// Type alias for Externalities implementation used in tests. #[cfg(feature = "std")] -pub type TestExternalities = sp_state_machine::TestExternalities; +pub type TestExternalities = sp_state_machine::TestExternalities; /// The host functions Substrate provides for the Wasm runtime environment. /// @@ -1531,6 +1701,7 @@ pub type SubstrateHostFunctions = ( crypto::HostFunctions, hashing::HostFunctions, allocator::HostFunctions, + panic_handler::HostFunctions, logging::HostFunctions, sandbox::HostFunctions, crate::trie::HostFunctions, @@ -1542,7 +1713,10 @@ pub type SubstrateHostFunctions = ( #[cfg(test)] mod tests { use super::*; - use sp_core::{map, storage::Storage, testing::TaskExecutor, traits::TaskExecutorExt}; + use sp_core::{ + crypto::UncheckedInto, map, storage::Storage, testing::TaskExecutor, + traits::TaskExecutorExt, + }; use sp_state_machine::BasicExternalities; use std::any::TypeId; @@ -1566,6 +1740,16 @@ mod tests { assert_eq!(storage::get(b"hello"), None); assert_eq!(storage::get(b"foo"), Some(b"bar".to_vec())); }); + + let value = vec![7u8; 35]; + let storage = + Storage { top: map![b"foo00".to_vec() => value.clone()], children_default: map![] }; + t = BasicExternalities::new(storage); + + t.execute_with(|| { + assert_eq!(storage::get(b"hello"), None); + assert_eq!(storage::get(b"foo00"), Some(value.clone())); + }); } #[test] @@ -1643,7 +1827,7 @@ mod tests { } // push invlaid - crypto::sr25519_batch_verify(&Default::default(), &Vec::new(), &Default::default()); + crypto::sr25519_batch_verify(&zero_sr_sig(), &Vec::new(), &zero_sr_pub()); assert!(!crypto::finish_batch_verify()); crypto::start_batch_verify(); @@ -1656,14 +1840,31 @@ mod tests { }); } + fn zero_ed_pub() -> ed25519::Public { + [0u8; 32].unchecked_into() + } + + fn zero_ed_sig() -> ed25519::Signature { + ed25519::Signature::from_raw([0u8; 64]) + } + + fn zero_sr_pub() -> sr25519::Public { + [0u8; 32].unchecked_into() + } + + fn zero_sr_sig() -> sr25519::Signature { + sr25519::Signature::from_raw([0u8; 64]) + } + #[test] fn batching_works() { let mut ext = BasicExternalities::default(); ext.register_extension(TaskExecutorExt::new(TaskExecutor::new())); + ext.execute_with(|| { // invalid ed25519 signature crypto::start_batch_verify(); - crypto::ed25519_batch_verify(&Default::default(), &Vec::new(), &Default::default()); + crypto::ed25519_batch_verify(&zero_ed_sig(), &Vec::new(), &zero_ed_pub()); assert!(!crypto::finish_batch_verify()); // 2 valid ed25519 signatures @@ -1689,7 +1890,7 @@ mod tests { let signature = pair.sign(msg); crypto::ed25519_batch_verify(&signature, msg, &pair.public()); - crypto::ed25519_batch_verify(&Default::default(), &Vec::new(), &Default::default()); + crypto::ed25519_batch_verify(&zero_ed_sig(), &Vec::new(), &zero_ed_pub()); assert!(!crypto::finish_batch_verify()); @@ -1721,7 +1922,7 @@ mod tests { let signature = pair.sign(msg); crypto::sr25519_batch_verify(&signature, msg, &pair.public()); - crypto::sr25519_batch_verify(&Default::default(), &Vec::new(), &Default::default()); + crypto::sr25519_batch_verify(&zero_sr_sig(), &Vec::new(), &zero_sr_pub()); assert!(!crypto::finish_batch_verify()); }); diff --git a/primitives/keyring/Cargo.toml b/primitives/keyring/Cargo.toml index a14e98d3d805..6186a51fef85 100644 --- a/primitives/keyring/Cargo.toml +++ b/primitives/keyring/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-keyring" -version = "4.0.0-dev" +version = "6.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Keyring support code for the runtime. A set of test accounts." documentation = "https://docs.rs/sp-keyring" @@ -13,9 +13,8 @@ readme = "README.md" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] - [dependencies] -sp-core = { version = "4.0.0-dev", path = "../core" } -sp-runtime = { version = "4.0.0-dev", path = "../runtime" } +sp-core = { version = "6.0.0", path = "../core" } +sp-runtime = { version = "6.0.0", path = "../runtime" } lazy_static = "1.4.0" -strum = { version = "0.20.0", features = ["derive"] } +strum = { version = "0.23.0", features = ["derive"] } diff --git a/primitives/keyring/src/ed25519.rs b/primitives/keyring/src/ed25519.rs index 65341a360579..6d56062d3473 100644 --- a/primitives/keyring/src/ed25519.rs +++ b/primitives/keyring/src/ed25519.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use lazy_static::lazy_static; pub use sp_core::ed25519; use sp_core::{ ed25519::{Pair, Public, Signature}, - Pair as PairT, Public as PublicT, H256, + ByteArray, Pair as PairT, H256, }; use sp_runtime::AccountId32; use std::{collections::HashMap, ops::Deref}; diff --git a/primitives/keyring/src/lib.rs b/primitives/keyring/src/lib.rs index d7fb7c4fd2f2..170a4c3d01b2 100644 --- a/primitives/keyring/src/lib.rs +++ b/primitives/keyring/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/keyring/src/sr25519.rs b/primitives/keyring/src/sr25519.rs index 6a7aa3635a43..86bcac9d58cb 100644 --- a/primitives/keyring/src/sr25519.rs +++ b/primitives/keyring/src/sr25519.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use lazy_static::lazy_static; pub use sp_core::sr25519; use sp_core::{ sr25519::{Pair, Public, Signature}, - Pair as PairT, Public as PublicT, H256, + ByteArray, Pair as PairT, H256, }; use sp_runtime::AccountId32; use std::{collections::HashMap, ops::Deref}; @@ -89,9 +89,20 @@ impl Keyring { pub fn public(self) -> Public { self.pair().public() } + pub fn to_seed(self) -> String { format!("//{}", self) } + + /// Create a crypto `Pair` from a numeric value. + pub fn numeric(idx: usize) -> Pair { + Pair::from_string(&format!("//{}", idx), None).expect("numeric values are known good; qed") + } + + /// Get account id of a `numeric` account. + pub fn numeric_id(idx: usize) -> AccountId32 { + (*Self::numeric(idx).public().as_array_ref()).into() + } } impl From for &'static str { diff --git a/primitives/keystore/Cargo.toml b/primitives/keystore/Cargo.toml index 35c66ef93f7a..f201cb8518bc 100644 --- a/primitives/keystore/Cargo.toml +++ b/primitives/keystore/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-keystore" -version = "0.10.0-dev" +version = "0.12.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Keystore primitives." documentation = "https://docs.rs/sp-core" @@ -14,21 +14,21 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] async-trait = "0.1.50" -derive_more = "0.99.2" -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -futures = { version = "0.3.1" } +futures = "0.3.21" +parking_lot = { version = "0.12.0", default-features = false } +serde = { version = "1.0", optional = true } +thiserror = "1.0" + +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } schnorrkel = { version = "0.9.1", features = ["preaudit_deprecated", "u64_backend"], default-features = false } merlin = { version = "2.0", default-features = false } -parking_lot = { version = "0.11.1", default-features = false } -serde = { version = "1.0", optional = true} -sp-core = { version = "4.0.0-dev", path = "../core" } -sp-externalities = { version = "0.10.0-dev", path = "../externalities", default-features = false } +sp-core = { version = "6.0.0", path = "../core" } +sp-externalities = { version = "0.12.0", path = "../externalities", default-features = false } [dev-dependencies] rand = "0.7.2" rand_chacha = "0.2.2" - [features] default = ["std"] std = [ diff --git a/primitives/keystore/src/lib.rs b/primitives/keystore/src/lib.rs index c45e8a6f5d2b..6540e71bc3fe 100644 --- a/primitives/keystore/src/lib.rs +++ b/primitives/keystore/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,19 +29,19 @@ use sp_core::{ use std::sync::Arc; /// CryptoStore error -#[derive(Debug, derive_more::Display)] +#[derive(Debug, thiserror::Error)] pub enum Error { /// Public key type is not supported - #[display(fmt = "Key not supported: {:?}", _0)] + #[error("Key not supported: {0:?}")] KeyNotSupported(KeyTypeId), /// Validation error - #[display(fmt = "Validation error: {}", _0)] + #[error("Validation error: {0}")] ValidationError(String), /// Keystore unavailable - #[display(fmt = "Keystore unavailable")] + #[error("Keystore unavailable")] Unavailable, /// Programming errors - #[display(fmt = "An unknown keystore error occurred: {}", _0)] + #[error("An unknown keystore error occurred: {0}")] Other(String), } diff --git a/primitives/keystore/src/testing.rs b/primitives/keystore/src/testing.rs index 718ba798dc0f..2723b743c10d 100644 --- a/primitives/keystore/src/testing.rs +++ b/primitives/keystore/src/testing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,7 @@ //! Types that should only be used for testing! use sp_core::{ - crypto::{CryptoTypePublicPair, KeyTypeId, Pair, Public}, + crypto::{ByteArray, CryptoTypePublicPair, KeyTypeId, Pair}, ecdsa, ed25519, sr25519, }; @@ -340,20 +340,20 @@ impl SyncCryptoStore for KeyStore { match key.0 { ed25519::CRYPTO_ID => { - let key_pair = - self.ed25519_key_pair(id, &ed25519::Public::from_slice(key.1.as_slice())); + let key_pair = self + .ed25519_key_pair(id, &ed25519::Public::from_slice(key.1.as_slice()).unwrap()); key_pair.map(|k| k.sign(msg).encode()).map(Ok).transpose() }, sr25519::CRYPTO_ID => { - let key_pair = - self.sr25519_key_pair(id, &sr25519::Public::from_slice(key.1.as_slice())); + let key_pair = self + .sr25519_key_pair(id, &sr25519::Public::from_slice(key.1.as_slice()).unwrap()); key_pair.map(|k| k.sign(msg).encode()).map(Ok).transpose() }, ecdsa::CRYPTO_ID => { let key_pair = - self.ecdsa_key_pair(id, &ecdsa::Public::from_slice(key.1.as_slice())); + self.ecdsa_key_pair(id, &ecdsa::Public::from_slice(key.1.as_slice()).unwrap()); key_pair.map(|k| k.sign(msg).encode()).map(Ok).transpose() }, @@ -482,9 +482,7 @@ mod tests { assert!(res.is_none()); // insert key, sign again - let res = - SyncCryptoStore::insert_unknown(&store, ECDSA, suri, pair.public().as_ref()).unwrap(); - assert_eq!((), res); + SyncCryptoStore::insert_unknown(&store, ECDSA, suri, pair.public().as_ref()).unwrap(); let res = SyncCryptoStore::ecdsa_sign_prehashed(&store, ECDSA, &pair.public(), &msg).unwrap(); diff --git a/primitives/keystore/src/vrf.rs b/primitives/keystore/src/vrf.rs index 383abb77e17c..7409353afe9f 100644 --- a/primitives/keystore/src/vrf.rs +++ b/primitives/keystore/src/vrf.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/maybe-compressed-blob/Cargo.toml b/primitives/maybe-compressed-blob/Cargo.toml index 6994ccf5486b..d8814356df4b 100644 --- a/primitives/maybe-compressed-blob/Cargo.toml +++ b/primitives/maybe-compressed-blob/Cargo.toml @@ -1,14 +1,15 @@ [package] name = "sp-maybe-compressed-blob" -version = "4.0.0-dev" +version = "4.1.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Handling of blobs, usually Wasm code, which may be compresed" documentation = "https://docs.rs/sp-maybe-compressed-blob" readme = "README.md" [dependencies] +thiserror = "1.0" zstd = { version = "0.9.0", default-features = false } diff --git a/primitives/maybe-compressed-blob/src/lib.rs b/primitives/maybe-compressed-blob/src/lib.rs index e8a7e42b4eac..99c12ed39bc0 100644 --- a/primitives/maybe-compressed-blob/src/lib.rs +++ b/primitives/maybe-compressed-blob/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,10 @@ //! Handling of blobs that may be compressed, based on an 8-byte magic identifier //! at the head. -use std::{borrow::Cow, io::Read}; +use std::{ + borrow::Cow, + io::{Read, Write}, +}; // An arbitrary prefix, that indicates a blob beginning with should be decompressed with // Zstd compression. @@ -34,25 +37,16 @@ const ZSTD_PREFIX: [u8; 8] = [82, 188, 83, 118, 70, 219, 142, 5]; pub const CODE_BLOB_BOMB_LIMIT: usize = 50 * 1024 * 1024; /// A possible bomb was encountered. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, thiserror::Error)] pub enum Error { /// Decoded size was too large, and the code payload may be a bomb. + #[error("Possible compression bomb encountered")] PossibleBomb, /// The compressed value had an invalid format. + #[error("Blob had invalid format")] Invalid, } -impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match *self { - Error::PossibleBomb => write!(f, "Possible compression bomb encountered"), - Error::Invalid => write!(f, "Blob had invalid format"), - } - } -} - -impl std::error::Error for Error {} - fn read_from_decoder( decoder: impl Read, blob_len: usize, @@ -90,8 +84,6 @@ pub fn decompress(blob: &[u8], bomb_limit: usize) -> Result, Error> { /// this will not compress the blob, as the decoder will not be able to be /// able to differentiate it from a compression bomb. pub fn compress(blob: &[u8], bomb_limit: usize) -> Option> { - use std::io::Write; - if blob.len() > bomb_limit { return None } @@ -109,7 +101,6 @@ pub fn compress(blob: &[u8], bomb_limit: usize) -> Option> { #[cfg(test)] mod tests { use super::*; - use std::io::Write; const BOMB_LIMIT: usize = 10; diff --git a/primitives/npos-elections/Cargo.toml b/primitives/npos-elections/Cargo.toml index b277df8f58f1..13edbfe90008 100644 --- a/primitives/npos-elections/Cargo.toml +++ b/primitives/npos-elections/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-npos-elections" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "NPoS election algorithm primitives" readme = "README.md" @@ -13,14 +13,13 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.126", optional = true, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -sp-npos-elections-solution-type = { version = "4.0.0-dev", path = "./solution-type" } -sp-arithmetic = { version = "4.0.0-dev", default-features = false, path = "../arithmetic" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } -sp-runtime = { version = "4.0.0-dev", path = "../runtime", default-features = false } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +serde = { version = "1.0.136", optional = true, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +sp-arithmetic = { version = "5.0.0", default-features = false, path = "../arithmetic" } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +sp-runtime = { version = "6.0.0", path = "../runtime", default-features = false } [dev-dependencies] substrate-test-utils = { version = "4.0.0-dev", path = "../../test-utils" } diff --git a/primitives/npos-elections/fuzzer/Cargo.lock b/primitives/npos-elections/fuzzer/Cargo.lock deleted file mode 100644 index cd172421aeb2..000000000000 --- a/primitives/npos-elections/fuzzer/Cargo.lock +++ /dev/null @@ -1,1602 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -[[package]] -name = "Inflector" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" -dependencies = [ - "lazy_static", - "regex", -] - -[[package]] -name = "ahash" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f33b5018f120946c1dcf279194f238a9f146725593ead1c08fa47ff22b0b5d3" -dependencies = [ - "const-random", -] - -[[package]] -name = "aho-corasick" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" -dependencies = [ - "memchr", -] - -[[package]] -name = "arbitrary" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cf76cb6e2222ed0ea86b2b0ee2f71c96ec6edd5af42e84d59160e91b836ec4" - -[[package]] -name = "arrayref" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" - -[[package]] -name = "arrayvec" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" -dependencies = [ - "nodrop", -] - -[[package]] -name = "arrayvec" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" - -[[package]] -name = "autocfg" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" - -[[package]] -name = "autocfg" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" - -[[package]] -name = "backtrace" -version = "0.3.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad235dabf00f36301792cfe82499880ba54c6486be094d1047b02bacb67c14e8" -dependencies = [ - "backtrace-sys", - "cfg-if", - "libc", - "rustc-demangle", -] - -[[package]] -name = "backtrace-sys" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca797db0057bae1a7aa2eef3283a874695455cecf08a43bfb8507ee0ebc1ed69" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "base58" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" - -[[package]] -name = "bitflags" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" - -[[package]] -name = "bitvec" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a993f74b4c99c1908d156b8d2e0fb6277736b0ecbd833982fd1241d39b2766a6" - -[[package]] -name = "blake2-rfc" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" -dependencies = [ - "arrayvec 0.4.12", - "constant_time_eq", -] - -[[package]] -name = "block-buffer" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" -dependencies = [ - "block-padding", - "byte-tools", - "byteorder", - "generic-array", -] - -[[package]] -name = "block-padding" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -dependencies = [ - "byte-tools", -] - -[[package]] -name = "byte-slice-cast" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0a5e3906bcbf133e33c1d4d95afc664ad37fbdb9f6568d8043e7ea8c27d93d3" - -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" - -[[package]] -name = "byteorder" -version = "1.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" - -[[package]] -name = "cc" -version = "1.0.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" - -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - -[[package]] -name = "clear_on_drop" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97276801e127ffb46b66ce23f35cc96bd454fa311294bced4bbace7baa8b1d17" -dependencies = [ - "cc", -] - -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags", -] - -[[package]] -name = "const-random" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f1af9ac737b2dd2d577701e59fd09ba34822f6f2ebdb30a7647405d9e55e16a" -dependencies = [ - "const-random-macro", - "proc-macro-hack", -] - -[[package]] -name = "const-random-macro" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e4c606eb459dd29f7c57b2e0879f2b6f14ee130918c2b78ccb58a9624e6c7a" -dependencies = [ - "getrandom", - "proc-macro-hack", -] - -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - -[[package]] -name = "crypto-mac" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" -dependencies = [ - "generic-array", - "subtle 1.0.0", -] - -[[package]] -name = "curve25519-dalek" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b7dcd30ba50cdf88b55b033456138b7c0ac4afdc436d82e1b79f370f24cc66d" -dependencies = [ - "byteorder", - "clear_on_drop", - "digest", - "rand_core 0.3.1", - "subtle 2.2.2", -] - -[[package]] -name = "curve25519-dalek" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26778518a7f6cffa1d25a44b602b62b979bd88adb9e99ffec546998cf3404839" -dependencies = [ - "byteorder", - "digest", - "rand_core 0.5.1", - "subtle 2.2.2", - "zeroize 1.1.0", -] - -[[package]] -name = "derive_more" -version = "0.99.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a806e96c59a76a5ba6e18735b6cf833344671e61e7863f2edb5c518ea2cac95c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "digest" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" -dependencies = [ - "generic-array", -] - -[[package]] -name = "ed25519-dalek" -version = "1.0.0-pre.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978710b352437433c97b2bff193f2fb1dfd58a093f863dd95e225a19baa599a2" -dependencies = [ - "clear_on_drop", - "curve25519-dalek 2.0.0", - "rand 0.7.3", - "sha2", -] - -[[package]] -name = "environmental" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516aa8d7a71cb00a1c4146f0798549b93d083d4f189b3ced8f3de6b8f11ee6c4" - -[[package]] -name = "failure" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8529c2421efa3066a5cbd8063d2244603824daccb6936b079010bb2aa89464b" -dependencies = [ - "backtrace", - "failure_derive", -] - -[[package]] -name = "failure_derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" - -[[package]] -name = "fixed-hash" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3367952ceb191f4ab95dd5685dc163ac539e36202f9fcfd0cb22f9f9c542fefc" -dependencies = [ - "byteorder", - "libc", - "rand 0.7.3", - "rustc-hex", - "static_assertions", -] - -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - -[[package]] -name = "generic-array" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" -dependencies = [ - "typenum", -] - -[[package]] -name = "getrandom" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "hash-db" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a" - -[[package]] -name = "hash256-std-hasher" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92c171d55b98633f4ed3860808f004099b36c1cc29c42cfc53aa8591b21efcf2" -dependencies = [ - "crunchy", -] - -[[package]] -name = "hashbrown" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead" -dependencies = [ - "ahash", - "autocfg 0.1.7", -] - -[[package]] -name = "hex" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" - -[[package]] -name = "hmac" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" -dependencies = [ - "crypto-mac", - "digest", -] - -[[package]] -name = "hmac-drbg" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e570451493f10f6581b48cdd530413b63ea9e780f544bfd3bdcaa0d89d1a7b" -dependencies = [ - "digest", - "generic-array", - "hmac", -] - -[[package]] -name = "honggfuzz" -version = "0.5.45" -dependencies = [ - "arbitrary", - "lazy_static", - "memmap", -] - -[[package]] -name = "impl-codec" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1be51a921b067b0eaca2fad532d9400041561aa922221cc65f95a85641c6bf53" -dependencies = [ - "parity-scale-codec", -] - -[[package]] -name = "impl-serde" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e3cae7e99c7ff5a995da2cf78dd0a5383740eda71d98cf7b1910c301ac69b8" -dependencies = [ - "serde", -] - -[[package]] -name = "impl-serde" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bbe9ea9b182f0fb1cabbd61f4ff9b7b7b9197955e95a7e4c27de5055eb29ff8" -dependencies = [ - "serde", -] - -[[package]] -name = "impl-trait-for-tuples" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "integer-sqrt" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65877bf7d44897a473350b1046277941cee20b263397e90869c50b6e766088b" - -[[package]] -name = "keccak" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.67" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" - -[[package]] -name = "libsecp256k1" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc1e2c808481a63dc6da2074752fdd4336a3c8fcc68b83db6f1fd5224ae7962" -dependencies = [ - "arrayref", - "crunchy", - "digest", - "hmac-drbg", - "rand 0.7.3", - "sha2", - "subtle 2.2.2", - "typenum", -] - -[[package]] -name = "lock_api" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" -dependencies = [ - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - -[[package]] -name = "memchr" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" - -[[package]] -name = "memmap" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "memory-db" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "198831fe8722331a395bc199a5d08efbc197497ef354cb4c77b969c02ffc0fc4" -dependencies = [ - "ahash", - "hash-db", - "hashbrown", - "parity-util-mem", -] - -[[package]] -name = "memory_units" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" - -[[package]] -name = "merlin" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0942b357c1b4d0dc43ba724674ec89c3218e6ca2b3e8269e7cb53bcecd2f6e" -dependencies = [ - "byteorder", - "keccak", - "rand_core 0.4.2", - "zeroize 1.1.0", -] - -[[package]] -name = "nodrop" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" - -[[package]] -name = "num-bigint" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" -dependencies = [ - "autocfg 1.0.0", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" -dependencies = [ - "autocfg 1.0.0", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3" -dependencies = [ - "autocfg 1.0.0", - "num-bigint", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" -dependencies = [ - "autocfg 1.0.0", -] - -[[package]] -name = "once_cell" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c601810575c99596d4afc46f78a678c80105117c379eb3650cf99b8a21ce5b" -dependencies = [ - "parking_lot 0.9.0", -] - -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - -[[package]] -name = "parity-scale-codec" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f509c5e67ca0605ee17dcd3f91ef41cadd685c75a298fb6261b781a5acb3f910" -dependencies = [ - "arrayvec 0.5.1", - "bitvec", - "byte-slice-cast", - "parity-scale-codec-derive", - "serde", -] - -[[package]] -name = "parity-scale-codec-derive" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a0ec292e92e8ec7c58e576adacc1e3f399c597c8f263c42f18420abe58e7245" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "parity-util-mem" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef1476e40bf8f5c6776e9600983435821ca86eb9819d74a6207cca69d091406a" -dependencies = [ - "cfg-if", - "impl-trait-for-tuples", - "parity-util-mem-derive", - "parking_lot 0.10.0", - "primitive-types", - "winapi", -] - -[[package]] -name = "parity-util-mem-derive" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" -dependencies = [ - "proc-macro2", - "syn", - "synstructure", -] - -[[package]] -name = "parity-wasm" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" - -[[package]] -name = "parking_lot" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" -dependencies = [ - "lock_api", - "parking_lot_core 0.6.2", - "rustc_version", -] - -[[package]] -name = "parking_lot" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" -dependencies = [ - "lock_api", - "parking_lot_core 0.7.0", -] - -[[package]] -name = "parking_lot_core" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" -dependencies = [ - "cfg-if", - "cloudabi", - "libc", - "redox_syscall", - "rustc_version", - "smallvec 0.6.13", - "winapi", -] - -[[package]] -name = "parking_lot_core" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1" -dependencies = [ - "cfg-if", - "cloudabi", - "libc", - "redox_syscall", - "smallvec 1.3.0", - "winapi", -] - -[[package]] -name = "paste" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e1afe738d71b1ebab5f1207c055054015427dbfc7bbe9ee1266894156ec046" -dependencies = [ - "paste-impl", - "proc-macro-hack", -] - -[[package]] -name = "paste-impl" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d4dc4a7f6f743211c5aab239640a65091535d97d43d92a52bca435a640892bb" -dependencies = [ - "proc-macro-hack", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pbkdf2" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" -dependencies = [ - "byteorder", - "crypto-mac", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" - -[[package]] -name = "primitive-types" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4336f4f5d5524fa60bcbd6fe626f9223d8142a50e7053e979acdf0da41ab975" -dependencies = [ - "fixed-hash", - "impl-codec", - "impl-serde 0.3.0", - "uint", -] - -[[package]] -name = "proc-macro-crate" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" -dependencies = [ - "toml", -] - -[[package]] -name = "proc-macro-hack" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "proc-macro2" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435" -dependencies = [ - "unicode-xid", -] - -[[package]] -name = "quote" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -dependencies = [ - "autocfg 0.1.7", - "libc", - "rand_chacha 0.1.1", - "rand_core 0.4.2", - "rand_hc 0.1.0", - "rand_isaac", - "rand_jitter", - "rand_os", - "rand_pcg", - "rand_xorshift", - "winapi", -] - -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc 0.2.0", -] - -[[package]] -name = "rand_chacha" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -dependencies = [ - "autocfg 0.1.7", - "rand_core 0.3.1", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rand_hc" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_isaac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_jitter" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -dependencies = [ - "libc", - "rand_core 0.4.2", - "winapi", -] - -[[package]] -name = "rand_os" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -dependencies = [ - "cloudabi", - "fuchsia-cprng", - "libc", - "rand_core 0.4.2", - "rdrand", - "winapi", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg 0.1.7", - "rand_core 0.4.2", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "redox_syscall" -version = "0.1.56" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" - -[[package]] -name = "regex" -version = "1.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", - "thread_local", -] - -[[package]] -name = "regex-syntax" -version = "0.6.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1132f845907680735a84409c3bebc64d1364a5683ffbce899550cd09d5eaefc1" - -[[package]] -name = "rustc-demangle" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" - -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustc-hex" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" - -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver", -] - -[[package]] -name = "schnorrkel" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eacd8381b3c37840c9c9f40472af529e49975bdcbc24f83c31059fd6539023d3" -dependencies = [ - "curve25519-dalek 1.2.3", - "failure", - "merlin", - "rand 0.6.5", - "rand_core 0.4.2", - "rand_os", - "sha2", - "subtle 2.2.2", - "zeroize 0.9.3", -] - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - -[[package]] -name = "serde" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "sha2" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0" -dependencies = [ - "block-buffer", - "digest", - "fake-simd", - "opaque-debug", -] - -[[package]] -name = "smallvec" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" -dependencies = [ - "maybe-uninit", -] - -[[package]] -name = "smallvec" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" - -[[package]] -name = "sp-application-crypto" -version = "2.0.0-alpha.3" -dependencies = [ - "parity-scale-codec", - "serde", - "sp-core", - "sp-io", - "sp-std", -] - -[[package]] -name = "sp-arithmetic" -version = "2.0.0-alpha.3" -dependencies = [ - "integer-sqrt", - "num-traits", - "parity-scale-codec", - "serde", - "sp-debug-derive", - "sp-std", -] - -[[package]] -name = "sp-core" -version = "2.0.0-alpha.3" -dependencies = [ - "base58", - "blake2-rfc", - "byteorder", - "ed25519-dalek", - "hash-db", - "hash256-std-hasher", - "hex", - "impl-serde 0.3.0", - "lazy_static", - "libsecp256k1", - "log", - "num-traits", - "parity-scale-codec", - "parity-util-mem", - "parking_lot 0.10.0", - "primitive-types", - "rand 0.7.3", - "regex", - "rustc-hex", - "schnorrkel", - "serde", - "sha2", - "sp-debug-derive", - "sp-externalities", - "sp-runtime-interface", - "sp-std", - "sp-storage", - "substrate-bip39", - "tiny-bip39", - "tiny-keccak", - "twox-hash", - "wasmi", - "zeroize 1.1.0", -] - -[[package]] -name = "sp-debug-derive" -version = "2.0.0-alpha.3" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "sp-externalities" -version = "0.8.0-alpha.3" -dependencies = [ - "environmental", - "sp-std", - "sp-storage", -] - -[[package]] -name = "sp-inherents" -version = "2.0.0-alpha.3" -dependencies = [ - "derive_more", - "parity-scale-codec", - "parking_lot 0.10.0", - "sp-core", - "sp-std", -] - -[[package]] -name = "sp-io" -version = "2.0.0-alpha.3" -dependencies = [ - "hash-db", - "libsecp256k1", - "log", - "parity-scale-codec", - "sp-core", - "sp-externalities", - "sp-runtime-interface", - "sp-state-machine", - "sp-std", - "sp-trie", - "sp-wasm-interface", -] - -[[package]] -name = "sp-panic-handler" -version = "2.0.0-alpha.3" -dependencies = [ - "backtrace", - "log", -] - -[[package]] -name = "sp-npos-elections" -version = "2.0.0-alpha.3" -dependencies = [ - "parity-scale-codec", - "serde", - "sp-core", - "sp-npos-elections-compact", - "sp-runtime", - "sp-std", -] - -[[package]] -name = "sp-npos-elections-compact" -version = "2.0.0-rc3" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "sp-npos-elections-fuzzer" -version = "2.0.0" -dependencies = [ - "honggfuzz", - "rand 0.7.3", - "sp-npos-elections", -] - -[[package]] -name = "sp-runtime" -version = "2.0.0-alpha.3" -dependencies = [ - "hash256-std-hasher", - "impl-trait-for-tuples", - "log", - "parity-scale-codec", - "parity-util-mem", - "paste", - "rand 0.7.3", - "serde", - "sp-application-crypto", - "sp-arithmetic", - "sp-core", - "sp-inherents", - "sp-io", - "sp-std", -] - -[[package]] -name = "sp-runtime-interface" -version = "2.0.0-alpha.3" -dependencies = [ - "parity-scale-codec", - "primitive-types", - "sp-externalities", - "sp-runtime-interface-proc-macro", - "sp-std", - "sp-wasm-interface", - "static_assertions", -] - -[[package]] -name = "sp-runtime-interface-proc-macro" -version = "2.0.0-alpha.3" -dependencies = [ - "Inflector", - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "sp-state-machine" -version = "0.8.0-alpha.3" -dependencies = [ - "hash-db", - "log", - "num-traits", - "parity-scale-codec", - "parking_lot 0.10.0", - "rand 0.7.3", - "sp-core", - "sp-externalities", - "sp-panic-handler", - "sp-trie", - "trie-db", - "trie-root", -] - -[[package]] -name = "sp-std" -version = "2.0.0-alpha.3" - -[[package]] -name = "sp-storage" -version = "2.0.0-alpha.3" -dependencies = [ - "impl-serde 0.2.3", - "serde", - "sp-debug-derive", - "sp-std", -] - -[[package]] -name = "sp-trie" -version = "2.0.0-alpha.3" -dependencies = [ - "hash-db", - "memory-db", - "parity-scale-codec", - "sp-core", - "sp-std", - "trie-db", - "trie-root", -] - -[[package]] -name = "sp-wasm-interface" -version = "2.0.0-alpha.3" -dependencies = [ - "impl-trait-for-tuples", - "parity-scale-codec", - "sp-std", - "wasmi", -] - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "substrate-bip39" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be511be555a3633e71739a79e4ddff6a6aaa6579fa6114182a51d72c3eb93c5" -dependencies = [ - "hmac", - "pbkdf2", - "schnorrkel", - "sha2", -] - -[[package]] -name = "subtle" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" - -[[package]] -name = "subtle" -version = "2.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" - -[[package]] -name = "syn" -version = "1.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "123bd9499cfb380418d509322d7a6d52e5315f064fe4b3ad18a53d6b92c07859" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - -[[package]] -name = "synstructure" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", -] - -[[package]] -name = "thread_local" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "tiny-bip39" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6848cd8f566953ce1e8faeba12ee23cbdbb0437754792cd857d44628b5685e3" -dependencies = [ - "failure", - "hmac", - "once_cell", - "pbkdf2", - "rand 0.7.3", - "rustc-hash", - "sha2", - "unicode-normalization", -] - -[[package]] -name = "tiny-keccak" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2953ca5148619bc99695c1274cb54c5275bbb913c6adad87e72eaf8db9787f69" -dependencies = [ - "crunchy", -] - -[[package]] -name = "toml" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" -dependencies = [ - "serde", -] - -[[package]] -name = "trie-db" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9222c50cc325855621271157c973da27a0dcd26fa06f8edf81020bd2333df0" -dependencies = [ - "hash-db", - "hashbrown", - "log", - "rustc-hex", - "smallvec 1.3.0", -] - -[[package]] -name = "trie-root" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "652931506d2c1244d7217a70b99f56718a7b4161b37f04e7cd868072a99f68cd" -dependencies = [ - "hash-db", -] - -[[package]] -name = "twox-hash" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" -dependencies = [ - "rand 0.7.3", -] - -[[package]] -name = "typenum" -version = "1.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" - -[[package]] -name = "uint" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75a4cdd7b87b28840dba13c483b9a88ee6bbf16ba5c951ee1ecfcf723078e0d" -dependencies = [ - "byteorder", - "crunchy", - "rustc-hex", - "static_assertions", -] - -[[package]] -name = "unicode-normalization" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" -dependencies = [ - "smallvec 1.3.0", -] - -[[package]] -name = "unicode-xid" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" - -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - -[[package]] -name = "wasmi" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf617d864d25af3587aa745529f7aaa541066c876d57e050c0d0c85c61c92aff" -dependencies = [ - "libc", - "memory_units", - "num-rational", - "num-traits", - "parity-wasm", - "wasmi-validation", -] - -[[package]] -name = "wasmi-validation" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93" -dependencies = [ - "parity-wasm", -] - -[[package]] -name = "winapi" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "zeroize" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45af6a010d13e4cf5b54c94ba5a2b2eba5596b9e46bf5875612d332a1f2b3f86" - -[[package]] -name = "zeroize" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] diff --git a/primitives/npos-elections/fuzzer/Cargo.toml b/primitives/npos-elections/fuzzer/Cargo.toml index d6fcc09c8b58..71c0c07c3032 100644 --- a/primitives/npos-elections/fuzzer/Cargo.toml +++ b/primitives/npos-elections/fuzzer/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-npos-elections-fuzzer" version = "2.0.0-alpha.5" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Fuzzer for phragmén implementation." documentation = "https://docs.rs/sp-npos-elections-fuzzer" @@ -14,13 +14,14 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +clap = { version = "3.1.6", features = ["derive"] } honggfuzz = "0.5" -rand = { version = "0.7.3", features = ["std", "small_rng"] } +rand = { version = "0.8", features = ["std", "small_rng"] } + +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } sp-npos-elections = { version = "4.0.0-dev", path = ".." } -sp-runtime = { version = "4.0.0-dev", path = "../../runtime" } -structopt = "0.3.21" +sp-runtime = { version = "6.0.0", path = "../../runtime" } [[bin]] name = "reduce" @@ -34,10 +35,6 @@ path = "src/phragmen_balancing.rs" name = "phragmms_balancing" path = "src/phragmms_balancing.rs" -[[bin]] -name = "compact" -path = "src/compact.rs" - [[bin]] name = "phragmen_pjr" path = "src/phragmen_pjr.rs" diff --git a/primitives/npos-elections/fuzzer/src/common.rs b/primitives/npos-elections/fuzzer/src/common.rs index e97f7f7df8b1..1bef899d5e54 100644 --- a/primitives/npos-elections/fuzzer/src/common.rs +++ b/primitives/npos-elections/fuzzer/src/common.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -68,7 +68,7 @@ pub fn generate_random_npos_inputs( // always generate a sensible desired number of candidates: elections are uninteresting if we // desire 0 candidates, or a number of candidates >= the actual number of candidates present - let rounds = rng.gen_range(1, candidate_count); + let rounds = rng.gen_range(1..candidate_count); // candidates are easy: just a completely random set of IDs let mut candidates: Vec = Vec::with_capacity(candidate_count); @@ -95,7 +95,7 @@ pub fn generate_random_npos_inputs( let vote_weight = rng.gen(); // it's not interesting if a voter chooses 0 or all candidates, so rule those cases out. - let n_candidates_chosen = rng.gen_range(1, candidates.len()); + let n_candidates_chosen = rng.gen_range(1..candidates.len()); let mut chosen_candidates = Vec::with_capacity(n_candidates_chosen); chosen_candidates.extend(candidates.choose_multiple(&mut rng, n_candidates_chosen)); @@ -132,25 +132,25 @@ pub fn generate_random_npos_result( (1..=target_count).for_each(|acc| { candidates.push(acc); - let stake_var = rng.gen_range(ed, 100 * ed); + let stake_var = rng.gen_range(ed..100 * ed); stake_of.insert(acc, base_stake + stake_var); }); let mut voters = Vec::with_capacity(voter_count as usize); (prefix..=(prefix + voter_count)).for_each(|acc| { - let edge_per_this_voter = rng.gen_range(1, candidates.len()); + let edge_per_this_voter = rng.gen_range(1..candidates.len()); // all possible targets let mut all_targets = candidates.clone(); // we remove and pop into `targets` `edge_per_this_voter` times. let targets = (0..edge_per_this_voter) .map(|_| { let upper = all_targets.len() - 1; - let idx = rng.gen_range(0, upper); + let idx = rng.gen_range(0..upper); all_targets.remove(idx) }) .collect::>(); - let stake_var = rng.gen_range(ed, 100 * ed); + let stake_var = rng.gen_range(ed..100 * ed); let stake = base_stake + stake_var; stake_of.insert(acc, stake); voters.push((acc, stake, targets)); @@ -158,20 +158,10 @@ pub fn generate_random_npos_result( ( match election_type { - ElectionType::Phragmen(conf) => seq_phragmen::( - to_elect, - candidates.clone(), - voters.clone(), - conf, - ) - .unwrap(), - ElectionType::Phragmms(conf) => phragmms::( - to_elect, - candidates.clone(), - voters.clone(), - conf, - ) - .unwrap(), + ElectionType::Phragmen(conf) => + seq_phragmen(to_elect, candidates.clone(), voters.clone(), conf).unwrap(), + ElectionType::Phragmms(conf) => + phragmms(to_elect, candidates.clone(), voters.clone(), conf).unwrap(), }, candidates, voters, diff --git a/primitives/npos-elections/fuzzer/src/phragmen_balancing.rs b/primitives/npos-elections/fuzzer/src/phragmen_balancing.rs index 0c140a8ce6fa..76641fc2c79f 100644 --- a/primitives/npos-elections/fuzzer/src/phragmen_balancing.rs +++ b/primitives/npos-elections/fuzzer/src/phragmen_balancing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,7 +23,7 @@ use common::*; use honggfuzz::fuzz; use rand::{self, SeedableRng}; use sp_npos_elections::{ - assignment_ratio_to_staked_normalized, is_score_better, seq_phragmen, to_supports, + assignment_ratio_to_staked_normalized, seq_phragmen, to_supports, ElectionResult, EvaluateSupport, VoteWeight, }; use sp_runtime::Perbill; @@ -60,7 +60,7 @@ fn main() { .unwrap(); let score = to_supports(staked.as_ref()).evaluate(); - if score[0] == 0 { + if score.minimal_stake == 0 { // such cases cannot be improved by balancing. return } @@ -68,13 +68,8 @@ fn main() { }; if iterations > 0 { - let balanced = seq_phragmen::( - to_elect, - candidates, - voters, - Some((iterations, 0)), - ) - .unwrap(); + let balanced: ElectionResult = + seq_phragmen(to_elect, candidates, voters, Some((iterations, 0))).unwrap(); let balanced_score = { let staked = assignment_ratio_to_staked_normalized( @@ -85,7 +80,8 @@ fn main() { to_supports(staked.as_ref()).evaluate() }; - let enhance = is_score_better(balanced_score, unbalanced_score, Perbill::zero()); + let enhance = + balanced_score.strict_threshold_better(unbalanced_score, Perbill::zero()); println!( "iter = {} // {:?} -> {:?} [{}]", @@ -95,9 +91,9 @@ fn main() { // The only guarantee of balancing is such that the first and third element of the // score cannot decrease. assert!( - balanced_score[0] >= unbalanced_score[0] && - balanced_score[1] == unbalanced_score[1] && - balanced_score[2] <= unbalanced_score[2] + balanced_score.minimal_stake >= unbalanced_score.minimal_stake && + balanced_score.sum_stake == unbalanced_score.sum_stake && + balanced_score.sum_stake_squared <= unbalanced_score.sum_stake_squared ); } }); diff --git a/primitives/npos-elections/fuzzer/src/phragmen_pjr.rs b/primitives/npos-elections/fuzzer/src/phragmen_pjr.rs index f1110da8ef8b..2396fdfa3b40 100644 --- a/primitives/npos-elections/fuzzer/src/phragmen_pjr.rs +++ b/primitives/npos-elections/fuzzer/src/phragmen_pjr.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -42,7 +42,7 @@ use honggfuzz::fuzz; #[cfg(not(fuzzing))] -use structopt::StructOpt; +use clap::Parser; mod common; use common::{generate_random_npos_inputs, to_range}; @@ -67,24 +67,25 @@ fn main() { } #[cfg(not(fuzzing))] -#[derive(Debug, StructOpt)] +#[derive(Debug, Parser)] +#[clap(author, version, about)] struct Opt { /// How many candidates participate in this election - #[structopt(short, long)] + #[clap(short, long)] candidates: Option, /// How many voters participate in this election - #[structopt(short, long)] + #[clap(short, long)] voters: Option, /// Random seed to use in this election - #[structopt(long)] + #[clap(long)] seed: Option, } #[cfg(not(fuzzing))] fn main() { - let opt = Opt::from_args(); + let opt = Opt::parse(); // candidates and voters by default use the maxima, which turn out to be one less than // the constant. iteration( diff --git a/primitives/npos-elections/fuzzer/src/phragmms_balancing.rs b/primitives/npos-elections/fuzzer/src/phragmms_balancing.rs index 7b2aacfa8588..09daf3f34d32 100644 --- a/primitives/npos-elections/fuzzer/src/phragmms_balancing.rs +++ b/primitives/npos-elections/fuzzer/src/phragmms_balancing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,7 +23,7 @@ use common::*; use honggfuzz::fuzz; use rand::{self, SeedableRng}; use sp_npos_elections::{ - assignment_ratio_to_staked_normalized, is_score_better, phragmms, to_supports, EvaluateSupport, + assignment_ratio_to_staked_normalized, phragmms, to_supports, ElectionResult, EvaluateSupport, VoteWeight, }; use sp_runtime::Perbill; @@ -60,20 +60,15 @@ fn main() { .unwrap(); let score = to_supports(&staked).evaluate(); - if score[0] == 0 { + if score.minimal_stake == 0 { // such cases cannot be improved by balancing. return } score }; - let balanced = phragmms::( - to_elect, - candidates, - voters, - Some((iterations, 0)), - ) - .unwrap(); + let balanced: ElectionResult = + phragmms(to_elect, candidates, voters, Some((iterations, 0))).unwrap(); let balanced_score = { let staked = @@ -82,7 +77,7 @@ fn main() { to_supports(staked.as_ref()).evaluate() }; - let enhance = is_score_better(balanced_score, unbalanced_score, Perbill::zero()); + let enhance = balanced_score.strict_threshold_better(unbalanced_score, Perbill::zero()); println!( "iter = {} // {:?} -> {:?} [{}]", @@ -92,9 +87,9 @@ fn main() { // The only guarantee of balancing is such that the first and third element of the score // cannot decrease. assert!( - balanced_score[0] >= unbalanced_score[0] && - balanced_score[1] == unbalanced_score[1] && - balanced_score[2] <= unbalanced_score[2] + balanced_score.minimal_stake >= unbalanced_score.minimal_stake && + balanced_score.sum_stake == unbalanced_score.sum_stake && + balanced_score.sum_stake_squared <= unbalanced_score.sum_stake_squared ); }); } diff --git a/primitives/npos-elections/fuzzer/src/reduce.rs b/primitives/npos-elections/fuzzer/src/reduce.rs index 5f8a4f0e1384..a77b40ca56d5 100644 --- a/primitives/npos-elections/fuzzer/src/reduce.rs +++ b/primitives/npos-elections/fuzzer/src/reduce.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -79,8 +79,7 @@ fn generate_random_phragmen_assignment( let mut targets_to_chose_from = all_targets.clone(); let targets_to_chose = if edge_per_voter_var > 0 { rng.gen_range( - avg_edge_per_voter - edge_per_voter_var, - avg_edge_per_voter + edge_per_voter_var, + avg_edge_per_voter - edge_per_voter_var..avg_edge_per_voter + edge_per_voter_var, ) } else { avg_edge_per_voter @@ -89,11 +88,11 @@ fn generate_random_phragmen_assignment( let distribution = (0..targets_to_chose) .map(|_| { let target = - targets_to_chose_from.remove(rng.gen_range(0, targets_to_chose_from.len())); - if winners.iter().find(|w| **w == target).is_none() { + targets_to_chose_from.remove(rng.gen_range(0..targets_to_chose_from.len())); + if winners.iter().all(|w| *w != target) { winners.push(target.clone()); } - (target, rng.gen_range(1 * KSM, 100 * KSM)) + (target, rng.gen_range(1 * KSM..100 * KSM)) }) .collect::>(); diff --git a/primitives/npos-elections/solution-type/Cargo.toml b/primitives/npos-elections/solution-type/Cargo.toml deleted file mode 100644 index cbe6750266f0..000000000000 --- a/primitives/npos-elections/solution-type/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "sp-npos-elections-solution-type" -version = "4.0.0-dev" -authors = ["Parity Technologies "] -edition = "2018" -license = "Apache-2.0" -homepage = "https://substrate.dev" -repository = "https://github.com/paritytech/substrate/" -description = "NPoS Solution Type" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[lib] -proc-macro = true - -[dependencies] -syn = { version = "1.0.58", features = ["full", "visit"] } -quote = "1.0" -proc-macro2 = "1.0.29" -proc-macro-crate = "1.0.0" - -[dev-dependencies] -parity-scale-codec = "2.0.1" -scale-info = "1.0" -sp-arithmetic = { path = "../../arithmetic", version = "4.0.0-dev" } -# used by generate_solution_type: -sp-npos-elections = { path = "..", version = "4.0.0-dev" } -trybuild = "1.0.43" diff --git a/primitives/npos-elections/solution-type/tests/ui/fail/no_annotations.rs b/primitives/npos-elections/solution-type/tests/ui/fail/no_annotations.rs deleted file mode 100644 index cfca2841db63..000000000000 --- a/primitives/npos-elections/solution-type/tests/ui/fail/no_annotations.rs +++ /dev/null @@ -1,9 +0,0 @@ -use sp_npos_elections_solution_type::generate_solution_type; - -generate_solution_type!(pub struct TestSolution::< - u16, - u8, - Perbill, ->(8)); - -fn main() {} diff --git a/primitives/npos-elections/src/assignments.rs b/primitives/npos-elections/src/assignments.rs index bdd1e2cd281b..9422ccdf6588 100644 --- a/primitives/npos-elections/src/assignments.rs +++ b/primitives/npos-elections/src/assignments.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +17,7 @@ //! Structs and helpers for distributing a voter's stake among various winners. -use crate::{Error, ExtendedBalance, IdentifierT, PerThing128, __OrInvalidIndex}; +use crate::{ExtendedBalance, IdentifierT, PerThing128}; #[cfg(feature = "std")] use codec::{Decode, Encode}; use sp_arithmetic::{ @@ -166,44 +166,3 @@ impl StakedAssignment { self.distribution.iter().fold(Zero::zero(), |a, b| a.saturating_add(b.1)) } } -/// The [`IndexAssignment`] type is an intermediate between the assignments list -/// ([`&[Assignment]`][Assignment]) and `SolutionOf`. -/// -/// The voter and target identifiers have already been replaced with appropriate indices, -/// making it fast to repeatedly encode into a `SolutionOf`. This property turns out -/// to be important when trimming for solution length. -#[derive(RuntimeDebug, Clone, Default)] -#[cfg_attr(feature = "std", derive(PartialEq, Eq, Encode, Decode))] -pub struct IndexAssignment { - /// Index of the voter among the voters list. - pub who: VoterIndex, - /// The distribution of the voter's stake among winning targets. - /// - /// Targets are identified by their index in the canonical list. - pub distribution: Vec<(TargetIndex, P)>, -} - -impl IndexAssignment { - pub fn new( - assignment: &Assignment, - voter_index: impl Fn(&AccountId) -> Option, - target_index: impl Fn(&AccountId) -> Option, - ) -> Result { - Ok(Self { - who: voter_index(&assignment.who).or_invalid_index()?, - distribution: assignment - .distribution - .iter() - .map(|(target, proportion)| Some((target_index(target)?, proportion.clone()))) - .collect::>>() - .or_invalid_index()?, - }) - } -} - -/// A type alias for [`IndexAssignment`] made from [`crate::Solution`]. -pub type IndexAssignmentOf = IndexAssignment< - ::VoterIndex, - ::TargetIndex, - ::Accuracy, ->; diff --git a/primitives/npos-elections/src/balancing.rs b/primitives/npos-elections/src/balancing.rs index 63164049e526..98f193e9e611 100644 --- a/primitives/npos-elections/src/balancing.rs +++ b/primitives/npos-elections/src/balancing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/npos-elections/src/helpers.rs b/primitives/npos-elections/src/helpers.rs index ca97aeb996e4..76cfd59de6fb 100644 --- a/primitives/npos-elections/src/helpers.rs +++ b/primitives/npos-elections/src/helpers.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/npos-elections/src/lib.rs b/primitives/npos-elections/src/lib.rs index afe85ef53b3a..93fb24eb4a3c 100644 --- a/primitives/npos-elections/src/lib.rs +++ b/primitives/npos-elections/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -74,11 +74,14 @@ #![cfg_attr(not(feature = "std"), no_std)] +use scale_info::TypeInfo; use sp_arithmetic::{traits::Zero, Normalizable, PerThing, Rational128, ThresholdOrd}; use sp_core::RuntimeDebug; -use sp_std::{cell::RefCell, cmp::Ordering, collections::btree_map::BTreeMap, prelude::*, rc::Rc}; +use sp_std::{ + cell::RefCell, cmp::Ordering, collections::btree_map::BTreeMap, prelude::*, rc::Rc, vec, +}; -use codec::{Decode, Encode}; +use codec::{Decode, Encode, MaxEncodedLen}; #[cfg(feature = "std")] use serde::{Deserialize, Serialize}; @@ -97,29 +100,16 @@ pub mod pjr; pub mod reduce; pub mod traits; -pub use assignments::{Assignment, IndexAssignment, IndexAssignmentOf, StakedAssignment}; +pub use assignments::{Assignment, StakedAssignment}; pub use balancing::*; pub use helpers::*; pub use phragmen::*; pub use phragmms::*; pub use pjr::*; pub use reduce::reduce; -pub use traits::{IdentifierT, NposSolution, PerThing128, __OrInvalidIndex}; - -// re-export for the solution macro, with the dependencies of the macro. -#[doc(hidden)] -pub use codec; -#[doc(hidden)] -pub use scale_info; -#[doc(hidden)] -pub use sp_arithmetic; -#[doc(hidden)] -pub use sp_std; - -// re-export the solution type macro. -pub use sp_npos_elections_solution_type::generate_solution_type; - -/// The errors that might occur in the this crate and solution-type. +pub use traits::{IdentifierT, PerThing128}; + +/// The errors that might occur in this crate and `frame-election-provider-solution-type`. #[derive(Eq, PartialEq, RuntimeDebug)] pub enum Error { /// While going from solution indices to ratio, the weight of all the edges has gone above the @@ -129,12 +119,14 @@ pub enum Error { SolutionTargetOverflow, /// One of the index functions returned none. SolutionInvalidIndex, - /// One of the page indices was invalid + /// One of the page indices was invalid. SolutionInvalidPageIndex, /// An error occurred in some arithmetic operation. ArithmeticError(&'static str), /// The data provided to create support map was invalid. InvalidSupportEdge, + /// The number of voters is bigger than the `MaxVoters` bound. + TooManyVoters, } /// A type which is used in the API of this crate as a numeric weight of a vote, most often the @@ -144,9 +136,86 @@ pub type VoteWeight = u64; /// A type in which performing operations on vote weights are safe. pub type ExtendedBalance = u128; -/// The score of an assignment. This can be computed from the support map via -/// [`EvaluateSupport::evaluate`]. -pub type ElectionScore = [ExtendedBalance; 3]; +/// The score of an election. This is the main measure of an election's quality. +/// +/// By definition, the order of significance in [`ElectionScore`] is: +/// +/// 1. `minimal_stake`. +/// 2. `sum_stake`. +/// 3. `sum_stake_squared`. +#[derive(Clone, Copy, PartialEq, Eq, Encode, Decode, MaxEncodedLen, TypeInfo, Debug, Default)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +pub struct ElectionScore { + /// The minimal winner, in terms of total backing stake. + /// + /// This parameter should be maximized. + pub minimal_stake: ExtendedBalance, + /// The sum of the total backing of all winners. + /// + /// This parameter should maximized + pub sum_stake: ExtendedBalance, + /// The sum squared of the total backing of all winners, aka. the variance. + /// + /// Ths parameter should be minimized. + pub sum_stake_squared: ExtendedBalance, +} + +impl ElectionScore { + /// Iterate over the inner items, first visiting the most significant one. + fn iter_by_significance(self) -> impl Iterator { + [self.minimal_stake, self.sum_stake, self.sum_stake_squared].into_iter() + } + + /// Compares two sets of election scores based on desirability, returning true if `self` is + /// strictly `threshold` better than `other`. In other words, each element of `self` must be + /// `self * threshold` better than `other`. + /// + /// Evaluation is done based on the order of significance of the fields of [`ElectionScore`]. + pub fn strict_threshold_better(self, other: Self, threshold: impl PerThing) -> bool { + match self + .iter_by_significance() + .zip(other.iter_by_significance()) + .map(|(this, that)| (this.ge(&that), this.tcmp(&that, threshold.mul_ceil(that)))) + .collect::>() + .as_slice() + { + // threshold better in the `score.minimal_stake`, accept. + [(x, Ordering::Greater), _, _] => { + debug_assert!(x); + true + }, + + // less than threshold better in `score.minimal_stake`, but more than threshold better + // in `score.sum_stake`. + [(true, Ordering::Equal), (_, Ordering::Greater), _] => true, + + // less than threshold better in `score.minimal_stake` and `score.sum_stake`, but more + // than threshold better in `score.sum_stake_squared`. + [(true, Ordering::Equal), (true, Ordering::Equal), (_, Ordering::Less)] => true, + + // anything else is not a good score. + _ => false, + } + } +} + +impl sp_std::cmp::Ord for ElectionScore { + fn cmp(&self, other: &Self) -> Ordering { + // we delegate this to the lexicographic cmp of slices`, and to incorporate that we want the + // third element to be minimized, we swap them. + [self.minimal_stake, self.sum_stake, other.sum_stake_squared].cmp(&[ + other.minimal_stake, + other.sum_stake, + self.sum_stake_squared, + ]) + } +} + +impl sp_std::cmp::PartialOrd for ElectionScore { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} /// A pointer to a candidate struct with interior mutability. pub type CandidatePtr = Rc>>; @@ -178,7 +247,7 @@ impl Candidate { } /// A vote being casted by a [`Voter`] to a [`Candidate`] is an `Edge`. -#[derive(Clone, Default)] +#[derive(Clone)] pub struct Edge { /// Identifier of the target. /// @@ -193,6 +262,15 @@ pub struct Edge { weight: ExtendedBalance, } +#[cfg(test)] +impl Edge { + fn new(candidate: Candidate, weight: ExtendedBalance) -> Self { + let who = candidate.who.clone(); + let candidate = Rc::new(RefCell::new(candidate)); + Self { weight, who, candidate, load: Default::default() } + } +} + #[cfg(feature = "std")] impl sp_std::fmt::Debug for Edge { fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { @@ -223,7 +301,12 @@ impl std::fmt::Debug for Voter { impl Voter { /// Create a new `Voter`. pub fn new(who: AccountId) -> Self { - Self { who, ..Default::default() } + Self { + who, + edges: Default::default(), + budget: Default::default(), + load: Default::default(), + } } /// Returns `true` if `self` votes for `target`. @@ -339,7 +422,7 @@ pub struct ElectionResult { /// /// This, at the current version, resembles the `Exposure` defined in the Staking pallet, yet they /// do not necessarily have to be the same. -#[derive(Default, RuntimeDebug, Encode, Decode, Clone, Eq, PartialEq, scale_info::TypeInfo)] +#[derive(RuntimeDebug, Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] pub struct Support { /// Total support. @@ -348,6 +431,12 @@ pub struct Support { pub voters: Vec<(AccountId, ExtendedBalance)>, } +impl Default for Support { + fn default() -> Self { + Self { total: Default::default(), voters: vec![] } + } +} + /// A target-major representation of the the election outcome. /// /// Essentially a flat variant of [`SupportMap`]. @@ -398,49 +487,22 @@ pub trait EvaluateSupport { impl EvaluateSupport for Supports { fn evaluate(&self) -> ElectionScore { - let mut min_support = ExtendedBalance::max_value(); - let mut sum: ExtendedBalance = Zero::zero(); + let mut minimal_stake = ExtendedBalance::max_value(); + let mut sum_stake: ExtendedBalance = Zero::zero(); // NOTE: The third element might saturate but fine for now since this will run on-chain and // need to be fast. - let mut sum_squared: ExtendedBalance = Zero::zero(); + let mut sum_stake_squared: ExtendedBalance = Zero::zero(); + for (_, support) in self { - sum = sum.saturating_add(support.total); + sum_stake = sum_stake.saturating_add(support.total); let squared = support.total.saturating_mul(support.total); - sum_squared = sum_squared.saturating_add(squared); - if support.total < min_support { - min_support = support.total; + sum_stake_squared = sum_stake_squared.saturating_add(squared); + if support.total < minimal_stake { + minimal_stake = support.total; } } - [min_support, sum, sum_squared] - } -} -/// Compares two sets of election scores based on desirability and returns true if `this` is better -/// than `that`. -/// -/// Evaluation is done in a lexicographic manner, and if each element of `this` is `that * epsilon` -/// greater or less than `that`. -/// -/// Note that the third component should be minimized. -pub fn is_score_better(this: ElectionScore, that: ElectionScore, epsilon: P) -> bool { - match this - .iter() - .zip(that.iter()) - .map(|(thi, tha)| (thi.ge(&tha), thi.tcmp(&tha, epsilon.mul_ceil(*tha)))) - .collect::>() - .as_slice() - { - // epsilon better in the score[0], accept. - [(_, Ordering::Greater), _, _] => true, - - // less than epsilon better in score[0], but more than epsilon better in the second. - [(true, Ordering::Equal), (_, Ordering::Greater), _] => true, - - // less than epsilon better in score[0, 1], but more than epsilon better in the third - [(true, Ordering::Equal), (true, Ordering::Equal), (_, Ordering::Less)] => true, - - // anything else is not a good score. - _ => false, + ElectionScore { minimal_stake, sum_stake, sum_stake_squared } } } @@ -451,7 +513,7 @@ pub fn is_score_better(this: ElectionScore, that: ElectionScore, ep /// - It drops duplicate targets within a voter. pub fn setup_inputs( initial_candidates: Vec, - initial_voters: Vec<(AccountId, VoteWeight, Vec)>, + initial_voters: Vec<(AccountId, VoteWeight, impl IntoIterator)>, ) -> (Vec>, Vec>) { // used to cache and access candidates index. let mut c_idx_cache = BTreeMap::::new(); @@ -461,14 +523,22 @@ pub fn setup_inputs( .enumerate() .map(|(idx, who)| { c_idx_cache.insert(who.clone(), idx); - Candidate { who, ..Default::default() }.to_ptr() + Candidate { + who, + score: Default::default(), + approval_stake: Default::default(), + backed_stake: Default::default(), + elected: Default::default(), + round: Default::default(), + } + .to_ptr() }) .collect::>>(); let voters = initial_voters .into_iter() .filter_map(|(who, voter_stake, votes)| { - let mut edges: Vec> = Vec::with_capacity(votes.len()); + let mut edges: Vec> = Vec::new(); for v in votes { if edges.iter().any(|e| e.who == v) { // duplicate edge. @@ -482,7 +552,8 @@ pub fn setup_inputs( edges.push(Edge { who: v.clone(), candidate: Rc::clone(&candidates[*idx]), - ..Default::default() + load: Default::default(), + weight: Default::default(), }); } // else {} would be wrong votes. We don't really care about it. } diff --git a/primitives/npos-elections/src/mock.rs b/primitives/npos-elections/src/mock.rs index 36fd78b5757e..dd85ce9b6dfa 100644 --- a/primitives/npos-elections/src/mock.rs +++ b/primitives/npos-elections/src/mock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,13 +19,6 @@ #![cfg(test)] -use std::{ - collections::{HashMap, HashSet}, - convert::TryInto, - hash::Hash, -}; - -use rand::{self, seq::SliceRandom, Rng}; use sp_arithmetic::{ traits::{One, SaturatedConversion, Zero}, PerThing, @@ -37,27 +30,6 @@ use crate::{seq_phragmen, Assignment, ElectionResult, ExtendedBalance, PerThing1 pub type AccountId = u64; -/// The candidate mask allows easy disambiguation between voters and candidates: accounts -/// for which this bit is set are candidates, and without it, are voters. -pub const CANDIDATE_MASK: AccountId = 1 << ((std::mem::size_of::() * 8) - 1); - -pub type TestAccuracy = sp_runtime::Perbill; - -crate::generate_solution_type! { - pub struct TestSolution::< - VoterIndex = u32, - TargetIndex = u16, - Accuracy = TestAccuracy, - >(16) -} - -pub fn p(p: u8) -> TestAccuracy { - TestAccuracy::from_percent(p.into()) -} - -pub type MockAssignment = crate::Assignment; -pub type Voter = (AccountId, VoteWeight, Vec); - #[derive(Default, Debug)] pub(crate) struct _Candidate { who: A, @@ -344,7 +316,7 @@ pub(crate) fn run_and_compare( FS: Fn(&AccountId) -> VoteWeight, { // run fixed point code. - let ElectionResult { winners, assignments } = seq_phragmen::<_, Output>( + let ElectionResult::<_, Output> { winners, assignments } = seq_phragmen( to_elect, candidates.clone(), voters @@ -412,136 +384,3 @@ pub(crate) fn build_support_map_float( } supports } - -/// Generate voter and assignment lists. Makes no attempt to be realistic about winner or assignment -/// fairness. -/// -/// Maintains these invariants: -/// -/// - candidate ids have `CANDIDATE_MASK` bit set -/// - voter ids do not have `CANDIDATE_MASK` bit set -/// - assignments have the same ordering as voters -/// - `assignments.distribution.iter().map(|(_, frac)| frac).sum() == One::one()` -/// - a coherent set of winners is chosen. -/// - the winner set is a subset of the candidate set. -/// - `assignments.distribution.iter().all(|(who, _)| winners.contains(who))` -pub fn generate_random_votes( - candidate_count: usize, - voter_count: usize, - mut rng: impl Rng, -) -> (Vec, Vec, Vec) { - // cache for fast generation of unique candidate and voter ids - let mut used_ids = HashSet::with_capacity(candidate_count + voter_count); - - // candidates are easy: just a completely random set of IDs - let mut candidates: Vec = Vec::with_capacity(candidate_count); - while candidates.len() < candidate_count { - let mut new = || rng.gen::() | CANDIDATE_MASK; - let mut id = new(); - // insert returns `false` when the value was already present - while !used_ids.insert(id) { - id = new(); - } - candidates.push(id); - } - - // voters are random ids, random weights, random selection from the candidates - let mut voters = Vec::with_capacity(voter_count); - while voters.len() < voter_count { - let mut new = || rng.gen::() & !CANDIDATE_MASK; - let mut id = new(); - // insert returns `false` when the value was already present - while !used_ids.insert(id) { - id = new(); - } - - let vote_weight = rng.gen(); - - // it's not interesting if a voter chooses 0 or all candidates, so rule those cases out. - // also, let's not generate any cases which result in a compact overflow. - let n_candidates_chosen = - rng.gen_range(1, candidates.len().min(::LIMIT)); - - let mut chosen_candidates = Vec::with_capacity(n_candidates_chosen); - chosen_candidates.extend(candidates.choose_multiple(&mut rng, n_candidates_chosen)); - voters.push((id, vote_weight, chosen_candidates)); - } - - // always generate a sensible number of winners: elections are uninteresting if nobody wins, - // or everybody wins - let num_winners = rng.gen_range(1, candidate_count); - let mut winners: HashSet = HashSet::with_capacity(num_winners); - winners.extend(candidates.choose_multiple(&mut rng, num_winners)); - assert_eq!(winners.len(), num_winners); - - let mut assignments = Vec::with_capacity(voters.len()); - for (voter_id, _, votes) in voters.iter() { - let chosen_winners = votes.iter().filter(|vote| winners.contains(vote)).cloned(); - let num_chosen_winners = chosen_winners.clone().count(); - - // distribute the available stake randomly - let stake_distribution = if num_chosen_winners == 0 { - continue - } else { - let mut available_stake = 1000; - let mut stake_distribution = Vec::with_capacity(num_chosen_winners); - for _ in 0..num_chosen_winners - 1 { - let stake = rng.gen_range(0, available_stake).min(1); - stake_distribution.push(TestAccuracy::from_perthousand(stake)); - available_stake -= stake; - } - stake_distribution.push(TestAccuracy::from_perthousand(available_stake)); - stake_distribution.shuffle(&mut rng); - stake_distribution - }; - - assignments.push(MockAssignment { - who: *voter_id, - distribution: chosen_winners.zip(stake_distribution).collect(), - }); - } - - (voters, assignments, candidates) -} - -fn generate_cache(voters: Voters) -> HashMap -where - Voters: Iterator, - Item: Hash + Eq + Copy, -{ - let mut cache = HashMap::new(); - for (idx, voter_id) in voters.enumerate() { - cache.insert(voter_id, idx); - } - cache -} - -/// Create a function that returns the index of a voter in the voters list. -pub fn make_voter_fn(voters: &[Voter]) -> impl Fn(&AccountId) -> Option -where - usize: TryInto, -{ - let cache = generate_cache(voters.iter().map(|(id, _, _)| *id)); - move |who| { - if cache.get(who).is_none() { - println!("WARNING: voter {} will raise InvalidIndex", who); - } - cache.get(who).cloned().and_then(|i| i.try_into().ok()) - } -} - -/// Create a function that returns the index of a candidate in the candidates list. -pub fn make_target_fn( - candidates: &[AccountId], -) -> impl Fn(&AccountId) -> Option -where - usize: TryInto, -{ - let cache = generate_cache(candidates.iter().cloned()); - move |who| { - if cache.get(who).is_none() { - println!("WARNING: target {} will raise InvalidIndex", who); - } - cache.get(who).cloned().and_then(|i| i.try_into().ok()) - } -} diff --git a/primitives/npos-elections/src/node.rs b/primitives/npos-elections/src/node.rs index 62b728d52258..6642a9ae3973 100644 --- a/primitives/npos-elections/src/node.rs +++ b/primitives/npos-elections/src/node.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/npos-elections/src/phragmen.rs b/primitives/npos-elections/src/phragmen.rs index 5ed472284351..e8e925935f77 100644 --- a/primitives/npos-elections/src/phragmen.rs +++ b/primitives/npos-elections/src/phragmen.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -70,7 +70,7 @@ const DEN: ExtendedBalance = ExtendedBalance::max_value(); pub fn seq_phragmen( to_elect: usize, candidates: Vec, - voters: Vec<(AccountId, VoteWeight, Vec)>, + voters: Vec<(AccountId, VoteWeight, impl IntoIterator)>, balancing: Option<(usize, ExtendedBalance)>, ) -> Result, crate::Error> { let (candidates, voters) = setup_inputs(candidates, voters); diff --git a/primitives/npos-elections/src/phragmms.rs b/primitives/npos-elections/src/phragmms.rs index e9135a13190c..6220cacd157b 100644 --- a/primitives/npos-elections/src/phragmms.rs +++ b/primitives/npos-elections/src/phragmms.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -44,7 +44,7 @@ use sp_std::{prelude::*, rc::Rc}; pub fn phragmms( to_elect: usize, candidates: Vec, - voters: Vec<(AccountId, VoteWeight, Vec)>, + voters: Vec<(AccountId, VoteWeight, impl IntoIterator)>, balancing: Option<(usize, ExtendedBalance)>, ) -> Result, crate::Error> { let (candidates, mut voters) = setup_inputs(candidates, voters); @@ -351,8 +351,8 @@ mod tests { let candidates = vec![1, 2, 3]; let voters = vec![(10, 10, vec![1, 2]), (20, 20, vec![1, 3]), (30, 30, vec![2, 3])]; - let ElectionResult { winners, assignments } = - phragmms::<_, Perbill>(2, candidates, voters, Some((2, 0))).unwrap(); + let ElectionResult::<_, Perbill> { winners, assignments } = + phragmms(2, candidates, voters, Some((2, 0))).unwrap(); assert_eq!(winners, vec![(3, 30), (2, 30)]); assert_eq!( assignments, @@ -383,8 +383,8 @@ mod tests { (130, 1000, vec![61, 71]), ]; - let ElectionResult { winners, assignments: _ } = - phragmms::<_, Perbill>(4, candidates, voters, Some((2, 0))).unwrap(); + let ElectionResult::<_, Perbill> { winners, assignments: _ } = + phragmms(4, candidates, voters, Some((2, 0))).unwrap(); assert_eq!(winners, vec![(11, 3000), (31, 2000), (51, 1500), (61, 1500),]); } @@ -396,8 +396,8 @@ mod tests { // give a bit more to 1 and 3. voters.push((2, u64::MAX, vec![1, 3])); - let ElectionResult { winners, assignments: _ } = - phragmms::<_, Perbill>(2, candidates, voters, Some((2, 0))).unwrap(); + let ElectionResult::<_, Perbill> { winners, assignments: _ } = + phragmms(2, candidates, voters, Some((2, 0))).unwrap(); assert_eq!(winners.into_iter().map(|(w, _)| w).collect::>(), vec![1u32, 3]); } } diff --git a/primitives/npos-elections/src/pjr.rs b/primitives/npos-elections/src/pjr.rs index e27acf1408f9..2d58ca49c8a2 100644 --- a/primitives/npos-elections/src/pjr.rs +++ b/primitives/npos-elections/src/pjr.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -287,7 +287,15 @@ fn prepare_pjr_input( let elected = maybe_support.is_some(); let backed_stake = maybe_support.map(|support| support.total).unwrap_or_default(); - Candidate { who, elected, backed_stake, ..Default::default() }.to_ptr() + Candidate { + who, + elected, + backed_stake, + score: Default::default(), + approval_stake: Default::default(), + round: Default::default(), + } + .to_ptr() }) .collect::>(); @@ -315,14 +323,14 @@ fn prepare_pjr_input( who: t, candidate: Rc::clone(&candidates[*idx]), weight, - ..Default::default() + load: Default::default(), }); } } let who = v; let budget: ExtendedBalance = w.into(); - Voter { who, budget, edges, ..Default::default() } + Voter { who, budget, edges, load: Default::default() } }) .collect::>(); @@ -387,7 +395,14 @@ mod tests { .into_iter() .map(|(t, w, e)| { budget += w; - Candidate { who: t, elected: e, backed_stake: w, ..Default::default() } + Candidate { + who: t, + elected: e, + backed_stake: w, + score: Default::default(), + approval_stake: Default::default(), + round: Default::default(), + } }) .collect::>(); let edges = candidates @@ -396,7 +411,7 @@ mod tests { who: c.who, weight: c.backed_stake, candidate: c.to_ptr(), - ..Default::default() + load: Default::default(), }) .collect::>(); voter.edges = edges; @@ -432,7 +447,15 @@ mod tests { // will give 10 slack. let v3 = setup_voter(30, vec![(1, 20, true), (2, 20, true), (3, 0, false)]); - let unelected = Candidate { who: 3u32, elected: false, ..Default::default() }.to_ptr(); + let unelected = Candidate { + who: 3u32, + elected: false, + score: Default::default(), + approval_stake: Default::default(), + backed_stake: Default::default(), + round: Default::default(), + } + .to_ptr(); let score = pre_score(unelected, &vec![v1, v2, v3], 15); assert_eq!(score, 15); diff --git a/primitives/npos-elections/src/reduce.rs b/primitives/npos-elections/src/reduce.rs index 8b90796af85c..f089a37e3fff 100644 --- a/primitives/npos-elections/src/reduce.rs +++ b/primitives/npos-elections/src/reduce.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -55,7 +55,6 @@ use sp_arithmetic::traits::{Bounded, Zero}; use sp_std::{ collections::btree_map::{BTreeMap, Entry::*}, prelude::*, - vec, }; /// Map type used for reduce_4. Can be easily swapped with HashMap. @@ -276,9 +275,8 @@ fn reduce_4(assignments: &mut Vec>) -> u32 { }); // remove either one of them. - let who_removed = remove_indices.iter().find(|i| **i < 2usize).is_some(); - let other_removed = - remove_indices.into_iter().find(|i| *i >= 2usize).is_some(); + let who_removed = remove_indices.iter().any(|i| *i < 2usize); + let other_removed = remove_indices.into_iter().any(|i| i >= 2usize); match (who_removed, other_removed) { (false, true) => { @@ -356,7 +354,7 @@ fn reduce_all(assignments: &mut Vec>) -> u32 .or_insert_with(|| Node::new(target_id).into_ref()) .clone(); - // If one exists but the other one doesn't, or if both does not, then set the existing + // If one exists but the other one doesn't, or if both do not, then set the existing // one as the parent of the non-existing one and move on. Else, continue with the rest // of the code. match (voter_exists, target_exists) { @@ -390,39 +388,44 @@ fn reduce_all(assignments: &mut Vec>) -> u32 let common_count = trailing_common(&voter_root_path, &target_root_path); // because roots are the same. - #[cfg(feature = "std")] - debug_assert_eq!(target_root_path.last().unwrap(), voter_root_path.last().unwrap()); + //debug_assert_eq!(target_root_path.last().unwrap(), + // voter_root_path.last().unwrap()); TODO: @kian + // the common path must be non-void.. debug_assert!(common_count > 0); + // and smaller than btoh + debug_assert!(common_count <= voter_root_path.len()); + debug_assert!(common_count <= target_root_path.len()); // cycle part of each path will be `path[path.len() - common_count - 1 : 0]` // NOTE: the order of chaining is important! it is always build from [target, ..., // voter] let cycle = target_root_path .iter() - .take(target_root_path.len() - common_count + 1) + .take(target_root_path.len().saturating_sub(common_count).saturating_add(1)) .cloned() .chain( voter_root_path .iter() - .take(voter_root_path.len() - common_count) + .take(voter_root_path.len().saturating_sub(common_count)) .rev() .cloned(), ) .collect::>>(); // a cycle's length shall always be multiple of two. - #[cfg(feature = "std")] debug_assert_eq!(cycle.len() % 2, 0); // find minimum of cycle. let mut min_value: ExtendedBalance = Bounded::max_value(); - // The voter and the target pair that create the min edge. - let mut min_target: A = Default::default(); - let mut min_voter: A = Default::default(); + // The voter and the target pair that create the min edge. These MUST be set by the + // end of this code block, otherwise we skip. + let mut maybe_min_target: Option = None; + let mut maybe_min_voter: Option = None; // The index of the min in opaque cycle list. - let mut min_index = 0usize; + let mut maybe_min_index: Option = None; // 1 -> next // 0 -> prev - let mut min_direction = 0u32; + let mut maybe_min_direction: Option = None; + // helpers let next_index = |i| { if i < (cycle.len() - 1) { @@ -438,6 +441,7 @@ fn reduce_all(assignments: &mut Vec>) -> u32 cycle.len() - 1 } }; + for i in 0..cycle.len() { if cycle[i].borrow().id.role == NodeRole::Voter { // NOTE: sadly way too many clones since I don't want to make A: Copy @@ -448,10 +452,10 @@ fn reduce_all(assignments: &mut Vec>) -> u32 ass.distribution.iter().find(|d| d.0 == next).map(|(_, w)| { if *w < min_value { min_value = *w; - min_target = next.clone(); - min_voter = current.clone(); - min_index = i; - min_direction = 1; + maybe_min_target = Some(next.clone()); + maybe_min_voter = Some(current.clone()); + maybe_min_index = Some(i); + maybe_min_direction = Some(1); } }) }); @@ -459,16 +463,39 @@ fn reduce_all(assignments: &mut Vec>) -> u32 ass.distribution.iter().find(|d| d.0 == prev).map(|(_, w)| { if *w < min_value { min_value = *w; - min_target = prev.clone(); - min_voter = current.clone(); - min_index = i; - min_direction = 0; + maybe_min_target = Some(prev.clone()); + maybe_min_voter = Some(current.clone()); + maybe_min_index = Some(i); + maybe_min_direction = Some(0); } }) }); } } + // all of these values must be set by now, we assign them to un-mut values, no + // longer being optional either. + let (min_value, min_target, min_voter, min_index, min_direction) = + match ( + min_value, + maybe_min_target, + maybe_min_voter, + maybe_min_index, + maybe_min_direction, + ) { + ( + min_value, + Some(min_target), + Some(min_voter), + Some(min_index), + Some(min_direction), + ) => (min_value, min_target, min_voter, min_index, min_direction), + _ => { + sp_runtime::print("UNREACHABLE code reached in `reduce` algorithm. This must be a bug."); + break + }, + }; + // if the min edge is in the voter's sub-chain. // [target, ..., X, Y, ... voter] let target_chunk = target_root_path.len() - common_count; @@ -624,8 +651,8 @@ fn reduce_all(assignments: &mut Vec>) -> u32 num_changed } -/// Reduce the given [`Vec>`]. This removes redundant edges from -/// without changing the overall backing of any of the elected candidates. +/// Reduce the given [`Vec>`]. This removes redundant edges without +/// changing the overall backing of any of the elected candidates. /// /// Returns the number of edges removed. /// diff --git a/primitives/npos-elections/src/tests.rs b/primitives/npos-elections/src/tests.rs index bf9ca57677ef..1cf5ea8a2492 100644 --- a/primitives/npos-elections/src/tests.rs +++ b/primitives/npos-elections/src/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,13 +18,10 @@ //! Tests for npos-elections. use crate::{ - balancing, helpers::*, is_score_better, mock::*, seq_phragmen, seq_phragmen_core, setup_inputs, - to_support_map, Assignment, ElectionResult, ExtendedBalance, IndexAssignment, NposSolution, - StakedAssignment, Support, Voter, + balancing, helpers::*, mock::*, seq_phragmen, seq_phragmen_core, setup_inputs, to_support_map, + Assignment, ElectionResult, ExtendedBalance, StakedAssignment, Support, Voter, }; -use rand::{self, SeedableRng}; use sp_arithmetic::{PerU16, Perbill, Percent, Permill}; -use std::convert::TryInto; use substrate_test_utils::assert_eq_uvec; #[test] @@ -192,16 +189,15 @@ fn balancing_core_works() { #[test] fn voter_normalize_ops_works() { use crate::{Candidate, Edge}; - use sp_std::{cell::RefCell, rc::Rc}; // normalize { let c1 = Candidate { who: 10, elected: false, ..Default::default() }; let c2 = Candidate { who: 20, elected: false, ..Default::default() }; let c3 = Candidate { who: 30, elected: false, ..Default::default() }; - let e1 = Edge { candidate: Rc::new(RefCell::new(c1)), weight: 30, ..Default::default() }; - let e2 = Edge { candidate: Rc::new(RefCell::new(c2)), weight: 33, ..Default::default() }; - let e3 = Edge { candidate: Rc::new(RefCell::new(c3)), weight: 30, ..Default::default() }; + let e1 = Edge::new(c1, 30); + let e2 = Edge::new(c2, 33); + let e3 = Edge::new(c3, 30); let mut v = Voter { who: 1, budget: 100, edges: vec![e1, e2, e3], ..Default::default() }; @@ -214,9 +210,9 @@ fn voter_normalize_ops_works() { let c2 = Candidate { who: 20, elected: true, ..Default::default() }; let c3 = Candidate { who: 30, elected: true, ..Default::default() }; - let e1 = Edge { candidate: Rc::new(RefCell::new(c1)), weight: 30, ..Default::default() }; - let e2 = Edge { candidate: Rc::new(RefCell::new(c2)), weight: 33, ..Default::default() }; - let e3 = Edge { candidate: Rc::new(RefCell::new(c3)), weight: 30, ..Default::default() }; + let e1 = Edge::new(c1, 30); + let e2 = Edge::new(c2, 33); + let e3 = Edge::new(c3, 30); let mut v = Voter { who: 1, budget: 100, edges: vec![e1, e2, e3], ..Default::default() }; @@ -231,7 +227,7 @@ fn phragmen_poc_works() { let voters = vec![(10, vec![1, 2]), (20, vec![1, 3]), (30, vec![2, 3])]; let stake_of = create_stake_of(&[(10, 10), (20, 20), (30, 30)]); - let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( + let ElectionResult::<_, Perbill> { winners, assignments } = seq_phragmen( 2, candidates, voters @@ -286,7 +282,7 @@ fn phragmen_poc_works_with_balancing() { let voters = vec![(10, vec![1, 2]), (20, vec![1, 3]), (30, vec![2, 3])]; let stake_of = create_stake_of(&[(10, 10), (20, 20), (30, 30)]); - let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( + let ElectionResult::<_, Perbill> { winners, assignments } = seq_phragmen( 2, candidates, voters @@ -373,7 +369,7 @@ fn phragmen_accuracy_on_large_scale_only_candidates() { (5, (u64::MAX - 2).into()), ]); - let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( + let ElectionResult::<_, Perbill> { winners, assignments } = seq_phragmen( 2, candidates.clone(), auto_generate_self_voters(&candidates) @@ -404,7 +400,7 @@ fn phragmen_accuracy_on_large_scale_voters_and_candidates() { (14, u64::MAX.into()), ]); - let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( + let ElectionResult::<_, Perbill> { winners, assignments } = seq_phragmen( 2, candidates, voters @@ -436,7 +432,7 @@ fn phragmen_accuracy_on_small_scale_self_vote() { let voters = auto_generate_self_voters(&candidates); let stake_of = create_stake_of(&[(40, 0), (10, 1), (20, 2), (30, 1)]); - let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( + let ElectionResult::<_, Perbill> { winners, assignments } = seq_phragmen( 3, candidates, voters @@ -466,7 +462,7 @@ fn phragmen_accuracy_on_small_scale_no_self_vote() { (3, 1), ]); - let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( + let ElectionResult::<_, Perbill> { winners, assignments } = seq_phragmen( 3, candidates, voters @@ -502,7 +498,7 @@ fn phragmen_large_scale_test() { (50, 990000000000000000), ]); - let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( + let ElectionResult::<_, Perbill> { winners, assignments } = seq_phragmen( 2, candidates, voters @@ -529,7 +525,7 @@ fn phragmen_large_scale_test_2() { let stake_of = create_stake_of(&[(2, c_budget.into()), (4, c_budget.into()), (50, nom_budget.into())]); - let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( + let ElectionResult::<_, Perbill> { winners, assignments } = seq_phragmen( 2, candidates, voters @@ -598,7 +594,7 @@ fn elect_has_no_entry_barrier() { let voters = vec![(1, vec![10]), (2, vec![20])]; let stake_of = create_stake_of(&[(1, 10), (2, 10)]); - let ElectionResult { winners, assignments: _ } = seq_phragmen::<_, Perbill>( + let ElectionResult::<_, Perbill> { winners, assignments: _ } = seq_phragmen( 3, candidates, voters @@ -619,7 +615,7 @@ fn phragmen_self_votes_should_be_kept() { let voters = vec![(5, vec![5]), (10, vec![10]), (20, vec![20]), (1, vec![10, 20])]; let stake_of = create_stake_of(&[(5, 5), (10, 10), (20, 20), (1, 8)]); - let result = seq_phragmen::<_, Perbill>( + let result: ElectionResult<_, Perbill> = seq_phragmen( 2, candidates, voters @@ -665,8 +661,8 @@ fn duplicate_target_is_ignored() { let candidates = vec![1, 2, 3]; let voters = vec![(10, 100, vec![1, 1, 2, 3]), (20, 100, vec![2, 3]), (30, 50, vec![1, 1, 2])]; - let ElectionResult { winners, assignments } = - seq_phragmen::<_, Perbill>(2, candidates, voters, None).unwrap(); + let ElectionResult::<_, Perbill> { winners, assignments } = + seq_phragmen(2, candidates, voters, None).unwrap(); assert_eq!(winners, vec![(2, 140), (3, 110)]); assert_eq!( @@ -683,8 +679,8 @@ fn duplicate_target_is_ignored_when_winner() { let candidates = vec![1, 2, 3]; let voters = vec![(10, 100, vec![1, 1, 2, 3]), (20, 100, vec![1, 2])]; - let ElectionResult { winners, assignments } = - seq_phragmen::<_, Perbill>(2, candidates, voters, None).unwrap(); + let ElectionResult::<_, Perbill> { winners, assignments } = + seq_phragmen(2, candidates, voters, None).unwrap(); assert_eq!(winners, vec![(1, 100), (2, 100)]); assert_eq!( @@ -793,6 +789,21 @@ mod assignment_convert_normalize { mod score { use super::*; + use crate::ElectionScore; + use sp_arithmetic::PerThing; + + /// NOTE: in tests, we still use the legacy [u128; 3] since it is more compact. Each `u128` + /// corresponds to element at the respective field index of `ElectionScore`. + impl From<[ExtendedBalance; 3]> for ElectionScore { + fn from(t: [ExtendedBalance; 3]) -> Self { + Self { minimal_stake: t[0], sum_stake: t[1], sum_stake_squared: t[2] } + } + } + + fn is_score_better(this: [u128; 3], that: [u128; 3], p: impl PerThing) -> bool { + ElectionScore::from(this).strict_threshold_better(ElectionScore::from(that), p) + } + #[test] fn score_comparison_is_lexicographical_no_epsilon() { let epsilon = Perbill::zero(); @@ -884,330 +895,24 @@ mod score { false, ); } -} - -mod solution_type { - use super::*; - use codec::{Decode, Encode}; - // these need to come from the same dev-dependency `sp-npos-elections`, not from the crate. - use crate::{generate_solution_type, Assignment, Error as NposError, NposSolution}; - use sp_std::{convert::TryInto, fmt::Debug}; - - #[allow(dead_code)] - mod __private { - // This is just to make sure that the solution can be generated in a scope without any - // imports. - use crate::generate_solution_type; - generate_solution_type!( - #[compact] - struct InnerTestSolutionIsolated::(12) - ); - } - - #[test] - fn solution_struct_works_with_and_without_compact() { - // we use u32 size to make sure compact is smaller. - let without_compact = { - generate_solution_type!( - pub struct InnerTestSolution::< - VoterIndex = u32, - TargetIndex = u32, - Accuracy = TestAccuracy, - >(16) - ); - let solution = InnerTestSolution { - votes1: vec![(2, 20), (4, 40)], - votes2: vec![(1, [(10, p(80))], 11), (5, [(50, p(85))], 51)], - ..Default::default() - }; - - solution.encode().len() - }; - - let with_compact = { - generate_solution_type!( - #[compact] - pub struct InnerTestSolutionCompact::< - VoterIndex = u32, - TargetIndex = u32, - Accuracy = TestAccuracy, - >(16) - ); - let compact = InnerTestSolutionCompact { - votes1: vec![(2, 20), (4, 40)], - votes2: vec![(1, [(10, p(80))], 11), (5, [(50, p(85))], 51)], - ..Default::default() - }; - - compact.encode().len() - }; - - assert!(with_compact < without_compact); - } - - #[test] - fn solution_struct_is_codec() { - let solution = TestSolution { - votes1: vec![(2, 20), (4, 40)], - votes2: vec![(1, [(10, p(80))], 11), (5, [(50, p(85))], 51)], - ..Default::default() - }; - - let encoded = solution.encode(); - - assert_eq!(solution, Decode::decode(&mut &encoded[..]).unwrap()); - assert_eq!(solution.voter_count(), 4); - assert_eq!(solution.edge_count(), 2 + 4); - assert_eq!(solution.unique_targets(), vec![10, 11, 20, 40, 50, 51]); - } - - #[test] - fn remove_voter_works() { - let mut solution = TestSolution { - votes1: vec![(0, 2), (1, 6)], - votes2: vec![(2, [(0, p(80))], 1), (3, [(7, p(85))], 8)], - votes3: vec![(4, [(3, p(50)), (4, p(25))], 5)], - ..Default::default() - }; - - assert!(!solution.remove_voter(11)); - assert!(solution.remove_voter(2)); - assert_eq!( - solution, - TestSolution { - votes1: vec![(0, 2), (1, 6)], - votes2: vec![(3, [(7, p(85))], 8)], - votes3: vec![(4, [(3, p(50)), (4, p(25))], 5,)], - ..Default::default() - }, - ); - - assert!(solution.remove_voter(4)); - assert_eq!( - solution, - TestSolution { - votes1: vec![(0, 2), (1, 6)], - votes2: vec![(3, [(7, p(85))], 8)], - ..Default::default() - }, - ); - - assert!(solution.remove_voter(1)); - assert_eq!( - solution, - TestSolution { - votes1: vec![(0, 2)], - votes2: vec![(3, [(7, p(85))], 8),], - ..Default::default() - }, - ); - } - - #[test] - fn from_and_into_assignment_works() { - let voters = vec![2 as AccountId, 4, 1, 5, 3]; - let targets = vec![ - 10 as AccountId, - 11, - 20, // 2 - 30, - 31, // 4 - 32, - 40, // 6 - 50, - 51, // 8 - ]; - - let assignments = vec![ - Assignment { who: 2 as AccountId, distribution: vec![(20u64, p(100))] }, - Assignment { who: 4, distribution: vec![(40, p(100))] }, - Assignment { who: 1, distribution: vec![(10, p(80)), (11, p(20))] }, - Assignment { who: 5, distribution: vec![(50, p(85)), (51, p(15))] }, - Assignment { who: 3, distribution: vec![(30, p(50)), (31, p(25)), (32, p(25))] }, - ]; - - let voter_index = |a: &AccountId| -> Option { - voters.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() - }; - let target_index = |a: &AccountId| -> Option { - targets.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() - }; - - let solution = - TestSolution::from_assignment(&assignments, voter_index, target_index).unwrap(); - - // basically number of assignments that it is encoding. - assert_eq!(solution.voter_count(), assignments.len()); - assert_eq!( - solution.edge_count(), - assignments.iter().fold(0, |a, b| a + b.distribution.len()), - ); - - assert_eq!( - solution, - TestSolution { - votes1: vec![(0, 2), (1, 6)], - votes2: vec![(2, [(0, p(80))], 1), (3, [(7, p(85))], 8)], - votes3: vec![(4, [(3, p(50)), (4, p(25))], 5)], - ..Default::default() - } - ); - - assert_eq!(solution.unique_targets(), vec![0, 1, 2, 3, 4, 5, 6, 7, 8]); - - let voter_at = |a: u32| -> Option { - voters.get(>::try_into(a).unwrap()).cloned() - }; - let target_at = |a: u16| -> Option { - targets.get(>::try_into(a).unwrap()).cloned() - }; - - assert_eq!(solution.into_assignment(voter_at, target_at).unwrap(), assignments); - } - - #[test] - fn unique_targets_len_edge_count_works() { - // we don't really care about voters here so all duplicates. This is not invalid per se. - let solution = TestSolution { - votes1: vec![(99, 1), (99, 2)], - votes2: vec![(99, [(3, p(10))], 7), (99, [(4, p(10))], 8)], - votes3: vec![(99, [(11, p(10)), (12, p(10))], 13)], - // ensure the last one is also counted. - votes16: vec![( - 99, - [ - (66, p(10)), - (66, p(10)), - (66, p(10)), - (66, p(10)), - (66, p(10)), - (66, p(10)), - (66, p(10)), - (66, p(10)), - (66, p(10)), - (66, p(10)), - (66, p(10)), - (66, p(10)), - (66, p(10)), - (66, p(10)), - (66, p(10)), - ], - 67, - )], - ..Default::default() - }; - - assert_eq!(solution.unique_targets(), vec![1, 2, 3, 4, 7, 8, 11, 12, 13, 66, 67]); - assert_eq!(solution.edge_count(), 2 + (2 * 2) + 3 + 16); - assert_eq!(solution.voter_count(), 6); - - // this one has some duplicates. - let solution = TestSolution { - votes1: vec![(99, 1), (99, 1)], - votes2: vec![(99, [(3, p(10))], 7), (99, [(4, p(10))], 8)], - votes3: vec![(99, [(11, p(10)), (11, p(10))], 13)], - ..Default::default() - }; - - assert_eq!(solution.unique_targets(), vec![1, 3, 4, 7, 8, 11, 13]); - assert_eq!(solution.edge_count(), 2 + (2 * 2) + 3); - assert_eq!(solution.voter_count(), 5); - } #[test] - fn solution_into_assignment_must_report_overflow() { - // in votes2 - let solution = TestSolution { - votes1: Default::default(), - votes2: vec![(0, [(1, p(100))], 2)], - ..Default::default() - }; - - let voter_at = |a: u32| -> Option { Some(a as AccountId) }; - let target_at = |a: u16| -> Option { Some(a as AccountId) }; - - assert_eq!( - solution.into_assignment(&voter_at, &target_at).unwrap_err(), - NposError::SolutionWeightOverflow, - ); - - // in votes3 onwards - let solution = TestSolution { - votes1: Default::default(), - votes2: Default::default(), - votes3: vec![(0, [(1, p(70)), (2, p(80))], 3)], - ..Default::default() - }; - - assert_eq!( - solution.into_assignment(&voter_at, &target_at).unwrap_err(), - NposError::SolutionWeightOverflow, - ); + fn ord_works() { + // equal only when all elements are equal + assert!(ElectionScore::from([10, 5, 15]) == ElectionScore::from([10, 5, 15])); + assert!(ElectionScore::from([10, 5, 15]) != ElectionScore::from([9, 5, 15])); + assert!(ElectionScore::from([10, 5, 15]) != ElectionScore::from([10, 5, 14])); + + // first element greater, rest don't matter + assert!(ElectionScore::from([10, 5, 15]) > ElectionScore::from([8, 5, 25])); + assert!(ElectionScore::from([10, 5, 15]) > ElectionScore::from([9, 20, 5])); + + // second element greater, rest don't matter + assert!(ElectionScore::from([10, 5, 15]) > ElectionScore::from([10, 4, 25])); + assert!(ElectionScore::from([10, 5, 15]) > ElectionScore::from([10, 4, 5])); + + // second element is less, rest don't matter. Note that this is swapped. + assert!(ElectionScore::from([10, 5, 15]) > ElectionScore::from([10, 5, 16])); + assert!(ElectionScore::from([10, 5, 15]) > ElectionScore::from([10, 5, 25])); } - - #[test] - fn target_count_overflow_is_detected() { - let voter_index = |a: &AccountId| -> Option { Some(*a as u32) }; - let target_index = |a: &AccountId| -> Option { Some(*a as u16) }; - - let assignments = vec![Assignment { - who: 1 as AccountId, - distribution: (10..27).map(|i| (i as AccountId, p(i as u8))).collect::>(), - }]; - - let solution = TestSolution::from_assignment(&assignments, voter_index, target_index); - assert_eq!(solution.unwrap_err(), NposError::SolutionTargetOverflow); - } - - #[test] - fn zero_target_count_is_ignored() { - let voters = vec![1 as AccountId, 2]; - let targets = vec![10 as AccountId, 11]; - - let assignments = vec![ - Assignment { who: 1 as AccountId, distribution: vec![(10, p(50)), (11, p(50))] }, - Assignment { who: 2, distribution: vec![] }, - ]; - - let voter_index = |a: &AccountId| -> Option { - voters.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() - }; - let target_index = |a: &AccountId| -> Option { - targets.iter().position(|x| x == a).map(TryInto::try_into).unwrap().ok() - }; - - let solution = - TestSolution::from_assignment(&assignments, voter_index, target_index).unwrap(); - - assert_eq!( - solution, - TestSolution { - votes1: Default::default(), - votes2: vec![(0, [(0, p(50))], 1)], - ..Default::default() - } - ); - } -} - -#[test] -fn index_assignments_generate_same_solution_as_plain_assignments() { - let rng = rand::rngs::SmallRng::seed_from_u64(0); - - let (voters, assignments, candidates) = generate_random_votes(1000, 2500, rng); - let voter_index = make_voter_fn(&voters); - let target_index = make_target_fn(&candidates); - - let solution = - TestSolution::from_assignment(&assignments, &voter_index, &target_index).unwrap(); - - let index_assignments = assignments - .into_iter() - .map(|assignment| IndexAssignment::new(&assignment, &voter_index, &target_index)) - .collect::, _>>() - .unwrap(); - - let index_compact = index_assignments.as_slice().try_into().unwrap(); - - assert_eq!(solution, index_compact); } diff --git a/primitives/npos-elections/src/traits.rs b/primitives/npos-elections/src/traits.rs index 597d7e648fd9..91026f9de4b6 100644 --- a/primitives/npos-elections/src/traits.rs +++ b/primitives/npos-elections/src/traits.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,141 +17,15 @@ //! Traits for the npos-election operations. -use crate::{ - Assignment, ElectionScore, Error, EvaluateSupport, ExtendedBalance, IndexAssignmentOf, - VoteWeight, -}; -use codec::Encode; -use scale_info::TypeInfo; -use sp_arithmetic::{ - traits::{Bounded, UniqueSaturatedInto}, - PerThing, -}; -use sp_std::{ - convert::{TryFrom, TryInto}, - fmt::Debug, - ops::Mul, - prelude::*, -}; +use crate::ExtendedBalance; +use sp_arithmetic::PerThing; +use sp_std::{fmt::Debug, ops::Mul, prelude::*}; /// an aggregator trait for a generic type of a voter/target identifier. This usually maps to /// substrate's account id. -pub trait IdentifierT: Clone + Eq + Default + Ord + Debug + codec::Codec {} -impl IdentifierT for T {} +pub trait IdentifierT: Clone + Eq + Ord + Debug + codec::Codec {} +impl IdentifierT for T {} /// Aggregator trait for a PerThing that can be multiplied by u128 (ExtendedBalance). pub trait PerThing128: PerThing + Mul {} impl> PerThing128 for T {} - -/// Simple Extension trait to easily convert `None` from index closures to `Err`. -/// -/// This is only generated and re-exported for the solution code to use. -#[doc(hidden)] -pub trait __OrInvalidIndex { - fn or_invalid_index(self) -> Result; -} - -impl __OrInvalidIndex for Option { - fn or_invalid_index(self) -> Result { - self.ok_or(Error::SolutionInvalidIndex) - } -} - -/// An opaque index-based, NPoS solution type. -pub trait NposSolution -where - Self: Sized + for<'a> sp_std::convert::TryFrom<&'a [IndexAssignmentOf], Error = Error>, -{ - /// The maximum number of votes that are allowed. - const LIMIT: usize; - - /// The voter type. Needs to be an index (convert to usize). - type VoterIndex: UniqueSaturatedInto - + TryInto - + TryFrom - + Debug - + Copy - + Clone - + Bounded - + Encode - + TypeInfo; - - /// The target type. Needs to be an index (convert to usize). - type TargetIndex: UniqueSaturatedInto - + TryInto - + TryFrom - + Debug - + Copy - + Clone - + Bounded - + Encode - + TypeInfo; - - /// The weight/accuracy type of each vote. - type Accuracy: PerThing128; - - /// Get the length of all the voters that this type is encoding. - /// - /// This is basically the same as the number of assignments, or number of active voters. - fn voter_count(&self) -> usize; - - /// Get the total count of edges. - /// - /// This is effectively in the range of {[`Self::voter_count`], [`Self::voter_count`] * - /// [`Self::LIMIT`]}. - fn edge_count(&self) -> usize; - - /// Get the number of unique targets in the whole struct. - /// - /// Once presented with a list of winners, this set and the set of winners must be - /// equal. - fn unique_targets(&self) -> Vec; - - /// Get the average edge count. - fn average_edge_count(&self) -> usize { - self.edge_count().checked_div(self.voter_count()).unwrap_or(0) - } - - /// Compute the score of this solution type. - fn score( - self, - stake_of: FS, - voter_at: impl Fn(Self::VoterIndex) -> Option, - target_at: impl Fn(Self::TargetIndex) -> Option, - ) -> Result - where - for<'r> FS: Fn(&'r A) -> VoteWeight, - A: IdentifierT, - { - let ratio = self.into_assignment(voter_at, target_at)?; - let staked = crate::helpers::assignment_ratio_to_staked_normalized(ratio, stake_of)?; - let supports = crate::to_supports(&staked); - Ok(supports.evaluate()) - } - - /// Remove a certain voter. - /// - /// This will only search until the first instance of `to_remove`, and return true. If - /// no instance is found (no-op), then it returns false. - /// - /// In other words, if this return true, exactly **one** element must have been removed self. - fn remove_voter(&mut self, to_remove: Self::VoterIndex) -> bool; - - /// Build self from a list of assignments. - fn from_assignment( - assignments: &[Assignment], - voter_index: FV, - target_index: FT, - ) -> Result - where - A: IdentifierT, - for<'r> FV: Fn(&'r A) -> Option, - for<'r> FT: Fn(&'r A) -> Option; - - /// Convert self into a `Vec>` - fn into_assignment( - self, - voter_at: impl Fn(Self::VoterIndex) -> Option, - target_at: impl Fn(Self::TargetIndex) -> Option, - ) -> Result>, Error>; -} diff --git a/primitives/offchain/Cargo.toml b/primitives/offchain/Cargo.toml index dd54147b6c62..ff647f0e7420 100644 --- a/primitives/offchain/Cargo.toml +++ b/primitives/offchain/Cargo.toml @@ -4,8 +4,8 @@ name = "sp-offchain" version = "4.0.0-dev" license = "Apache-2.0" authors = ["Parity Technologies "] -edition = "2018" -homepage = "https://substrate.dev" +edition = "2021" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -13,9 +13,9 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } sp-api = { version = "4.0.0-dev", default-features = false, path = "../api" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } [features] default = ["std"] diff --git a/primitives/offchain/src/lib.rs b/primitives/offchain/src/lib.rs index 72ceca80cfbf..3e967f16d2a8 100644 --- a/primitives/offchain/src/lib.rs +++ b/primitives/offchain/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/panic-handler/Cargo.toml b/primitives/panic-handler/Cargo.toml index ad03baca24eb..7b2023eff0bf 100644 --- a/primitives/panic-handler/Cargo.toml +++ b/primitives/panic-handler/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-panic-handler" -version = "3.0.0" +version = "4.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Custom panic hook with bug report link" documentation = "https://docs.rs/sp-panic-handler" @@ -14,4 +14,6 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -backtrace = "0.3.38" +backtrace = "0.3.64" +regex = "1.5.5" +lazy_static = "1.4.0" diff --git a/primitives/panic-handler/src/lib.rs b/primitives/panic-handler/src/lib.rs index 75b057cebf3e..df1f78da1cbe 100644 --- a/primitives/panic-handler/src/lib.rs +++ b/primitives/panic-handler/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,6 +25,7 @@ //! temporarily be disabled by using an [`AbortGuard`]. use backtrace::Backtrace; +use regex::Regex; use std::{ cell::Cell, io::{self, Write}, @@ -125,6 +126,24 @@ impl Drop for AbortGuard { } } +// NOTE: When making any changes here make sure to also change this function in `sc-tracing`. +fn strip_control_codes(input: &str) -> std::borrow::Cow { + lazy_static::lazy_static! { + static ref RE: Regex = Regex::new(r#"(?x) + \x1b\[[^m]+m| # VT100 escape codes + [ + \x00-\x09\x0B-\x1F # ASCII control codes / Unicode C0 control codes, except \n + \x7F # ASCII delete + \u{80}-\u{9F} # Unicode C1 control codes + \u{202A}-\u{202E} # Unicode left-to-right / right-to-left control characters + \u{2066}-\u{2069} # Same as above + ] + "#).expect("regex parsing doesn't fail; qed"); + } + + RE.replace_all(input, "") +} + /// Function being called when a panic happens. fn panic_hook(info: &PanicInfo, report_url: &str, version: &str) { let location = info.location(); @@ -139,6 +158,8 @@ fn panic_hook(info: &PanicInfo, report_url: &str, version: &str) { }, }; + let msg = strip_control_codes(&msg); + let thread = thread::current(); let name = thread.name().unwrap_or(""); @@ -181,4 +202,44 @@ mod tests { let _guard = AbortGuard::force_abort(); std::panic::catch_unwind(|| panic!()).ok(); } + + fn run_test_in_another_process( + test_name: &str, + test_body: impl FnOnce(), + ) -> Option { + if std::env::var("RUN_FORKED_TEST").is_ok() { + test_body(); + None + } else { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .arg(test_name) + .env("RUN_FORKED_TEST", "1") + .output() + .unwrap(); + + assert!(output.status.success()); + Some(output) + } + } + + #[test] + fn control_characters_are_always_stripped_out_from_the_panic_messages() { + const RAW_LINE: &str = "$$START$$\x1B[1;32mIn\u{202a}\u{202e}\u{2066}\u{2069}ner\n\r\x7ftext!\u{80}\u{9f}\x1B[0m$$END$$"; + const SANITIZED_LINE: &str = "$$START$$Inner\ntext!$$END$$"; + + let output = run_test_in_another_process( + "control_characters_are_always_stripped_out_from_the_panic_messages", + || { + set("test", "1.2.3"); + let _guard = AbortGuard::force_unwind(); + let _ = std::panic::catch_unwind(|| panic!("{}", RAW_LINE)); + }, + ); + + if let Some(output) = output { + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(!stderr.contains(RAW_LINE)); + assert!(stderr.contains(SANITIZED_LINE)); + } + } } diff --git a/primitives/rpc/Cargo.toml b/primitives/rpc/Cargo.toml index 8e1b91a9acb2..dcfd48558de2 100644 --- a/primitives/rpc/Cargo.toml +++ b/primitives/rpc/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-rpc" -version = "4.0.0-dev" +version = "6.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate RPC primitives and utilities." readme = "README.md" @@ -13,9 +13,9 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -serde = { version = "1.0.126", features = ["derive"] } -sp-core = { version = "4.0.0-dev", path = "../core" } +serde = { version = "1.0.136", features = ["derive"] } +sp-core = { version = "6.0.0", path = "../core" } rustc-hash = "1.1.0" [dev-dependencies] -serde_json = "1.0.68" +serde_json = "1.0.79" diff --git a/primitives/rpc/src/lib.rs b/primitives/rpc/src/lib.rs index 0d716d5a07c1..915482e5fca7 100644 --- a/primitives/rpc/src/lib.rs +++ b/primitives/rpc/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/rpc/src/list.rs b/primitives/rpc/src/list.rs index b3d0a4f546e9..7ec4d3491c7e 100644 --- a/primitives/rpc/src/list.rs +++ b/primitives/rpc/src/list.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/rpc/src/number.rs b/primitives/rpc/src/number.rs index 916f2c3d8326..81084a09d4ac 100644 --- a/primitives/rpc/src/number.rs +++ b/primitives/rpc/src/number.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,10 +20,7 @@ use serde::{Deserialize, Serialize}; use sp_core::U256; -use std::{ - convert::{TryFrom, TryInto}, - fmt::Debug, -}; +use std::fmt::Debug; /// A number type that can be serialized both as a number or a string that encodes a number in a /// string. diff --git a/primitives/rpc/src/tracing.rs b/primitives/rpc/src/tracing.rs index 737ace241037..a923ffcb69e0 100644 --- a/primitives/rpc/src/tracing.rs +++ b/primitives/rpc/src/tracing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime-interface/Cargo.toml b/primitives/runtime-interface/Cargo.toml index dd1b84eabfe9..102592288af1 100644 --- a/primitives/runtime-interface/Cargo.toml +++ b/primitives/runtime-interface/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-runtime-interface" -version = "4.0.0-dev" +version = "6.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate runtime interface" documentation = "https://docs.rs/sp-runtime-interface/" @@ -14,24 +14,24 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-wasm-interface = { version = "4.0.0-dev", path = "../wasm-interface", default-features = false } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -sp-tracing = { version = "4.0.0-dev", default-features = false, path = "../tracing" } -sp-runtime-interface-proc-macro = { version = "4.0.0-dev", path = "proc-macro" } -sp-externalities = { version = "0.10.0-dev", optional = true, path = "../externalities" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } +sp-wasm-interface = { version = "6.0.0", path = "../wasm-interface", default-features = false } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +sp-tracing = { version = "5.0.0", default-features = false, path = "../tracing" } +sp-runtime-interface-proc-macro = { version = "5.0.0", path = "proc-macro" } +sp-externalities = { version = "0.12.0", optional = true, path = "../externalities" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } static_assertions = "1.0.0" -primitive-types = { version = "0.10.1", default-features = false } -sp-storage = { version = "4.0.0-dev", default-features = false, path = "../storage" } -impl-trait-for-tuples = "0.2.1" +primitive-types = { version = "0.11.1", default-features = false } +sp-storage = { version = "6.0.0", default-features = false, path = "../storage" } +impl-trait-for-tuples = "0.2.2" [dev-dependencies] sp-runtime-interface-test-wasm = { version = "2.0.0", path = "test-wasm" } -sp-state-machine = { version = "0.10.0-dev", path = "../state-machine" } -sp-core = { version = "4.0.0-dev", path = "../core" } -sp-io = { version = "4.0.0-dev", path = "../io" } -rustversion = "1.0.0" -trybuild = "1.0.43" +sp-state-machine = { version = "0.12.0", path = "../state-machine" } +sp-core = { version = "6.0.0", path = "../core" } +sp-io = { version = "6.0.0", path = "../io" } +rustversion = "1.0.6" +trybuild = "1.0.53" [features] default = [ "std" ] diff --git a/primitives/runtime-interface/proc-macro/Cargo.toml b/primitives/runtime-interface/proc-macro/Cargo.toml index 1eb3bdd9039d..ef59e0119f9f 100644 --- a/primitives/runtime-interface/proc-macro/Cargo.toml +++ b/primitives/runtime-interface/proc-macro/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-runtime-interface-proc-macro" -version = "4.0.0-dev" +version = "5.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "This crate provides procedural macros for usage within the context of the Substrate runtime interface." documentation = "https://docs.rs/sp-runtime-interface-proc-macro" @@ -16,8 +16,8 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -syn = { version = "1.0.58", features = ["full", "visit", "fold", "extra-traits"] } -quote = "1.0.3" -proc-macro2 = "1.0.29" +syn = { version = "1.0.82", features = ["full", "visit", "fold", "extra-traits"] } +quote = "1.0.10" +proc-macro2 = "1.0.36" Inflector = "0.11.4" -proc-macro-crate = "1.0.0" +proc-macro-crate = "1.1.3" diff --git a/primitives/runtime-interface/proc-macro/src/lib.rs b/primitives/runtime-interface/proc-macro/src/lib.rs index 6b0669a298e1..afba38993fe7 100644 --- a/primitives/runtime-interface/proc-macro/src/lib.rs +++ b/primitives/runtime-interface/proc-macro/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime-interface/proc-macro/src/pass_by/codec.rs b/primitives/runtime-interface/proc-macro/src/pass_by/codec.rs index 2be455d17a47..4259b137d67c 100644 --- a/primitives/runtime-interface/proc-macro/src/pass_by/codec.rs +++ b/primitives/runtime-interface/proc-macro/src/pass_by/codec.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime-interface/proc-macro/src/pass_by/enum_.rs b/primitives/runtime-interface/proc-macro/src/pass_by/enum_.rs index f614e4d9f294..7c3f066f6c83 100644 --- a/primitives/runtime-interface/proc-macro/src/pass_by/enum_.rs +++ b/primitives/runtime-interface/proc-macro/src/pass_by/enum_.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -51,7 +51,7 @@ pub fn derive_impl(input: DeriveInput) -> Result { type PassBy = #crate_::pass_by::Enum<#ident>; } - impl #crate_::sp_std::convert::TryFrom for #ident { + impl TryFrom for #ident { type Error = (); fn try_from(inner: u8) -> #crate_::sp_std::result::Result { diff --git a/primitives/runtime-interface/proc-macro/src/pass_by/inner.rs b/primitives/runtime-interface/proc-macro/src/pass_by/inner.rs index 6eaa689d6293..7a527af12946 100644 --- a/primitives/runtime-interface/proc-macro/src/pass_by/inner.rs +++ b/primitives/runtime-interface/proc-macro/src/pass_by/inner.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime-interface/proc-macro/src/pass_by/mod.rs b/primitives/runtime-interface/proc-macro/src/pass_by/mod.rs index 80ac3396759f..e32c2beb8b72 100644 --- a/primitives/runtime-interface/proc-macro/src/pass_by/mod.rs +++ b/primitives/runtime-interface/proc-macro/src/pass_by/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs b/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs index c951dedb6771..b5745e25deb4 100644 --- a/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs +++ b/primitives/runtime-interface/proc-macro/src/runtime_interface/bare_function_interface.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -32,11 +32,12 @@ use crate::utils::{ create_exchangeable_host_function_ident, create_function_ident_with_version, generate_crate_access, get_function_argument_names, get_function_arguments, - get_runtime_interface, + get_runtime_interface, RuntimeInterfaceFunction, }; use syn::{ - parse_quote, spanned::Spanned, FnArg, Ident, ItemTrait, Result, Signature, TraitItemMethod, + parse_quote, spanned::Spanned, FnArg, Ident, ItemTrait, Result, Signature, Token, + TraitItemMethod, }; use proc_macro2::{Span, TokenStream}; @@ -52,7 +53,7 @@ pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool, tracing: bool) -> Res let runtime_interface = get_runtime_interface(trait_def)?; // latest version dispatch - let token_stream: Result = runtime_interface.latest_versions().try_fold( + let token_stream: Result = runtime_interface.latest_versions_to_call().try_fold( TokenStream::new(), |mut t, (latest_version, method)| { t.extend(function_for_method(method, latest_version, is_wasm_only)?); @@ -74,14 +75,14 @@ pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool, tracing: bool) -> Res /// Generates the bare function implementation for the given method for the host and wasm side. fn function_for_method( - method: &TraitItemMethod, + method: &RuntimeInterfaceFunction, latest_version: u32, is_wasm_only: bool, ) -> Result { let std_impl = if !is_wasm_only { function_std_latest_impl(method, latest_version)? } else { quote!() }; - let no_std_impl = function_no_std_impl(method)?; + let no_std_impl = function_no_std_impl(method, is_wasm_only)?; Ok(quote! { #std_impl @@ -91,20 +92,46 @@ fn function_for_method( } /// Generates the bare function implementation for `cfg(not(feature = "std"))`. -fn function_no_std_impl(method: &TraitItemMethod) -> Result { +fn function_no_std_impl( + method: &RuntimeInterfaceFunction, + is_wasm_only: bool, +) -> Result { let function_name = &method.sig.ident; let host_function_name = create_exchangeable_host_function_ident(&method.sig.ident); let args = get_function_arguments(&method.sig); let arg_names = get_function_argument_names(&method.sig); - let return_value = &method.sig.output; + let return_value = if method.should_trap_on_return() { + syn::ReturnType::Type( + ]>::default(), + Box::new(syn::TypeNever { bang_token: ::default() }.into()), + ) + } else { + method.sig.output.clone() + }; + let maybe_unreachable = if method.should_trap_on_return() { + quote! { + ; core::arch::wasm32::unreachable(); + } + } else { + quote! {} + }; + let attrs = method.attrs.iter().filter(|a| !a.path.is_ident("version")); + let cfg_wasm_only = if is_wasm_only { + quote! { #[cfg(target_arch = "wasm32")] } + } else { + quote! {} + }; + Ok(quote! { + #cfg_wasm_only #[cfg(not(feature = "std"))] #( #attrs )* pub fn #function_name( #( #args, )* ) #return_value { // Call the host function #host_function_name.get()( #( #arg_names, )* ) + #maybe_unreachable } }) } diff --git a/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs b/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs index 75498c09c18c..03da0bed5981 100644 --- a/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs +++ b/primitives/runtime-interface/proc-macro/src/runtime_interface/host_function_interface.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,7 +26,7 @@ use crate::utils::{ create_host_function_ident, generate_crate_access, get_function_argument_names, get_function_argument_names_and_types_without_ref, get_function_argument_types, get_function_argument_types_ref_and_mut, get_function_argument_types_without_ref, - get_function_arguments, get_runtime_interface, + get_function_arguments, get_runtime_interface, RuntimeInterfaceFunction, }; use syn::{ @@ -35,29 +35,28 @@ use syn::{ use proc_macro2::{Span, TokenStream}; -use quote::{quote, ToTokens}; +use quote::quote; use inflector::Inflector; -use std::iter::{self, Iterator}; +use std::iter::Iterator; /// Generate the extern host functions for wasm and the `HostFunctions` struct that provides the /// implementations for the host functions on the host. pub fn generate(trait_def: &ItemTrait, is_wasm_only: bool) -> Result { let trait_name = &trait_def.ident; - let extern_host_function_impls = get_runtime_interface(trait_def)?.latest_versions().try_fold( - TokenStream::new(), - |mut t, (version, method)| { + let extern_host_function_impls = get_runtime_interface(trait_def)? + .latest_versions_to_call() + .try_fold(TokenStream::new(), |mut t, (version, method)| { t.extend(generate_extern_host_function(method, version, trait_name)?); Ok::<_, Error>(t) - }, - )?; + })?; let exchangeable_host_functions = get_runtime_interface(trait_def)? - .latest_versions() + .latest_versions_to_call() .try_fold(TokenStream::new(), |mut t, (_, m)| { - t.extend(generate_exchangeable_host_function(m)?); - Ok::<_, Error>(t) - })?; + t.extend(generate_exchangeable_host_function(m)?); + Ok::<_, Error>(t) + })?; let host_functions_struct = generate_host_functions_struct(trait_def, is_wasm_only)?; Ok(quote! { @@ -163,14 +162,20 @@ fn generate_host_functions_struct( ) -> Result { let crate_ = generate_crate_access(); - let host_functions = get_runtime_interface(trait_def)? - .all_versions() - .map(|(version, method)| { - generate_host_function_implementation(&trait_def.ident, method, version, is_wasm_only) - }) - .collect::>>()?; + let mut host_function_impls = Vec::new(); + let mut host_function_names = Vec::new(); + let mut register_bodies = Vec::new(); + for (version, method) in get_runtime_interface(trait_def)?.all_versions() { + let (implementation, name, register_body) = + generate_host_function_implementation(&trait_def.ident, method, version, is_wasm_only)?; + host_function_impls.push(implementation); + host_function_names.push(name); + register_bodies.push(register_body); + } Ok(quote! { + #(#host_function_impls)* + /// Provides implementations for the extern host functions. #[cfg(feature = "std")] pub struct HostFunctions; @@ -178,7 +183,16 @@ fn generate_host_functions_struct( #[cfg(feature = "std")] impl #crate_::sp_wasm_interface::HostFunctions for HostFunctions { fn host_functions() -> Vec<&'static dyn #crate_::sp_wasm_interface::Function> { - vec![ #( #host_functions ),* ] + vec![ #( &#host_function_names as &dyn #crate_::sp_wasm_interface::Function ),* ] + } + + #crate_::sp_wasm_interface::if_wasmtime_is_enabled! { + fn register_static(registry: &mut T) -> core::result::Result<(), T::Error> + where T: #crate_::sp_wasm_interface::HostFunctionRegistry + { + #(#register_bodies)* + Ok(()) + } } } }) @@ -191,50 +205,201 @@ fn generate_host_functions_struct( /// implementation of the function. fn generate_host_function_implementation( trait_name: &Ident, - method: &TraitItemMethod, + method: &RuntimeInterfaceFunction, version: u32, is_wasm_only: bool, -) -> Result { +) -> Result<(TokenStream, Ident, TokenStream)> { let name = create_host_function_ident(&method.sig.ident, version, trait_name).to_string(); let struct_name = Ident::new(&name.to_pascal_case(), Span::call_site()); let crate_ = generate_crate_access(); let signature = generate_wasm_interface_signature_for_host_function(&method.sig)?; - let wasm_to_ffi_values = - generate_wasm_to_ffi_values(&method.sig, trait_name).collect::>>()?; - let ffi_to_host_values = generate_ffi_to_host_value(&method.sig).collect::>>()?; - let host_function_call = generate_host_function_call(&method.sig, version, is_wasm_only); - let into_preallocated_ffi_value = generate_into_preallocated_ffi_value(&method.sig)?; - let convert_return_value = generate_return_value_into_wasm_value(&method.sig); - Ok(quote! { - { - struct #struct_name; + let fn_name = create_function_ident_with_version(&method.sig.ident, version); + let ref_and_mut = get_function_argument_types_ref_and_mut(&method.sig); + + // List of variable names containing WASM FFI-compatible arguments. + let mut ffi_names = Vec::new(); + + // List of `$name: $ty` tokens containing WASM FFI-compatible arguments. + let mut ffi_args_prototype = Vec::new(); + + // List of variable names containing arguments already converted into native Rust types. + // Also includes the preceding `&` or `&mut`. To be used to call the actual implementation of + // the host function. + let mut host_names_with_ref = Vec::new(); + + // List of code snippets to copy over the results returned from a host function through + // any `&mut` arguments back into WASM's linear memory. + let mut copy_data_into_ref_mut_args = Vec::new(); + + // List of code snippets to convert dynamic FFI args (`Value` enum) into concrete static FFI + // types (`u32`, etc.). + let mut convert_args_dynamic_ffi_to_static_ffi = Vec::new(); + + // List of code snippets to convert static FFI args (`u32`, etc.) into native Rust types. + let mut convert_args_static_ffi_to_host = Vec::new(); + + for ((host_name, host_ty), ref_and_mut) in + get_function_argument_names_and_types_without_ref(&method.sig).zip(ref_and_mut) + { + let ffi_name = generate_ffi_value_var_name(&host_name)?; + let host_name_ident = match *host_name { + Pat::Ident(ref pat_ident) => pat_ident.ident.clone(), + _ => unreachable!("`generate_ffi_value_var_name` above would return an error on `Pat` != `Ident`; qed"), + }; + + let ffi_ty = quote! { <#host_ty as #crate_::RIType>::FFIType }; + ffi_args_prototype.push(quote! { #ffi_name: #ffi_ty }); + ffi_names.push(quote! { #ffi_name }); + + let convert_arg_error = format!( + "could not marshal the '{}' argument through the WASM FFI boundary while executing '{}' from interface '{}'", + host_name_ident, + method.sig.ident, + trait_name + ); + convert_args_static_ffi_to_host.push(quote! { + let mut #host_name = <#host_ty as #crate_::host::FromFFIValue>::from_ffi_value(__function_context__, #ffi_name) + .map_err(|err| format!("{}: {}", err, #convert_arg_error))?; + }); - impl #crate_::sp_wasm_interface::Function for #struct_name { - fn name(&self) -> &str { - #name - } + let ref_and_mut_tokens = + ref_and_mut.map(|(token_ref, token_mut)| quote!(#token_ref #token_mut)); - fn signature(&self) -> #crate_::sp_wasm_interface::Signature { - #signature - } + host_names_with_ref.push(quote! { #ref_and_mut_tokens #host_name }); - fn execute( - &self, - __function_context__: &mut dyn #crate_::sp_wasm_interface::FunctionContext, - args: &mut dyn Iterator, - ) -> std::result::Result, String> { - #( #wasm_to_ffi_values )* - #( #ffi_to_host_values )* - #host_function_call - #into_preallocated_ffi_value - #convert_return_value - } + if ref_and_mut.map(|(_, token_mut)| token_mut.is_some()).unwrap_or(false) { + copy_data_into_ref_mut_args.push(quote! { + <#host_ty as #crate_::host::IntoPreallocatedFFIValue>::into_preallocated_ffi_value( + #host_name, + __function_context__, + #ffi_name, + )?; + }); + } + + let arg_count_mismatch_error = format!( + "missing argument '{}': number of arguments given to '{}' from interface '{}' does not match the expected number of arguments", + host_name_ident, + method.sig.ident, + trait_name + ); + convert_args_dynamic_ffi_to_static_ffi.push(quote! { + let #ffi_name = args.next().ok_or_else(|| #arg_count_mismatch_error.to_owned())?; + let #ffi_name: #ffi_ty = #crate_::sp_wasm_interface::TryFromValue::try_from_value(#ffi_name) + .ok_or_else(|| #convert_arg_error.to_owned())?; + }); + } + + let ffi_return_ty = match &method.sig.output { + ReturnType::Type(_, ty) => quote! { <#ty as #crate_::RIType>::FFIType }, + ReturnType::Default => quote! { () }, + }; + + let convert_return_value_host_to_static_ffi = match &method.sig.output { + ReturnType::Type(_, ty) => quote! { + let __result__ = <#ty as #crate_::host::IntoFFIValue>::into_ffi_value( + __result__, + __function_context__ + ); + }, + ReturnType::Default => quote! { + let __result__ = Ok(__result__); + }, + }; + + let convert_return_value_static_ffi_to_dynamic_ffi = match &method.sig.output { + ReturnType::Type(_, _) => quote! { + let __result__ = Ok(Some(#crate_::sp_wasm_interface::IntoValue::into_value(__result__))); + }, + ReturnType::Default => quote! { + let __result__ = Ok(None); + }, + }; + + if is_wasm_only { + host_names_with_ref.push(quote! { + __function_context__ + }); + } + + let implementation = quote! { + #[cfg(feature = "std")] + struct #struct_name; + + #[cfg(feature = "std")] + impl #struct_name { + fn call( + __function_context__: &mut dyn #crate_::sp_wasm_interface::FunctionContext, + #(#ffi_args_prototype),* + ) -> std::result::Result<#ffi_return_ty, String> { + #(#convert_args_static_ffi_to_host)* + let __result__ = #fn_name(#(#host_names_with_ref),*); + #(#copy_data_into_ref_mut_args)* + #convert_return_value_host_to_static_ffi + __result__ } + } - &#struct_name as &dyn #crate_::sp_wasm_interface::Function + #[cfg(feature = "std")] + impl #crate_::sp_wasm_interface::Function for #struct_name { + fn name(&self) -> &str { + #name + } + + fn signature(&self) -> #crate_::sp_wasm_interface::Signature { + #signature + } + + fn execute( + &self, + __function_context__: &mut dyn #crate_::sp_wasm_interface::FunctionContext, + args: &mut dyn Iterator, + ) -> std::result::Result, String> { + #(#convert_args_dynamic_ffi_to_static_ffi)* + let __result__ = Self::call( + __function_context__, + #(#ffi_names),* + )?; + #convert_return_value_static_ffi_to_dynamic_ffi + __result__ + } } - }) + }; + + let register_body = quote! { + registry.register_static( + #crate_::sp_wasm_interface::Function::name(&#struct_name), + |mut caller: #crate_::sp_wasm_interface::wasmtime::Caller, #(#ffi_args_prototype),*| + -> std::result::Result<#ffi_return_ty, #crate_::sp_wasm_interface::wasmtime::Trap> + { + T::with_function_context(caller, move |__function_context__| { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + #struct_name::call( + __function_context__, + #(#ffi_names,)* + ).map_err(#crate_::sp_wasm_interface::wasmtime::Trap::new) + })); + match result { + Ok(result) => result, + Err(panic) => { + let message = + if let Some(message) = panic.downcast_ref::() { + format!("host code panicked while being called by the runtime: {}", message) + } else if let Some(message) = panic.downcast_ref::<&'static str>() { + format!("host code panicked while being called by the runtime: {}", message) + } else { + "host code panicked while being called by the runtime".to_owned() + }; + return Err(#crate_::sp_wasm_interface::wasmtime::Trap::new(message)); + } + } + }) + } + )?; + }; + + Ok((implementation, struct_name, register_body)) } /// Generate the `wasm_interface::Signature` for the given host function `sig`. @@ -260,86 +425,6 @@ fn generate_wasm_interface_signature_for_host_function(sig: &Signature) -> Resul }) } -/// Generate the code that converts the wasm values given to `HostFunctions::execute` into the FFI -/// values. -fn generate_wasm_to_ffi_values<'a>( - sig: &'a Signature, - trait_name: &'a Ident, -) -> impl Iterator> + 'a { - let crate_ = generate_crate_access(); - let function_name = &sig.ident; - let error_message = format!( - "Number of arguments given to `{}` does not match the expected number of arguments!", - function_name, - ); - - get_function_argument_names_and_types_without_ref(sig).map(move |(name, ty)| { - let try_from_error = format!( - "Could not instantiate `{}` from wasm value while executing `{}` from interface `{}`!", - name.to_token_stream(), - function_name, - trait_name, - ); - - let var_name = generate_ffi_value_var_name(&name)?; - - Ok(quote! { - let val = args.next().ok_or_else(|| #error_message)?; - let #var_name = < - <#ty as #crate_::RIType>::FFIType as #crate_::sp_wasm_interface::TryFromValue - >::try_from_value(val).ok_or_else(|| #try_from_error)?; - }) - }) -} - -/// Generate the code to convert the ffi values on the host to the host values using `FromFFIValue`. -fn generate_ffi_to_host_value<'a>( - sig: &'a Signature, -) -> impl Iterator> + 'a { - let mut_access = get_function_argument_types_ref_and_mut(sig); - let crate_ = generate_crate_access(); - - get_function_argument_names_and_types_without_ref(sig) - .zip(mut_access.map(|v| v.and_then(|m| m.1))) - .map(move |((name, ty), mut_access)| { - let ffi_value_var_name = generate_ffi_value_var_name(&name)?; - - Ok(quote! { - let #mut_access #name = <#ty as #crate_::host::FromFFIValue>::from_ffi_value( - __function_context__, - #ffi_value_var_name, - )?; - }) - }) -} - -/// Generate the code to call the host function and the ident that stores the result. -fn generate_host_function_call(sig: &Signature, version: u32, is_wasm_only: bool) -> TokenStream { - let host_function_name = create_function_ident_with_version(&sig.ident, version); - let result_var_name = generate_host_function_result_var_name(&sig.ident); - let ref_and_mut = - get_function_argument_types_ref_and_mut(sig).map(|ram| ram.map(|(vr, vm)| quote!(#vr #vm))); - let names = get_function_argument_names(sig); - - let var_access = names - .zip(ref_and_mut) - .map(|(n, ref_and_mut)| quote!( #ref_and_mut #n )) - // If this is a wasm only interface, we add the function context as last parameter. - .chain( - iter::from_fn(|| if is_wasm_only { Some(quote!(__function_context__)) } else { None }) - .take(1), - ); - - quote! { - let #result_var_name = #host_function_name ( #( #var_access ),* ); - } -} - -/// Generate the variable name that stores the result of the host function. -fn generate_host_function_result_var_name(name: &Ident) -> Ident { - Ident::new(&format!("{}_result", name), Span::call_site()) -} - /// Generate the variable name that stores the FFI value. fn generate_ffi_value_var_name(pat: &Pat) -> Result { match pat { @@ -354,49 +439,3 @@ fn generate_ffi_value_var_name(pat: &Pat) -> Result { _ => Err(Error::new(pat.span(), "Not supported as variable name!")), } } - -/// Generate code that copies data from the host back to preallocated wasm memory. -/// -/// Any argument that is given as `&mut` is interpreted as preallocated memory and it is expected -/// that the type implements `IntoPreAllocatedFFIValue`. -fn generate_into_preallocated_ffi_value(sig: &Signature) -> Result { - let crate_ = generate_crate_access(); - let ref_and_mut = get_function_argument_types_ref_and_mut(sig) - .map(|ram| ram.and_then(|(vr, vm)| vm.map(|v| (vr, v)))); - let names_and_types = get_function_argument_names_and_types_without_ref(sig); - - ref_and_mut - .zip(names_and_types) - .filter_map(|(ram, (name, ty))| ram.map(|_| (name, ty))) - .map(|(name, ty)| { - let ffi_var_name = generate_ffi_value_var_name(&name)?; - - Ok(quote! { - <#ty as #crate_::host::IntoPreallocatedFFIValue>::into_preallocated_ffi_value( - #name, - __function_context__, - #ffi_var_name, - )?; - }) - }) - .collect() -} - -/// Generate the code that converts the return value into the appropriate wasm value. -fn generate_return_value_into_wasm_value(sig: &Signature) -> TokenStream { - let crate_ = generate_crate_access(); - - match &sig.output { - ReturnType::Default => quote!(Ok(None)), - ReturnType::Type(_, ty) => { - let result_var_name = generate_host_function_result_var_name(&sig.ident); - - quote! { - <#ty as #crate_::host::IntoFFIValue>::into_ffi_value( - #result_var_name, - __function_context__, - ).map(#crate_::sp_wasm_interface::IntoValue::into_value).map(Some) - } - }, - } -} diff --git a/primitives/runtime-interface/proc-macro/src/runtime_interface/mod.rs b/primitives/runtime-interface/proc-macro/src/runtime_interface/mod.rs index 78feda663850..d14c1f67ecff 100644 --- a/primitives/runtime-interface/proc-macro/src/runtime_interface/mod.rs +++ b/primitives/runtime-interface/proc-macro/src/runtime_interface/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs b/primitives/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs index c62e3ba87ccd..0ae0f5260286 100644 --- a/primitives/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs +++ b/primitives/runtime-interface/proc-macro/src/runtime_interface/trait_decl_impl.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -153,7 +153,7 @@ fn impl_trait_for_externalities(trait_def: &ItemTrait, is_wasm_only: bool) -> Re let crate_ = generate_crate_access(); let interface = get_runtime_interface(trait_def)?; let methods = interface.all_versions().map(|(version, method)| { - let mut cloned = method.clone(); + let mut cloned = (*method).clone(); cloned.attrs.retain(|a| !a.path.is_ident("version")); cloned.sig.ident = create_function_ident_with_version(&cloned.sig.ident, version); cloned diff --git a/primitives/runtime-interface/proc-macro/src/utils.rs b/primitives/runtime-interface/proc-macro/src/utils.rs index 42ce09c57393..19f7fea023c3 100644 --- a/primitives/runtime-interface/proc-macro/src/utils.rs +++ b/primitives/runtime-interface/proc-macro/src/utils.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,8 +20,8 @@ use proc_macro2::{Span, TokenStream}; use syn::{ - parse_quote, spanned::Spanned, token, Attribute, Error, FnArg, Ident, ItemTrait, Lit, Meta, - NestedMeta, Pat, PatType, Result, Signature, TraitItem, TraitItemMethod, Type, + parse::Parse, parse_quote, spanned::Spanned, token, Error, FnArg, Ident, ItemTrait, LitInt, + Pat, PatType, Result, Signature, TraitItem, TraitItemMethod, Type, }; use proc_macro_crate::{crate_name, FoundCrate}; @@ -35,49 +35,134 @@ use quote::quote; use inflector::Inflector; -/// Runtime interface function with all associated versions of this function. -pub struct RuntimeInterfaceFunction<'a> { - latest_version: u32, - versions: BTreeMap, +mod attributes { + syn::custom_keyword!(register_only); } -impl<'a> RuntimeInterfaceFunction<'a> { - fn new(version: u32, trait_item: &'a TraitItemMethod) -> Self { - Self { - latest_version: version, - versions: { - let mut res = BTreeMap::new(); - res.insert(version, trait_item); - res - }, +/// A concrete, specific version of a runtime interface function. +pub struct RuntimeInterfaceFunction { + item: TraitItemMethod, + should_trap_on_return: bool, +} + +impl std::ops::Deref for RuntimeInterfaceFunction { + type Target = TraitItemMethod; + fn deref(&self) -> &Self::Target { + &self.item + } +} + +impl RuntimeInterfaceFunction { + fn new(item: &TraitItemMethod) -> Result { + let mut item = item.clone(); + let mut should_trap_on_return = false; + item.attrs.retain(|attr| { + if attr.path.is_ident("trap_on_return") { + should_trap_on_return = true; + false + } else { + true + } + }); + + if should_trap_on_return { + if !matches!(item.sig.output, syn::ReturnType::Default) { + return Err(Error::new( + item.sig.ident.span(), + "Methods marked as #[trap_on_return] cannot return anything", + )) + } } + + Ok(Self { item, should_trap_on_return }) + } + + pub fn should_trap_on_return(&self) -> bool { + self.should_trap_on_return + } +} + +/// Runtime interface function with all associated versions of this function. +struct RuntimeInterfaceFunctionSet { + latest_version_to_call: Option, + versions: BTreeMap, +} + +impl RuntimeInterfaceFunctionSet { + fn new(version: VersionAttribute, trait_item: &TraitItemMethod) -> Result { + Ok(Self { + latest_version_to_call: version.is_callable().then(|| version.version), + versions: BTreeMap::from([( + version.version, + RuntimeInterfaceFunction::new(trait_item)?, + )]), + }) } - pub fn latest_version(&self) -> (u32, &TraitItemMethod) { - ( - self.latest_version, - self.versions.get(&self.latest_version).expect( - "If latest_version has a value, the key with this value is in the versions; qed", + /// Returns the latest version of this runtime interface function plus the actual function + /// implementation. + /// + /// This isn't required to be the latest version, because a runtime interface function can be + /// annotated with `register_only` to ensure that the host exposes the host function but it + /// isn't used when compiling the runtime. + pub fn latest_version_to_call(&self) -> Option<(u32, &RuntimeInterfaceFunction)> { + self.latest_version_to_call.map(|v| { + ( + v, + self.versions.get(&v).expect( + "If latest_version_to_call has a value, the key with this value is in the versions; qed", ), ) + }) + } + + /// Add a different version of the function. + fn add_version( + &mut self, + version: VersionAttribute, + trait_item: &TraitItemMethod, + ) -> Result<()> { + if let Some(existing_item) = self.versions.get(&version.version) { + let mut err = Error::new(trait_item.span(), "Duplicated version attribute"); + err.combine(Error::new( + existing_item.span(), + "Previous version with the same number defined here", + )); + + return Err(err) + } + + self.versions + .insert(version.version, RuntimeInterfaceFunction::new(trait_item)?); + if self.latest_version_to_call.map_or(true, |v| v < version.version) && + version.is_callable() + { + self.latest_version_to_call = Some(version.version); + } + + Ok(()) } } /// All functions of a runtime interface grouped by the function names. -pub struct RuntimeInterface<'a> { - items: BTreeMap>, +pub struct RuntimeInterface { + items: BTreeMap, } -impl<'a> RuntimeInterface<'a> { - pub fn latest_versions(&self) -> impl Iterator { - self.items.iter().map(|(_, item)| item.latest_version()) +impl RuntimeInterface { + /// Returns an iterator over all runtime interface function + /// [`latest_version_to_call`](RuntimeInterfaceFunctionSet::latest_version). + pub fn latest_versions_to_call( + &self, + ) -> impl Iterator { + self.items.iter().filter_map(|(_, item)| item.latest_version_to_call()) } - pub fn all_versions(&self) -> impl Iterator { + pub fn all_versions(&self) -> impl Iterator { self.items .iter() .flat_map(|(_, item)| item.versions.iter()) - .map(|(v, i)| (*v, *i)) + .map(|(v, i)| (*v, i)) } } @@ -199,67 +284,76 @@ fn get_trait_methods<'a>(trait_def: &'a ItemTrait) -> impl Iterator Result { - let meta = version.parse_meta()?; - - let err = Err(Error::new( - meta.span(), - "Unexpected `version` attribute. The supported format is `#[version(1)]`", - )); - - match meta { - Meta::List(list) => - if list.nested.len() != 1 { - err - } else if let Some(NestedMeta::Lit(Lit::Int(i))) = list.nested.first() { - i.base10_parse() - } else { - err - }, - _ => err, +/// Supports the following formats: +/// - `#[version(1)]` +/// - `#[version(1, register_only)]` +/// +/// While this struct is only for parsing the inner parts inside the `()`. +struct VersionAttribute { + version: u32, + register_only: Option, +} + +impl VersionAttribute { + /// Is this function version callable? + fn is_callable(&self) -> bool { + self.register_only.is_none() } } -/// Return item version (`#[version(X)]`) attribute, if present. -fn get_item_version(item: &TraitItemMethod) -> Result> { +impl Default for VersionAttribute { + fn default() -> Self { + Self { version: 1, register_only: None } + } +} + +impl Parse for VersionAttribute { + fn parse(input: syn::parse::ParseStream) -> Result { + let version: LitInt = input.parse()?; + let register_only = if input.peek(token::Comma) { + let _ = input.parse::(); + Some(input.parse()?) + } else { + if !input.is_empty() { + return Err(Error::new(input.span(), "Unexpected token, expected `,`.")) + } + + None + }; + + Ok(Self { version: version.base10_parse()?, register_only }) + } +} + +/// Return [`VersionAttribute`], if present. +fn get_item_version(item: &TraitItemMethod) -> Result> { item.attrs .iter() .find(|attr| attr.path.is_ident("version")) - .map(|attr| parse_version_attribute(attr)) + .map(|attr| attr.parse_args()) .transpose() } /// Returns all runtime interface members, with versions. -pub fn get_runtime_interface<'a>(trait_def: &'a ItemTrait) -> Result> { - let mut functions: BTreeMap> = BTreeMap::new(); +pub fn get_runtime_interface(trait_def: &ItemTrait) -> Result { + let mut functions: BTreeMap = BTreeMap::new(); for item in get_trait_methods(trait_def) { let name = item.sig.ident.clone(); - let version = get_item_version(item)?.unwrap_or(1); + let version = get_item_version(item)?.unwrap_or_default(); + + if version.version < 1 { + return Err(Error::new(item.span(), "Version needs to be at least `1`.")) + } match functions.entry(name.clone()) { Entry::Vacant(entry) => { - entry.insert(RuntimeInterfaceFunction::new(version, item)); + entry.insert(RuntimeInterfaceFunctionSet::new(version, item)?); }, Entry::Occupied(mut entry) => { - if let Some(existing_item) = entry.get().versions.get(&version) { - let mut err = Error::new(item.span(), "Duplicated version attribute"); - err.combine(Error::new( - existing_item.span(), - "Previous version with the same number defined here", - )); - - return Err(err) - } - - let interface_item = entry.get_mut(); - if interface_item.latest_version < version { - interface_item.latest_version = version; - } - interface_item.versions.insert(version, item); + entry.get_mut().add_version(version, item)?; }, } } diff --git a/primitives/runtime-interface/src/host.rs b/primitives/runtime-interface/src/host.rs index a6ea96af9004..36492430266a 100644 --- a/primitives/runtime-interface/src/host.rs +++ b/primitives/runtime-interface/src/host.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime-interface/src/impls.rs b/primitives/runtime-interface/src/impls.rs index 40f8e90479f9..3c1927d6b361 100644 --- a/primitives/runtime-interface/src/impls.rs +++ b/primitives/runtime-interface/src/impls.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -95,36 +95,36 @@ macro_rules! impl_traits_for_primitives { } impl_traits_for_primitives! { - u8, u8, - u16, u16, + u8, u32, + u16, u32, u32, u32, u64, u64, - i8, i8, - i16, i16, + i8, i32, + i16, i32, i32, i32, i64, i64, } -/// `bool` is passed as `u8`. +/// `bool` is passed as `u32`. /// /// - `1`: true /// - `0`: false impl RIType for bool { - type FFIType = u8; + type FFIType = u32; } #[cfg(not(feature = "std"))] impl IntoFFIValue for bool { type Owned = (); - fn into_ffi_value(&self) -> WrappedFFIValue { + fn into_ffi_value(&self) -> WrappedFFIValue { if *self { 1 } else { 0 }.into() } } #[cfg(not(feature = "std"))] impl FromFFIValue for bool { - fn from_ffi_value(arg: u8) -> bool { + fn from_ffi_value(arg: u32) -> bool { arg == 1 } } @@ -133,14 +133,14 @@ impl FromFFIValue for bool { impl FromFFIValue for bool { type SelfInstance = bool; - fn from_ffi_value(_: &mut dyn FunctionContext, arg: u8) -> Result { + fn from_ffi_value(_: &mut dyn FunctionContext, arg: u32) -> Result { Ok(arg == 1) } } #[cfg(feature = "std")] impl IntoFFIValue for bool { - fn into_ffi_value(self, _: &mut dyn FunctionContext) -> Result { + fn into_ffi_value(self, _: &mut dyn FunctionContext) -> Result { Ok(if self { 1 } else { 0 }) } } @@ -318,9 +318,8 @@ macro_rules! impl_traits_for_arrays { type SelfInstance = [u8; $n]; fn from_ffi_value(context: &mut dyn FunctionContext, arg: u32) -> Result<[u8; $n]> { - let data = context.read_memory(Pointer::new(arg), $n)?; let mut res = [0u8; $n]; - res.copy_from_slice(&data); + context.read_memory_into(Pointer::new(arg), &mut res)?; Ok(res) } } @@ -514,10 +513,8 @@ macro_rules! for_u128_i128 { type SelfInstance = $type; fn from_ffi_value(context: &mut dyn FunctionContext, arg: u32) -> Result<$type> { - let data = - context.read_memory(Pointer::new(arg), mem::size_of::<$type>() as u32)?; let mut res = [0u8; mem::size_of::<$type>()]; - res.copy_from_slice(&data); + context.read_memory_into(Pointer::new(arg), &mut res)?; Ok(<$type>::from_le_bytes(res)) } } @@ -547,3 +544,7 @@ impl PassBy for sp_wasm_interface::Value { impl PassBy for sp_storage::TrackedStorageKey { type PassBy = Codec; } + +impl PassBy for sp_storage::StateVersion { + type PassBy = Enum; +} diff --git a/primitives/runtime-interface/src/lib.rs b/primitives/runtime-interface/src/lib.rs index 27c4422ed900..f9bf8825f948 100644 --- a/primitives/runtime-interface/src/lib.rs +++ b/primitives/runtime-interface/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -80,17 +80,17 @@ //! //! | Type | FFI type | Conversion | //! |----|----|----| -//! | `u8` | `u8` | `Identity` | -//! | `u16` | `u16` | `Identity` | +//! | `u8` | `u32` | zero-extended to 32-bits | +//! | `u16` | `u32` | zero-extended to 32-bits | //! | `u32` | `u32` | `Identity` | //! | `u64` | `u64` | `Identity` | //! | `i128` | `u32` | `v.as_ptr()` (pointer to a 16 byte array) | -//! | `i8` | `i8` | `Identity` | -//! | `i16` | `i16` | `Identity` | +//! | `i8` | `i32` | sign-extended to 32-bits | +//! | `i16` | `i32` | sign-extended to 32-bits | //! | `i32` | `i32` | `Identity` | //! | `i64` | `i64` | `Identity` | //! | `u128` | `u32` | `v.as_ptr()` (pointer to a 16 byte array) | -//! | `bool` | `u8` | `if v { 1 } else { 0 }` | +//! | `bool` | `u32` | `if v { 1 } else { 0 }` | //! | `&str` | `u64` | v.len() 32bit << 32 | v.as_ptr() 32bit | //! | `&[u8]` | `u64` | v.len() 32bit << 32 | v.as_ptr() 32bit | //! | `Vec` | `u64` | v.len() 32bit << 32 | v.as_ptr() 32bit | @@ -153,6 +153,22 @@ pub use sp_std; /// [17].to_vec() /// } /// +/// /// Call function, different version and only being registered. +/// /// +/// /// This `register_only` version is only being registered, aka exposed to the runtime, +/// /// but the runtime will still use the version 2 of this function. This is useful for when +/// /// new host functions should be introduced. Adding new host functions requires that all +/// /// nodes have the host functions available, because otherwise they fail at instantiation +/// /// of the runtime. With `register_only` the function will not be used when compiling the +/// /// runtime, but it will already be there for a future version of the runtime that will +/// /// switch to using these host function. +/// #[version(3, register_only)] +/// fn call(data: &[u8]) -> Vec { +/// // Here you could call some rather complex code that only compiles on native or +/// // is way faster in native than executing it in wasm. +/// [18].to_vec() +/// } +/// /// /// A function can take a `&self` or `&mut self` argument to get access to the /// /// `Externalities`. (The generated method does not require /// /// this argument, so the function can be called just with the `optional` argument) @@ -177,12 +193,14 @@ pub use sp_std; /// trait Interface { /// fn call_version_1(data: &[u8]) -> Vec; /// fn call_version_2(data: &[u8]) -> Vec; +/// fn call_version_3(data: &[u8]) -> Vec; /// fn set_or_clear_version_1(&mut self, optional: Option>); /// } /// /// impl Interface for &mut dyn sp_externalities::Externalities { /// fn call_version_1(data: &[u8]) -> Vec { Vec::new() } /// fn call_version_2(data: &[u8]) -> Vec { [17].to_vec() } +/// fn call_version_3(data: &[u8]) -> Vec { [18].to_vec() } /// fn set_or_clear_version_1(&mut self, optional: Option>) { /// match optional { /// Some(value) => self.set_storage([1, 2, 3, 4].to_vec(), value), @@ -204,6 +222,10 @@ pub use sp_std; /// <&mut dyn sp_externalities::Externalities as Interface>::call_version_2(data) /// } /// +/// fn call_version_3(data: &[u8]) -> Vec { +/// <&mut dyn sp_externalities::Externalities as Interface>::call_version_3(data) +/// } +/// /// pub fn set_or_clear(optional: Option>) { /// set_or_clear_version_1(optional) /// } @@ -285,8 +307,8 @@ pub use sp_std; /// This instructs the macro to make two significant changes to the generated code: /// /// 1. The generated functions are not callable from the native side. -/// 2. The trait as shown above is not implemented for `Externalities` and is instead -/// implemented for `FunctionExecutor` (from `sp-wasm-interface`). +/// 2. The trait as shown above is not implemented for [`Externalities`] and is instead +/// implemented for `FunctionExecutor` (from `sp-wasm-interface`). /// /// # Disable tracing /// By addding `no_tracing` to the list of options you can prevent the wasm-side interface from @@ -325,7 +347,9 @@ pub use util::{pack_ptr_and_len, unpack_ptr_and_len}; pub trait RIType { /// The ffi type that is used to represent `Self`. #[cfg(feature = "std")] - type FFIType: sp_wasm_interface::IntoValue + sp_wasm_interface::TryFromValue; + type FFIType: sp_wasm_interface::IntoValue + + sp_wasm_interface::TryFromValue + + sp_wasm_interface::WasmTy; #[cfg(not(feature = "std"))] type FFIType; } diff --git a/primitives/runtime-interface/src/pass_by.rs b/primitives/runtime-interface/src/pass_by.rs index 7324e9363804..5d895ff5b3f8 100644 --- a/primitives/runtime-interface/src/pass_by.rs +++ b/primitives/runtime-interface/src/pass_by.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,7 +33,7 @@ use crate::wasm::*; #[cfg(feature = "std")] use sp_wasm_interface::{FunctionContext, Pointer, Result}; -use sp_std::{convert::TryFrom, marker::PhantomData}; +use sp_std::marker::PhantomData; #[cfg(not(feature = "std"))] use sp_std::vec::Vec; @@ -382,7 +382,7 @@ impl, I: RIType> RIType for Inner { /// } /// } /// -/// impl std::convert::TryFrom for Test { +/// impl TryFrom for Test { /// type Error = (); /// /// fn try_from(val: u8) -> Result { @@ -403,11 +403,11 @@ pub struct Enum + TryFrom>(PhantomData); #[cfg(feature = "std")] impl + TryFrom> PassByImpl for Enum { fn into_ffi_value(instance: T, _: &mut dyn FunctionContext) -> Result { - Ok(instance.into()) + Ok(instance.into() as u32) } fn from_ffi_value(_: &mut dyn FunctionContext, arg: Self::FFIType) -> Result { - T::try_from(arg).map_err(|_| format!("Invalid enum discriminant: {}", arg)) + T::try_from(arg as u8).map_err(|_| format!("Invalid enum discriminant: {}", arg)) } } @@ -417,17 +417,17 @@ impl + TryFrom> PassByImpl for Enum { fn into_ffi_value(instance: &T) -> WrappedFFIValue { let value: u8 = (*instance).into(); - value.into() + (value as u32).into() } fn from_ffi_value(arg: Self::FFIType) -> T { - T::try_from(arg).expect("Host to wasm provides a valid enum discriminant; qed") + T::try_from(arg as u8).expect("Host to wasm provides a valid enum discriminant; qed") } } -/// The type is passed as `u8`. +/// The type is passed as `u32`. /// /// The value is corresponds to the discriminant of the variant. impl + TryFrom> RIType for Enum { - type FFIType = u8; + type FFIType = u32; } diff --git a/primitives/runtime-interface/src/util.rs b/primitives/runtime-interface/src/util.rs index 31045c83c9dc..fe8afe99508a 100644 --- a/primitives/runtime-interface/src/util.rs +++ b/primitives/runtime-interface/src/util.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime-interface/src/wasm.rs b/primitives/runtime-interface/src/wasm.rs index 28613f81a68b..4ed27687a10a 100644 --- a/primitives/runtime-interface/src/wasm.rs +++ b/primitives/runtime-interface/src/wasm.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml b/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml index 3ae5d78b0ef9..1b0c9f57e5c4 100644 --- a/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml +++ b/primitives/runtime-interface/test-wasm-deprecated/Cargo.toml @@ -2,10 +2,10 @@ name = "sp-runtime-interface-test-wasm-deprecated" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" build = "build.rs" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" publish = false @@ -13,10 +13,10 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-runtime-interface = { version = "4.0.0-dev", default-features = false, path = "../" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../io" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../core" } +sp-runtime-interface = { version = "6.0.0", default-features = false, path = "../" } +sp-std = { version = "4.0.0", default-features = false, path = "../../std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../io" } +sp-core = { version = "6.0.0", default-features = false, path = "../../core" } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } diff --git a/primitives/runtime-interface/test-wasm-deprecated/build.rs b/primitives/runtime-interface/test-wasm-deprecated/build.rs index a1c4b2d892cf..b773ed9cf6fb 100644 --- a/primitives/runtime-interface/test-wasm-deprecated/build.rs +++ b/primitives/runtime-interface/test-wasm-deprecated/build.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime-interface/test-wasm-deprecated/src/lib.rs b/primitives/runtime-interface/test-wasm-deprecated/src/lib.rs index 4a59e4fe8aa5..512d52214c12 100644 --- a/primitives/runtime-interface/test-wasm-deprecated/src/lib.rs +++ b/primitives/runtime-interface/test-wasm-deprecated/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime-interface/test-wasm/Cargo.toml b/primitives/runtime-interface/test-wasm/Cargo.toml index 7c7d3e10b2d0..4dfa0fd015d8 100644 --- a/primitives/runtime-interface/test-wasm/Cargo.toml +++ b/primitives/runtime-interface/test-wasm/Cargo.toml @@ -2,10 +2,10 @@ name = "sp-runtime-interface-test-wasm" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" build = "build.rs" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" publish = false @@ -13,10 +13,10 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-runtime-interface = { version = "4.0.0-dev", default-features = false, path = "../" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../io" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../core" } +sp-runtime-interface = { version = "6.0.0", default-features = false, path = "../" } +sp-std = { version = "4.0.0", default-features = false, path = "../../std" } +sp-io = { version = "6.0.0", default-features = false, path = "../../io" } +sp-core = { version = "6.0.0", default-features = false, path = "../../core" } [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../../utils/wasm-builder" } diff --git a/primitives/runtime-interface/test-wasm/build.rs b/primitives/runtime-interface/test-wasm/build.rs index a1c4b2d892cf..b773ed9cf6fb 100644 --- a/primitives/runtime-interface/test-wasm/build.rs +++ b/primitives/runtime-interface/test-wasm/build.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime-interface/test-wasm/src/lib.rs b/primitives/runtime-interface/test-wasm/src/lib.rs index 72acdd4ff8d6..f518a2e17498 100644 --- a/primitives/runtime-interface/test-wasm/src/lib.rs +++ b/primitives/runtime-interface/test-wasm/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,7 @@ use sp_runtime_interface::runtime_interface; #[cfg(not(feature = "std"))] -use sp_std::{convert::TryFrom, mem, prelude::*}; +use sp_std::{mem, prelude::*}; use sp_core::{sr25519::Public, wasm_export_functions}; @@ -123,6 +123,15 @@ pub trait TestApi { data == 42 } + fn test_versionning_register_only(&self, data: u32) -> bool { + data == 80 + } + + #[version(2, register_only)] + fn test_versionning_register_only(&self, data: u32) -> bool { + data == 42 + } + /// Returns the input values as tuple. fn return_input_as_tuple( a: Vec, @@ -271,6 +280,13 @@ wasm_export_functions! { assert!(!test_api::test_versionning(102)); } + fn test_versionning_register_only_works() { + // Ensure that we will import the version of the runtime interface function that + // isn't tagged with `register_only`. + assert!(!test_api::test_versionning_register_only(42)); + assert!(test_api::test_versionning_register_only(80)); + } + fn test_return_input_as_tuple() { let a = vec![1, 3, 4, 5]; let b = 10000; diff --git a/primitives/runtime-interface/test/Cargo.toml b/primitives/runtime-interface/test/Cargo.toml index 377729521fcf..3b51088ea377 100644 --- a/primitives/runtime-interface/test/Cargo.toml +++ b/primitives/runtime-interface/test/Cargo.toml @@ -2,23 +2,23 @@ name = "sp-runtime-interface-test" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" publish = false -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-runtime-interface = { version = "4.0.0-dev", path = "../" } +sp-runtime-interface = { version = "6.0.0", path = "../" } sc-executor = { version = "0.10.0-dev", path = "../../../client/executor" } sc-executor-common = { version = "0.10.0-dev", path = "../../../client/executor/common" } sp-runtime-interface-test-wasm = { version = "2.0.0", path = "../test-wasm" } sp-runtime-interface-test-wasm-deprecated = { version = "2.0.0", path = "../test-wasm-deprecated" } -sp-state-machine = { version = "0.10.0-dev", path = "../../state-machine" } -sp-runtime = { version = "4.0.0-dev", path = "../../runtime" } -sp-io = { version = "4.0.0-dev", path = "../../io" } -tracing = "0.1.25" +sp-state-machine = { version = "0.12.0", path = "../../state-machine" } +sp-runtime = { version = "6.0.0", path = "../../runtime" } +sp-io = { version = "6.0.0", path = "../../io" } +tracing = "0.1.29" tracing-core = "0.1.17" diff --git a/primitives/runtime-interface/test/src/lib.rs b/primitives/runtime-interface/test/src/lib.rs index 82c50fffeb8d..1ab8dbfbbff2 100644 --- a/primitives/runtime-interface/test/src/lib.rs +++ b/primitives/runtime-interface/test/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,14 +24,14 @@ use sp_runtime_interface_test_wasm::{test_api::HostFunctions, wasm_binary_unwrap use sp_runtime_interface_test_wasm_deprecated::wasm_binary_unwrap as wasm_binary_deprecated_unwrap; use sc_executor_common::runtime_blob::RuntimeBlob; -use sp_wasm_interface::HostFunctions as HostFunctionsT; +use sp_wasm_interface::{ExtendedHostFunctions, HostFunctions as HostFunctionsT}; use std::{ collections::HashSet, sync::{Arc, Mutex}, }; -type TestExternalities = sp_state_machine::TestExternalities; +type TestExternalities = sp_state_machine::TestExternalities; fn call_wasm_method_with_result( binary: &[u8], @@ -39,16 +39,11 @@ fn call_wasm_method_with_result( ) -> Result { let mut ext = TestExternalities::default(); let mut ext_ext = ext.ext(); - let mut host_functions = HF::host_functions(); - host_functions.extend(sp_io::SubstrateHostFunctions::host_functions()); - - let executor = sc_executor::WasmExecutor::new( - sc_executor::WasmExecutionMethod::Interpreted, - Some(8), - host_functions, - 8, - None, - ); + + let executor = sc_executor::WasmExecutor::< + ExtendedHostFunctions, + >::new(sc_executor::WasmExecutionMethod::Interpreted, Some(8), 8, None, 2); + executor .uncached_call( RuntimeBlob::uncompress_if_needed(binary).expect("Failed to parse binary"), @@ -67,17 +62,17 @@ fn call_wasm_method(binary: &[u8], method: &str) -> TestExte #[test] fn test_return_data() { - call_wasm_method::(&wasm_binary_unwrap()[..], "test_return_data"); + call_wasm_method::(wasm_binary_unwrap(), "test_return_data"); } #[test] fn test_return_option_data() { - call_wasm_method::(&wasm_binary_unwrap()[..], "test_return_option_data"); + call_wasm_method::(wasm_binary_unwrap(), "test_return_option_data"); } #[test] fn test_set_storage() { - let mut ext = call_wasm_method::(&wasm_binary_unwrap()[..], "test_set_storage"); + let mut ext = call_wasm_method::(wasm_binary_unwrap(), "test_set_storage"); let expected = "world"; assert_eq!(expected.as_bytes(), &ext.ext().storage("hello".as_bytes()).unwrap()[..]); @@ -86,30 +81,30 @@ fn test_set_storage() { #[test] fn test_return_value_into_mutable_reference() { call_wasm_method::( - &wasm_binary_unwrap()[..], + wasm_binary_unwrap(), "test_return_value_into_mutable_reference", ); } #[test] fn test_get_and_return_array() { - call_wasm_method::(&wasm_binary_unwrap()[..], "test_get_and_return_array"); + call_wasm_method::(wasm_binary_unwrap(), "test_get_and_return_array"); } #[test] fn test_array_as_mutable_reference() { - call_wasm_method::(&wasm_binary_unwrap()[..], "test_array_as_mutable_reference"); + call_wasm_method::(wasm_binary_unwrap(), "test_array_as_mutable_reference"); } #[test] fn test_return_input_public_key() { - call_wasm_method::(&wasm_binary_unwrap()[..], "test_return_input_public_key"); + call_wasm_method::(wasm_binary_unwrap(), "test_return_input_public_key"); } #[test] fn host_function_not_found() { - let err = call_wasm_method_with_result::<()>(&wasm_binary_unwrap()[..], "test_return_data") - .unwrap_err(); + let err = + call_wasm_method_with_result::<()>(wasm_binary_unwrap(), "test_return_data").unwrap_err(); assert!(err.contains("Instantiation: Export ")); assert!(err.contains(" not found")); @@ -119,7 +114,7 @@ fn host_function_not_found() { #[should_panic(expected = "Invalid utf8 data provided")] fn test_invalid_utf8_data_should_return_an_error() { call_wasm_method::( - &wasm_binary_unwrap()[..], + wasm_binary_unwrap(), "test_invalid_utf8_data_should_return_an_error", ); } @@ -127,7 +122,7 @@ fn test_invalid_utf8_data_should_return_an_error() { #[test] fn test_overwrite_native_function_implementation() { call_wasm_method::( - &wasm_binary_unwrap()[..], + wasm_binary_unwrap(), "test_overwrite_native_function_implementation", ); } @@ -135,7 +130,7 @@ fn test_overwrite_native_function_implementation() { #[test] fn test_u128_i128_as_parameter_and_return_value() { call_wasm_method::( - &wasm_binary_unwrap()[..], + wasm_binary_unwrap(), "test_u128_i128_as_parameter_and_return_value", ); } @@ -143,7 +138,7 @@ fn test_u128_i128_as_parameter_and_return_value() { #[test] fn test_vec_return_value_memory_is_freed() { call_wasm_method::( - &wasm_binary_unwrap()[..], + wasm_binary_unwrap(), "test_vec_return_value_memory_is_freed", ); } @@ -151,7 +146,7 @@ fn test_vec_return_value_memory_is_freed() { #[test] fn test_encoded_return_value_memory_is_freed() { call_wasm_method::( - &wasm_binary_unwrap()[..], + wasm_binary_unwrap(), "test_encoded_return_value_memory_is_freed", ); } @@ -159,7 +154,7 @@ fn test_encoded_return_value_memory_is_freed() { #[test] fn test_array_return_value_memory_is_freed() { call_wasm_method::( - &wasm_binary_unwrap()[..], + wasm_binary_unwrap(), "test_array_return_value_memory_is_freed", ); } @@ -167,14 +162,16 @@ fn test_array_return_value_memory_is_freed() { #[test] fn test_versionining_with_new_host_works() { // We call to the new wasm binary with new host function. - call_wasm_method::(&wasm_binary_unwrap()[..], "test_versionning_works"); + call_wasm_method::(wasm_binary_unwrap(), "test_versionning_works"); // we call to the old wasm binary with a new host functions // old versions of host functions should be called and test should be ok! - call_wasm_method::( - &wasm_binary_deprecated_unwrap()[..], - "test_versionning_works", - ); + call_wasm_method::(wasm_binary_deprecated_unwrap(), "test_versionning_works"); +} + +#[test] +fn test_versionining_register_only() { + call_wasm_method::(wasm_binary_unwrap(), "test_versionning_register_only_works"); } #[test] @@ -229,7 +226,7 @@ fn test_tracing() { let _guard = tracing::subscriber::set_default(subscriber.clone()); // Call some method to generate a trace - call_wasm_method::(&wasm_binary_unwrap()[..], "test_return_data"); + call_wasm_method::(wasm_binary_unwrap(), "test_return_data"); let inner = subscriber.0.lock().unwrap(); assert!(inner.spans.contains("return_input_version_1")); @@ -237,5 +234,5 @@ fn test_tracing() { #[test] fn test_return_input_as_tuple() { - call_wasm_method::(&wasm_binary_unwrap()[..], "test_return_input_as_tuple"); + call_wasm_method::(wasm_binary_unwrap(), "test_return_input_as_tuple"); } diff --git a/primitives/runtime-interface/tests/ui.rs b/primitives/runtime-interface/tests/ui.rs index 5a6025f463af..9e3af145dc56 100644 --- a/primitives/runtime-interface/tests/ui.rs +++ b/primitives/runtime-interface/tests/ui.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime/Cargo.toml b/primitives/runtime/Cargo.toml index 5ac5bcf1963e..22d2ea62ae45 100644 --- a/primitives/runtime/Cargo.toml +++ b/primitives/runtime/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-runtime" -version = "4.0.0-dev" +version = "6.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Runtime Modules shared primitive types." documentation = "https://docs.rs/sp-runtime" @@ -15,29 +15,30 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] -serde = { version = "1.0.126", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "2.2.0", default-features = false, features = ["derive", "max-encoded-len"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../application-crypto" } -sp-arithmetic = { version = "4.0.0-dev", default-features = false, path = "../arithmetic" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../io" } +serde = { version = "1.0.136", optional = true, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../application-crypto" } +sp-arithmetic = { version = "5.0.0", default-features = false, path = "../arithmetic" } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +sp-io = { version = "6.0.0", default-features = false, path = "../io" } log = { version = "0.4.14", default-features = false } paste = "1.0" rand = { version = "0.7.2", optional = true } -impl-trait-for-tuples = "0.2.1" -parity-util-mem = { version = "0.10.0", default-features = false, features = ["primitive-types"] } +impl-trait-for-tuples = "0.2.2" +parity-util-mem = { version = "0.11.0", default-features = false, features = ["primitive-types"] } hash256-std-hasher = { version = "0.15.2", default-features = false } either = { version = "1.5", default-features = false } [dev-dependencies] -serde_json = "1.0.68" +serde_json = "1.0.79" rand = "0.7.2" -sp-state-machine = { version = "0.10.0-dev", path = "../state-machine" } +sp-state-machine = { version = "0.12.0", path = "../state-machine" } sp-api = { version = "4.0.0-dev", path = "../api" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } +sp-tracing = { version = "5.0.0", path = "../../primitives/tracing" } +zstd = "0.9" [features] bench = [] diff --git a/primitives/runtime/src/curve.rs b/primitives/runtime/src/curve.rs index d6bd94c2bff7..c6bfa6601787 100644 --- a/primitives/runtime/src/curve.rs +++ b/primitives/runtime/src/curve.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -110,8 +110,6 @@ where #[test] fn test_multiply_by_rational_saturating() { - use std::convert::TryInto; - let div = 100u32; for value in 0..=div { for p in 0..=div { @@ -132,8 +130,6 @@ fn test_multiply_by_rational_saturating() { #[test] fn test_calculate_for_fraction_times_denominator() { - use std::convert::TryInto; - let curve = PiecewiseLinear { points: &[ (Perbill::from_parts(0_000_000_000), Perbill::from_parts(0_500_000_000)), diff --git a/primitives/runtime/src/generic/block.rs b/primitives/runtime/src/generic/block.rs index 21a01933bc69..2cd350b2c5ba 100644 --- a/primitives/runtime/src/generic/block.rs +++ b/primitives/runtime/src/generic/block.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -35,7 +35,7 @@ use sp_core::RuntimeDebug; use sp_std::prelude::*; /// Something to identify a block. -#[derive(PartialEq, Eq, Clone, RuntimeDebug)] +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Serialize))] #[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] #[cfg_attr(feature = "std", serde(deny_unknown_fields))] diff --git a/primitives/runtime/src/generic/checked_extrinsic.rs b/primitives/runtime/src/generic/checked_extrinsic.rs index b2044a6cf74f..5d6c657a6897 100644 --- a/primitives/runtime/src/generic/checked_extrinsic.rs +++ b/primitives/runtime/src/generic/checked_extrinsic.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -70,20 +70,26 @@ where info: &DispatchInfoOf, len: usize, ) -> crate::ApplyExtrinsicResultWithInfo> { - let (maybe_who, pre) = if let Some((id, extra)) = self.signed { + let (maybe_who, maybe_pre) = if let Some((id, extra)) = self.signed { let pre = Extra::pre_dispatch(extra, &id, &self.function, info, len)?; - (Some(id), pre) + (Some(id), Some(pre)) } else { - let pre = Extra::pre_dispatch_unsigned(&self.function, info, len)?; + Extra::pre_dispatch_unsigned(&self.function, info, len)?; U::pre_dispatch(&self.function)?; - (None, pre) + (None, None) }; let res = self.function.dispatch(Origin::from(maybe_who)); let post_info = match res { Ok(info) => info, Err(err) => err.post_info, }; - Extra::post_dispatch(pre, info, &post_info, len, &res.map(|_| ()).map_err(|e| e.error))?; + Extra::post_dispatch( + maybe_pre, + info, + &post_info, + len, + &res.map(|_| ()).map_err(|e| e.error), + )?; Ok(res) } } diff --git a/primitives/runtime/src/generic/digest.rs b/primitives/runtime/src/generic/digest.rs index 87af9bc77a5f..55e0d69fad33 100644 --- a/primitives/runtime/src/generic/digest.rs +++ b/primitives/runtime/src/generic/digest.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,59 +26,43 @@ use crate::{ codec::{Decode, Encode, Error, Input}, scale_info::{ build::{Fields, Variants}, - meta_type, Path, Type, TypeInfo, TypeParameter, + Path, Type, TypeInfo, }, ConsensusEngineId, }; -use sp_core::{ChangesTrieConfiguration, RuntimeDebug}; +use sp_core::RuntimeDebug; /// Generic header digest. -#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] +#[derive(PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo, Default)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize, parity_util_mem::MallocSizeOf))] -pub struct Digest { +pub struct Digest { /// A list of logs in the digest. - #[cfg_attr( - feature = "std", - serde(bound(serialize = "Hash: codec::Codec", deserialize = "Hash: codec::Codec")) - )] - pub logs: Vec>, + pub logs: Vec, } -impl Default for Digest { - fn default() -> Self { - Self { logs: Vec::new() } - } -} - -impl Digest { +impl Digest { /// Get reference to all digest items. - pub fn logs(&self) -> &[DigestItem] { + pub fn logs(&self) -> &[DigestItem] { &self.logs } /// Push new digest item. - pub fn push(&mut self, item: DigestItem) { + pub fn push(&mut self, item: DigestItem) { self.logs.push(item); } /// Pop a digest item. - pub fn pop(&mut self) -> Option> { + pub fn pop(&mut self) -> Option { self.logs.pop() } /// Get reference to the first digest item that matches the passed predicate. - pub fn log) -> Option<&T>>( - &self, - predicate: F, - ) -> Option<&T> { + pub fn log Option<&T>>(&self, predicate: F) -> Option<&T> { self.logs().iter().find_map(predicate) } /// Get a conversion of the first digest item that successfully converts using the function. - pub fn convert_first) -> Option>( - &self, - predicate: F, - ) -> Option { + pub fn convert_first Option>(&self, predicate: F) -> Option { self.logs().iter().find_map(predicate) } } @@ -87,12 +71,7 @@ impl Digest { /// provide opaque access to other items. #[derive(PartialEq, Eq, Clone, RuntimeDebug)] #[cfg_attr(feature = "std", derive(parity_util_mem::MallocSizeOf))] -pub enum DigestItem { - /// System digest item that contains the root of changes trie at given - /// block. It is created for every block iff runtime supports changes - /// trie creation. - ChangesTrieRoot(Hash), - +pub enum DigestItem { /// A pre-runtime digest. /// /// These are messages from the consensus engine to the runtime, although @@ -116,10 +95,6 @@ pub enum DigestItem { /// by runtimes. Seal(ConsensusEngineId, Vec), - /// Digest item that contains signal from changes tries manager to the - /// native code. - ChangesTrieSignal(ChangesTrieSignal), - /// Some other thing. Unsupported and experimental. Other(Vec), @@ -132,25 +107,8 @@ pub enum DigestItem { RuntimeEnvironmentUpdated, } -/// Available changes trie signals. -#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo)] -#[cfg_attr(feature = "std", derive(Debug, parity_util_mem::MallocSizeOf))] -pub enum ChangesTrieSignal { - /// New changes trie configuration is enacted, starting from **next block**. - /// - /// The block that emits this signal will contain changes trie (CT) that covers - /// blocks range [BEGIN; current block], where BEGIN is (order matters): - /// - LAST_TOP_LEVEL_DIGEST_BLOCK+1 if top level digest CT has ever been created using current - /// configuration AND the last top level digest CT has been created at block - /// LAST_TOP_LEVEL_DIGEST_BLOCK; - /// - LAST_CONFIGURATION_CHANGE_BLOCK+1 if there has been CT configuration change before and - /// the last configuration change happened at block LAST_CONFIGURATION_CHANGE_BLOCK; - /// - 1 otherwise. - NewConfiguration(Option), -} - #[cfg(feature = "std")] -impl serde::Serialize for DigestItem { +impl serde::Serialize for DigestItem { fn serialize(&self, seq: S) -> Result where S: serde::Serializer, @@ -160,7 +118,7 @@ impl serde::Serialize for DigestItem { } #[cfg(feature = "std")] -impl<'a, Hash: Decode> serde::Deserialize<'a> for DigestItem { +impl<'a> serde::Deserialize<'a> for DigestItem { fn deserialize(de: D) -> Result where D: serde::Deserializer<'a>, @@ -171,94 +129,64 @@ impl<'a, Hash: Decode> serde::Deserialize<'a> for DigestItem { } } -impl TypeInfo for DigestItem -where - Hash: TypeInfo + 'static, -{ +impl TypeInfo for DigestItem { type Identity = Self; fn type_info() -> Type { - Type::builder() - .path(Path::new("DigestItem", module_path!())) - .type_params(vec![TypeParameter::new("Hash", Some(meta_type::()))]) - .variant( - Variants::new() - .variant("ChangesTrieRoot", |v| { - v.index(DigestItemType::ChangesTrieRoot as u8) - .fields(Fields::unnamed().field(|f| f.ty::().type_name("Hash"))) - }) - .variant("PreRuntime", |v| { - v.index(DigestItemType::PreRuntime as u8).fields( - Fields::unnamed() - .field(|f| { - f.ty::().type_name("ConsensusEngineId") - }) - .field(|f| f.ty::>().type_name("Vec")), - ) - }) - .variant("Consensus", |v| { - v.index(DigestItemType::Consensus as u8).fields( - Fields::unnamed() - .field(|f| { - f.ty::().type_name("ConsensusEngineId") - }) - .field(|f| f.ty::>().type_name("Vec")), - ) - }) - .variant("Seal", |v| { - v.index(DigestItemType::Seal as u8).fields( - Fields::unnamed() - .field(|f| { - f.ty::().type_name("ConsensusEngineId") - }) - .field(|f| f.ty::>().type_name("Vec")), - ) - }) - .variant("ChangesTrieSignal", |v| { - v.index(DigestItemType::ChangesTrieSignal as u8).fields( - Fields::unnamed().field(|f| { - f.ty::().type_name("ChangesTrieSignal") - }), - ) - }) - .variant("Other", |v| { - v.index(DigestItemType::Other as u8).fields( - Fields::unnamed().field(|f| f.ty::>().type_name("Vec")), - ) - }) - .variant("RuntimeEnvironmentUpdated", |v| { - v.index(DigestItemType::RuntimeEnvironmentUpdated as u8) - .fields(Fields::unit()) - }), - ) + Type::builder().path(Path::new("DigestItem", module_path!())).variant( + Variants::new() + .variant("PreRuntime", |v| { + v.index(DigestItemType::PreRuntime as u8).fields( + Fields::unnamed() + .field(|f| f.ty::().type_name("ConsensusEngineId")) + .field(|f| f.ty::>().type_name("Vec")), + ) + }) + .variant("Consensus", |v| { + v.index(DigestItemType::Consensus as u8).fields( + Fields::unnamed() + .field(|f| f.ty::().type_name("ConsensusEngineId")) + .field(|f| f.ty::>().type_name("Vec")), + ) + }) + .variant("Seal", |v| { + v.index(DigestItemType::Seal as u8).fields( + Fields::unnamed() + .field(|f| f.ty::().type_name("ConsensusEngineId")) + .field(|f| f.ty::>().type_name("Vec")), + ) + }) + .variant("Other", |v| { + v.index(DigestItemType::Other as u8) + .fields(Fields::unnamed().field(|f| f.ty::>().type_name("Vec"))) + }) + .variant("RuntimeEnvironmentUpdated", |v| { + v.index(DigestItemType::RuntimeEnvironmentUpdated as u8).fields(Fields::unit()) + }), + ) } } /// A 'referencing view' for digest item. Does not own its contents. Used by /// final runtime implementations for encoding/decoding its log items. #[derive(PartialEq, Eq, Clone, RuntimeDebug)] -pub enum DigestItemRef<'a, Hash: 'a> { - /// Reference to `DigestItem::ChangesTrieRoot`. - ChangesTrieRoot(&'a Hash), +pub enum DigestItemRef<'a> { /// A pre-runtime digest. /// /// These are messages from the consensus engine to the runtime, although /// the consensus engine can (and should) read them itself to avoid /// code and state duplication. It is erroneous for a runtime to produce /// these, but this is not (yet) checked. - PreRuntime(&'a ConsensusEngineId, &'a Vec), + PreRuntime(&'a ConsensusEngineId, &'a [u8]), /// A message from the runtime to the consensus engine. This should *never* /// be generated by the native code of any consensus engine, but this is not /// checked (yet). - Consensus(&'a ConsensusEngineId, &'a Vec), + Consensus(&'a ConsensusEngineId, &'a [u8]), /// Put a Seal on it. This is only used by native code, and is never seen /// by runtimes. - Seal(&'a ConsensusEngineId, &'a Vec), - /// Digest item that contains signal from changes tries manager to the - /// native code. - ChangesTrieSignal(&'a ChangesTrieSignal), + Seal(&'a ConsensusEngineId, &'a [u8]), /// Any 'non-system' digest item, opaque to the native code. - Other(&'a Vec), + Other(&'a [u8]), /// Runtime code or heap pages updated. RuntimeEnvironmentUpdated, } @@ -271,11 +199,9 @@ pub enum DigestItemRef<'a, Hash: 'a> { #[derive(Encode, Decode)] pub enum DigestItemType { Other = 0, - ChangesTrieRoot = 2, Consensus = 4, Seal = 5, PreRuntime = 6, - ChangesTrieSignal = 7, RuntimeEnvironmentUpdated = 8, } @@ -293,25 +219,18 @@ pub enum OpaqueDigestItemId<'a> { Other, } -impl DigestItem { +impl DigestItem { /// Returns a 'referencing view' for this digest item. - pub fn dref(&self) -> DigestItemRef { + pub fn dref(&self) -> DigestItemRef { match *self { - Self::ChangesTrieRoot(ref v) => DigestItemRef::ChangesTrieRoot(v), Self::PreRuntime(ref v, ref s) => DigestItemRef::PreRuntime(v, s), Self::Consensus(ref v, ref s) => DigestItemRef::Consensus(v, s), Self::Seal(ref v, ref s) => DigestItemRef::Seal(v, s), - Self::ChangesTrieSignal(ref s) => DigestItemRef::ChangesTrieSignal(s), Self::Other(ref v) => DigestItemRef::Other(v), Self::RuntimeEnvironmentUpdated => DigestItemRef::RuntimeEnvironmentUpdated, } } - /// Returns `Some` if the entry is the `ChangesTrieRoot` entry. - pub fn as_changes_trie_root(&self) -> Option<&Hash> { - self.dref().as_changes_trie_root() - } - /// Returns `Some` if this entry is the `PreRuntime` entry. pub fn as_pre_runtime(&self) -> Option<(ConsensusEngineId, &[u8])> { self.dref().as_pre_runtime() @@ -327,11 +246,6 @@ impl DigestItem { self.dref().as_seal() } - /// Returns `Some` if the entry is the `ChangesTrieSignal` entry. - pub fn as_changes_trie_signal(&self) -> Option<&ChangesTrieSignal> { - self.dref().as_changes_trie_signal() - } - /// Returns Some if `self` is a `DigestItem::Other`. pub fn as_other(&self) -> Option<&[u8]> { self.dref().as_other() @@ -372,20 +286,19 @@ impl DigestItem { } } -impl Encode for DigestItem { +impl Encode for DigestItem { fn encode(&self) -> Vec { self.dref().encode() } } -impl codec::EncodeLike for DigestItem {} +impl codec::EncodeLike for DigestItem {} -impl Decode for DigestItem { +impl Decode for DigestItem { #[allow(deprecated)] fn decode(input: &mut I) -> Result { let item_type: DigestItemType = Decode::decode(input)?; match item_type { - DigestItemType::ChangesTrieRoot => Ok(Self::ChangesTrieRoot(Decode::decode(input)?)), DigestItemType::PreRuntime => { let vals: (ConsensusEngineId, Vec) = Decode::decode(input)?; Ok(Self::PreRuntime(vals.0, vals.1)) @@ -398,23 +311,13 @@ impl Decode for DigestItem { let vals: (ConsensusEngineId, Vec) = Decode::decode(input)?; Ok(Self::Seal(vals.0, vals.1)) }, - DigestItemType::ChangesTrieSignal => - Ok(Self::ChangesTrieSignal(Decode::decode(input)?)), DigestItemType::Other => Ok(Self::Other(Decode::decode(input)?)), DigestItemType::RuntimeEnvironmentUpdated => Ok(Self::RuntimeEnvironmentUpdated), } } } -impl<'a, Hash> DigestItemRef<'a, Hash> { - /// Cast this digest item into `ChangesTrieRoot`. - pub fn as_changes_trie_root(&self) -> Option<&'a Hash> { - match *self { - Self::ChangesTrieRoot(ref changes_trie_root) => Some(changes_trie_root), - _ => None, - } - } - +impl<'a> DigestItemRef<'a> { /// Cast this digest item into `PreRuntime` pub fn as_pre_runtime(&self) -> Option<(ConsensusEngineId, &'a [u8])> { match *self { @@ -439,14 +342,6 @@ impl<'a, Hash> DigestItemRef<'a, Hash> { } } - /// Cast this digest item into `ChangesTrieSignal`. - pub fn as_changes_trie_signal(&self) -> Option<&'a ChangesTrieSignal> { - match *self { - Self::ChangesTrieSignal(ref changes_trie_signal) => Some(changes_trie_signal), - _ => None, - } - } - /// Cast this digest item into `PreRuntime` pub fn as_other(&self) -> Option<&'a [u8]> { match *self { @@ -463,8 +358,8 @@ impl<'a, Hash> DigestItemRef<'a, Hash> { (OpaqueDigestItemId::Seal(w), &Self::Seal(v, s)) | (OpaqueDigestItemId::PreRuntime(w), &Self::PreRuntime(v, s)) if v == w => - Some(&s[..]), - (OpaqueDigestItemId::Other, &Self::Other(s)) => Some(&s[..]), + Some(s), + (OpaqueDigestItemId::Other, &Self::Other(s)) => Some(s), _ => None, } } @@ -508,15 +403,11 @@ impl<'a, Hash> DigestItemRef<'a, Hash> { } } -impl<'a, Hash: Encode> Encode for DigestItemRef<'a, Hash> { +impl<'a> Encode for DigestItemRef<'a> { fn encode(&self) -> Vec { let mut v = Vec::new(); match *self { - Self::ChangesTrieRoot(changes_trie_root) => { - DigestItemType::ChangesTrieRoot.encode_to(&mut v); - changes_trie_root.encode_to(&mut v); - }, Self::Consensus(val, data) => { DigestItemType::Consensus.encode_to(&mut v); (val, data).encode_to(&mut v); @@ -529,10 +420,6 @@ impl<'a, Hash: Encode> Encode for DigestItemRef<'a, Hash> { DigestItemType::PreRuntime.encode_to(&mut v); (val, data).encode_to(&mut v); }, - Self::ChangesTrieSignal(changes_trie_signal) => { - DigestItemType::ChangesTrieSignal.encode_to(&mut v); - changes_trie_signal.encode_to(&mut v); - }, Self::Other(val) => { DigestItemType::Other.encode_to(&mut v); val.encode_to(&mut v); @@ -546,16 +433,7 @@ impl<'a, Hash: Encode> Encode for DigestItemRef<'a, Hash> { } } -impl ChangesTrieSignal { - /// Try to cast this signal to NewConfiguration. - pub fn as_new_configuration(&self) -> Option<&Option> { - match self { - Self::NewConfiguration(config) => Some(config), - } - } -} - -impl<'a, Hash: Encode> codec::EncodeLike for DigestItemRef<'a, Hash> {} +impl<'a> codec::EncodeLike for DigestItemRef<'a> {} #[cfg(test)] mod tests { @@ -564,22 +442,18 @@ mod tests { #[test] fn should_serialize_digest() { let digest = Digest { - logs: vec![ - DigestItem::ChangesTrieRoot(4), - DigestItem::Other(vec![1, 2, 3]), - DigestItem::Seal(*b"test", vec![1, 2, 3]), - ], + logs: vec![DigestItem::Other(vec![1, 2, 3]), DigestItem::Seal(*b"test", vec![1, 2, 3])], }; assert_eq!( serde_json::to_string(&digest).unwrap(), - r#"{"logs":["0x0204000000","0x000c010203","0x05746573740c010203"]}"# + r#"{"logs":["0x000c010203","0x05746573740c010203"]}"# ); } #[test] fn digest_item_type_info() { - let type_info = DigestItem::::type_info(); + let type_info = DigestItem::type_info(); let variants = if let scale_info::TypeDef::Variant(variant) = type_info.type_def() { variant.variants() } else { @@ -589,21 +463,13 @@ mod tests { // ensure that all variants are covered by manual TypeInfo impl let check = |digest_item_type: DigestItemType| { let (variant_name, digest_item) = match digest_item_type { - DigestItemType::Other => ("Other", DigestItem::::Other(Default::default())), - DigestItemType::ChangesTrieRoot => - ("ChangesTrieRoot", DigestItem::ChangesTrieRoot(Default::default())), + DigestItemType::Other => ("Other", DigestItem::Other(Default::default())), DigestItemType::Consensus => ("Consensus", DigestItem::Consensus(Default::default(), Default::default())), DigestItemType::Seal => ("Seal", DigestItem::Seal(Default::default(), Default::default())), DigestItemType::PreRuntime => ("PreRuntime", DigestItem::PreRuntime(Default::default(), Default::default())), - DigestItemType::ChangesTrieSignal => ( - "ChangesTrieSignal", - DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration( - Default::default(), - )), - ), DigestItemType::RuntimeEnvironmentUpdated => ("RuntimeEnvironmentUpdated", DigestItem::RuntimeEnvironmentUpdated), }; @@ -617,11 +483,9 @@ mod tests { }; check(DigestItemType::Other); - check(DigestItemType::ChangesTrieRoot); check(DigestItemType::Consensus); check(DigestItemType::Seal); check(DigestItemType::PreRuntime); - check(DigestItemType::ChangesTrieSignal); check(DigestItemType::RuntimeEnvironmentUpdated); } } diff --git a/primitives/runtime/src/generic/era.rs b/primitives/runtime/src/generic/era.rs index 9d831b679c5e..2ca50b12b2e1 100644 --- a/primitives/runtime/src/generic/era.rs +++ b/primitives/runtime/src/generic/era.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime/src/generic/header.rs b/primitives/runtime/src/generic/header.rs index 82f081c0d70b..a7b43608f2b7 100644 --- a/primitives/runtime/src/generic/header.rs +++ b/primitives/runtime/src/generic/header.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,7 +29,7 @@ use crate::{ #[cfg(feature = "std")] use serde::{Deserialize, Serialize}; use sp_core::U256; -use sp_std::{convert::TryFrom, fmt::Debug}; +use sp_std::fmt::Debug; /// Abstraction over a block header for a substrate chain. #[derive(Encode, Decode, PartialEq, Eq, Clone, sp_core::RuntimeDebug, TypeInfo)] @@ -51,7 +51,7 @@ pub struct Header + TryFrom, Hash: HashT> { /// The merkle root of the extrinsics. pub extrinsics_root: Hash::Output, /// A chain-specific digest of data useful for light clients or referencing auxiliary data. - pub digest: Digest, + pub digest: Digest, } #[cfg(feature = "std")] @@ -150,11 +150,11 @@ where self.parent_hash = hash } - fn digest(&self) -> &Digest { + fn digest(&self) -> &Digest { &self.digest } - fn digest_mut(&mut self) -> &mut Digest { + fn digest_mut(&mut self) -> &mut Digest { #[cfg(feature = "std")] log::debug!(target: "header", "Retrieving mutable reference to digest"); &mut self.digest @@ -165,7 +165,7 @@ where extrinsics_root: Self::Hash, state_root: Self::Hash, parent_hash: Self::Hash, - digest: Digest, + digest: Digest, ) -> Self { Self { number, extrinsics_root, state_root, parent_hash, digest } } @@ -235,10 +235,7 @@ mod tests { state_root: BlakeTwo256::hash(b"3"), extrinsics_root: BlakeTwo256::hash(b"4"), digest: crate::generic::Digest { - logs: vec![ - crate::generic::DigestItem::ChangesTrieRoot(BlakeTwo256::hash(b"5")), - crate::generic::DigestItem::Other(b"6".to_vec()), - ], + logs: vec![crate::generic::DigestItem::Other(b"6".to_vec())], }, }; @@ -251,9 +248,7 @@ mod tests { 72, 51, 123, 15, 62, 20, 134, 32, 23, 61, 170, 165, 249, 77, 0, 216, 129, 112, 93, 203, 240, 170, 131, 239, 218, 186, 97, 210, 237, 225, 235, 134, 73, 33, 73, 151, 87, 78, 32, 196, 100, 56, 138, 23, 36, 32, 210, 84, 3, 104, 43, 187, 184, 12, 73, - 104, 49, 200, 204, 31, 143, 13, 8, 2, 112, 178, 1, 53, 47, 36, 191, 28, 151, 112, - 185, 159, 143, 113, 32, 24, 33, 65, 28, 244, 20, 55, 124, 155, 140, 45, 188, 238, - 97, 219, 135, 214, 0, 4, 54 + 104, 49, 200, 204, 31, 143, 13, 4, 0, 4, 54 ], ); assert_eq!(header, Header::::decode(&mut &header_encoded[..]).unwrap()); @@ -264,10 +259,7 @@ mod tests { state_root: BlakeTwo256::hash(b"3000"), extrinsics_root: BlakeTwo256::hash(b"4000"), digest: crate::generic::Digest { - logs: vec![ - crate::generic::DigestItem::Other(b"5000".to_vec()), - crate::generic::DigestItem::ChangesTrieRoot(BlakeTwo256::hash(b"6000")), - ], + logs: vec![crate::generic::DigestItem::Other(b"5000".to_vec())], }, }; @@ -280,9 +272,7 @@ mod tests { 47, 12, 107, 88, 153, 146, 55, 21, 226, 186, 110, 48, 167, 187, 67, 183, 228, 232, 118, 136, 30, 254, 11, 87, 48, 112, 7, 97, 31, 82, 146, 110, 96, 87, 152, 68, 98, 162, 227, 222, 78, 14, 244, 194, 120, 154, 112, 97, 222, 144, 174, 101, 220, 44, - 111, 126, 54, 34, 155, 220, 253, 124, 8, 0, 16, 53, 48, 48, 48, 2, 42, 105, 109, - 150, 206, 223, 24, 44, 164, 77, 27, 137, 177, 220, 25, 170, 140, 35, 156, 246, 233, - 112, 26, 23, 192, 61, 226, 14, 84, 219, 144, 252 + 111, 126, 54, 34, 155, 220, 253, 124, 4, 0, 16, 53, 48, 48, 48 ], ); assert_eq!(header, Header::::decode(&mut &header_encoded[..]).unwrap()); diff --git a/primitives/runtime/src/generic/mod.rs b/primitives/runtime/src/generic/mod.rs index 71127e88ec32..049b0e1624e7 100644 --- a/primitives/runtime/src/generic/mod.rs +++ b/primitives/runtime/src/generic/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,7 +31,7 @@ mod unchecked_extrinsic; pub use self::{ block::{Block, BlockId, SignedBlock}, checked_extrinsic::CheckedExtrinsic, - digest::{ChangesTrieSignal, Digest, DigestItem, DigestItemRef, OpaqueDigestItemId}, + digest::{Digest, DigestItem, DigestItemRef, OpaqueDigestItemId}, era::{Era, Phase}, header::Header, unchecked_extrinsic::{SignedPayload, UncheckedExtrinsic}, diff --git a/primitives/runtime/src/generic/tests.rs b/primitives/runtime/src/generic/tests.rs index 095bcb717bb1..d0536a567312 100644 --- a/primitives/runtime/src/generic/tests.rs +++ b/primitives/runtime/src/generic/tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,29 +19,26 @@ use super::DigestItem; use crate::codec::{Decode, Encode}; -use sp_core::H256; #[test] fn system_digest_item_encoding() { - let item = DigestItem::ChangesTrieRoot::(H256::default()); + let item = DigestItem::Consensus([1, 2, 3, 4], vec![5, 6, 7, 8]); let encoded = item.encode(); assert_eq!( encoded, vec![ - // type = DigestItemType::ChangesTrieRoot - 2, // trie root - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, + 4, // type = DigestItemType::Consensus + 1, 2, 3, 4, 16, 5, 6, 7, 8, ] ); - let decoded: DigestItem = Decode::decode(&mut &encoded[..]).unwrap(); + let decoded: DigestItem = Decode::decode(&mut &encoded[..]).unwrap(); assert_eq!(item, decoded); } #[test] fn non_system_digest_item_encoding() { - let item = DigestItem::Other::(vec![10, 20, 30]); + let item = DigestItem::Other(vec![10, 20, 30]); let encoded = item.encode(); assert_eq!( encoded, @@ -53,6 +50,6 @@ fn non_system_digest_item_encoding() { ] ); - let decoded: DigestItem = Decode::decode(&mut &encoded[..]).unwrap(); + let decoded: DigestItem = Decode::decode(&mut &encoded[..]).unwrap(); assert_eq!(item, decoded); } diff --git a/primitives/runtime/src/generic/unchecked_extrinsic.rs b/primitives/runtime/src/generic/unchecked_extrinsic.rs index 95f4f2f3584d..fb333abd6ac6 100644 --- a/primitives/runtime/src/generic/unchecked_extrinsic.rs +++ b/primitives/runtime/src/generic/unchecked_extrinsic.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,8 +31,12 @@ use scale_info::{build::Fields, meta_type, Path, StaticTypeInfo, Type, TypeInfo, use sp_io::hashing::blake2_256; use sp_std::{fmt, prelude::*}; -/// Current version of the [`UncheckedExtrinsic`] format. -const EXTRINSIC_VERSION: u8 = 4; +/// Current version of the [`UncheckedExtrinsic`] encoded format. +/// +/// This version needs to be bumped if the encoded representation changes. +/// It ensures that if the representation is changed and the format is not known, +/// the decoding fails. +const EXTRINSIC_FORMAT_VERSION: u8 = 4; /// A extrinsic right from the external world. This is unchecked and so /// can contain a signature. @@ -164,7 +168,7 @@ impl ExtrinsicMetadata where Extra: SignedExtension, { - const VERSION: u8 = EXTRINSIC_VERSION; + const VERSION: u8 = EXTRINSIC_FORMAT_VERSION; type SignedExtensions = Extra; } @@ -235,23 +239,33 @@ where { fn decode(input: &mut I) -> Result { // This is a little more complicated than usual since the binary format must be compatible - // with substrate's generic `Vec` type. Basically this just means accepting that there - // will be a prefix of vector length (we don't need - // to use this). - let _length_do_not_remove_me_see_above: Compact = Decode::decode(input)?; + // with SCALE's generic `Vec` type. Basically this just means accepting that there + // will be a prefix of vector length. + let expected_length: Compact = Decode::decode(input)?; + let before_length = input.remaining_len()?; let version = input.read_byte()?; let is_signed = version & 0b1000_0000 != 0; let version = version & 0b0111_1111; - if version != EXTRINSIC_VERSION { + if version != EXTRINSIC_FORMAT_VERSION { return Err("Invalid transaction version".into()) } - Ok(Self { - signature: if is_signed { Some(Decode::decode(input)?) } else { None }, - function: Decode::decode(input)?, - }) + let signature = is_signed.then(|| Decode::decode(input)).transpose()?; + let function = Decode::decode(input)?; + + if let Some((before_length, after_length)) = + input.remaining_len()?.and_then(|a| before_length.map(|b| (b, a))) + { + let length = before_length.saturating_sub(after_length); + + if length != expected_length.0 as usize { + return Err("Invalid length prefix".into()) + } + } + + Ok(Self { signature, function }) } } @@ -268,11 +282,11 @@ where // 1 byte version id. match self.signature.as_ref() { Some(s) => { - tmp.push(EXTRINSIC_VERSION | 0b1000_0000); + tmp.push(EXTRINSIC_FORMAT_VERSION | 0b1000_0000); s.encode_to(&mut tmp); }, None => { - tmp.push(EXTRINSIC_VERSION & 0b0111_1111); + tmp.push(EXTRINSIC_FORMAT_VERSION & 0b0111_1111); }, } self.function.encode_to(&mut tmp); @@ -364,7 +378,7 @@ mod tests { use crate::{ codec::{Decode, Encode}, testing::TestSignature as TestSig, - traits::{IdentityLookup, SignedExtension}, + traits::{DispatchInfoOf, IdentityLookup, SignedExtension}, }; use sp_io::hashing::blake2_256; @@ -387,6 +401,16 @@ mod tests { fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } + + fn pre_dispatch( + self, + who: &Self::AccountId, + call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> Result { + self.validate(who, call, info, len).map(|_| ()) + } } type Ex = UncheckedExtrinsic; @@ -399,6 +423,17 @@ mod tests { assert_eq!(Ex::decode(&mut &encoded[..]), Ok(ux)); } + #[test] + fn invalid_length_prefix_is_detected() { + let ux = Ex::new_unsigned(vec![0u8; 0]); + let mut encoded = ux.encode(); + + let length = Compact::::decode(&mut &encoded[..]).unwrap(); + Compact(length.0 + 10).encode_to(&mut &mut encoded[..1]); + + assert_eq!(Ex::decode(&mut &encoded[..]), Err("Invalid length prefix".into())); + } + #[test] fn signed_codec_should_work() { let ux = Ex::new_signed( diff --git a/frame/support/test/tests/reserved_keyword.rs b/primitives/runtime/src/legacy.rs similarity index 64% rename from frame/support/test/tests/reserved_keyword.rs rename to primitives/runtime/src/legacy.rs index d29b0477c383..7bc7c88a7e10 100644 --- a/frame/support/test/tests/reserved_keyword.rs +++ b/primitives/runtime/src/legacy.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,12 +15,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[rustversion::attr(not(stable), ignore)] -#[test] -fn reserved_keyword() { - // As trybuild is using `cargo check`, we don't need the real WASM binaries. - std::env::set_var("SKIP_WASM_BUILD", "1"); +//! Runtime types that existed in old API versions. - let t = trybuild::TestCases::new(); - t.compile_fail("tests/reserved_keyword/*.rs"); -} +pub mod byte_sized_error; diff --git a/primitives/runtime/src/legacy/byte_sized_error.rs b/primitives/runtime/src/legacy/byte_sized_error.rs new file mode 100644 index 000000000000..049abff69ff1 --- /dev/null +++ b/primitives/runtime/src/legacy/byte_sized_error.rs @@ -0,0 +1,100 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Runtime types that existed prior to BlockBuilder API version 6. + +use crate::{ArithmeticError, TokenError}; +use codec::{Decode, Encode}; +use scale_info::TypeInfo; +#[cfg(feature = "std")] +use serde::{Deserialize, Serialize}; + +/// [`ModuleError`] type definition before BlockBuilder API version 6. +#[derive(Eq, Clone, Copy, Encode, Decode, Debug, TypeInfo)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +pub struct ModuleError { + /// Module index, matching the metadata module index. + pub index: u8, + /// Module specific error value. + pub error: u8, + /// Optional error message. + #[codec(skip)] + #[cfg_attr(feature = "std", serde(skip_deserializing))] + pub message: Option<&'static str>, +} + +impl PartialEq for ModuleError { + fn eq(&self, other: &Self) -> bool { + (self.index == other.index) && (self.error == other.error) + } +} + +/// [`DispatchError`] type definition before BlockBuilder API version 6. +#[derive(Eq, Clone, Copy, Encode, Decode, Debug, TypeInfo, PartialEq)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +pub enum DispatchError { + /// Some error occurred. + Other( + #[codec(skip)] + #[cfg_attr(feature = "std", serde(skip_deserializing))] + &'static str, + ), + /// Failed to lookup some data. + CannotLookup, + /// A bad origin. + BadOrigin, + /// A custom error in a module. + Module(ModuleError), + /// At least one consumer is remaining so the account cannot be destroyed. + ConsumerRemaining, + /// There are no providers so the account cannot be created. + NoProviders, + /// There are too many consumers so the account cannot be created. + TooManyConsumers, + /// An error to do with tokens. + Token(TokenError), + /// An arithmetic error. + Arithmetic(ArithmeticError), +} + +/// [`DispatchOutcome`] type definition before BlockBuilder API version 6. +pub type DispatchOutcome = Result<(), DispatchError>; + +/// [`ApplyExtrinsicResult`] type definition before BlockBuilder API version 6. +pub type ApplyExtrinsicResult = + Result; + +/// Convert the legacy `ApplyExtrinsicResult` type to the latest version. +pub fn convert_to_latest(old: ApplyExtrinsicResult) -> crate::ApplyExtrinsicResult { + old.map(|outcome| { + outcome.map_err(|e| match e { + DispatchError::Other(s) => crate::DispatchError::Other(s), + DispatchError::CannotLookup => crate::DispatchError::CannotLookup, + DispatchError::BadOrigin => crate::DispatchError::BadOrigin, + DispatchError::Module(err) => crate::DispatchError::Module(crate::ModuleError { + index: err.index, + error: [err.error, 0, 0, 0], + message: err.message, + }), + DispatchError::ConsumerRemaining => crate::DispatchError::ConsumerRemaining, + DispatchError::NoProviders => crate::DispatchError::NoProviders, + DispatchError::TooManyConsumers => crate::DispatchError::TooManyConsumers, + DispatchError::Token(err) => crate::DispatchError::Token(err), + DispatchError::Arithmetic(err) => crate::DispatchError::Arithmetic(err), + }) + }) +} diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 80293fe73484..39e606eb9b5f 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,22 +40,24 @@ pub use paste; #[doc(hidden)] pub use sp_application_crypto as app_crypto; +pub use sp_core::storage::StateVersion; #[cfg(feature = "std")] pub use sp_core::storage::{Storage, StorageChild}; use sp_core::{ - crypto::{self, Public}, + crypto::{self, ByteArray}, ecdsa, ed25519, hash::{H256, H512}, sr25519, }; -use sp_std::{convert::TryFrom, prelude::*}; +use sp_std::prelude::*; -use codec::{Decode, Encode}; +use codec::{Decode, Encode, MaxEncodedLen}; use scale_info::TypeInfo; pub mod curve; pub mod generic; +pub mod legacy; mod multiaddress; pub mod offchain; pub mod runtime_logger; @@ -96,6 +98,10 @@ pub use sp_arithmetic::{ pub use either::Either; +/// The number of bytes of the module-specific `error` field defined in [`ModuleError`]. +/// In FRAME, this is the maximum encoded size of a pallet error type. +pub const MAX_MODULE_ERROR_ENCODED_SIZE: usize = 4; + /// An abstraction over justification for a block's validity under a consensus algorithm. /// /// Essentially a finality proof. The exact formulation will vary between consensus @@ -223,7 +229,7 @@ pub type ConsensusEngineId = [u8; 4]; /// Signature verify that can work with any known signature types.. #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] -#[derive(Eq, PartialEq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] +#[derive(Eq, PartialEq, Clone, Encode, Decode, MaxEncodedLen, RuntimeDebug, TypeInfo)] pub enum MultiSignature { /// An Ed25519 signature. Ed25519(ed25519::Signature), @@ -284,12 +290,6 @@ impl TryFrom for ecdsa::Signature { } } -impl Default for MultiSignature { - fn default() -> Self { - Self::Ed25519(Default::default()) - } -} - /// Public key for any known crypto algorithm. #[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] @@ -302,12 +302,6 @@ pub enum MultiSigner { Ecdsa(ecdsa::Public), } -impl Default for MultiSigner { - fn default() -> Self { - Self::Ed25519(Default::default()) - } -} - /// NOTE: This implementations is required by `SimpleAddressDeterminer`, /// we convert the hash into some AccountId, it's fine to use any scheme. impl> crypto::UncheckedFrom for MultiSigner { @@ -403,10 +397,14 @@ impl Verify for MultiSignature { type Signer = MultiSigner; fn verify>(&self, mut msg: L, signer: &AccountId32) -> bool { match (self, signer) { - (Self::Ed25519(ref sig), who) => - sig.verify(msg, &ed25519::Public::from_slice(who.as_ref())), - (Self::Sr25519(ref sig), who) => - sig.verify(msg, &sr25519::Public::from_slice(who.as_ref())), + (Self::Ed25519(ref sig), who) => match ed25519::Public::from_slice(who.as_ref()) { + Ok(signer) => sig.verify(msg, &signer), + Err(()) => false, + }, + (Self::Sr25519(ref sig), who) => match sr25519::Public::from_slice(who.as_ref()) { + Ok(signer) => sig.verify(msg, &signer), + Err(()) => false, + }, (Self::Ecdsa(ref sig), who) => { let m = sp_io::hashing::blake2_256(msg.get()); match sp_io::crypto::secp256k1_ecdsa_recover_compressed(sig.as_ref(), &m) { @@ -433,7 +431,10 @@ impl Verify for AnySignature { .map(|s| s.verify(msg, signer)) .unwrap_or(false) || ed25519::Signature::try_from(self.0.as_fixed_bytes().as_ref()) - .map(|s| s.verify(msg, &ed25519::Public::from_slice(signer.as_ref()))) + .map(|s| match ed25519::Public::from_slice(signer.as_ref()) { + Err(()) => false, + Ok(signer) => s.verify(msg, &signer), + }) .unwrap_or(false) } } @@ -465,9 +466,54 @@ pub type DispatchResult = sp_std::result::Result<(), DispatchError>; /// about the `Dispatchable` that is only known post dispatch. pub type DispatchResultWithInfo = sp_std::result::Result>; -/// Reason why a dispatch call failed. +/// Reason why a pallet call failed. #[derive(Eq, Clone, Copy, Encode, Decode, Debug, TypeInfo)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +pub struct ModuleError { + /// Module index, matching the metadata module index. + pub index: u8, + /// Module specific error value. + pub error: [u8; MAX_MODULE_ERROR_ENCODED_SIZE], + /// Optional error message. + #[codec(skip)] + #[cfg_attr(feature = "std", serde(skip_deserializing))] + pub message: Option<&'static str>, +} + +impl PartialEq for ModuleError { + fn eq(&self, other: &Self) -> bool { + (self.index == other.index) && (self.error == other.error) + } +} + +/// Errors related to transactional storage layers. +#[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, Debug, TypeInfo)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] +pub enum TransactionalError { + /// Too many transactional layers have been spawned. + LimitReached, + /// A transactional layer was expected, but does not exist. + NoLayer, +} + +impl From for &'static str { + fn from(e: TransactionalError) -> &'static str { + match e { + TransactionalError::LimitReached => "Too many transactional layers have been spawned", + TransactionalError::NoLayer => "A transactional layer was expected, but does not exist", + } + } +} + +impl From for DispatchError { + fn from(e: TransactionalError) -> DispatchError { + Self::Transactional(e) + } +} + +/// Reason why a dispatch call failed. +#[derive(Eq, Clone, Copy, Encode, Decode, Debug, TypeInfo, PartialEq)] +#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] pub enum DispatchError { /// Some error occurred. Other( @@ -480,29 +526,25 @@ pub enum DispatchError { /// A bad origin. BadOrigin, /// A custom error in a module. - Module { - /// Module index, matching the metadata module index. - index: u8, - /// Module specific error value. - error: u8, - /// Optional error message. - #[codec(skip)] - #[cfg_attr(feature = "std", serde(skip_deserializing))] - message: Option<&'static str>, - }, + Module(ModuleError), /// At least one consumer is remaining so the account cannot be destroyed. ConsumerRemaining, /// There are no providers so the account cannot be created. NoProviders, + /// There are too many consumers so the account cannot be created. + TooManyConsumers, /// An error to do with tokens. Token(TokenError), /// An arithmetic error. Arithmetic(ArithmeticError), + /// The number of transactional layers has been reached, or we are not in a transactional + /// layer. + Transactional(TransactionalError), } /// Result of a `Dispatchable` which contains the `DispatchResult` and additional information about /// the `Dispatchable` that is only known post dispatch. -#[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, RuntimeDebug)] +#[derive(Eq, PartialEq, Clone, Copy, Encode, Decode, RuntimeDebug, TypeInfo)] pub struct DispatchErrorWithPostInfo where Info: Eq + PartialEq + Clone + Copy + Encode + Decode + traits::Printable, @@ -517,8 +559,8 @@ impl DispatchError { /// Return the same error but without the attached message. pub fn stripped(self) -> Self { match self { - DispatchError::Module { index, error, message: Some(_) } => - DispatchError::Module { index, error, message: None }, + DispatchError::Module(ModuleError { index, error, message: Some(_) }) => + DispatchError::Module(ModuleError { index, error, message: None }), m => m, } } @@ -626,11 +668,14 @@ impl From for &'static str { DispatchError::Other(msg) => msg, DispatchError::CannotLookup => "Cannot lookup", DispatchError::BadOrigin => "Bad origin", - DispatchError::Module { message, .. } => message.unwrap_or("Unknown module error"), + DispatchError::Module(ModuleError { message, .. }) => + message.unwrap_or("Unknown module error"), DispatchError::ConsumerRemaining => "Consumer remaining", DispatchError::NoProviders => "No providers", + DispatchError::TooManyConsumers => "Too many consumers", DispatchError::Token(e) => e.into(), DispatchError::Arithmetic(e) => e.into(), + DispatchError::Transactional(e) => e.into(), } } } @@ -651,7 +696,7 @@ impl traits::Printable for DispatchError { Self::Other(err) => err.print(), Self::CannotLookup => "Cannot lookup".print(), Self::BadOrigin => "Bad origin".print(), - Self::Module { index, error, message } => { + Self::Module(ModuleError { index, error, message }) => { index.print(); error.print(); if let Some(msg) = message { @@ -660,6 +705,7 @@ impl traits::Printable for DispatchError { }, Self::ConsumerRemaining => "Consumer remaining".print(), Self::NoProviders => "No providers".print(), + Self::TooManyConsumers => "Too many consumers".print(), Self::Token(e) => { "Token error: ".print(); <&'static str>::from(*e).print(); @@ -668,6 +714,10 @@ impl traits::Printable for DispatchError { "Arithmetic error: ".print(); <&'static str>::from(*e).print(); }, + Self::Transactional(e) => { + "Transactional error: ".print(); + <&'static str>::from(*e).print(); + }, } } } @@ -683,30 +733,6 @@ where } } -impl PartialEq for DispatchError { - fn eq(&self, other: &Self) -> bool { - use DispatchError::*; - - match (self, other) { - (CannotLookup, CannotLookup) | - (BadOrigin, BadOrigin) | - (ConsumerRemaining, ConsumerRemaining) | - (NoProviders, NoProviders) => true, - - (Token(l), Token(r)) => l == r, - (Other(l), Other(r)) => l == r, - (Arithmetic(l), Arithmetic(r)) => l == r, - - ( - Module { index: index_l, error: error_l, .. }, - Module { index: index_r, error: error_r, .. }, - ) => (index_l == index_r) && (error_l == error_r), - - _ => false, - } - } -} - /// This type specifies the outcome of dispatching a call to a module. /// /// In case of failure an error specific to the module is returned. @@ -801,7 +827,7 @@ macro_rules! assert_eq_error_rate { /// Simple blob to hold an extrinsic without committing to its format and ensure it is serialized /// correctly. -#[derive(PartialEq, Eq, Clone, Default, Encode, Decode)] +#[derive(PartialEq, Eq, Clone, Default, Encode, Decode, TypeInfo)] pub struct OpaqueExtrinsic(Vec); impl OpaqueExtrinsic { @@ -916,9 +942,13 @@ impl TransactionOutcome { #[cfg(test)] mod tests { + use crate::traits::BlakeTwo256; + use super::*; use codec::{Decode, Encode}; - use sp_core::crypto::Pair; + use sp_core::crypto::{Pair, UncheckedFrom}; + use sp_io::TestExternalities; + use sp_state_machine::create_proof_check_backend; #[test] fn opaque_extrinsic_serialization() { @@ -928,11 +958,18 @@ mod tests { #[test] fn dispatch_error_encoding() { - let error = DispatchError::Module { index: 1, error: 2, message: Some("error message") }; + let error = DispatchError::Module(ModuleError { + index: 1, + error: [2, 0, 0, 0], + message: Some("error message"), + }); let encoded = error.encode(); let decoded = DispatchError::decode(&mut &encoded[..]).unwrap(); - assert_eq!(encoded, vec![3, 1, 2]); - assert_eq!(decoded, DispatchError::Module { index: 1, error: 2, message: None }); + assert_eq!(encoded, vec![3, 1, 2, 0, 0, 0]); + assert_eq!( + decoded, + DispatchError::Module(ModuleError { index: 1, error: [2, 0, 0, 0], message: None }) + ); } #[test] @@ -944,9 +981,9 @@ mod tests { Other("bar"), CannotLookup, BadOrigin, - Module { index: 1, error: 1, message: None }, - Module { index: 1, error: 2, message: None }, - Module { index: 2, error: 1, message: None }, + Module(ModuleError { index: 1, error: [1, 0, 0, 0], message: None }), + Module(ModuleError { index: 1, error: [2, 0, 0, 0], message: None }), + Module(ModuleError { index: 2, error: [1, 0, 0, 0], message: None }), ConsumerRemaining, NoProviders, Token(TokenError::NoFunds), @@ -971,8 +1008,8 @@ mod tests { // Ignores `message` field in `Module` variant. assert_eq!( - Module { index: 1, error: 1, message: Some("foo") }, - Module { index: 1, error: 1, message: None }, + Module(ModuleError { index: 1, error: [1, 0, 0, 0], message: Some("foo") }), + Module(ModuleError { index: 1, error: [1, 0, 0, 0], message: None }), ); } @@ -1002,7 +1039,9 @@ mod tests { ext.execute_with(|| { let _batching = SignatureBatching::start(); - sp_io::crypto::sr25519_verify(&Default::default(), &Vec::new(), &Default::default()); + let dummy = UncheckedFrom::unchecked_from([1; 32]); + let dummy_sig = UncheckedFrom::unchecked_from([1; 64]); + sp_io::crypto::sr25519_verify(&dummy_sig, &Vec::new(), &dummy); }); } @@ -1019,4 +1058,47 @@ mod tests { panic!("Hey, I'm an error"); }); } + + #[test] + fn execute_and_generate_proof_works() { + use codec::Encode; + use sp_state_machine::Backend; + let mut ext = TestExternalities::default(); + + ext.insert(b"a".to_vec(), vec![1u8; 33]); + ext.insert(b"b".to_vec(), vec![2u8; 33]); + ext.insert(b"c".to_vec(), vec![3u8; 33]); + ext.insert(b"d".to_vec(), vec![4u8; 33]); + + let pre_root = ext.backend.root().clone(); + let (_, proof) = ext.execute_and_prove(|| { + sp_io::storage::get(b"a"); + sp_io::storage::get(b"b"); + sp_io::storage::get(b"v"); + sp_io::storage::get(b"d"); + }); + + let compact_proof = proof.clone().into_compact_proof::(pre_root).unwrap(); + let compressed_proof = zstd::stream::encode_all(&compact_proof.encode()[..], 0).unwrap(); + + // just an example of how you'd inspect the size of the proof. + println!("proof size: {:?}", proof.encoded_size()); + println!("compact proof size: {:?}", compact_proof.encoded_size()); + println!("zstd-compressed compact proof size: {:?}", &compressed_proof.len()); + + // create a new trie-backed from the proof and make sure it contains everything + let proof_check = create_proof_check_backend::(pre_root, proof).unwrap(); + assert_eq!(proof_check.storage(b"a",).unwrap().unwrap(), vec![1u8; 33]); + + let _ = ext.execute_and_prove(|| { + sp_io::storage::set(b"a", &vec![1u8; 44]); + }); + + // ensure that these changes are propagated to the backend. + + ext.execute_with(|| { + assert_eq!(sp_io::storage::get(b"a").unwrap(), vec![1u8; 44]); + assert_eq!(sp_io::storage::get(b"b").unwrap(), vec![2u8; 33]); + }); + } } diff --git a/primitives/runtime/src/multiaddress.rs b/primitives/runtime/src/multiaddress.rs index 46d80608352d..b2d46fb106cb 100644 --- a/primitives/runtime/src/multiaddress.rs +++ b/primitives/runtime/src/multiaddress.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -62,9 +62,3 @@ impl From for MultiAddress Default for MultiAddress { - fn default() -> Self { - Self::Id(Default::default()) - } -} diff --git a/primitives/runtime/src/offchain/http.rs b/primitives/runtime/src/offchain/http.rs index 469f2fb5aff3..c479062de5f1 100644 --- a/primitives/runtime/src/offchain/http.rs +++ b/primitives/runtime/src/offchain/http.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -535,6 +535,38 @@ mod tests { }) } + #[test] + fn should_send_huge_response() { + let (offchain, state) = testing::TestOffchainExt::new(); + let mut t = TestExternalities::default(); + t.register_extension(OffchainWorkerExt::new(offchain)); + + t.execute_with(|| { + let request: Request = Request::get("http://localhost:1234"); + let pending = request.add_header("X-Auth", "hunter2").send().unwrap(); + // make sure it's sent correctly + state.write().fulfill_pending_request( + 0, + testing::PendingRequest { + method: "GET".into(), + uri: "http://localhost:1234".into(), + headers: vec![("X-Auth".into(), "hunter2".into())], + sent: true, + ..Default::default() + }, + vec![0; 5923], + None, + ); + + // wait + let response = pending.wait().unwrap(); + + let body = response.body(); + assert_eq!(body.clone().collect::>(), vec![0; 5923]); + assert_eq!(body.error(), &None); + }) + } + #[test] fn should_send_a_post_request() { let (offchain, state) = testing::TestOffchainExt::new(); diff --git a/primitives/runtime/src/offchain/mod.rs b/primitives/runtime/src/offchain/mod.rs index 35f9352d81b0..eb4c222ceb56 100644 --- a/primitives/runtime/src/offchain/mod.rs +++ b/primitives/runtime/src/offchain/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime/src/offchain/storage.rs b/primitives/runtime/src/offchain/storage.rs index 3bc5b10f161f..38ad268e5894 100644 --- a/primitives/runtime/src/offchain/storage.rs +++ b/primitives/runtime/src/offchain/storage.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime/src/offchain/storage_lock.rs b/primitives/runtime/src/offchain/storage_lock.rs index b4833bf345fc..90f8df794557 100644 --- a/primitives/runtime/src/offchain/storage_lock.rs +++ b/primitives/runtime/src/offchain/storage_lock.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime/src/runtime_logger.rs b/primitives/runtime/src/runtime_logger.rs index ff0e531ed814..8b5a15762e1f 100644 --- a/primitives/runtime/src/runtime_logger.rs +++ b/primitives/runtime/src/runtime_logger.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/runtime/src/runtime_string.rs b/primitives/runtime/src/runtime_string.rs index 179e88145181..fcbdd2e787ff 100644 --- a/primitives/runtime/src/runtime_string.rs +++ b/primitives/runtime/src/runtime_string.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -92,6 +92,18 @@ impl AsRef<[u8]> for RuntimeString { } } +#[cfg(feature = "std")] +impl std::ops::Deref for RuntimeString { + type Target = str; + + fn deref(&self) -> &str { + match self { + Self::Borrowed(val) => &val, + Self::Owned(val) => &val, + } + } +} + impl Encode for RuntimeString { fn encode(&self) -> Vec { match self { diff --git a/primitives/runtime/src/testing.rs b/primitives/runtime/src/testing.rs index fe9ba588adb8..003fa62c9e08 100644 --- a/primitives/runtime/src/testing.rs +++ b/primitives/runtime/src/testing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,7 +30,7 @@ use crate::{ }; use serde::{de::Error as DeError, Deserialize, Deserializer, Serialize, Serializer}; use sp_core::{ - crypto::{key_types, CryptoType, Dummy, Public}, + crypto::{key_types, ByteArray, CryptoType, Dummy}, U256, }; pub use sp_core::{sr25519, H256}; @@ -77,10 +77,10 @@ impl From for u64 { } impl UintAuthorityId { - /// Convert this authority id into a public key. - pub fn to_public_key(&self) -> T { + /// Convert this authority ID into a public key. + pub fn to_public_key(&self) -> T { let bytes: [u8; 32] = U256::from(self.0).into(); - T::from_slice(&bytes) + T::from_slice(&bytes).unwrap() } } @@ -182,10 +182,10 @@ impl traits::Verify for TestSignature { } /// Digest item -pub type DigestItem = generic::DigestItem; +pub type DigestItem = generic::DigestItem; /// Header Digest -pub type Digest = generic::Digest; +pub type Digest = generic::Digest; /// Block Header pub type Header = generic::Header; diff --git a/primitives/runtime/src/traits.rs b/primitives/runtime/src/traits.rs index 6d79d740dc4e..9c71b2023d3f 100644 --- a/primitives/runtime/src/traits.rs +++ b/primitives/runtime/src/traits.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,7 @@ use crate::{ codec::{Codec, Decode, Encode, MaxEncodedLen}, - generic::{Digest, DigestItem}, + generic::Digest, scale_info::{MetaType, StaticTypeInfo, TypeInfo}, transaction_validity::{ TransactionSource, TransactionValidity, TransactionValidityError, UnknownTransaction, @@ -36,14 +36,8 @@ pub use sp_arithmetic::traits::{ CheckedShr, CheckedSub, IntegerSquareRoot, One, SaturatedConversion, Saturating, UniqueSaturatedFrom, UniqueSaturatedInto, Zero, }; -use sp_core::{self, Hasher, RuntimeDebug, TypeId}; -use sp_std::{ - self, - convert::{TryFrom, TryInto}, - fmt::Debug, - marker::PhantomData, - prelude::*, -}; +use sp_core::{self, storage::StateVersion, Hasher, RuntimeDebug, TypeId}; +use sp_std::{self, fmt::Debug, marker::PhantomData, prelude::*}; #[cfg(feature = "std")] use std::fmt::Display; #[cfg(feature = "std")] @@ -459,10 +453,10 @@ pub trait Hash: } /// The ordered Patricia tree root of the given `input`. - fn ordered_trie_root(input: Vec>) -> Self::Output; + fn ordered_trie_root(input: Vec>, state_version: StateVersion) -> Self::Output; /// The Patricia tree root of the given mapping. - fn trie_root(input: Vec<(Vec, Vec)>) -> Self::Output; + fn trie_root(input: Vec<(Vec, Vec)>, state_version: StateVersion) -> Self::Output; } /// Blake2-256 Hash implementation. @@ -483,12 +477,12 @@ impl Hasher for BlakeTwo256 { impl Hash for BlakeTwo256 { type Output = sp_core::H256; - fn trie_root(input: Vec<(Vec, Vec)>) -> Self::Output { - sp_io::trie::blake2_256_root(input) + fn trie_root(input: Vec<(Vec, Vec)>, version: StateVersion) -> Self::Output { + sp_io::trie::blake2_256_root(input, version) } - fn ordered_trie_root(input: Vec>) -> Self::Output { - sp_io::trie::blake2_256_ordered_root(input) + fn ordered_trie_root(input: Vec>, version: StateVersion) -> Self::Output { + sp_io::trie::blake2_256_ordered_root(input, version) } } @@ -510,12 +504,12 @@ impl Hasher for Keccak256 { impl Hash for Keccak256 { type Output = sp_core::H256; - fn trie_root(input: Vec<(Vec, Vec)>) -> Self::Output { - sp_io::trie::keccak_256_root(input) + fn trie_root(input: Vec<(Vec, Vec)>, version: StateVersion) -> Self::Output { + sp_io::trie::keccak_256_root(input, version) } - fn ordered_trie_root(input: Vec>) -> Self::Output { - sp_io::trie::keccak_256_ordered_root(input) + fn ordered_trie_root(input: Vec>, version: StateVersion) -> Self::Output { + sp_io::trie::keccak_256_ordered_root(input, version) } } @@ -548,10 +542,7 @@ impl CheckEqual for sp_core::H256 { } } -impl CheckEqual for super::generic::DigestItem -where - H: Encode, -{ +impl CheckEqual for super::generic::DigestItem { #[cfg(feature = "std")] fn check_equal(&self, other: &Self) { if self != other { @@ -642,7 +633,7 @@ pub trait Header: extrinsics_root: Self::Hash, state_root: Self::Hash, parent_hash: Self::Hash, - digest: Digest, + digest: Digest, ) -> Self; /// Returns a reference to the header number. @@ -666,9 +657,9 @@ pub trait Header: fn set_parent_hash(&mut self, hash: Self::Hash); /// Returns a reference to the digest. - fn digest(&self) -> &Digest; + fn digest(&self) -> &Digest; /// Get a mutable reference to the digest. - fn digest_mut(&mut self) -> &mut Digest; + fn digest_mut(&mut self) -> &mut Digest; /// Returns the hash of the header. fn hash(&self) -> Self::Hash { @@ -751,7 +742,9 @@ pub trait Extrinsic: Sized + MaybeMallocSizeOf { /// Implementor is an [`Extrinsic`] and provides metadata about this extrinsic. pub trait ExtrinsicMetadata { - /// The version of the `Extrinsic`. + /// The format version of the `Extrinsic`. + /// + /// By format is meant the encoded representation of the `Extrinsic`. const VERSION: u8; /// Signed extensions attached to this `Extrinsic`. @@ -763,9 +756,6 @@ pub type HashFor = <::Header as Header>::Hashing; /// Extract the number type for a block. pub type NumberFor = <::Header as Header>::Number; /// Extract the digest type for a block. -pub type DigestFor = Digest<<::Header as Header>::Hash>; -/// Extract the digest item type for a block. -pub type DigestItemFor = DigestItem<<::Header as Header>::Hash>; /// A "checkable" piece of information, used by the standard Substrate Executive in order to /// check the validity of a piece of extrinsic information, usually by verifying the signature. @@ -857,7 +847,7 @@ pub trait SignedExtension: type AdditionalSigned: Encode + TypeInfo; /// The type that encodes information that can be passed from pre_dispatch to post-dispatch. - type Pre: Default; + type Pre; /// Construct any additional data that should be in the signed payload of the transaction. Can /// also perform any pre-signature-verification checks and return an error if needed. @@ -896,11 +886,7 @@ pub trait SignedExtension: call: &Self::Call, info: &DispatchInfoOf, len: usize, - ) -> Result { - self.validate(who, call, info, len) - .map(|_| Self::Pre::default()) - .map_err(Into::into) - } + ) -> Result; /// Validate an unsigned transaction for the transaction queue. /// @@ -930,14 +916,15 @@ pub trait SignedExtension: call: &Self::Call, info: &DispatchInfoOf, len: usize, - ) -> Result { - Self::validate_unsigned(call, info, len) - .map(|_| Self::Pre::default()) - .map_err(Into::into) + ) -> Result<(), TransactionValidityError> { + Self::validate_unsigned(call, info, len).map(|_| ()).map_err(Into::into) } /// Do any post-flight stuff for an extrinsic. /// + /// If the transaction is signed, then `_pre` will contain the output of `pre_dispatch`, + /// and `None` otherwise. + /// /// This gets given the `DispatchResult` `_result` from the extrinsic and can, if desired, /// introduce a `TransactionValidityError`, causing the block to become invalid for including /// it. @@ -950,7 +937,7 @@ pub trait SignedExtension: /// introduced by the current block author; generally this implies that it is an inherent and /// will come from either an offchain-worker or via `InherentData`. fn post_dispatch( - _pre: Self::Pre, + _pre: Option, _info: &DispatchInfoOf, _post_info: &PostDispatchInfoOf, _len: usize, @@ -1035,18 +1022,26 @@ impl SignedExtension for Tuple { call: &Self::Call, info: &DispatchInfoOf, len: usize, - ) -> Result { - Ok(for_tuples!( ( #( Tuple::pre_dispatch_unsigned(call, info, len)? ),* ) )) + ) -> Result<(), TransactionValidityError> { + for_tuples!( #( Tuple::pre_dispatch_unsigned(call, info, len)?; )* ); + Ok(()) } fn post_dispatch( - pre: Self::Pre, + pre: Option, info: &DispatchInfoOf, post_info: &PostDispatchInfoOf, len: usize, result: &DispatchResult, ) -> Result<(), TransactionValidityError> { - for_tuples!( #( Tuple::post_dispatch(pre.Tuple, info, post_info, len, result)?; )* ); + match pre { + Some(x) => { + for_tuples!( #( Tuple::post_dispatch(Some(x.Tuple), info, post_info, len, result)?; )* ); + }, + None => { + for_tuples!( #( Tuple::post_dispatch(None, info, post_info, len, result)?; )* ); + }, + } Ok(()) } @@ -1068,6 +1063,15 @@ impl SignedExtension for () { fn additional_signed(&self) -> sp_std::result::Result<(), TransactionValidityError> { Ok(()) } + fn pre_dispatch( + self, + who: &Self::AccountId, + call: &Self::Call, + info: &DispatchInfoOf, + len: usize, + ) -> Result { + self.validate(who, call, info, len).map(|_| ()) + } } /// An "executable" piece of information, used by the standard Substrate Executive in order to @@ -1218,6 +1222,11 @@ impl<'a> TrailingZeroInput<'a> { pub fn new(data: &'a [u8]) -> Self { Self(data) } + + /// Create a new instance which only contains zeroes as input. + pub fn zeroes() -> Self { + Self::new(&[][..]) + } } impl<'a> codec::Input for TrailingZeroInput<'a> { @@ -1266,11 +1275,11 @@ pub trait AccountIdConversion: Sized { /// Format is TYPE_ID ++ encode(parachain ID) ++ 00.... where 00... is indefinite trailing zeroes to /// fill AccountId. -impl AccountIdConversion for Id { +impl AccountIdConversion for Id { fn into_sub_account(&self, sub: S) -> T { (Id::TYPE_ID, self, sub) .using_encoded(|b| T::decode(&mut TrailingZeroInput(b))) - .unwrap_or_default() + .expect("`AccountId` type is never greater than 32 bytes; qed") } fn try_from_sub_account(x: &T) -> Option<(Self, S)> { @@ -1323,7 +1332,7 @@ macro_rules! impl_opaque_keys_inner { ) => { $( #[ $attr ] )* #[derive( - Default, Clone, PartialEq, Eq, + Clone, PartialEq, Eq, $crate::codec::Encode, $crate::codec::Decode, $crate::scale_info::TypeInfo, @@ -1539,6 +1548,12 @@ impl Printable for &[u8] { } } +impl Printable for [u8; N] { + fn print(&self) { + sp_io::misc::print_hex(&self[..]); + } +} + impl Printable for &str { fn print(&self) { sp_io::misc::print_utf8(self.as_bytes()); @@ -1572,7 +1587,7 @@ impl Printable for Tuple { #[cfg(feature = "std")] pub trait BlockIdTo { /// The error type that will be returned by the functions. - type Error: std::fmt::Debug; + type Error: std::error::Error; /// Convert the given `block_id` to the corresponding block hash. fn to_hash( @@ -1622,7 +1637,10 @@ pub trait BlockNumberProvider { mod tests { use super::*; use crate::codec::{Decode, Encode, Input}; - use sp_core::{crypto::Pair, ecdsa}; + use sp_core::{ + crypto::{Pair, UncheckedFrom}, + ecdsa, + }; mod t { use sp_application_crypto::{app_crypto, sr25519}; @@ -1635,8 +1653,8 @@ mod tests { use super::AppVerify; use t::*; - let s = Signature::default(); - let _ = s.verify(&[0u8; 100][..], &Public::default()); + let s = Signature::try_from(vec![0; 64]).unwrap(); + let _ = s.verify(&[0u8; 100][..], &Public::unchecked_from([0; 32])); } #[derive(Encode, Decode, Default, PartialEq, Debug)] diff --git a/primitives/runtime/src/transaction_validity.rs b/primitives/runtime/src/transaction_validity.rs index e114bb598546..29c8b542319b 100644 --- a/primitives/runtime/src/transaction_validity.rs +++ b/primitives/runtime/src/transaction_validity.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -79,6 +79,8 @@ pub enum InvalidTransaction { /// A transaction with a mandatory dispatch. This is invalid; only inherent extrinsics are /// allowed to have mandatory dispatches. MandatoryDispatch, + /// The sending address is disabled or known to be invalid. + BadSigner, } impl InvalidTransaction { @@ -109,6 +111,7 @@ impl From for &'static str { InvalidTransaction::MandatoryDispatch => "Transaction dispatch is mandatory; transactions may not have mandatory dispatches.", InvalidTransaction::Custom(_) => "InvalidTransaction custom error", + InvalidTransaction::BadSigner => "Invalid signing address", } } } diff --git a/primitives/sandbox/Cargo.toml b/primitives/sandbox/Cargo.toml old mode 100755 new mode 100644 index a4d4a4d5d031..22d295f31370 --- a/primitives/sandbox/Cargo.toml +++ b/primitives/sandbox/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-sandbox" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "This crate provides means to instantiate and execute wasm modules." readme = "README.md" @@ -13,18 +13,18 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [target.'cfg(target_arch = "wasm32")'.dependencies] -wasmi = { version = "0.9.0", default-features = false, features = ["core"] } +wasmi = { version = "0.9.1", default-features = false, features = ["core"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] wasmi = "0.9.0" [dependencies] wasmi = { version = "0.9.0", optional = true } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../io" } -sp-wasm-interface = { version = "4.0.0-dev", default-features = false, path = "../wasm-interface" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +sp-io = { version = "6.0.0", default-features = false, path = "../io" } +sp-wasm-interface = { version = "6.0.0", default-features = false, path = "../wasm-interface" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } log = { version = "0.4", default-features = false } [dev-dependencies] diff --git a/primitives/sandbox/embedded_executor.rs b/primitives/sandbox/src/embedded_executor.rs similarity index 90% rename from primitives/sandbox/embedded_executor.rs rename to primitives/sandbox/src/embedded_executor.rs index 678da3c3aeaf..43967a0a3898 100755 --- a/primitives/sandbox/embedded_executor.rs +++ b/primitives/sandbox/src/embedded_executor.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! An embedded WASM executor utilizing `wasmi`. + use super::{Error, HostError, HostFuncType, ReturnValue, Value, TARGET}; use alloc::string::String; use log::debug; @@ -27,13 +29,14 @@ use wasmi::{ RuntimeArgs, RuntimeValue, Signature, TableDescriptor, TableRef, Trap, TrapKind, }; +/// The linear memory used by the sandbox. #[derive(Clone)] pub struct Memory { memref: MemoryRef, } -impl Memory { - pub fn new(initial: u32, maximum: Option) -> Result { +impl super::SandboxMemory for Memory { + fn new(initial: u32, maximum: Option) -> Result { Ok(Memory { memref: MemoryInstance::alloc( Pages(initial as usize), @@ -43,12 +46,12 @@ impl Memory { }) } - pub fn get(&self, ptr: u32, buf: &mut [u8]) -> Result<(), Error> { + fn get(&self, ptr: u32, buf: &mut [u8]) -> Result<(), Error> { self.memref.get_into(ptr, buf).map_err(|_| Error::OutOfBounds)?; Ok(()) } - pub fn set(&self, ptr: u32, value: &[u8]) -> Result<(), Error> { + fn set(&self, ptr: u32, value: &[u8]) -> Result<(), Error> { self.memref.set(ptr, value).map_err(|_| Error::OutOfBounds)?; Ok(()) } @@ -118,20 +121,21 @@ enum ExternVal { Memory(Memory), } +/// A builder for the environment of the sandboxed WASM module. pub struct EnvironmentDefinitionBuilder { map: BTreeMap<(Vec, Vec), ExternVal>, defined_host_functions: DefinedHostFunctions, } -impl EnvironmentDefinitionBuilder { - pub fn new() -> EnvironmentDefinitionBuilder { +impl super::SandboxEnvironmentBuilder for EnvironmentDefinitionBuilder { + fn new() -> EnvironmentDefinitionBuilder { EnvironmentDefinitionBuilder { map: BTreeMap::new(), defined_host_functions: DefinedHostFunctions::new(), } } - pub fn add_host_func(&mut self, module: N1, field: N2, f: HostFuncType) + fn add_host_func(&mut self, module: N1, field: N2, f: HostFuncType) where N1: Into>, N2: Into>, @@ -140,7 +144,7 @@ impl EnvironmentDefinitionBuilder { self.map.insert((module.into(), field.into()), ExternVal::HostFunc(idx)); } - pub fn add_memory(&mut self, module: N1, field: N2, mem: Memory) + fn add_memory(&mut self, module: N1, field: N2, mem: Memory) where N1: Into>, N2: Into>, @@ -213,14 +217,18 @@ impl ImportResolver for EnvironmentDefinitionBuilder { } } +/// Sandboxed instance of a WASM module. pub struct Instance { instance: ModuleRef, defined_host_functions: DefinedHostFunctions, _marker: PhantomData, } -impl Instance { - pub fn new( +impl super::SandboxInstance for Instance { + type Memory = Memory; + type EnvironmentBuilder = EnvironmentDefinitionBuilder; + + fn new( code: &[u8], env_def_builder: &EnvironmentDefinitionBuilder, state: &mut T, @@ -241,12 +249,7 @@ impl Instance { Ok(Instance { instance, defined_host_functions, _marker: PhantomData:: }) } - pub fn invoke( - &mut self, - name: &str, - args: &[Value], - state: &mut T, - ) -> Result { + fn invoke(&mut self, name: &str, args: &[Value], state: &mut T) -> Result { let args = args.iter().cloned().map(to_wasmi).collect::>(); let mut externals = @@ -260,7 +263,7 @@ impl Instance { } } - pub fn get_global_val(&self, name: &str) -> Option { + fn get_global_val(&self, name: &str) -> Option { let global = self.instance.export_by_name(name)?.as_global()?.get(); Some(to_interface(global)) @@ -289,7 +292,8 @@ fn to_interface(value: RuntimeValue) -> Value { #[cfg(test)] mod tests { - use crate::{EnvironmentDefinitionBuilder, Error, HostError, Instance, ReturnValue, Value}; + use super::{EnvironmentDefinitionBuilder, Instance}; + use crate::{Error, HostError, ReturnValue, SandboxEnvironmentBuilder, SandboxInstance, Value}; use assert_matches::assert_matches; fn execute_sandboxed(code: &[u8], args: &[Value]) -> Result { diff --git a/primitives/sandbox/host_executor.rs b/primitives/sandbox/src/host_executor.rs similarity index 78% rename from primitives/sandbox/host_executor.rs rename to primitives/sandbox/src/host_executor.rs index d2836e2ffd1e..83721f40e3d9 100755 --- a/primitives/sandbox/host_executor.rs +++ b/primitives/sandbox/src/host_executor.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,15 +15,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! A WASM executor utilizing the sandbox runtime interface of the host. + +use super::{Error, HostFuncType, ReturnValue, Value}; use codec::{Decode, Encode}; use sp_core::sandbox as sandbox_primitives; use sp_io::sandbox; -use sp_std::{prelude::*, slice, marker, mem, vec, rc::Rc}; -use super::{Error, Value, ReturnValue, HostFuncType}; +use sp_std::{marker, mem, prelude::*, rc::Rc, slice, vec}; mod ffi { - use sp_std::mem; use super::HostFuncType; + use sp_std::mem; /// Index into the default table that points to a `HostFuncType`. pub type HostFuncIndex = usize; @@ -38,8 +40,9 @@ mod ffi { pub unsafe fn coerce_host_index_to_func(idx: HostFuncIndex) -> HostFuncType { // We need to ensure that sizes of a callable function pointer and host function index is // indeed equal. - // We can't use `static_assertions` create because it makes compiler panic, fallback to runtime assert. - // const_assert!(mem::size_of::() == mem::size_of::>()); + // We can't use `static_assertions` create because it makes compiler panic, fallback to + // runtime assert. const_assert!(mem::size_of::() == + // mem::size_of::>()); assert!(mem::size_of::() == mem::size_of::>()); mem::transmute::>(idx) } @@ -55,6 +58,7 @@ impl Drop for MemoryHandle { } } +/// The linear memory used by the sandbox. #[derive(Clone)] pub struct Memory { // Handle to memory instance is wrapped to add reference-counting semantics @@ -62,29 +66,20 @@ pub struct Memory { handle: Rc, } -impl Memory { - pub fn new(initial: u32, maximum: Option) -> Result { - let maximum = if let Some(maximum) = maximum { - maximum - } else { - sandbox_primitives::MEM_UNLIMITED - }; +impl super::SandboxMemory for Memory { + fn new(initial: u32, maximum: Option) -> Result { + let maximum = + if let Some(maximum) = maximum { maximum } else { sandbox_primitives::MEM_UNLIMITED }; match sandbox::memory_new(initial, maximum) { sandbox_primitives::ERR_MODULE => Err(Error::Module), - memory_idx => Ok(Memory { - handle: Rc::new(MemoryHandle { memory_idx, }), - }), + memory_idx => Ok(Memory { handle: Rc::new(MemoryHandle { memory_idx }) }), } } - pub fn get(&self, offset: u32, buf: &mut [u8]) -> Result<(), Error> { - let result = sandbox::memory_get( - self.handle.memory_idx, - offset, - buf.as_mut_ptr(), - buf.len() as u32, - ); + fn get(&self, offset: u32, buf: &mut [u8]) -> Result<(), Error> { + let result = + sandbox::memory_get(self.handle.memory_idx, offset, buf.as_mut_ptr(), buf.len() as u32); match result { sandbox_primitives::ERR_OK => Ok(()), sandbox_primitives::ERR_OUT_OF_BOUNDS => Err(Error::OutOfBounds), @@ -92,11 +87,11 @@ impl Memory { } } - pub fn set(&self, offset: u32, val: &[u8]) -> Result<(), Error> { + fn set(&self, offset: u32, val: &[u8]) -> Result<(), Error> { let result = sandbox::memory_set( self.handle.memory_idx, offset, - val.as_ptr() as _ , + val.as_ptr() as _, val.len() as u32, ); match result { @@ -107,6 +102,7 @@ impl Memory { } } +/// A builder for the environment of the sandboxed WASM module. pub struct EnvironmentDefinitionBuilder { env_def: sandbox_primitives::EnvironmentDefinition, retained_memories: Vec, @@ -114,16 +110,6 @@ pub struct EnvironmentDefinitionBuilder { } impl EnvironmentDefinitionBuilder { - pub fn new() -> EnvironmentDefinitionBuilder { - EnvironmentDefinitionBuilder { - env_def: sandbox_primitives::EnvironmentDefinition { - entries: Vec::new(), - }, - retained_memories: Vec::new(), - _marker: marker::PhantomData::, - } - } - fn add_entry( &mut self, module: N1, @@ -140,8 +126,18 @@ impl EnvironmentDefinitionBuilder { }; self.env_def.entries.push(entry); } +} - pub fn add_host_func(&mut self, module: N1, field: N2, f: HostFuncType) +impl super::SandboxEnvironmentBuilder for EnvironmentDefinitionBuilder { + fn new() -> EnvironmentDefinitionBuilder { + EnvironmentDefinitionBuilder { + env_def: sandbox_primitives::EnvironmentDefinition { entries: Vec::new() }, + retained_memories: Vec::new(), + _marker: marker::PhantomData::, + } + } + + fn add_host_func(&mut self, module: N1, field: N2, f: HostFuncType) where N1: Into>, N2: Into>, @@ -150,7 +146,7 @@ impl EnvironmentDefinitionBuilder { self.add_entry(module, field, f); } - pub fn add_memory(&mut self, module: N1, field: N2, mem: Memory) + fn add_memory(&mut self, module: N1, field: N2, mem: Memory) where N1: Into>, N2: Into>, @@ -163,6 +159,7 @@ impl EnvironmentDefinitionBuilder { } } +/// Sandboxed instance of a WASM module. pub struct Instance { instance_idx: u32, _retained_memories: Vec, @@ -211,8 +208,11 @@ extern "C" fn dispatch_thunk( } } -impl Instance { - pub fn new( +impl super::SandboxInstance for Instance { + type Memory = Memory; + type EnvironmentBuilder = EnvironmentDefinitionBuilder; + + fn new( code: &[u8], env_def_builder: &EnvironmentDefinitionBuilder, state: &mut T, @@ -242,12 +242,7 @@ impl Instance { }) } - pub fn invoke( - &mut self, - name: &str, - args: &[Value], - state: &mut T, - ) -> Result { + fn invoke(&mut self, name: &str, args: &[Value], state: &mut T) -> Result { let serialized_args = args.to_vec().encode(); let mut return_val = vec![0u8; ReturnValue::ENCODED_MAX_SIZE]; @@ -262,16 +257,16 @@ impl Instance { match result { sandbox_primitives::ERR_OK => { - let return_val = ReturnValue::decode(&mut &return_val[..]) - .map_err(|_| Error::Execution)?; + let return_val = + ReturnValue::decode(&mut &return_val[..]).map_err(|_| Error::Execution)?; Ok(return_val) - } + }, sandbox_primitives::ERR_EXECUTION => Err(Error::Execution), _ => unreachable!(), } } - pub fn get_global_val(&self, name: &str) -> Option { + fn get_global_val(&self, name: &str) -> Option { sandbox::get_global_val(self.instance_idx, name) } } diff --git a/primitives/sandbox/src/lib.rs b/primitives/sandbox/src/lib.rs index 1724b4152ff3..537c7cbb31b6 100755 --- a/primitives/sandbox/src/lib.rs +++ b/primitives/sandbox/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -48,13 +48,15 @@ pub use sp_wasm_interface::{ReturnValue, Value}; /// The target used for logging. const TARGET: &str = "runtime::sandbox"; -mod imp { - #[cfg(all(feature = "wasmer-sandbox", not(feature = "std")))] - include!("../host_executor.rs"); +pub mod embedded_executor; +#[cfg(not(feature = "std"))] +pub mod host_executor; - #[cfg(not(all(feature = "wasmer-sandbox", not(feature = "std"))))] - include!("../embedded_executor.rs"); -} +#[cfg(all(feature = "wasmer-sandbox", not(feature = "std")))] +pub use host_executor as default_executor; + +#[cfg(not(all(feature = "wasmer-sandbox", not(feature = "std"))))] +pub use embedded_executor as default_executor; /// Error that can occur while using this crate. #[derive(sp_core::RuntimeDebug)] @@ -87,13 +89,8 @@ pub type HostFuncType = fn(&mut T, &[Value]) -> Result) -> Result { - Ok(Memory { inner: imp::Memory::new(initial, maximum)? }) - } + fn new(initial: u32, maximum: Option) -> Result; /// Read a memory area at the address `ptr` with the size of the provided slice `buf`. /// /// Returns `Err` if the range is out-of-bounds. - pub fn get(&self, ptr: u32, buf: &mut [u8]) -> Result<(), Error> { - self.inner.get(ptr, buf) - } + fn get(&self, ptr: u32, buf: &mut [u8]) -> Result<(), Error>; /// Write a memory area at the address `ptr` with contents of the provided slice `buf`. /// /// Returns `Err` if the range is out-of-bounds. - pub fn set(&self, ptr: u32, value: &[u8]) -> Result<(), Error> { - self.inner.set(ptr, value) - } + fn set(&self, ptr: u32, value: &[u8]) -> Result<(), Error>; } /// Struct that can be used for defining an environment for a sandboxed module. /// /// The sandboxed module can access only the entities which were defined and passed /// to the module at the instantiation time. -pub struct EnvironmentDefinitionBuilder { - inner: imp::EnvironmentDefinitionBuilder, -} - -impl EnvironmentDefinitionBuilder { +pub trait SandboxEnvironmentBuilder: Sized { /// Construct a new `EnvironmentDefinitionBuilder`. - pub fn new() -> EnvironmentDefinitionBuilder { - EnvironmentDefinitionBuilder { inner: imp::EnvironmentDefinitionBuilder::new() } - } + fn new() -> Self; /// Register a host function in this environment definition. /// @@ -143,32 +128,28 @@ impl EnvironmentDefinitionBuilder { /// can import function passed here with any signature it wants. It can even import /// the same function (i.e. with same `module` and `field`) several times. It's up to /// the user code to check or constrain the types of signatures. - pub fn add_host_func(&mut self, module: N1, field: N2, f: HostFuncType) + fn add_host_func(&mut self, module: N1, field: N2, f: HostFuncType) where N1: Into>, - N2: Into>, - { - self.inner.add_host_func(module, field, f); - } + N2: Into>; /// Register a memory in this environment definition. - pub fn add_memory(&mut self, module: N1, field: N2, mem: Memory) + fn add_memory(&mut self, module: N1, field: N2, mem: Memory) where N1: Into>, - N2: Into>, - { - self.inner.add_memory(module, field, mem.inner); - } + N2: Into>; } /// Sandboxed instance of a wasm module. /// /// This instance can be used for invoking exported functions. -pub struct Instance { - inner: imp::Instance, -} +pub trait SandboxInstance: Sized { + /// The memory type used for this sandbox. + type Memory: SandboxMemory; + + /// The environment builder used to construct this sandbox. + type EnvironmentBuilder: SandboxEnvironmentBuilder; -impl Instance { /// Instantiate a module with the given [`EnvironmentDefinitionBuilder`]. It will /// run the `start` function (if it is present in the module) with the given `state`. /// @@ -177,13 +158,11 @@ impl Instance { /// will be returned. /// /// [`EnvironmentDefinitionBuilder`]: struct.EnvironmentDefinitionBuilder.html - pub fn new( + fn new( code: &[u8], - env_def_builder: &EnvironmentDefinitionBuilder, - state: &mut T, - ) -> Result, Error> { - Ok(Instance { inner: imp::Instance::new(code, &env_def_builder.inner, state)? }) - } + env_def_builder: &Self::EnvironmentBuilder, + state: &mut State, + ) -> Result; /// Invoke an exported function with the given name. /// @@ -196,19 +175,15 @@ impl Instance { /// - If types of the arguments passed to the function doesn't match function signature then /// trap occurs (as if the exported function was called via call_indirect), /// - Trap occurred at the execution time. - pub fn invoke( + fn invoke( &mut self, name: &str, args: &[Value], - state: &mut T, - ) -> Result { - self.inner.invoke(name, args, state) - } + state: &mut State, + ) -> Result; /// Get the value from a global with the given `name`. /// /// Returns `Some(_)` if the global could be found. - pub fn get_global_val(&self, name: &str) -> Option { - self.inner.get_global_val(name) - } + fn get_global_val(&self, name: &str) -> Option; } diff --git a/primitives/serializer/Cargo.toml b/primitives/serializer/Cargo.toml index 2200274e0628..c81f1cd10a82 100644 --- a/primitives/serializer/Cargo.toml +++ b/primitives/serializer/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-serializer" -version = "3.0.0" +version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate customizable serde serializer." documentation = "https://docs.rs/sp-serializer" @@ -14,5 +14,5 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -serde = "1.0.126" -serde_json = "1.0.68" +serde = "1.0.136" +serde_json = "1.0.79" diff --git a/primitives/serializer/src/lib.rs b/primitives/serializer/src/lib.rs index ccdbbf27f179..d1e364a133ba 100644 --- a/primitives/serializer/src/lib.rs +++ b/primitives/serializer/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/session/Cargo.toml b/primitives/session/Cargo.toml index 8e1e2464e49e..476bacc88ae3 100644 --- a/primitives/session/Cargo.toml +++ b/primitives/session/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-session" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Primitives for sessions" readme = "README.md" @@ -13,13 +13,13 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } sp-api = { version = "4.0.0-dev", default-features = false, path = "../api" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../staking" } -sp-runtime = { version = "4.0.0-dev", optional = true, path = "../runtime" } +sp-runtime = { version = "6.0.0", optional = true, path = "../runtime" } [features] default = [ "std" ] diff --git a/primitives/session/src/lib.rs b/primitives/session/src/lib.rs index d85b6af4349e..1b25d285e3bc 100644 --- a/primitives/session/src/lib.rs +++ b/primitives/session/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/staking/Cargo.toml b/primitives/staking/Cargo.toml index 9e852319ede4..7761519dabef 100644 --- a/primitives/staking/Cargo.toml +++ b/primitives/staking/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-staking" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "A crate which contains primitives that are useful for implementation that uses staking approaches in general. Definitions related to sessions, slashing, etc go here." readme = "README.md" @@ -13,10 +13,10 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } [features] default = ["std"] diff --git a/primitives/staking/src/lib.rs b/primitives/staking/src/lib.rs index 4bb8ed93f88a..15208df62cc6 100644 --- a/primitives/staking/src/lib.rs +++ b/primitives/staking/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,3 +24,6 @@ pub mod offence; /// Simple index type with which we can count sessions. pub type SessionIndex = u32; + +/// Counter for the number of eras that have passed. +pub type EraIndex = u32; diff --git a/primitives/staking/src/offence.rs b/primitives/staking/src/offence.rs index a91cb47c117b..4261063993a5 100644 --- a/primitives/staking/src/offence.rs +++ b/primitives/staking/src/offence.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -37,6 +37,29 @@ pub type Kind = [u8; 16]; /// so that we can slash it accordingly. pub type OffenceCount = u32; +/// In case of an offence, which conditions get an offending validator disabled. +#[derive( + Clone, + Copy, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + Encode, + Decode, + sp_runtime::RuntimeDebug, + scale_info::TypeInfo, +)] +pub enum DisableStrategy { + /// Independently of slashing, this offence will not disable the offender. + Never, + /// Only disable the offender if it is also slashed. + WhenSlashed, + /// Independently of slashing, this offence will always disable the offender. + Always, +} + /// A trait implemented by an offence report. /// /// This trait assumes that the offence is legitimate and was validated already. @@ -79,6 +102,11 @@ pub trait Offence { /// number. Note that for GRANDPA the round number is reset each epoch. fn time_slot(&self) -> Self::TimeSlot; + /// In which cases this offence needs to disable offenders until the next era starts. + fn disable_strategy(&self) -> DisableStrategy { + DisableStrategy::WhenSlashed + } + /// A slash fraction of the total exposure that should be slashed for this /// particular offence kind for the given parameters that happened at a singular `TimeSlot`. /// @@ -150,12 +178,15 @@ pub trait OnOffenceHandler { /// /// The `session` parameter is the session index of the offence. /// + /// The `disable_strategy` parameter decides if the offenders need to be disabled immediately. + /// /// The receiver might decide to not accept this offence. In this case, the call site is /// responsible for queuing the report and re-submitting again. fn on_offence( offenders: &[OffenceDetails], slash_fraction: &[Perbill], session: SessionIndex, + disable_strategy: DisableStrategy, ) -> Res; } @@ -164,6 +195,7 @@ impl OnOffenceHandler _offenders: &[OffenceDetails], _slash_fraction: &[Perbill], _session: SessionIndex, + _disable_strategy: DisableStrategy, ) -> Res { Default::default() } diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index e444ae223a74..80651130575e 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "sp-state-machine" -version = "0.10.0-dev" +version = "0.12.0" authors = ["Parity Technologies "] description = "Substrate State Machine" -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sp-state-machine" readme = "README.md" @@ -15,26 +15,25 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] log = { version = "0.4.11", optional = true } -thiserror = { version = "1.0.21", optional = true } -parking_lot = { version = "0.11.1", optional = true } +thiserror = { version = "1.0.30", optional = true } +parking_lot = { version = "0.12.0", optional = true } hash-db = { version = "0.15.2", default-features = false } -trie-db = { version = "0.22.6", default-features = false } -trie-root = { version = "0.16.0", default-features = false } -sp-trie = { version = "4.0.0-dev", path = "../trie", default-features = false } -sp-core = { version = "4.0.0-dev", path = "../core", default-features = false } -sp-panic-handler = { version = "3.0.0", path = "../panic-handler", optional = true } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } +trie-root = { version = "0.17.0", default-features = false } +sp-trie = { version = "6.0.0", path = "../trie", default-features = false } +sp-core = { version = "6.0.0", path = "../core", default-features = false } +sp-panic-handler = { version = "4.0.0", path = "../panic-handler", optional = true } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } num-traits = { version = "0.2.8", default-features = false } rand = { version = "0.7.2", optional = true } -sp-externalities = { version = "0.10.0-dev", path = "../externalities", default-features = false } -smallvec = "1.7.0" -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -tracing = { version = "0.1.22", optional = true } +sp-externalities = { version = "0.12.0", path = "../externalities", default-features = false } +smallvec = "1.8.0" +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +tracing = { version = "0.1.29", optional = true } [dev-dependencies] -hex-literal = "0.3.1" -sp-runtime = { version = "4.0.0-dev", path = "../runtime" } -pretty_assertions = "0.6.1" +hex-literal = "0.3.4" +sp-runtime = { version = "6.0.0", path = "../runtime" } +pretty_assertions = "1.0.0" rand = "0.7.2" [features] @@ -47,7 +46,6 @@ std = [ "sp-externalities/std", "sp-std/std", "sp-trie/std", - "trie-db/std", "trie-root/std", "log", "thiserror", diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 7dcf92b06de0..8d0ac2d1369c 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,7 +23,7 @@ use crate::{ }; use codec::Encode; use hash_db::Hasher; -use sp_core::storage::{ChildInfo, TrackedStorageKey}; +use sp_core::storage::{ChildInfo, StateVersion, TrackedStorageKey}; #[cfg(feature = "std")] use sp_core::traits::RuntimeCode; use sp_std::vec::Vec; @@ -40,7 +40,7 @@ pub trait Backend: sp_std::fmt::Debug { type Transaction: Consolidate + Default + Send; /// Type of trie backend storage. - type TrieBackendStorage: TrieBackendStorage; + type TrieBackendStorage: TrieBackendStorage; /// Get keyed storage or None if there is nothing associated. fn storage(&self, key: &[u8]) -> Result, Self::Error>; @@ -140,6 +140,7 @@ pub trait Backend: sp_std::fmt::Debug { fn storage_root<'a>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (H::Out, Self::Transaction) where H::Out: Ord; @@ -151,6 +152,7 @@ pub trait Backend: sp_std::fmt::Debug { &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord; @@ -176,7 +178,6 @@ pub trait Backend: sp_std::fmt::Debug { fn as_trie_backend(&self) -> Option<&TrieBackend> { None } - /// Calculate the storage root, with given delta over what is already stored /// in the backend, and produce a "transaction" that can be used to commit. /// Does include child storage updates. @@ -186,6 +187,7 @@ pub trait Backend: sp_std::fmt::Debug { child_deltas: impl Iterator< Item = (&'a ChildInfo, impl Iterator)>), >, + state_version: StateVersion, ) -> (H::Out, Self::Transaction) where H::Out: Ord + Encode, @@ -194,7 +196,8 @@ pub trait Backend: sp_std::fmt::Debug { let mut child_roots: Vec<_> = Default::default(); // child first for (child_info, child_delta) in child_deltas { - let (child_root, empty, child_txs) = self.child_storage_root(&child_info, child_delta); + let (child_root, empty, child_txs) = + self.child_storage_root(&child_info, child_delta, state_version); let prefixed_storage_key = child_info.prefixed_storage_key(); txs.consolidate(child_txs); if empty { @@ -207,6 +210,7 @@ pub trait Backend: sp_std::fmt::Debug { delta .map(|(k, v)| (k, v.as_ref().map(|v| &v[..]))) .chain(child_roots.iter().map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..])))), + state_version, ); txs.consolidate(parent_txs); (root, txs) @@ -286,36 +290,14 @@ impl Consolidate for Vec<(Option, StorageCollection)> { } } -impl> Consolidate for sp_trie::GenericMemoryDB { - fn consolidate(&mut self, other: Self) { - sp_trie::GenericMemoryDB::consolidate(self, other) - } -} - -/// Insert input pairs into memory db. -#[cfg(test)] -pub(crate) fn insert_into_memory_db( - mdb: &mut sp_trie::MemoryDB, - input: I, -) -> Option +impl Consolidate for sp_trie::GenericMemoryDB where H: Hasher, - I: IntoIterator, + KF: sp_trie::KeyFunction, { - use sp_trie::{trie_types::TrieDBMut, TrieMut}; - - let mut root = ::Out::default(); - { - let mut trie = TrieDBMut::::new(mdb, &mut root); - for (key, value) in input { - if let Err(e) = trie.insert(&key, &value) { - log::warn!(target: "trie", "Failed to write to trie: {}", e); - return None - } - } + fn consolidate(&mut self, other: Self) { + sp_trie::GenericMemoryDB::consolidate(self, other) } - - Some(root) } /// Wrapper to create a [`RuntimeCode`] from a type that implements [`Backend`]. diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 0bbd2d0a8e8e..1f257550fbf7 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,13 +23,14 @@ use hash_db::Hasher; use log::warn; use sp_core::{ storage::{ - well_known_keys::is_child_storage_key, ChildInfo, Storage, StorageChild, TrackedStorageKey, + well_known_keys::is_child_storage_key, ChildInfo, StateVersion, Storage, StorageChild, + TrackedStorageKey, }, traits::Externalities, Blake2Hasher, }; use sp_externalities::{Extension, Extensions}; -use sp_trie::{empty_child_trie_root, trie_types::Layout, TrieConfiguration}; +use sp_trie::{empty_child_trie_root, LayoutV0, LayoutV1, TrieConfiguration}; use std::{ any::{Any, TypeId}, collections::BTreeMap, @@ -273,7 +274,7 @@ impl Externalities for BasicExternalities { crate::ext::StorageAppend::new(current).append(value); } - fn storage_root(&mut self) -> Vec { + fn storage_root(&mut self, state_version: StateVersion) -> Vec { let mut top = self.inner.top.clone(); let prefixed_keys: Vec<_> = self .inner @@ -284,9 +285,9 @@ impl Externalities for BasicExternalities { // Single child trie implementation currently allows using the same child // empty root for all child trie. Using null storage key until multiple // type of child trie support. - let empty_hash = empty_child_trie_root::>(); + let empty_hash = empty_child_trie_root::>(); for (prefixed_storage_key, child_info) in prefixed_keys { - let child_root = self.child_storage_root(&child_info); + let child_root = self.child_storage_root(&child_info, state_version); if &empty_hash[..] == &child_root[..] { top.remove(prefixed_storage_key.as_slice()); } else { @@ -294,25 +295,30 @@ impl Externalities for BasicExternalities { } } - Layout::::trie_root(self.inner.top.clone()).as_ref().into() + match state_version { + StateVersion::V0 => + LayoutV0::::trie_root(self.inner.top.clone()).as_ref().into(), + StateVersion::V1 => + LayoutV1::::trie_root(self.inner.top.clone()).as_ref().into(), + } } - fn child_storage_root(&mut self, child_info: &ChildInfo) -> Vec { + fn child_storage_root( + &mut self, + child_info: &ChildInfo, + state_version: StateVersion, + ) -> Vec { if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); crate::in_memory_backend::new_in_mem::() - .child_storage_root(&child.child_info, delta) + .child_storage_root(&child.child_info, delta, state_version) .0 } else { - empty_child_trie_root::>() + empty_child_trie_root::>() } .encode() } - fn storage_changes_root(&mut self, _parent: &[u8]) -> Result>, ()> { - Ok(None) - } - fn storage_start_transaction(&mut self) { unimplemented!("Transactions are not supported by BasicExternalities"); } @@ -393,7 +399,7 @@ mod tests { const ROOT: [u8; 32] = hex!("39245109cef3758c2eed2ccba8d9b370a917850af3824bc8348d505df2c298fa"); - assert_eq!(&ext.storage_root()[..], &ROOT); + assert_eq!(&ext.storage_root(StateVersion::default())[..], &ROOT); } #[test] diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs deleted file mode 100644 index d3c6c12122c4..000000000000 --- a/primitives/state-machine/src/changes_trie/build.rs +++ /dev/null @@ -1,1083 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Structures and functions required to build changes trie for given block. - -use crate::{ - backend::Backend, - changes_trie::{ - build_iterator::digest_build_iterator, - input::{ChildIndex, DigestIndex, ExtrinsicIndex, InputKey, InputPair}, - AnchorBlockId, BlockNumber, ConfigurationRange, Storage, - }, - overlayed_changes::{OverlayedChanges, OverlayedValue}, - trie_backend_essence::TrieBackendEssence, - StorageKey, -}; -use codec::{Decode, Encode}; -use hash_db::Hasher; -use num_traits::One; -use sp_core::storage::{ChildInfo, PrefixedStorageKey}; -use std::collections::{btree_map::Entry, BTreeMap}; - -/// Prepare input pairs for building a changes trie of given block. -/// -/// Returns Err if storage error has occurred OR if storage haven't returned -/// required data. -pub(crate) fn prepare_input<'a, B, H, Number>( - backend: &'a B, - storage: &'a dyn Storage, - config: ConfigurationRange<'a, Number>, - overlay: &'a OverlayedChanges, - parent: &'a AnchorBlockId, -) -> Result< - ( - impl Iterator> + 'a, - Vec<(ChildIndex, impl Iterator> + 'a)>, - Vec, - ), - String, -> -where - B: Backend, - H: Hasher + 'a, - H::Out: Encode, - Number: BlockNumber, -{ - let number = parent.number.clone() + One::one(); - let (extrinsics_input, children_extrinsics_input) = - prepare_extrinsics_input(backend, &number, overlay)?; - let (digest_input, mut children_digest_input, digest_input_blocks) = - prepare_digest_input::(parent, config, number, storage)?; - - let mut children_digest = Vec::with_capacity(children_extrinsics_input.len()); - for (child_index, ext_iter) in children_extrinsics_input.into_iter() { - let dig_iter = children_digest_input.remove(&child_index); - children_digest.push(( - child_index, - Some(ext_iter).into_iter().flatten().chain(dig_iter.into_iter().flatten()), - )); - } - for (child_index, dig_iter) in children_digest_input.into_iter() { - children_digest.push(( - child_index, - None.into_iter().flatten().chain(Some(dig_iter).into_iter().flatten()), - )); - } - - Ok((extrinsics_input.chain(digest_input), children_digest, digest_input_blocks)) -} -/// Prepare ExtrinsicIndex input pairs. -fn prepare_extrinsics_input<'a, B, H, Number>( - backend: &'a B, - block: &Number, - overlay: &'a OverlayedChanges, -) -> Result< - ( - impl Iterator> + 'a, - BTreeMap, impl Iterator> + 'a>, - ), - String, -> -where - B: Backend, - H: Hasher + 'a, - Number: BlockNumber, -{ - let mut children_result = BTreeMap::new(); - - for (child_changes, child_info) in overlay.children() { - let child_index = ChildIndex:: { - block: block.clone(), - storage_key: child_info.prefixed_storage_key(), - }; - - let iter = prepare_extrinsics_input_inner( - backend, - block, - overlay, - Some(child_info.clone()), - child_changes, - )?; - children_result.insert(child_index, iter); - } - - let top = prepare_extrinsics_input_inner(backend, block, overlay, None, overlay.changes())?; - - Ok((top, children_result)) -} - -fn prepare_extrinsics_input_inner<'a, B, H, Number>( - backend: &'a B, - block: &Number, - overlay: &'a OverlayedChanges, - child_info: Option, - changes: impl Iterator, -) -> Result> + 'a, String> -where - B: Backend, - H: Hasher, - Number: BlockNumber, -{ - changes - .filter_map(|(k, v)| { - let extrinsics = v.extrinsics(); - if !extrinsics.is_empty() { - Some((k, extrinsics)) - } else { - None - } - }) - .try_fold( - BTreeMap::new(), - |mut map: BTreeMap<&[u8], (ExtrinsicIndex, Vec)>, (k, extrinsics)| { - match map.entry(k) { - Entry::Vacant(entry) => { - // ignore temporary values (values that have null value at the end of - // operation AND are not in storage at the beginning of operation - if let Some(child_info) = child_info.as_ref() { - if !overlay - .child_storage(child_info, k) - .map(|v| v.is_some()) - .unwrap_or_default() - { - if !backend - .exists_child_storage(&child_info, k) - .map_err(|e| format!("{}", e))? - { - return Ok(map) - } - } - } else { - if !overlay.storage(k).map(|v| v.is_some()).unwrap_or_default() { - if !backend.exists_storage(k).map_err(|e| format!("{}", e))? { - return Ok(map) - } - } - }; - - let extrinsics = extrinsics.into_iter().collect(); - entry.insert(( - ExtrinsicIndex { block: block.clone(), key: k.to_vec() }, - extrinsics, - )); - }, - Entry::Occupied(mut entry) => { - // we do not need to check for temporary values here, because entry is - // Occupied AND we are checking it before insertion - let entry_extrinsics = &mut entry.get_mut().1; - entry_extrinsics.extend(extrinsics.into_iter()); - entry_extrinsics.sort(); - }, - } - - Ok(map) - }, - ) - .map(|pairs| pairs.into_iter().map(|(_, (k, v))| InputPair::ExtrinsicIndex(k, v))) -} - -/// Prepare DigestIndex input pairs. -fn prepare_digest_input<'a, H, Number>( - parent: &'a AnchorBlockId, - config: ConfigurationRange, - block: Number, - storage: &'a dyn Storage, -) -> Result< - ( - impl Iterator> + 'a, - BTreeMap, impl Iterator> + 'a>, - Vec, - ), - String, -> -where - H: Hasher, - H::Out: 'a + Encode, - Number: BlockNumber, -{ - let build_skewed_digest = config.end.as_ref() == Some(&block); - let block_for_digest = if build_skewed_digest { - config - .config - .next_max_level_digest_range(config.zero.clone(), block.clone()) - .map(|(_, end)| end) - .unwrap_or_else(|| block.clone()) - } else { - block.clone() - }; - - let digest_input_blocks = digest_build_iterator(config, block_for_digest).collect::>(); - digest_input_blocks - .clone() - .into_iter() - .try_fold( - (BTreeMap::new(), BTreeMap::new()), - move |(mut map, mut child_map), digest_build_block| { - let extrinsic_prefix = - ExtrinsicIndex::key_neutral_prefix(digest_build_block.clone()); - let digest_prefix = DigestIndex::key_neutral_prefix(digest_build_block.clone()); - let child_prefix = ChildIndex::key_neutral_prefix(digest_build_block.clone()); - let trie_root = storage.root(parent, digest_build_block.clone())?; - let trie_root = trie_root.ok_or_else(|| { - format!("No changes trie root for block {}", digest_build_block.clone()) - })?; - - let insert_to_map = |map: &mut BTreeMap<_, _>, key: StorageKey| { - match map.entry(key.clone()) { - Entry::Vacant(entry) => { - entry.insert(( - DigestIndex { block: block.clone(), key }, - vec![digest_build_block.clone()], - )); - }, - Entry::Occupied(mut entry) => { - // DigestIndexValue must be sorted. Here we are relying on the fact that - // digest_build_iterator() returns blocks in ascending order => we only - // need to check for duplicates - // - // is_dup_block could be true when key has been changed in both digest - // block AND other blocks that it covers - let is_dup_block = entry.get().1.last() == Some(&digest_build_block); - if !is_dup_block { - entry.get_mut().1.push(digest_build_block.clone()); - } - }, - } - }; - - // try to get all updated keys from cache - let populated_from_cache = - storage.with_cached_changed_keys(&trie_root, &mut |changed_keys| { - for (storage_key, changed_keys) in changed_keys { - let map = match storage_key { - Some(storage_key) => child_map - .entry(ChildIndex:: { - block: block.clone(), - storage_key: storage_key.clone(), - }) - .or_default(), - None => &mut map, - }; - for changed_key in changed_keys.iter().cloned() { - insert_to_map(map, changed_key); - } - } - }); - if populated_from_cache { - return Ok((map, child_map)) - } - - let mut children_roots = BTreeMap::::new(); - { - let trie_storage = TrieBackendEssence::<_, H>::new( - crate::changes_trie::TrieBackendStorageAdapter(storage), - trie_root, - ); - - trie_storage.for_key_values_with_prefix(&child_prefix, |mut key, mut value| { - if let Ok(InputKey::ChildIndex::(trie_key)) = - Decode::decode(&mut key) - { - if let Ok(value) = >::decode(&mut value) { - let mut trie_root = ::Out::default(); - trie_root.as_mut().copy_from_slice(&value[..]); - children_roots.insert(trie_key.storage_key, trie_root); - } - } - }); - - trie_storage.for_keys_with_prefix(&extrinsic_prefix, |mut key| { - if let Ok(InputKey::ExtrinsicIndex::(trie_key)) = - Decode::decode(&mut key) - { - insert_to_map(&mut map, trie_key.key); - } - }); - - trie_storage.for_keys_with_prefix(&digest_prefix, |mut key| { - if let Ok(InputKey::DigestIndex::(trie_key)) = - Decode::decode(&mut key) - { - insert_to_map(&mut map, trie_key.key); - } - }); - } - - for (storage_key, trie_root) in children_roots.into_iter() { - let child_index = ChildIndex:: { block: block.clone(), storage_key }; - - let mut map = child_map.entry(child_index).or_default(); - let trie_storage = TrieBackendEssence::<_, H>::new( - crate::changes_trie::TrieBackendStorageAdapter(storage), - trie_root, - ); - trie_storage.for_keys_with_prefix(&extrinsic_prefix, |mut key| { - if let Ok(InputKey::ExtrinsicIndex::(trie_key)) = - Decode::decode(&mut key) - { - insert_to_map(&mut map, trie_key.key); - } - }); - - trie_storage.for_keys_with_prefix(&digest_prefix, |mut key| { - if let Ok(InputKey::DigestIndex::(trie_key)) = - Decode::decode(&mut key) - { - insert_to_map(&mut map, trie_key.key); - } - }); - } - Ok((map, child_map)) - }, - ) - .map(|(pairs, child_pairs)| { - ( - pairs.into_iter().map(|(_, (k, v))| InputPair::DigestIndex(k, v)), - child_pairs - .into_iter() - .map(|(sk, pairs)| { - (sk, pairs.into_iter().map(|(_, (k, v))| InputPair::DigestIndex(k, v))) - }) - .collect(), - digest_input_blocks, - ) - }) -} - -#[cfg(test)] -mod test { - use super::*; - use crate::{ - changes_trie::{ - build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}, - storage::InMemoryStorage, - Configuration, RootsStorage, - }, - InMemoryBackend, - }; - use sp_core::Blake2Hasher; - - fn prepare_for_build( - zero: u64, - ) -> ( - InMemoryBackend, - InMemoryStorage, - OverlayedChanges, - Configuration, - ) { - let child_info_1 = ChildInfo::new_default(b"storage_key1"); - let child_info_2 = ChildInfo::new_default(b"storage_key2"); - let backend: InMemoryBackend<_> = vec![ - (vec![100], vec![255]), - (vec![101], vec![255]), - (vec![102], vec![255]), - (vec![103], vec![255]), - (vec![104], vec![255]), - (vec![105], vec![255]), - ] - .into_iter() - .collect::>() - .into(); - let prefixed_child_trie_key1 = child_info_1.prefixed_storage_key(); - let storage = InMemoryStorage::with_inputs( - vec![ - ( - zero + 1, - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 1, key: vec![100] }, - vec![1, 3], - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 1, key: vec![101] }, - vec![0, 2], - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 1, key: vec![105] }, - vec![0, 2, 4], - ), - ], - ), - ( - zero + 2, - vec![InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 2, key: vec![102] }, - vec![0], - )], - ), - ( - zero + 3, - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 3, key: vec![100] }, - vec![0], - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 3, key: vec![105] }, - vec![1], - ), - ], - ), - ( - zero + 4, - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4, key: vec![100] }, - vec![0, 2, 3], - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4, key: vec![101] }, - vec![1], - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4, key: vec![103] }, - vec![0, 1], - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![100] }, - vec![zero + 1, zero + 3], - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![101] }, - vec![zero + 1], - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![102] }, - vec![zero + 2], - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![105] }, - vec![zero + 1, zero + 3], - ), - ], - ), - (zero + 5, Vec::new()), - ( - zero + 6, - vec![InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 6, key: vec![105] }, - vec![2], - )], - ), - (zero + 7, Vec::new()), - ( - zero + 8, - vec![InputPair::DigestIndex( - DigestIndex { block: zero + 8, key: vec![105] }, - vec![zero + 6], - )], - ), - (zero + 9, Vec::new()), - (zero + 10, Vec::new()), - (zero + 11, Vec::new()), - (zero + 12, Vec::new()), - (zero + 13, Vec::new()), - (zero + 14, Vec::new()), - (zero + 15, Vec::new()), - ], - vec![( - prefixed_child_trie_key1.clone(), - vec![ - ( - zero + 1, - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 1, key: vec![100] }, - vec![1, 3], - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 1, key: vec![101] }, - vec![0, 2], - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 1, key: vec![105] }, - vec![0, 2, 4], - ), - ], - ), - ( - zero + 2, - vec![InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 2, key: vec![102] }, - vec![0], - )], - ), - ( - zero + 4, - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 2, key: vec![102] }, - vec![0, 3], - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![102] }, - vec![zero + 2], - ), - ], - ), - ], - )], - ); - - let mut changes = OverlayedChanges::default(); - changes.set_collect_extrinsics(true); - - changes.start_transaction(); - - changes.set_extrinsic_index(1); - changes.set_storage(vec![101], Some(vec![203])); - - changes.set_extrinsic_index(3); - changes.set_storage(vec![100], Some(vec![202])); - changes.set_child_storage(&child_info_1, vec![100], Some(vec![202])); - - changes.commit_transaction().unwrap(); - - changes.set_extrinsic_index(0); - changes.set_storage(vec![100], Some(vec![0])); - changes.set_extrinsic_index(2); - changes.set_storage(vec![100], Some(vec![200])); - - changes.set_extrinsic_index(0); - changes.set_storage(vec![103], Some(vec![0])); - changes.set_extrinsic_index(1); - changes.set_storage(vec![103], None); - - changes.set_extrinsic_index(0); - changes.set_child_storage(&child_info_1, vec![100], Some(vec![0])); - changes.set_extrinsic_index(2); - changes.set_child_storage(&child_info_1, vec![100], Some(vec![200])); - - changes.set_extrinsic_index(0); - changes.set_child_storage(&child_info_2, vec![100], Some(vec![0])); - changes.set_extrinsic_index(2); - changes.set_child_storage(&child_info_2, vec![100], Some(vec![200])); - - changes.set_extrinsic_index(1); - - let config = Configuration { digest_interval: 4, digest_levels: 2 }; - - (backend, storage, changes, config) - } - - fn configuration_range<'a>( - config: &'a Configuration, - zero: u64, - ) -> ConfigurationRange<'a, u64> { - ConfigurationRange { config, zero, end: None } - } - - #[test] - fn build_changes_trie_nodes_on_non_digest_block() { - fn test_with_zero(zero: u64) { - let child_trie_key1 = ChildInfo::new_default(b"storage_key1").prefixed_storage_key(); - let child_trie_key2 = ChildInfo::new_default(b"storage_key2").prefixed_storage_key(); - let (backend, storage, changes, config) = prepare_for_build(zero); - let parent = AnchorBlockId { hash: Default::default(), number: zero + 4 }; - let changes_trie_nodes = prepare_input( - &backend, - &storage, - configuration_range(&config, zero), - &changes, - &parent, - ) - .unwrap(); - assert_eq!( - changes_trie_nodes.0.collect::>>(), - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 5, key: vec![100] }, - vec![0, 2, 3] - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 5, key: vec![101] }, - vec![1] - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 5, key: vec![103] }, - vec![0, 1] - ), - ] - ); - assert_eq!( - changes_trie_nodes - .1 - .into_iter() - .map(|(k, v)| (k, v.collect::>())) - .collect::>(), - vec![ - ( - ChildIndex { block: zero + 5u64, storage_key: child_trie_key1 }, - vec![InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 5u64, key: vec![100] }, - vec![0, 2, 3] - ),] - ), - ( - ChildIndex { block: zero + 5, storage_key: child_trie_key2 }, - vec![InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 5, key: vec![100] }, - vec![0, 2] - ),] - ), - ] - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } - - #[test] - fn build_changes_trie_nodes_on_digest_block_l1() { - fn test_with_zero(zero: u64) { - let child_trie_key1 = ChildInfo::new_default(b"storage_key1").prefixed_storage_key(); - let child_trie_key2 = ChildInfo::new_default(b"storage_key2").prefixed_storage_key(); - let (backend, storage, changes, config) = prepare_for_build(zero); - let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 }; - let changes_trie_nodes = prepare_input( - &backend, - &storage, - configuration_range(&config, zero), - &changes, - &parent, - ) - .unwrap(); - assert_eq!( - changes_trie_nodes.0.collect::>>(), - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4, key: vec![100] }, - vec![0, 2, 3] - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4, key: vec![101] }, - vec![1] - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4, key: vec![103] }, - vec![0, 1] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![100] }, - vec![zero + 1, zero + 3] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![101] }, - vec![zero + 1] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![102] }, - vec![zero + 2] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![105] }, - vec![zero + 1, zero + 3] - ), - ] - ); - assert_eq!( - changes_trie_nodes - .1 - .into_iter() - .map(|(k, v)| (k, v.collect::>())) - .collect::>(), - vec![ - ( - ChildIndex { block: zero + 4u64, storage_key: child_trie_key1.clone() }, - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4u64, key: vec![100] }, - vec![0, 2, 3] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![100] }, - vec![zero + 1] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![101] }, - vec![zero + 1] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![102] }, - vec![zero + 2] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![105] }, - vec![zero + 1] - ), - ] - ), - ( - ChildIndex { block: zero + 4, storage_key: child_trie_key2.clone() }, - vec![InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4, key: vec![100] }, - vec![0, 2] - ),] - ), - ] - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } - - #[test] - fn build_changes_trie_nodes_on_digest_block_l2() { - fn test_with_zero(zero: u64) { - let child_trie_key1 = ChildInfo::new_default(b"storage_key1").prefixed_storage_key(); - let child_trie_key2 = ChildInfo::new_default(b"storage_key2").prefixed_storage_key(); - let (backend, storage, changes, config) = prepare_for_build(zero); - let parent = AnchorBlockId { hash: Default::default(), number: zero + 15 }; - let changes_trie_nodes = prepare_input( - &backend, - &storage, - configuration_range(&config, zero), - &changes, - &parent, - ) - .unwrap(); - assert_eq!( - changes_trie_nodes.0.collect::>>(), - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 16, key: vec![100] }, - vec![0, 2, 3] - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 16, key: vec![101] }, - vec![1] - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 16, key: vec![103] }, - vec![0, 1] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 16, key: vec![100] }, - vec![zero + 4] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 16, key: vec![101] }, - vec![zero + 4] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 16, key: vec![102] }, - vec![zero + 4] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 16, key: vec![103] }, - vec![zero + 4] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 16, key: vec![105] }, - vec![zero + 4, zero + 8] - ), - ] - ); - assert_eq!( - changes_trie_nodes - .1 - .into_iter() - .map(|(k, v)| (k, v.collect::>())) - .collect::>(), - vec![ - ( - ChildIndex { block: zero + 16u64, storage_key: child_trie_key1.clone() }, - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 16u64, key: vec![100] }, - vec![0, 2, 3] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 16, key: vec![102] }, - vec![zero + 4] - ), - ] - ), - ( - ChildIndex { block: zero + 16, storage_key: child_trie_key2.clone() }, - vec![InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 16, key: vec![100] }, - vec![0, 2] - ),] - ), - ] - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } - - #[test] - fn build_changes_trie_nodes_on_skewed_digest_block() { - fn test_with_zero(zero: u64) { - let (backend, storage, changes, config) = prepare_for_build(zero); - let parent = AnchorBlockId { hash: Default::default(), number: zero + 10 }; - - let mut configuration_range = configuration_range(&config, zero); - let changes_trie_nodes = - prepare_input(&backend, &storage, configuration_range.clone(), &changes, &parent) - .unwrap(); - assert_eq!( - changes_trie_nodes.0.collect::>>(), - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 11, key: vec![100] }, - vec![0, 2, 3] - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 11, key: vec![101] }, - vec![1] - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 11, key: vec![103] }, - vec![0, 1] - ), - ] - ); - - configuration_range.end = Some(zero + 11); - let changes_trie_nodes = - prepare_input(&backend, &storage, configuration_range, &changes, &parent).unwrap(); - assert_eq!( - changes_trie_nodes.0.collect::>>(), - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 11, key: vec![100] }, - vec![0, 2, 3] - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 11, key: vec![101] }, - vec![1] - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 11, key: vec![103] }, - vec![0, 1] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 11, key: vec![100] }, - vec![zero + 4] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 11, key: vec![101] }, - vec![zero + 4] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 11, key: vec![102] }, - vec![zero + 4] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 11, key: vec![103] }, - vec![zero + 4] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 11, key: vec![105] }, - vec![zero + 4, zero + 8] - ), - ] - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } - - #[test] - fn build_changes_trie_nodes_ignores_temporary_storage_values() { - fn test_with_zero(zero: u64) { - let child_trie_key1 = ChildInfo::new_default(b"storage_key1").prefixed_storage_key(); - let child_trie_key2 = ChildInfo::new_default(b"storage_key2").prefixed_storage_key(); - let (backend, storage, mut changes, config) = prepare_for_build(zero); - - // 110: missing from backend, set to None in overlay - changes.set_storage(vec![110], None); - - let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 }; - let changes_trie_nodes = prepare_input( - &backend, - &storage, - configuration_range(&config, zero), - &changes, - &parent, - ) - .unwrap(); - assert_eq!( - changes_trie_nodes.0.collect::>>(), - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4, key: vec![100] }, - vec![0, 2, 3] - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4, key: vec![101] }, - vec![1] - ), - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4, key: vec![103] }, - vec![0, 1] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![100] }, - vec![zero + 1, zero + 3] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![101] }, - vec![zero + 1] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![102] }, - vec![zero + 2] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![105] }, - vec![zero + 1, zero + 3] - ), - ] - ); - assert_eq!( - changes_trie_nodes - .1 - .into_iter() - .map(|(k, v)| (k, v.collect::>())) - .collect::>(), - vec![ - ( - ChildIndex { block: zero + 4u64, storage_key: child_trie_key1.clone() }, - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4u64, key: vec![100] }, - vec![0, 2, 3] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![100] }, - vec![zero + 1] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![101] }, - vec![zero + 1] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![102] }, - vec![zero + 2] - ), - InputPair::DigestIndex( - DigestIndex { block: zero + 4, key: vec![105] }, - vec![zero + 1] - ), - ] - ), - ( - ChildIndex { block: zero + 4, storage_key: child_trie_key2.clone() }, - vec![InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: zero + 4, key: vec![100] }, - vec![0, 2] - ),] - ), - ] - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } - - #[test] - fn cache_is_used_when_changes_trie_is_built() { - let child_trie_key1 = ChildInfo::new_default(b"storage_key1").prefixed_storage_key(); - let child_trie_key2 = ChildInfo::new_default(b"storage_key2").prefixed_storage_key(); - let (backend, mut storage, changes, config) = prepare_for_build(0); - let parent = AnchorBlockId { hash: Default::default(), number: 15 }; - - // override some actual values from storage with values from the cache - // - // top-level storage: - // (keys 100, 101, 103, 105 are now missing from block#4 => they do not appear - // in l2 digest at block 16) - // - // "1" child storage: - // key 102 is now missing from block#4 => it doesn't appear in l2 digest at block 16 - // (keys 103, 104) are now added to block#4 => they appear in l2 digest at block 16 - // - // "2" child storage: - // (keys 105, 106) are now added to block#4 => they appear in l2 digest at block 16 - let trie_root4 = storage.root(&parent, 4).unwrap().unwrap(); - let cached_data4 = IncompleteCacheAction::CacheBuildData(IncompleteCachedBuildData::new()) - .set_digest_input_blocks(vec![1, 2, 3]) - .insert(None, vec![vec![100], vec![102]].into_iter().collect()) - .insert(Some(child_trie_key1.clone()), vec![vec![103], vec![104]].into_iter().collect()) - .insert(Some(child_trie_key2.clone()), vec![vec![105], vec![106]].into_iter().collect()) - .complete(4, &trie_root4); - storage.cache_mut().perform(cached_data4); - - let (root_changes_trie_nodes, child_changes_tries_nodes, _) = - prepare_input(&backend, &storage, configuration_range(&config, 0), &changes, &parent) - .unwrap(); - assert_eq!( - root_changes_trie_nodes.collect::>>(), - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: 16, key: vec![100] }, - vec![0, 2, 3] - ), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![101] }, vec![1]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![103] }, vec![0, 1]), - InputPair::DigestIndex(DigestIndex { block: 16, key: vec![100] }, vec![4]), - InputPair::DigestIndex(DigestIndex { block: 16, key: vec![102] }, vec![4]), - InputPair::DigestIndex(DigestIndex { block: 16, key: vec![105] }, vec![8]), - ] - ); - - let child_changes_tries_nodes = child_changes_tries_nodes - .into_iter() - .map(|(k, i)| (k, i.collect::>())) - .collect::>(); - assert_eq!( - child_changes_tries_nodes - .get(&ChildIndex { block: 16u64, storage_key: child_trie_key1.clone() }) - .unwrap(), - &vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: 16u64, key: vec![100] }, - vec![0, 2, 3] - ), - InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![103] }, vec![4]), - InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![104] }, vec![4]), - ], - ); - assert_eq!( - child_changes_tries_nodes - .get(&ChildIndex { block: 16u64, storage_key: child_trie_key2.clone() }) - .unwrap(), - &vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: 16u64, key: vec![100] }, - vec![0, 2] - ), - InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![105] }, vec![4]), - InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![106] }, vec![4]), - ], - ); - } -} diff --git a/primitives/state-machine/src/changes_trie/build_cache.rs b/primitives/state-machine/src/changes_trie/build_cache.rs deleted file mode 100644 index 04820242d9d0..000000000000 --- a/primitives/state-machine/src/changes_trie/build_cache.rs +++ /dev/null @@ -1,278 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Changes tries build cache. - -use std::collections::{HashMap, HashSet}; - -use crate::StorageKey; -use sp_core::storage::PrefixedStorageKey; - -/// Changes trie build cache. -/// -/// Helps to avoid read of changes tries from the database when digest trie -/// is built. It holds changed keys for every block (indexed by changes trie -/// root) that could be referenced by future digest items. For digest entries -/// it also holds keys covered by this digest. Entries for top level digests -/// are never created, because they'll never be used to build other digests. -/// -/// Entries are pruned from the cache once digest block that is using this entry -/// is inserted (because digest block will includes all keys from this entry). -/// When there's a fork, entries are pruned when first changes trie is inserted. -pub struct BuildCache { - /// Map of block (implies changes trie) number => changes trie root. - roots_by_number: HashMap, - /// Map of changes trie root => set of storage keys that are in this trie. - /// The `Option>` in inner `HashMap` stands for the child storage key. - /// If it is `None`, then the `HashSet` contains keys changed in top-level storage. - /// If it is `Some`, then the `HashSet` contains keys changed in child storage, identified by - /// the key. - changed_keys: HashMap, HashSet>>, -} - -/// The action to perform when block-with-changes-trie is imported. -#[derive(Debug, PartialEq)] -pub enum CacheAction { - /// Cache data that has been collected when CT has been built. - CacheBuildData(CachedBuildData), - /// Clear cache from all existing entries. - Clear, -} - -/// The data that has been cached during changes trie building. -#[derive(Debug, PartialEq)] -pub struct CachedBuildData { - block: N, - trie_root: H, - digest_input_blocks: Vec, - changed_keys: HashMap, HashSet>, -} - -/// The action to perform when block-with-changes-trie is imported. -#[derive(Debug, PartialEq)] -pub(crate) enum IncompleteCacheAction { - /// Cache data that has been collected when CT has been built. - CacheBuildData(IncompleteCachedBuildData), - /// Clear cache from all existing entries. - Clear, -} - -/// The data (without changes trie root) that has been cached during changes trie building. -#[derive(Debug, PartialEq)] -pub(crate) struct IncompleteCachedBuildData { - digest_input_blocks: Vec, - changed_keys: HashMap, HashSet>, -} - -impl BuildCache -where - N: Eq + ::std::hash::Hash, - H: Eq + ::std::hash::Hash + Clone, -{ - /// Create new changes trie build cache. - pub fn new() -> Self { - BuildCache { roots_by_number: HashMap::new(), changed_keys: HashMap::new() } - } - - /// Get cached changed keys for changes trie with given root. - pub fn get( - &self, - root: &H, - ) -> Option<&HashMap, HashSet>> { - self.changed_keys.get(&root) - } - - /// Execute given functor with cached entry for given block. - /// Returns true if the functor has been called and false otherwise. - pub fn with_changed_keys( - &self, - root: &H, - functor: &mut dyn FnMut(&HashMap, HashSet>), - ) -> bool { - match self.changed_keys.get(&root) { - Some(changed_keys) => { - functor(changed_keys); - true - }, - None => false, - } - } - - /// Insert data into cache. - pub fn perform(&mut self, action: CacheAction) { - match action { - CacheAction::CacheBuildData(data) => { - self.roots_by_number.insert(data.block, data.trie_root.clone()); - self.changed_keys.insert(data.trie_root, data.changed_keys); - - for digest_input_block in data.digest_input_blocks { - let digest_input_block_hash = self.roots_by_number.remove(&digest_input_block); - if let Some(digest_input_block_hash) = digest_input_block_hash { - self.changed_keys.remove(&digest_input_block_hash); - } - } - }, - CacheAction::Clear => { - self.roots_by_number.clear(); - self.changed_keys.clear(); - }, - } - } -} - -impl IncompleteCacheAction { - /// Returns true if we need to collect changed keys for this action. - pub fn collects_changed_keys(&self) -> bool { - match *self { - IncompleteCacheAction::CacheBuildData(_) => true, - IncompleteCacheAction::Clear => false, - } - } - - /// Complete cache action with computed changes trie root. - pub(crate) fn complete(self, block: N, trie_root: &H) -> CacheAction { - match self { - IncompleteCacheAction::CacheBuildData(build_data) => - CacheAction::CacheBuildData(build_data.complete(block, trie_root.clone())), - IncompleteCacheAction::Clear => CacheAction::Clear, - } - } - - /// Set numbers of blocks that are superseded by this new entry. - /// - /// If/when this build data is committed to the cache, entries for these blocks - /// will be removed from the cache. - pub(crate) fn set_digest_input_blocks(self, digest_input_blocks: Vec) -> Self { - match self { - IncompleteCacheAction::CacheBuildData(build_data) => - IncompleteCacheAction::CacheBuildData( - build_data.set_digest_input_blocks(digest_input_blocks), - ), - IncompleteCacheAction::Clear => IncompleteCacheAction::Clear, - } - } - - /// Insert changed keys of given storage into cached data. - pub(crate) fn insert( - self, - storage_key: Option, - changed_keys: HashSet, - ) -> Self { - match self { - IncompleteCacheAction::CacheBuildData(build_data) => - IncompleteCacheAction::CacheBuildData(build_data.insert(storage_key, changed_keys)), - IncompleteCacheAction::Clear => IncompleteCacheAction::Clear, - } - } -} - -impl IncompleteCachedBuildData { - /// Create new cached data. - pub(crate) fn new() -> Self { - IncompleteCachedBuildData { digest_input_blocks: Vec::new(), changed_keys: HashMap::new() } - } - - fn complete(self, block: N, trie_root: H) -> CachedBuildData { - CachedBuildData { - block, - trie_root, - digest_input_blocks: self.digest_input_blocks, - changed_keys: self.changed_keys, - } - } - - fn set_digest_input_blocks(mut self, digest_input_blocks: Vec) -> Self { - self.digest_input_blocks = digest_input_blocks; - self - } - - fn insert( - mut self, - storage_key: Option, - changed_keys: HashSet, - ) -> Self { - self.changed_keys.insert(storage_key, changed_keys); - self - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn updated_keys_are_stored_when_non_top_level_digest_is_built() { - let mut data = IncompleteCachedBuildData::::new(); - data = data.insert(None, vec![vec![1]].into_iter().collect()); - assert_eq!(data.changed_keys.len(), 1); - - let mut cache = BuildCache::new(); - cache.perform(CacheAction::CacheBuildData(data.complete(1, 1))); - assert_eq!(cache.changed_keys.len(), 1); - assert_eq!( - cache.get(&1).unwrap().clone(), - vec![(None, vec![vec![1]].into_iter().collect())].into_iter().collect(), - ); - } - - #[test] - fn obsolete_entries_are_purged_when_new_ct_is_built() { - let mut cache = BuildCache::::new(); - cache.perform(CacheAction::CacheBuildData( - IncompleteCachedBuildData::new() - .insert(None, vec![vec![1]].into_iter().collect()) - .complete(1, 1), - )); - cache.perform(CacheAction::CacheBuildData( - IncompleteCachedBuildData::new() - .insert(None, vec![vec![2]].into_iter().collect()) - .complete(2, 2), - )); - cache.perform(CacheAction::CacheBuildData( - IncompleteCachedBuildData::new() - .insert(None, vec![vec![3]].into_iter().collect()) - .complete(3, 3), - )); - - assert_eq!(cache.changed_keys.len(), 3); - - cache.perform(CacheAction::CacheBuildData( - IncompleteCachedBuildData::new() - .set_digest_input_blocks(vec![1, 2, 3]) - .complete(4, 4), - )); - - assert_eq!(cache.changed_keys.len(), 1); - - cache.perform(CacheAction::CacheBuildData( - IncompleteCachedBuildData::new() - .insert(None, vec![vec![8]].into_iter().collect()) - .complete(8, 8), - )); - cache.perform(CacheAction::CacheBuildData( - IncompleteCachedBuildData::new() - .insert(None, vec![vec![12]].into_iter().collect()) - .complete(12, 12), - )); - - assert_eq!(cache.changed_keys.len(), 3); - - cache.perform(CacheAction::Clear); - - assert_eq!(cache.changed_keys.len(), 0); - } -} diff --git a/primitives/state-machine/src/changes_trie/build_iterator.rs b/primitives/state-machine/src/changes_trie/build_iterator.rs deleted file mode 100644 index 62bb00a2f882..000000000000 --- a/primitives/state-machine/src/changes_trie/build_iterator.rs +++ /dev/null @@ -1,487 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Structures and functions to return blocks whose changes are to be included -//! in given block's changes trie. - -use crate::changes_trie::{BlockNumber, ConfigurationRange}; -use num_traits::Zero; - -/// Returns iterator of OTHER blocks that are required for inclusion into -/// changes trie of given block. Blocks are guaranteed to be returned in -/// ascending order. -/// -/// Skewed digest is built IF block >= config.end. -pub fn digest_build_iterator<'a, Number: BlockNumber>( - config: ConfigurationRange<'a, Number>, - block: Number, -) -> DigestBuildIterator { - // prepare digest build parameters - let (_, _, digest_step) = match config.config.digest_level_at_block(config.zero, block.clone()) - { - Some((current_level, digest_interval, digest_step)) => - (current_level, digest_interval, digest_step), - None => return DigestBuildIterator::empty(), - }; - - DigestBuildIterator::new( - block.clone(), - config.end.unwrap_or(block), - config.config.digest_interval, - digest_step, - ) -} - -/// Changes trie build iterator that returns numbers of OTHER blocks that are -/// required for inclusion into changes trie of given block. -#[derive(Debug)] -pub struct DigestBuildIterator { - /// Block we're building changes trie for. It could (logically) be a post-end block if we are - /// creating skewed digest. - block: Number, - /// Block that is a last block where current configuration is active. We have never yet created - /// anything after this block => digest that we're creating can't reference any blocks that are - /// >= end. - end: Number, - /// Interval of L1 digest blocks. - digest_interval: u32, - /// Max step that could be used when digest is created. - max_step: u32, - - // Mutable data below: - /// Step of current blocks range. - current_step: u32, - /// Reverse step of current blocks range. - current_step_reverse: u32, - /// Current blocks range. - current_range: Option>, - /// Last block that we have returned. - last_block: Option, -} - -impl DigestBuildIterator { - /// Create new digest build iterator. - pub fn new(block: Number, end: Number, digest_interval: u32, max_step: u32) -> Self { - DigestBuildIterator { - block, - end, - digest_interval, - max_step, - current_step: max_step, - current_step_reverse: 0, - current_range: None, - last_block: None, - } - } - - /// Create empty digest build iterator. - pub fn empty() -> Self { - Self::new(Zero::zero(), Zero::zero(), 0, 0) - } -} - -impl Iterator for DigestBuildIterator { - type Item = Number; - - fn next(&mut self) -> Option { - // when we're building skewed digest, we might want to skip some blocks if - // they're not covered by current configuration - loop { - if let Some(next) = self.current_range.as_mut().and_then(|iter| iter.next()) { - if next < self.end { - self.last_block = Some(next.clone()); - return Some(next) - } - } - - // we are safe to use non-checking mul/sub versions here because: - // DigestBuildIterator is created only by internal function that is checking - // that all multiplications/subtractions are safe within max_step limit - - let next_step_reverse = if self.current_step_reverse == 0 { - 1 - } else { - self.current_step_reverse * self.digest_interval - }; - if next_step_reverse > self.max_step { - return None - } - - self.current_step_reverse = next_step_reverse; - self.current_range = Some(BlocksRange::new( - match self.last_block.clone() { - Some(last_block) => last_block + self.current_step.into(), - None => - self.block.clone() - - (self.current_step * self.digest_interval - self.current_step).into(), - }, - self.block.clone(), - self.current_step.into(), - )); - - self.current_step = self.current_step / self.digest_interval; - if self.current_step == 0 { - self.current_step = 1; - } - } - } -} - -/// Blocks range iterator with builtin step_by support. -#[derive(Debug)] -struct BlocksRange { - current: Number, - end: Number, - step: Number, -} - -impl BlocksRange { - pub fn new(begin: Number, end: Number, step: Number) -> Self { - BlocksRange { current: begin, end, step } - } -} - -impl Iterator for BlocksRange { - type Item = Number; - - fn next(&mut self) -> Option { - if self.current >= self.end { - return None - } - - let current = Some(self.current.clone()); - self.current += self.step.clone(); - current - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::changes_trie::Configuration; - - fn digest_build_iterator( - digest_interval: u32, - digest_levels: u32, - zero: u64, - block: u64, - end: Option, - ) -> DigestBuildIterator { - super::digest_build_iterator( - ConfigurationRange { - config: &Configuration { digest_interval, digest_levels }, - zero, - end, - }, - block, - ) - } - - fn digest_build_iterator_basic( - digest_interval: u32, - digest_levels: u32, - zero: u64, - block: u64, - ) -> (u64, u32, u32) { - let iter = digest_build_iterator(digest_interval, digest_levels, zero, block, None); - (iter.block, iter.digest_interval, iter.max_step) - } - - fn digest_build_iterator_blocks( - digest_interval: u32, - digest_levels: u32, - zero: u64, - block: u64, - end: Option, - ) -> Vec { - digest_build_iterator(digest_interval, digest_levels, zero, block, end).collect() - } - - #[test] - fn suggest_digest_inclusion_returns_empty_iterator() { - fn test_with_zero(zero: u64) { - let empty = (0, 0, 0); - assert_eq!(digest_build_iterator_basic(4, 16, zero, zero + 0), empty, "block is 0"); - assert_eq!( - digest_build_iterator_basic(0, 16, zero, zero + 64), - empty, - "digest_interval is 0" - ); - assert_eq!( - digest_build_iterator_basic(1, 16, zero, zero + 64), - empty, - "digest_interval is 1" - ); - assert_eq!( - digest_build_iterator_basic(4, 0, zero, zero + 64), - empty, - "digest_levels is 0" - ); - assert_eq!( - digest_build_iterator_basic(4, 16, zero, zero + 1), - empty, - "digest is not required for this block", - ); - assert_eq!( - digest_build_iterator_basic(4, 16, zero, zero + 2), - empty, - "digest is not required for this block", - ); - assert_eq!( - digest_build_iterator_basic(4, 16, zero, zero + 15), - empty, - "digest is not required for this block", - ); - assert_eq!( - digest_build_iterator_basic(4, 16, zero, zero + 17), - empty, - "digest is not required for this block", - ); - assert_eq!( - digest_build_iterator_basic(::std::u32::MAX / 2 + 1, 16, zero, ::std::u64::MAX,), - empty, - "digest_interval * 2 is greater than u64::MAX" - ); - } - - test_with_zero(0); - test_with_zero(1); - test_with_zero(2); - test_with_zero(4); - test_with_zero(17); - } - - #[test] - fn suggest_digest_inclusion_returns_level1_iterator() { - fn test_with_zero(zero: u64) { - assert_eq!( - digest_build_iterator_basic(16, 1, zero, zero + 16), - (zero + 16, 16, 1), - "!(block % interval) && first digest level == block", - ); - assert_eq!( - digest_build_iterator_basic(16, 1, zero, zero + 256), - (zero + 256, 16, 1), - "!(block % interval^2), but there's only 1 digest level", - ); - assert_eq!( - digest_build_iterator_basic(16, 2, zero, zero + 32), - (zero + 32, 16, 1), - "second level digest is not required for this block", - ); - assert_eq!( - digest_build_iterator_basic(16, 3, zero, zero + 4080), - (zero + 4080, 16, 1), - "second && third level digest are not required for this block", - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } - - #[test] - fn suggest_digest_inclusion_returns_level2_iterator() { - fn test_with_zero(zero: u64) { - assert_eq!( - digest_build_iterator_basic(16, 2, zero, zero + 256), - (zero + 256, 16, 16), - "second level digest", - ); - assert_eq!( - digest_build_iterator_basic(16, 2, zero, zero + 4096), - (zero + 4096, 16, 16), - "!(block % interval^3), but there's only 2 digest levels", - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } - - #[test] - fn suggest_digest_inclusion_returns_level3_iterator() { - fn test_with_zero(zero: u64) { - assert_eq!( - digest_build_iterator_basic(16, 3, zero, zero + 4096), - (zero + 4096, 16, 256), - "third level digest: beginning", - ); - assert_eq!( - digest_build_iterator_basic(16, 3, zero, zero + 8192), - (zero + 8192, 16, 256), - "third level digest: next", - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } - - #[test] - fn digest_iterator_returns_level1_blocks() { - fn test_with_zero(zero: u64) { - assert_eq!( - digest_build_iterator_blocks(16, 1, zero, zero + 16, None), - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] - .iter() - .map(|item| zero + item) - .collect::>() - ); - assert_eq!( - digest_build_iterator_blocks(16, 1, zero, zero + 256, None), - [241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255] - .iter() - .map(|item| zero + item) - .collect::>() - ); - assert_eq!( - digest_build_iterator_blocks(16, 2, zero, zero + 32, None), - [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] - .iter() - .map(|item| zero + item) - .collect::>() - ); - assert_eq!( - digest_build_iterator_blocks(16, 3, zero, zero + 4080, None), - [ - 4065, 4066, 4067, 4068, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4076, 4077, - 4078, 4079 - ] - .iter() - .map(|item| zero + item) - .collect::>() - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } - - #[test] - fn digest_iterator_returns_level1_and_level2_blocks() { - fn test_with_zero(zero: u64) { - assert_eq!( - digest_build_iterator_blocks(16, 2, zero, zero + 256, None), - [ - // level2 points to previous 16-1 level1 digests: - 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, - // level2 is a level1 digest of 16-1 previous blocks: - 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, - ] - .iter() - .map(|item| zero + item) - .collect::>(), - ); - assert_eq!( - digest_build_iterator_blocks(16, 2, zero, zero + 4096, None), - [ - // level2 points to previous 16-1 level1 digests: - 3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048, - 4064, 4080, // level2 is a level1 digest of 16-1 previous blocks: - 4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, - 4094, 4095, - ] - .iter() - .map(|item| zero + item) - .collect::>(), - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } - - #[test] - fn digest_iterator_returns_level1_and_level2_and_level3_blocks() { - fn test_with_zero(zero: u64) { - assert_eq!( - digest_build_iterator_blocks(16, 3, zero, zero + 4096, None), - [ - // level3 points to previous 16-1 level2 digests: - 256, 512, 768, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, - 3840, // level3 points to previous 16-1 level1 digests: - 3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048, - 4064, 4080, // level3 is a level1 digest of 16-1 previous blocks: - 4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, - 4094, 4095, - ] - .iter() - .map(|item| zero + item) - .collect::>(), - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } - - #[test] - fn digest_iterator_returns_skewed_digest_blocks() { - fn test_with_zero(zero: u64) { - assert_eq!( - digest_build_iterator_blocks(16, 3, zero, zero + 4096, Some(zero + 1338)), - [ - // level3 MUST point to previous 16-1 level2 digests, BUT there are only 5: - 256, 512, 768, 1024, 1280, - // level3 MUST point to previous 16-1 level1 digests, BUT there are only 3: - 1296, 1312, 1328, - // level3 MUST be a level1 digest of 16-1 previous blocks, BUT there are only - // 9: - 1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, - ] - .iter() - .map(|item| zero + item) - .collect::>(), - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } - - #[test] - fn digest_iterator_returns_skewed_digest_blocks_skipping_level() { - fn test_with_zero(zero: u64) { - assert_eq!( - digest_build_iterator_blocks(16, 3, zero, zero + 4096, Some(zero + 1284)), - [ - // level3 MUST point to previous 16-1 level2 digests, BUT there are only 5: - 256, 512, 768, 1024, 1280, - // level3 MUST point to previous 16-1 level1 digests, BUT there are NO ANY - // L1-digests: level3 MUST be a level1 digest of 16-1 previous blocks, BUT - // there are only 3: - 1281, 1282, 1283, - ] - .iter() - .map(|item| zero + item) - .collect::>(), - ); - } - - test_with_zero(0); - test_with_zero(16); - test_with_zero(17); - } -} diff --git a/primitives/state-machine/src/changes_trie/changes_iterator.rs b/primitives/state-machine/src/changes_trie/changes_iterator.rs deleted file mode 100644 index 9343a226a3aa..000000000000 --- a/primitives/state-machine/src/changes_trie/changes_iterator.rs +++ /dev/null @@ -1,748 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Functions + iterator that traverses changes tries and returns all -//! (block, extrinsic) pairs where given key has been changed. - -use crate::{ - changes_trie::{ - input::{ChildIndex, DigestIndex, DigestIndexValue, ExtrinsicIndex, ExtrinsicIndexValue}, - storage::{InMemoryStorage, TrieBackendAdapter}, - surface_iterator::{surface_iterator, SurfaceIterator}, - AnchorBlockId, BlockNumber, ConfigurationRange, RootsStorage, Storage, - }, - proving_backend::ProvingBackendRecorder, - trie_backend_essence::TrieBackendEssence, -}; -use codec::{Codec, Decode, Encode}; -use hash_db::Hasher; -use num_traits::Zero; -use sp_core::storage::PrefixedStorageKey; -use sp_trie::Recorder; -use std::{cell::RefCell, collections::VecDeque}; - -/// Return changes of given key at given blocks range. -/// `max` is the number of best known block. -/// Changes are returned in descending order (i.e. last block comes first). -pub fn key_changes<'a, H: Hasher, Number: BlockNumber>( - config: ConfigurationRange<'a, Number>, - storage: &'a dyn Storage, - begin: Number, - end: &'a AnchorBlockId, - max: Number, - storage_key: Option<&'a PrefixedStorageKey>, - key: &'a [u8], -) -> Result, String> { - // we can't query any roots before root - let max = std::cmp::min(max, end.number.clone()); - - Ok(DrilldownIterator { - essence: DrilldownIteratorEssence { - storage_key, - key, - roots_storage: storage.as_roots_storage(), - storage, - begin: begin.clone(), - end, - config: config.clone(), - surface: surface_iterator(config, max, begin, end.number.clone())?, - - extrinsics: Default::default(), - blocks: Default::default(), - - _hasher: ::std::marker::PhantomData::::default(), - }, - }) -} - -/// Returns proof of changes of given key at given blocks range. -/// `max` is the number of best known block. -pub fn key_changes_proof<'a, H: Hasher, Number: BlockNumber>( - config: ConfigurationRange<'a, Number>, - storage: &dyn Storage, - begin: Number, - end: &AnchorBlockId, - max: Number, - storage_key: Option<&PrefixedStorageKey>, - key: &[u8], -) -> Result>, String> -where - H::Out: Codec, -{ - // we can't query any roots before root - let max = std::cmp::min(max, end.number.clone()); - - let mut iter = ProvingDrilldownIterator { - essence: DrilldownIteratorEssence { - storage_key, - key, - roots_storage: storage.as_roots_storage(), - storage, - begin: begin.clone(), - end, - config: config.clone(), - surface: surface_iterator(config, max, begin, end.number.clone())?, - - extrinsics: Default::default(), - blocks: Default::default(), - - _hasher: ::std::marker::PhantomData::::default(), - }, - proof_recorder: Default::default(), - }; - - // iterate to collect proof - while let Some(item) = iter.next() { - item?; - } - - Ok(iter.extract_proof()) -} - -/// Check key changes proof and return changes of the key at given blocks range. -/// `max` is the number of best known block. -/// Changes are returned in descending order (i.e. last block comes first). -pub fn key_changes_proof_check<'a, H: Hasher, Number: BlockNumber>( - config: ConfigurationRange<'a, Number>, - roots_storage: &dyn RootsStorage, - proof: Vec>, - begin: Number, - end: &AnchorBlockId, - max: Number, - storage_key: Option<&PrefixedStorageKey>, - key: &[u8], -) -> Result, String> -where - H::Out: Encode, -{ - key_changes_proof_check_with_db( - config, - roots_storage, - &InMemoryStorage::with_proof(proof), - begin, - end, - max, - storage_key, - key, - ) -} - -/// Similar to the `key_changes_proof_check` function, but works with prepared proof storage. -pub fn key_changes_proof_check_with_db<'a, H: Hasher, Number: BlockNumber>( - config: ConfigurationRange<'a, Number>, - roots_storage: &dyn RootsStorage, - proof_db: &InMemoryStorage, - begin: Number, - end: &AnchorBlockId, - max: Number, - storage_key: Option<&PrefixedStorageKey>, - key: &[u8], -) -> Result, String> -where - H::Out: Encode, -{ - // we can't query any roots before root - let max = std::cmp::min(max, end.number.clone()); - - DrilldownIterator { - essence: DrilldownIteratorEssence { - storage_key, - key, - roots_storage, - storage: proof_db, - begin: begin.clone(), - end, - config: config.clone(), - surface: surface_iterator(config, max, begin, end.number.clone())?, - - extrinsics: Default::default(), - blocks: Default::default(), - - _hasher: ::std::marker::PhantomData::::default(), - }, - } - .collect() -} - -/// Drilldown iterator - receives 'digest points' from surface iterator and explores -/// every point until extrinsic is found. -pub struct DrilldownIteratorEssence<'a, H, Number> -where - H: Hasher, - Number: BlockNumber, - H::Out: 'a, -{ - storage_key: Option<&'a PrefixedStorageKey>, - key: &'a [u8], - roots_storage: &'a dyn RootsStorage, - storage: &'a dyn Storage, - begin: Number, - end: &'a AnchorBlockId, - config: ConfigurationRange<'a, Number>, - surface: SurfaceIterator<'a, Number>, - - extrinsics: VecDeque<(Number, u32)>, - blocks: VecDeque<(Number, Option)>, - - _hasher: ::std::marker::PhantomData, -} - -impl<'a, H, Number> DrilldownIteratorEssence<'a, H, Number> -where - H: Hasher, - Number: BlockNumber, - H::Out: 'a, -{ - pub fn next(&mut self, trie_reader: F) -> Option> - where - F: FnMut(&dyn Storage, H::Out, &[u8]) -> Result>, String>, - { - match self.do_next(trie_reader) { - Ok(Some(res)) => Some(Ok(res)), - Ok(None) => None, - Err(err) => Some(Err(err)), - } - } - - fn do_next(&mut self, mut trie_reader: F) -> Result, String> - where - F: FnMut(&dyn Storage, H::Out, &[u8]) -> Result>, String>, - { - loop { - if let Some((block, extrinsic)) = self.extrinsics.pop_front() { - return Ok(Some((block, extrinsic))) - } - - if let Some((block, level)) = self.blocks.pop_front() { - // not having a changes trie root is an error because: - // we never query roots for future blocks - // AND trie roots for old blocks are known (both on full + light node) - let trie_root = - self.roots_storage.root(&self.end, block.clone())?.ok_or_else(|| { - format!("Changes trie root for block {} is not found", block.clone()) - })?; - let trie_root = if let Some(storage_key) = self.storage_key { - let child_key = - ChildIndex { block: block.clone(), storage_key: storage_key.clone() } - .encode(); - if let Some(trie_root) = trie_reader(self.storage, trie_root, &child_key)? - .and_then(|v| >::decode(&mut &v[..]).ok()) - .map(|v| { - let mut hash = H::Out::default(); - hash.as_mut().copy_from_slice(&v[..]); - hash - }) { - trie_root - } else { - continue - } - } else { - trie_root - }; - - // only return extrinsics for blocks before self.max - // most of blocks will be filtered out before pushing to `self.blocks` - // here we just throwing away changes at digest blocks we're processing - debug_assert!( - block >= self.begin, - "We shall not touch digests earlier than a range' begin" - ); - if block <= self.end.number { - let extrinsics_key = - ExtrinsicIndex { block: block.clone(), key: self.key.to_vec() }.encode(); - let extrinsics = trie_reader(self.storage, trie_root, &extrinsics_key); - if let Some(extrinsics) = extrinsics? { - if let Ok(extrinsics) = ExtrinsicIndexValue::decode(&mut &extrinsics[..]) { - self.extrinsics - .extend(extrinsics.into_iter().rev().map(|e| (block.clone(), e))); - } - } - } - - let blocks_key = - DigestIndex { block: block.clone(), key: self.key.to_vec() }.encode(); - let blocks = trie_reader(self.storage, trie_root, &blocks_key); - if let Some(blocks) = blocks? { - if let Ok(blocks) = >::decode(&mut &blocks[..]) { - // filter level0 blocks here because we tend to use digest blocks, - // AND digest block changes could also include changes for out-of-range - // blocks - let begin = self.begin.clone(); - let end = self.end.number.clone(); - let config = self.config.clone(); - self.blocks.extend( - blocks - .into_iter() - .rev() - .filter(|b| { - level.map(|level| level > 1).unwrap_or(true) || - (*b >= begin && *b <= end) - }) - .map(|b| { - let prev_level = - level.map(|level| Some(level - 1)).unwrap_or_else(|| { - Some( - config - .config - .digest_level_at_block( - config.zero.clone(), - b.clone(), - ) - .map(|(level, _, _)| level) - .unwrap_or_else(|| Zero::zero()), - ) - }); - (b, prev_level) - }), - ); - } - } - - continue - } - - match self.surface.next() { - Some(Ok(block)) => self.blocks.push_back(block), - Some(Err(err)) => return Err(err), - None => return Ok(None), - } - } - } -} - -/// Exploring drilldown operator. -pub struct DrilldownIterator<'a, H, Number> -where - Number: BlockNumber, - H: Hasher, - H::Out: 'a, -{ - essence: DrilldownIteratorEssence<'a, H, Number>, -} - -impl<'a, H: Hasher, Number: BlockNumber> Iterator for DrilldownIterator<'a, H, Number> -where - H::Out: Encode, -{ - type Item = Result<(Number, u32), String>; - - fn next(&mut self) -> Option { - self.essence.next(|storage, root, key| { - TrieBackendEssence::<_, H>::new(TrieBackendAdapter::new(storage), root).storage(key) - }) - } -} - -/// Proving drilldown iterator. -struct ProvingDrilldownIterator<'a, H, Number> -where - Number: BlockNumber, - H: Hasher, - H::Out: 'a, -{ - essence: DrilldownIteratorEssence<'a, H, Number>, - proof_recorder: RefCell>, -} - -impl<'a, H, Number> ProvingDrilldownIterator<'a, H, Number> -where - Number: BlockNumber, - H: Hasher, - H::Out: 'a, -{ - /// Consume the iterator, extracting the gathered proof in lexicographical order - /// by value. - pub fn extract_proof(self) -> Vec> { - self.proof_recorder - .into_inner() - .drain() - .into_iter() - .map(|n| n.data.to_vec()) - .collect() - } -} - -impl<'a, H, Number> Iterator for ProvingDrilldownIterator<'a, H, Number> -where - Number: BlockNumber, - H: Hasher, - H::Out: 'a + Codec, -{ - type Item = Result<(Number, u32), String>; - - fn next(&mut self) -> Option { - let proof_recorder = &mut *self - .proof_recorder - .try_borrow_mut() - .expect("only fails when already borrowed; storage() is non-reentrant; qed"); - self.essence.next(|storage, root, key| { - ProvingBackendRecorder::<_, H> { - backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root), - proof_recorder, - } - .storage(key) - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::changes_trie::{input::InputPair, storage::InMemoryStorage, Configuration}; - use sp_runtime::traits::BlakeTwo256; - use std::iter::FromIterator; - - fn child_key() -> PrefixedStorageKey { - let child_info = sp_core::storage::ChildInfo::new_default(&b"1"[..]); - child_info.prefixed_storage_key() - } - - fn prepare_for_drilldown() -> (Configuration, InMemoryStorage) { - let config = Configuration { digest_interval: 4, digest_levels: 2 }; - let backend = InMemoryStorage::with_inputs( - vec![ - // digest: 1..4 => [(3, 0)] - (1, vec![]), - (2, vec![]), - ( - 3, - vec![InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: 3, key: vec![42] }, - vec![0], - )], - ), - (4, vec![InputPair::DigestIndex(DigestIndex { block: 4, key: vec![42] }, vec![3])]), - // digest: 5..8 => [(6, 3), (8, 1+2)] - (5, vec![]), - ( - 6, - vec![InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: 6, key: vec![42] }, - vec![3], - )], - ), - (7, vec![]), - ( - 8, - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: 8, key: vec![42] }, - vec![1, 2], - ), - InputPair::DigestIndex(DigestIndex { block: 8, key: vec![42] }, vec![6]), - ], - ), - // digest: 9..12 => [] - (9, vec![]), - (10, vec![]), - (11, vec![]), - (12, vec![]), - // digest: 0..16 => [4, 8] - (13, vec![]), - (14, vec![]), - (15, vec![]), - ( - 16, - vec![InputPair::DigestIndex( - DigestIndex { block: 16, key: vec![42] }, - vec![4, 8], - )], - ), - ], - vec![( - child_key(), - vec![ - ( - 1, - vec![InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: 1, key: vec![42] }, - vec![0], - )], - ), - ( - 2, - vec![InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: 2, key: vec![42] }, - vec![3], - )], - ), - ( - 16, - vec![ - InputPair::ExtrinsicIndex( - ExtrinsicIndex { block: 16, key: vec![42] }, - vec![5], - ), - InputPair::DigestIndex( - DigestIndex { block: 16, key: vec![42] }, - vec![2], - ), - ], - ), - ], - )], - ); - - (config, backend) - } - - fn configuration_range<'a>( - config: &'a Configuration, - zero: u64, - ) -> ConfigurationRange<'a, u64> { - ConfigurationRange { config, zero, end: None } - } - - #[test] - fn drilldown_iterator_works() { - let (config, storage) = prepare_for_drilldown(); - let drilldown_result = key_changes::( - configuration_range(&config, 0), - &storage, - 1, - &AnchorBlockId { hash: Default::default(), number: 16 }, - 16, - None, - &[42], - ) - .and_then(Result::from_iter); - assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)])); - - let drilldown_result = key_changes::( - configuration_range(&config, 0), - &storage, - 1, - &AnchorBlockId { hash: Default::default(), number: 2 }, - 4, - None, - &[42], - ) - .and_then(Result::from_iter); - assert_eq!(drilldown_result, Ok(vec![])); - - let drilldown_result = key_changes::( - configuration_range(&config, 0), - &storage, - 1, - &AnchorBlockId { hash: Default::default(), number: 3 }, - 4, - None, - &[42], - ) - .and_then(Result::from_iter); - assert_eq!(drilldown_result, Ok(vec![(3, 0)])); - - let drilldown_result = key_changes::( - configuration_range(&config, 0), - &storage, - 1, - &AnchorBlockId { hash: Default::default(), number: 7 }, - 7, - None, - &[42], - ) - .and_then(Result::from_iter); - assert_eq!(drilldown_result, Ok(vec![(6, 3), (3, 0)])); - - let drilldown_result = key_changes::( - configuration_range(&config, 0), - &storage, - 7, - &AnchorBlockId { hash: Default::default(), number: 8 }, - 8, - None, - &[42], - ) - .and_then(Result::from_iter); - assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1)])); - - let drilldown_result = key_changes::( - configuration_range(&config, 0), - &storage, - 5, - &AnchorBlockId { hash: Default::default(), number: 7 }, - 8, - None, - &[42], - ) - .and_then(Result::from_iter); - assert_eq!(drilldown_result, Ok(vec![(6, 3)])); - } - - #[test] - fn drilldown_iterator_fails_when_storage_fails() { - let (config, storage) = prepare_for_drilldown(); - storage.clear_storage(); - - assert!(key_changes::( - configuration_range(&config, 0), - &storage, - 1, - &AnchorBlockId { hash: Default::default(), number: 100 }, - 1000, - None, - &[42], - ) - .and_then(|i| i.collect::, _>>()) - .is_err()); - - assert!(key_changes::( - configuration_range(&config, 0), - &storage, - 1, - &AnchorBlockId { hash: Default::default(), number: 100 }, - 1000, - Some(&child_key()), - &[42], - ) - .and_then(|i| i.collect::, _>>()) - .is_err()); - } - - #[test] - fn drilldown_iterator_fails_when_range_is_invalid() { - let (config, storage) = prepare_for_drilldown(); - assert!(key_changes::( - configuration_range(&config, 0), - &storage, - 1, - &AnchorBlockId { hash: Default::default(), number: 100 }, - 50, - None, - &[42], - ) - .is_err()); - assert!(key_changes::( - configuration_range(&config, 0), - &storage, - 20, - &AnchorBlockId { hash: Default::default(), number: 10 }, - 100, - None, - &[42], - ) - .is_err()); - } - - #[test] - fn proving_drilldown_iterator_works() { - // happens on remote full node: - - // create drilldown iterator that records all trie nodes during drilldown - let (remote_config, remote_storage) = prepare_for_drilldown(); - let remote_proof = key_changes_proof::( - configuration_range(&remote_config, 0), - &remote_storage, - 1, - &AnchorBlockId { hash: Default::default(), number: 16 }, - 16, - None, - &[42], - ) - .unwrap(); - - let (remote_config, remote_storage) = prepare_for_drilldown(); - let remote_proof_child = key_changes_proof::( - configuration_range(&remote_config, 0), - &remote_storage, - 1, - &AnchorBlockId { hash: Default::default(), number: 16 }, - 16, - Some(&child_key()), - &[42], - ) - .unwrap(); - - // happens on local light node: - - // create drilldown iterator that works the same, but only depends on trie - let (local_config, local_storage) = prepare_for_drilldown(); - local_storage.clear_storage(); - let local_result = key_changes_proof_check::( - configuration_range(&local_config, 0), - &local_storage, - remote_proof, - 1, - &AnchorBlockId { hash: Default::default(), number: 16 }, - 16, - None, - &[42], - ); - - let (local_config, local_storage) = prepare_for_drilldown(); - local_storage.clear_storage(); - let local_result_child = key_changes_proof_check::( - configuration_range(&local_config, 0), - &local_storage, - remote_proof_child, - 1, - &AnchorBlockId { hash: Default::default(), number: 16 }, - 16, - Some(&child_key()), - &[42], - ); - - // check that drilldown result is the same as if it was happening at the full node - assert_eq!(local_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)])); - assert_eq!(local_result_child, Ok(vec![(16, 5), (2, 3)])); - } - - #[test] - fn drilldown_iterator_works_with_skewed_digest() { - let config = Configuration { digest_interval: 4, digest_levels: 3 }; - let mut config_range = configuration_range(&config, 0); - config_range.end = Some(91); - - // when 4^3 deactivates at block 91: - // last L3 digest has been created at block#64 - // skewed digest covers: - // L2 digests at blocks: 80 - // L1 digests at blocks: 84, 88 - // regular blocks: 89, 90, 91 - let mut input = (1u64..92u64).map(|b| (b, vec![])).collect::>(); - // changed at block#63 and covered by L3 digest at block#64 - input[63 - 1] - .1 - .push(InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 63, key: vec![42] }, vec![0])); - input[64 - 1] - .1 - .push(InputPair::DigestIndex(DigestIndex { block: 64, key: vec![42] }, vec![63])); - // changed at block#79 and covered by L2 digest at block#80 + skewed digest at block#91 - input[79 - 1] - .1 - .push(InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 79, key: vec![42] }, vec![1])); - input[80 - 1] - .1 - .push(InputPair::DigestIndex(DigestIndex { block: 80, key: vec![42] }, vec![79])); - input[91 - 1] - .1 - .push(InputPair::DigestIndex(DigestIndex { block: 91, key: vec![42] }, vec![80])); - let storage = InMemoryStorage::with_inputs(input, vec![]); - - let drilldown_result = key_changes::( - config_range, - &storage, - 1, - &AnchorBlockId { hash: Default::default(), number: 91 }, - 100_000u64, - None, - &[42], - ) - .and_then(Result::from_iter); - assert_eq!(drilldown_result, Ok(vec![(79, 1), (63, 0)])); - } -} diff --git a/primitives/state-machine/src/changes_trie/input.rs b/primitives/state-machine/src/changes_trie/input.rs deleted file mode 100644 index af0a423e5726..000000000000 --- a/primitives/state-machine/src/changes_trie/input.rs +++ /dev/null @@ -1,207 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Different types of changes trie input pairs. - -use crate::{changes_trie::BlockNumber, StorageKey, StorageValue}; -use codec::{Decode, Encode, Error, Input, Output}; -use sp_core::storage::PrefixedStorageKey; - -/// Key of { changed key => set of extrinsic indices } mapping. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ExtrinsicIndex { - /// Block at which this key has been inserted in the trie. - pub block: Number, - /// Storage key this node is responsible for. - pub key: StorageKey, -} - -/// Value of { changed key => set of extrinsic indices } mapping. -pub type ExtrinsicIndexValue = Vec; - -/// Key of { changed key => block/digest block numbers } mapping. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct DigestIndex { - /// Block at which this key has been inserted in the trie. - pub block: Number, - /// Storage key this node is responsible for. - pub key: StorageKey, -} - -/// Key of { childtrie key => Childchange trie } mapping. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct ChildIndex { - /// Block at which this key has been inserted in the trie. - pub block: Number, - /// Storage key this node is responsible for. - pub storage_key: PrefixedStorageKey, -} - -/// Value of { changed key => block/digest block numbers } mapping. -pub type DigestIndexValue = Vec; - -/// Value of { changed key => block/digest block numbers } mapping. -/// That is the root of the child change trie. -pub type ChildIndexValue = Vec; - -/// Single input pair of changes trie. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum InputPair { - /// Element of { key => set of extrinsics where key has been changed } element mapping. - ExtrinsicIndex(ExtrinsicIndex, ExtrinsicIndexValue), - /// Element of { key => set of blocks/digest blocks where key has been changed } element - /// mapping. - DigestIndex(DigestIndex, DigestIndexValue), - /// Element of { childtrie key => Childchange trie } where key has been changed } element - /// mapping. - ChildIndex(ChildIndex, ChildIndexValue), -} - -/// Single input key of changes trie. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum InputKey { - /// Key of { key => set of extrinsics where key has been changed } element mapping. - ExtrinsicIndex(ExtrinsicIndex), - /// Key of { key => set of blocks/digest blocks where key has been changed } element mapping. - DigestIndex(DigestIndex), - /// Key of { childtrie key => Childchange trie } where key has been changed } element mapping. - ChildIndex(ChildIndex), -} - -impl InputPair { - /// Extract storage key that this pair corresponds to. - pub fn key(&self) -> Option<&[u8]> { - match *self { - InputPair::ExtrinsicIndex(ref key, _) => Some(&key.key), - InputPair::DigestIndex(ref key, _) => Some(&key.key), - InputPair::ChildIndex(_, _) => None, - } - } -} - -impl Into<(StorageKey, StorageValue)> for InputPair { - fn into(self) -> (StorageKey, StorageValue) { - match self { - InputPair::ExtrinsicIndex(key, value) => (key.encode(), value.encode()), - InputPair::DigestIndex(key, value) => (key.encode(), value.encode()), - InputPair::ChildIndex(key, value) => (key.encode(), value.encode()), - } - } -} - -impl Into> for InputPair { - fn into(self) -> InputKey { - match self { - InputPair::ExtrinsicIndex(key, _) => InputKey::ExtrinsicIndex(key), - InputPair::DigestIndex(key, _) => InputKey::DigestIndex(key), - InputPair::ChildIndex(key, _) => InputKey::ChildIndex(key), - } - } -} - -impl ExtrinsicIndex { - pub fn key_neutral_prefix(block: Number) -> Vec { - let mut prefix = vec![1]; - prefix.extend(block.encode()); - prefix - } -} - -impl Encode for ExtrinsicIndex { - fn encode_to(&self, dest: &mut W) { - dest.push_byte(1); - self.block.encode_to(dest); - self.key.encode_to(dest); - } -} - -impl codec::EncodeLike for ExtrinsicIndex {} - -impl DigestIndex { - pub fn key_neutral_prefix(block: Number) -> Vec { - let mut prefix = vec![2]; - prefix.extend(block.encode()); - prefix - } -} - -impl Encode for DigestIndex { - fn encode_to(&self, dest: &mut W) { - dest.push_byte(2); - self.block.encode_to(dest); - self.key.encode_to(dest); - } -} - -impl ChildIndex { - pub fn key_neutral_prefix(block: Number) -> Vec { - let mut prefix = vec![3]; - prefix.extend(block.encode()); - prefix - } -} - -impl Encode for ChildIndex { - fn encode_to(&self, dest: &mut W) { - dest.push_byte(3); - self.block.encode_to(dest); - self.storage_key.encode_to(dest); - } -} - -impl codec::EncodeLike for DigestIndex {} - -impl Decode for InputKey { - fn decode(input: &mut I) -> Result { - match input.read_byte()? { - 1 => Ok(InputKey::ExtrinsicIndex(ExtrinsicIndex { - block: Decode::decode(input)?, - key: Decode::decode(input)?, - })), - 2 => Ok(InputKey::DigestIndex(DigestIndex { - block: Decode::decode(input)?, - key: Decode::decode(input)?, - })), - 3 => Ok(InputKey::ChildIndex(ChildIndex { - block: Decode::decode(input)?, - storage_key: PrefixedStorageKey::new(Decode::decode(input)?), - })), - _ => Err("Invalid input key variant".into()), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn extrinsic_index_serialized_and_deserialized() { - let original = ExtrinsicIndex { block: 777u64, key: vec![42] }; - let serialized = original.encode(); - let deserialized: InputKey = Decode::decode(&mut &serialized[..]).unwrap(); - assert_eq!(InputKey::ExtrinsicIndex(original), deserialized); - } - - #[test] - fn digest_index_serialized_and_deserialized() { - let original = DigestIndex { block: 777u64, key: vec![42] }; - let serialized = original.encode(); - let deserialized: InputKey = Decode::decode(&mut &serialized[..]).unwrap(); - assert_eq!(InputKey::DigestIndex(original), deserialized); - } -} diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs deleted file mode 100644 index 40148095247d..000000000000 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ /dev/null @@ -1,428 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Changes trie related structures and functions. -//! -//! Changes trie is a trie built of { storage key => extrinsics } pairs -//! at the end of each block. For every changed storage key it contains -//! a pair, mapping key to the set of extrinsics where it has been changed. -//! -//! Optionally, every N blocks, additional level1-digest nodes are appended -//! to the changes trie, containing pairs { storage key => blocks }. For every -//! storage key that has been changed in PREVIOUS N-1 blocks (except for genesis -//! block) it contains a pair, mapping this key to the set of blocks where it -//! has been changed. -//! -//! Optionally, every N^digest_level (where digest_level > 1) blocks, additional -//! digest_level digest is created. It is built out of pairs { storage key => digest -//! block }, containing entries for every storage key that has been changed in -//! the last N*digest_level-1 blocks (except for genesis block), mapping these keys -//! to the set of lower-level digest blocks. -//! -//! Changes trie configuration could change within a time. The range of blocks, where -//! configuration has been active, is given by two blocks: zero and end. Zero block is -//! the block where configuration has been set. But the first changes trie that uses -//! this configuration will be built at the block zero+1. If configuration deactivates -//! at some block, this will be the end block of the configuration. It is also the -//! zero block of the next configuration. -//! -//! If configuration has the end block, it also means that 'skewed digest' has/should -//! been built at that block. If this is the block where max-level digest should have -//! been created, than it is simply max-level digest of this configuration. Otherwise, -//! it is the digest that covers all blocks since last max-level digest block was -//! created. -//! -//! Changes trie only contains the top level storage changes. Sub-level changes -//! are propagated through its storage root on the top level storage. - -mod build; -mod build_cache; -mod build_iterator; -mod changes_iterator; -mod input; -mod prune; -mod storage; -mod surface_iterator; - -pub use self::{ - build_cache::{BuildCache, CacheAction, CachedBuildData}, - changes_iterator::{ - key_changes, key_changes_proof, key_changes_proof_check, key_changes_proof_check_with_db, - }, - prune::prune, - storage::InMemoryStorage, -}; - -use crate::{ - backend::Backend, - changes_trie::{ - build::prepare_input, - build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}, - }, - overlayed_changes::OverlayedChanges, - StorageKey, -}; -use codec::{Decode, Encode}; -use hash_db::{Hasher, Prefix}; -use num_traits::{One, Zero}; -use sp_core::{self, storage::PrefixedStorageKey}; -use sp_trie::{trie_types::TrieDBMut, DBValue, MemoryDB, TrieMut}; -use std::{ - collections::{HashMap, HashSet}, - convert::TryInto, -}; - -/// Requirements for block number that can be used with changes tries. -pub trait BlockNumber: - Send - + Sync - + 'static - + std::fmt::Display - + Clone - + From - + TryInto - + One - + Zero - + PartialEq - + Ord - + std::hash::Hash - + std::ops::Add - + ::std::ops::Sub - + std::ops::Mul - + ::std::ops::Div - + std::ops::Rem - + std::ops::AddAssign - + num_traits::CheckedMul - + num_traits::CheckedSub - + Decode - + Encode -{ -} - -impl BlockNumber for T where - T: Send - + Sync - + 'static - + std::fmt::Display - + Clone - + From - + TryInto - + One - + Zero - + PartialEq - + Ord - + std::hash::Hash - + std::ops::Add - + ::std::ops::Sub - + std::ops::Mul - + ::std::ops::Div - + std::ops::Rem - + std::ops::AddAssign - + num_traits::CheckedMul - + num_traits::CheckedSub - + Decode - + Encode -{ -} - -/// Block identifier that could be used to determine fork of this block. -#[derive(Debug)] -pub struct AnchorBlockId { - /// Hash of this block. - pub hash: Hash, - /// Number of this block. - pub number: Number, -} - -/// Changes tries state at some block. -pub struct State<'a, H, Number> { - /// Configuration that is active at given block. - pub config: Configuration, - /// Configuration activation block number. Zero if it is the first configuration on the chain, - /// or number of the block that have emit NewConfiguration signal (thus activating - /// configuration starting from the **next** block). - pub zero: Number, - /// Underlying changes tries storage reference. - pub storage: &'a dyn Storage, -} - -/// Changes trie storage. Provides access to trie roots and trie nodes. -pub trait RootsStorage: Send + Sync { - /// Resolve hash of the block into anchor. - fn build_anchor(&self, hash: H::Out) -> Result, String>; - /// Get changes trie root for the block with given number which is an ancestor (or the block - /// itself) of the anchor_block (i.e. anchor_block.number >= block). - fn root( - &self, - anchor: &AnchorBlockId, - block: Number, - ) -> Result, String>; -} - -/// Changes trie storage. Provides access to trie roots and trie nodes. -pub trait Storage: RootsStorage { - /// Casts from self reference to RootsStorage reference. - fn as_roots_storage(&self) -> &dyn RootsStorage; - /// Execute given functor with cached entry for given trie root. - /// Returns true if the functor has been called (cache entry exists) and false otherwise. - fn with_cached_changed_keys( - &self, - root: &H::Out, - functor: &mut dyn FnMut(&HashMap, HashSet>), - ) -> bool; - /// Get a trie node. - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String>; -} - -/// Changes trie storage -> trie backend essence adapter. -pub struct TrieBackendStorageAdapter<'a, H: Hasher, Number: BlockNumber>( - pub &'a dyn Storage, -); - -impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage - for TrieBackendStorageAdapter<'a, H, N> -{ - type Overlay = sp_trie::MemoryDB; - - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { - self.0.get(key, prefix) - } -} - -/// Changes trie configuration. -pub type Configuration = sp_core::ChangesTrieConfiguration; - -/// Blocks range where configuration has been constant. -#[derive(Clone)] -pub struct ConfigurationRange<'a, N> { - /// Active configuration. - pub config: &'a Configuration, - /// Zero block of this configuration. The configuration is active starting from the next block. - pub zero: N, - /// End block of this configuration. It is the last block where configuration has been active. - pub end: Option, -} - -impl<'a, H, Number> State<'a, H, Number> { - /// Create state with given config and storage. - pub fn new(config: Configuration, zero: Number, storage: &'a dyn Storage) -> Self { - Self { config, zero, storage } - } -} - -impl<'a, H, Number: Clone> Clone for State<'a, H, Number> { - fn clone(&self) -> Self { - State { config: self.config.clone(), zero: self.zero.clone(), storage: self.storage } - } -} - -/// Create state where changes tries are disabled. -pub fn disabled_state<'a, H, Number>() -> Option> { - None -} - -/// Compute the changes trie root and transaction for given block. -/// Returns Err(()) if unknown `parent_hash` has been passed. -/// Returns Ok(None) if there's no data to perform computation. -/// Panics if background storage returns an error OR if insert to MemoryDB fails. -pub fn build_changes_trie<'a, B: Backend, H: Hasher, Number: BlockNumber>( - backend: &B, - state: Option<&'a State<'a, H, Number>>, - changes: &OverlayedChanges, - parent_hash: H::Out, - panic_on_storage_error: bool, -) -> Result, H::Out, CacheAction)>, ()> -where - H::Out: Ord + 'static + Encode, -{ - /// Panics when `res.is_err() && panic`, otherwise it returns `Err(())` on an error. - fn maybe_panic( - res: std::result::Result, - panic: bool, - ) -> std::result::Result { - res.map(Ok).unwrap_or_else(|e| { - if panic { - panic!( - "changes trie: storage access is not allowed to fail within runtime: {:?}", - e - ) - } else { - Err(()) - } - }) - } - - // when storage isn't provided, changes tries aren't created - let state = match state { - Some(state) => state, - None => return Ok(None), - }; - - // build_anchor error should not be considered fatal - let parent = state.storage.build_anchor(parent_hash).map_err(|_| ())?; - let block = parent.number.clone() + One::one(); - - // prepare configuration range - we already know zero block. Current block may be the end block - // if configuration has been changed in this block - let is_config_changed = - match changes.storage(sp_core::storage::well_known_keys::CHANGES_TRIE_CONFIG) { - Some(Some(new_config)) => new_config != &state.config.encode()[..], - Some(None) => true, - None => false, - }; - let config_range = ConfigurationRange { - config: &state.config, - zero: state.zero.clone(), - end: if is_config_changed { Some(block.clone()) } else { None }, - }; - - // storage errors are considered fatal (similar to situations when runtime fetches values from - // storage) - let (input_pairs, child_input_pairs, digest_input_blocks) = maybe_panic( - prepare_input::( - backend, - state.storage, - config_range.clone(), - changes, - &parent, - ), - panic_on_storage_error, - )?; - - // prepare cached data - let mut cache_action = prepare_cached_build_data(config_range, block.clone()); - let needs_changed_keys = cache_action.collects_changed_keys(); - cache_action = cache_action.set_digest_input_blocks(digest_input_blocks); - - let mut mdb = MemoryDB::default(); - let mut child_roots = Vec::with_capacity(child_input_pairs.len()); - for (child_index, input_pairs) in child_input_pairs { - let mut not_empty = false; - let mut root = Default::default(); - { - let mut trie = TrieDBMut::::new(&mut mdb, &mut root); - let mut storage_changed_keys = HashSet::new(); - for input_pair in input_pairs { - if needs_changed_keys { - if let Some(key) = input_pair.key() { - storage_changed_keys.insert(key.to_vec()); - } - } - - let (key, value) = input_pair.into(); - not_empty = true; - maybe_panic(trie.insert(&key, &value), panic_on_storage_error)?; - } - - cache_action = - cache_action.insert(Some(child_index.storage_key.clone()), storage_changed_keys); - } - if not_empty { - child_roots.push(input::InputPair::ChildIndex(child_index, root.as_ref().to_vec())); - } - } - let mut root = Default::default(); - { - let mut trie = TrieDBMut::::new(&mut mdb, &mut root); - for (key, value) in child_roots.into_iter().map(Into::into) { - maybe_panic(trie.insert(&key, &value), panic_on_storage_error)?; - } - - let mut storage_changed_keys = HashSet::new(); - for input_pair in input_pairs { - if needs_changed_keys { - if let Some(key) = input_pair.key() { - storage_changed_keys.insert(key.to_vec()); - } - } - - let (key, value) = input_pair.into(); - maybe_panic(trie.insert(&key, &value), panic_on_storage_error)?; - } - - cache_action = cache_action.insert(None, storage_changed_keys); - } - - let cache_action = cache_action.complete(block, &root); - Ok(Some((mdb, root, cache_action))) -} - -/// Prepare empty cached build data for given block. -fn prepare_cached_build_data( - config: ConfigurationRange, - block: Number, -) -> IncompleteCacheAction { - // when digests are not enabled in configuration, we do not need to cache anything - // because it'll never be used again for building other tries - // => let's clear the cache - if !config.config.is_digest_build_enabled() { - return IncompleteCacheAction::Clear - } - - // when this is the last block where current configuration is active - // => let's clear the cache - if config.end.as_ref() == Some(&block) { - return IncompleteCacheAction::Clear - } - - // we do not need to cache anything when top-level digest trie is created, because - // it'll never be used again for building other tries - // => let's clear the cache - match config.config.digest_level_at_block(config.zero.clone(), block) { - Some((digest_level, _, _)) if digest_level == config.config.digest_levels => - IncompleteCacheAction::Clear, - _ => IncompleteCacheAction::CacheBuildData(IncompleteCachedBuildData::new()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cache_is_cleared_when_digests_are_disabled() { - let config = Configuration { digest_interval: 0, digest_levels: 0 }; - let config_range = ConfigurationRange { zero: 0, end: None, config: &config }; - assert_eq!(prepare_cached_build_data(config_range, 8u32), IncompleteCacheAction::Clear); - } - - #[test] - fn build_data_is_cached_when_digests_are_enabled() { - let config = Configuration { digest_interval: 8, digest_levels: 2 }; - let config_range = ConfigurationRange { zero: 0, end: None, config: &config }; - assert!(prepare_cached_build_data(config_range.clone(), 4u32).collects_changed_keys()); - assert!(prepare_cached_build_data(config_range.clone(), 7u32).collects_changed_keys()); - assert!(prepare_cached_build_data(config_range, 8u32).collects_changed_keys()); - } - - #[test] - fn cache_is_cleared_when_digests_are_enabled_and_top_level_digest_is_built() { - let config = Configuration { digest_interval: 8, digest_levels: 2 }; - let config_range = ConfigurationRange { zero: 0, end: None, config: &config }; - assert_eq!(prepare_cached_build_data(config_range, 64u32), IncompleteCacheAction::Clear); - } - - #[test] - fn cache_is_cleared_when_end_block_of_configuration_is_built() { - let config = Configuration { digest_interval: 8, digest_levels: 2 }; - let config_range = ConfigurationRange { zero: 0, end: Some(4u32), config: &config }; - assert_eq!( - prepare_cached_build_data(config_range.clone(), 4u32), - IncompleteCacheAction::Clear - ); - } -} diff --git a/primitives/state-machine/src/changes_trie/prune.rs b/primitives/state-machine/src/changes_trie/prune.rs deleted file mode 100644 index 2ca540562b47..000000000000 --- a/primitives/state-machine/src/changes_trie/prune.rs +++ /dev/null @@ -1,204 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Changes trie pruning-related functions. - -use crate::{ - changes_trie::{ - input::{ChildIndex, InputKey}, - storage::TrieBackendAdapter, - AnchorBlockId, BlockNumber, Storage, - }, - proving_backend::ProvingBackendRecorder, - trie_backend_essence::TrieBackendEssence, -}; -use codec::{Codec, Decode}; -use hash_db::Hasher; -use log::warn; -use num_traits::One; -use sp_trie::Recorder; - -/// Prune obsolete changes tries. Pruning happens at the same block, where highest -/// level digest is created. Pruning guarantees to save changes tries for last -/// `min_blocks_to_keep` blocks. We only prune changes tries at `max_digest_interval` -/// ranges. -pub fn prune( - storage: &dyn Storage, - first: Number, - last: Number, - current_block: &AnchorBlockId, - mut remove_trie_node: F, -) where - H::Out: Codec, -{ - // delete changes trie for every block in range - let mut block = first; - loop { - if block >= last.clone() + One::one() { - break - } - - let prev_block = block.clone(); - block += One::one(); - - let block = prev_block; - let root = match storage.root(current_block, block.clone()) { - Ok(Some(root)) => root, - Ok(None) => continue, - Err(error) => { - // try to delete other tries - warn!(target: "trie", "Failed to read changes trie root from DB: {}", error); - continue - }, - }; - let children_roots = { - let trie_storage = TrieBackendEssence::<_, H>::new( - crate::changes_trie::TrieBackendStorageAdapter(storage), - root, - ); - let child_prefix = ChildIndex::key_neutral_prefix(block.clone()); - let mut children_roots = Vec::new(); - trie_storage.for_key_values_with_prefix(&child_prefix, |mut key, mut value| { - if let Ok(InputKey::ChildIndex::(_trie_key)) = Decode::decode(&mut key) { - if let Ok(value) = >::decode(&mut value) { - let mut trie_root = ::Out::default(); - trie_root.as_mut().copy_from_slice(&value[..]); - children_roots.push(trie_root); - } - } - }); - - children_roots - }; - for root in children_roots.into_iter() { - prune_trie(storage, root, &mut remove_trie_node); - } - - prune_trie(storage, root, &mut remove_trie_node); - } -} - -// Prune a trie. -fn prune_trie( - storage: &dyn Storage, - root: H::Out, - remove_trie_node: &mut F, -) where - H::Out: Codec, -{ - // enumerate all changes trie' keys, recording all nodes that have been 'touched' - // (effectively - all changes trie nodes) - let mut proof_recorder: Recorder = Default::default(); - { - let mut trie = ProvingBackendRecorder::<_, H> { - backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root), - proof_recorder: &mut proof_recorder, - }; - trie.record_all_keys(); - } - - // all nodes of this changes trie should be pruned - remove_trie_node(root); - for node in proof_recorder.drain().into_iter().map(|n| n.hash) { - remove_trie_node(node); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{backend::insert_into_memory_db, changes_trie::storage::InMemoryStorage}; - use codec::Encode; - use sp_core::H256; - use sp_runtime::traits::BlakeTwo256; - use sp_trie::MemoryDB; - use std::collections::HashSet; - - fn prune_by_collect( - storage: &dyn Storage, - first: u64, - last: u64, - current_block: u64, - ) -> HashSet { - let mut pruned_trie_nodes = HashSet::new(); - let anchor = AnchorBlockId { hash: Default::default(), number: current_block }; - prune(storage, first, last, &anchor, |node| { - pruned_trie_nodes.insert(node); - }); - pruned_trie_nodes - } - - #[test] - fn prune_works() { - fn prepare_storage() -> InMemoryStorage { - let child_info = sp_core::storage::ChildInfo::new_default(&b"1"[..]); - let child_key = - ChildIndex { block: 67u64, storage_key: child_info.prefixed_storage_key() } - .encode(); - let mut mdb1 = MemoryDB::::default(); - let root1 = - insert_into_memory_db::(&mut mdb1, vec![(vec![10], vec![20])]) - .unwrap(); - let mut mdb2 = MemoryDB::::default(); - let root2 = insert_into_memory_db::( - &mut mdb2, - vec![(vec![11], vec![21]), (vec![12], vec![22])], - ) - .unwrap(); - let mut mdb3 = MemoryDB::::default(); - let ch_root3 = - insert_into_memory_db::(&mut mdb3, vec![(vec![110], vec![120])]) - .unwrap(); - let root3 = insert_into_memory_db::( - &mut mdb3, - vec![ - (vec![13], vec![23]), - (vec![14], vec![24]), - (child_key, ch_root3.as_ref().encode()), - ], - ) - .unwrap(); - let mut mdb4 = MemoryDB::::default(); - let root4 = - insert_into_memory_db::(&mut mdb4, vec![(vec![15], vec![25])]) - .unwrap(); - let storage = InMemoryStorage::new(); - storage.insert(65, root1, mdb1); - storage.insert(66, root2, mdb2); - storage.insert(67, root3, mdb3); - storage.insert(68, root4, mdb4); - - storage - } - - let storage = prepare_storage(); - assert!(prune_by_collect(&storage, 20, 30, 90).is_empty()); - assert!(!storage.into_mdb().drain().is_empty()); - - let storage = prepare_storage(); - let prune60_65 = prune_by_collect(&storage, 60, 65, 90); - assert!(!prune60_65.is_empty()); - storage.remove_from_storage(&prune60_65); - assert!(!storage.into_mdb().drain().is_empty()); - - let storage = prepare_storage(); - let prune60_70 = prune_by_collect(&storage, 60, 70, 90); - assert!(!prune60_70.is_empty()); - storage.remove_from_storage(&prune60_70); - assert!(storage.into_mdb().drain().is_empty()); - } -} diff --git a/primitives/state-machine/src/changes_trie/storage.rs b/primitives/state-machine/src/changes_trie/storage.rs deleted file mode 100644 index bd5e3a32b565..000000000000 --- a/primitives/state-machine/src/changes_trie/storage.rs +++ /dev/null @@ -1,214 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Changes trie storage utilities. - -use crate::{ - changes_trie::{AnchorBlockId, BlockNumber, BuildCache, RootsStorage, Storage}, - trie_backend_essence::TrieBackendStorage, - StorageKey, -}; -use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; -use parking_lot::RwLock; -use sp_core::storage::PrefixedStorageKey; -use sp_trie::{DBValue, MemoryDB}; -use std::collections::{BTreeMap, HashMap, HashSet}; - -#[cfg(test)] -use crate::backend::insert_into_memory_db; -#[cfg(test)] -use crate::changes_trie::input::{ChildIndex, InputPair}; - -/// In-memory implementation of changes trie storage. -pub struct InMemoryStorage { - data: RwLock>, - cache: BuildCache, -} - -/// Adapter for using changes trie storage as a TrieBackendEssence' storage. -pub struct TrieBackendAdapter<'a, H: Hasher, Number: BlockNumber> { - storage: &'a dyn Storage, - _hasher: std::marker::PhantomData<(H, Number)>, -} - -struct InMemoryStorageData { - roots: BTreeMap, - mdb: MemoryDB, -} - -impl InMemoryStorage { - /// Creates storage from given in-memory database. - pub fn with_db(mdb: MemoryDB) -> Self { - Self { - data: RwLock::new(InMemoryStorageData { roots: BTreeMap::new(), mdb }), - cache: BuildCache::new(), - } - } - - /// Creates storage with empty database. - pub fn new() -> Self { - Self::with_db(Default::default()) - } - - /// Creates storage with given proof. - pub fn with_proof(proof: Vec>) -> Self { - use hash_db::HashDB; - - let mut proof_db = MemoryDB::::default(); - for item in proof { - proof_db.insert(EMPTY_PREFIX, &item); - } - Self::with_db(proof_db) - } - - /// Get mutable cache reference. - pub fn cache_mut(&mut self) -> &mut BuildCache { - &mut self.cache - } - - /// Create the storage with given blocks. - pub fn with_blocks(blocks: Vec<(Number, H::Out)>) -> Self { - Self { - data: RwLock::new(InMemoryStorageData { - roots: blocks.into_iter().collect(), - mdb: MemoryDB::default(), - }), - cache: BuildCache::new(), - } - } - - #[cfg(test)] - pub fn with_inputs( - mut top_inputs: Vec<(Number, Vec>)>, - children_inputs: Vec<(PrefixedStorageKey, Vec<(Number, Vec>)>)>, - ) -> Self { - let mut mdb = MemoryDB::default(); - let mut roots = BTreeMap::new(); - for (storage_key, child_input) in children_inputs { - for (block, pairs) in child_input { - let root = - insert_into_memory_db::(&mut mdb, pairs.into_iter().map(Into::into)); - - if let Some(root) = root { - let ix = if let Some(ix) = top_inputs.iter().position(|v| v.0 == block) { - ix - } else { - top_inputs.push((block.clone(), Default::default())); - top_inputs.len() - 1 - }; - top_inputs[ix].1.push(InputPair::ChildIndex( - ChildIndex { block: block.clone(), storage_key: storage_key.clone() }, - root.as_ref().to_vec(), - )); - } - } - } - - for (block, pairs) in top_inputs { - let root = insert_into_memory_db::(&mut mdb, pairs.into_iter().map(Into::into)); - if let Some(root) = root { - roots.insert(block, root); - } - } - - InMemoryStorage { - data: RwLock::new(InMemoryStorageData { roots, mdb }), - cache: BuildCache::new(), - } - } - - #[cfg(test)] - pub fn clear_storage(&self) { - self.data.write().mdb = MemoryDB::default(); // use new to be more correct - } - - #[cfg(test)] - pub fn remove_from_storage(&self, keys: &HashSet) { - let mut data = self.data.write(); - for key in keys { - data.mdb.remove_and_purge(key, hash_db::EMPTY_PREFIX); - } - } - - #[cfg(test)] - pub fn into_mdb(self) -> MemoryDB { - self.data.into_inner().mdb - } - - /// Insert changes trie for given block. - pub fn insert(&self, block: Number, changes_trie_root: H::Out, trie: MemoryDB) { - let mut data = self.data.write(); - data.roots.insert(block, changes_trie_root); - data.mdb.consolidate(trie); - } -} - -impl RootsStorage for InMemoryStorage { - fn build_anchor(&self, parent_hash: H::Out) -> Result, String> { - self.data - .read() - .roots - .iter() - .find(|(_, v)| **v == parent_hash) - .map(|(k, _)| AnchorBlockId { hash: parent_hash, number: k.clone() }) - .ok_or_else(|| format!("Can't find associated number for block {:?}", parent_hash)) - } - - fn root( - &self, - _anchor_block: &AnchorBlockId, - block: Number, - ) -> Result, String> { - Ok(self.data.read().roots.get(&block).cloned()) - } -} - -impl Storage for InMemoryStorage { - fn as_roots_storage(&self) -> &dyn RootsStorage { - self - } - - fn with_cached_changed_keys( - &self, - root: &H::Out, - functor: &mut dyn FnMut(&HashMap, HashSet>), - ) -> bool { - self.cache.with_changed_keys(root, functor) - } - - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { - MemoryDB::::get(&self.data.read().mdb, key, prefix) - } -} - -impl<'a, H: Hasher, Number: BlockNumber> TrieBackendAdapter<'a, H, Number> { - pub fn new(storage: &'a dyn Storage) -> Self { - Self { storage, _hasher: Default::default() } - } -} - -impl<'a, H, Number> TrieBackendStorage for TrieBackendAdapter<'a, H, Number> -where - Number: BlockNumber, - H: Hasher, -{ - type Overlay = MemoryDB; - - fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { - self.storage.get(key, prefix) - } -} diff --git a/primitives/state-machine/src/changes_trie/surface_iterator.rs b/primitives/state-machine/src/changes_trie/surface_iterator.rs deleted file mode 100644 index b3e5a490cd18..000000000000 --- a/primitives/state-machine/src/changes_trie/surface_iterator.rs +++ /dev/null @@ -1,326 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! The best way to understand how this iterator works is to imagine some 2D terrain that have some -//! mountains (digest changes tries) and valleys (changes tries for regular blocks). There are gems -//! (blocks) beneath the terrain. Given the request to find all gems in the range [X1; X2] this -//! iterator will return **minimal set** of points at the terrain (mountains and valleys) inside -//! this range that have to be drilled down to search for gems. - -use crate::changes_trie::{BlockNumber, ConfigurationRange}; -use num_traits::One; - -/// Returns surface iterator for given range of blocks. -/// -/// `max` is the number of best block, known to caller. We can't access any changes tries -/// that are built after this block, even though we may have them built already. -pub fn surface_iterator<'a, Number: BlockNumber>( - config: ConfigurationRange<'a, Number>, - max: Number, - begin: Number, - end: Number, -) -> Result, String> { - let (current, current_begin, digest_step, digest_level) = - lower_bound_max_digest(config.clone(), max.clone(), begin.clone(), end)?; - Ok(SurfaceIterator { - config, - begin, - max, - current: Some(current), - current_begin, - digest_step, - digest_level, - }) -} - -/// Surface iterator - only traverses top-level digests from given range and tries to find -/// all valid digest changes. -/// -/// Iterator item is the tuple of (last block of the current point + digest level of the current -/// point). Digest level is Some(0) when it is regular block, is Some(non-zero) when it is digest -/// block and None if it is skewed digest block. -pub struct SurfaceIterator<'a, Number: BlockNumber> { - config: ConfigurationRange<'a, Number>, - begin: Number, - max: Number, - current: Option, - current_begin: Number, - digest_step: u32, - digest_level: Option, -} - -impl<'a, Number: BlockNumber> Iterator for SurfaceIterator<'a, Number> { - type Item = Result<(Number, Option), String>; - - fn next(&mut self) -> Option { - let current = self.current.clone()?; - let digest_level = self.digest_level; - - if current < self.digest_step.into() { - self.current = None; - } else { - let next = current.clone() - self.digest_step.into(); - if next.is_zero() || next < self.begin { - self.current = None; - } else if next > self.current_begin { - self.current = Some(next); - } else { - let max_digest_interval = lower_bound_max_digest( - self.config.clone(), - self.max.clone(), - self.begin.clone(), - next, - ); - let (current, current_begin, digest_step, digest_level) = match max_digest_interval - { - Err(err) => return Some(Err(err)), - Ok(range) => range, - }; - - self.current = Some(current); - self.current_begin = current_begin; - self.digest_step = digest_step; - self.digest_level = digest_level; - } - } - - Some(Ok((current, digest_level))) - } -} - -/// Returns parameters of highest level digest block that includes the end of given range -/// and tends to include the whole range. -fn lower_bound_max_digest<'a, Number: BlockNumber>( - config: ConfigurationRange<'a, Number>, - max: Number, - begin: Number, - end: Number, -) -> Result<(Number, Number, u32, Option), String> { - if end > max || begin > end { - return Err(format!("invalid changes range: {}..{}/{}", begin, end, max)) - } - if begin <= config.zero || - config.end.as_ref().map(|config_end| end > *config_end).unwrap_or(false) - { - return Err(format!( - "changes trie range is not covered by configuration: {}..{}/{}..{}", - begin, - end, - config.zero, - match config.end.as_ref() { - Some(config_end) => format!("{}", config_end), - None => "None".into(), - } - )) - } - - let mut digest_level = 0u32; - let mut digest_step = 1u32; - let mut digest_interval = 0u32; - let mut current = end.clone(); - let mut current_begin = begin.clone(); - if current_begin != current { - while digest_level != config.config.digest_levels { - // try to use next level digest - let new_digest_level = digest_level + 1; - let new_digest_step = digest_step * config.config.digest_interval; - let new_digest_interval = config.config.digest_interval * { - if digest_interval == 0 { - 1 - } else { - digest_interval - } - }; - let new_digest_begin = config.zero.clone() + - ((current.clone() - One::one() - config.zero.clone()) / - new_digest_interval.into()) * - new_digest_interval.into(); - let new_digest_end = new_digest_begin.clone() + new_digest_interval.into(); - let new_current = new_digest_begin.clone() + new_digest_interval.into(); - - // check if we met skewed digest - if let Some(skewed_digest_end) = config.end.as_ref() { - if new_digest_end > *skewed_digest_end { - let skewed_digest_start = config.config.prev_max_level_digest_block( - config.zero.clone(), - skewed_digest_end.clone(), - ); - if let Some(skewed_digest_start) = skewed_digest_start { - let skewed_digest_range = (skewed_digest_end.clone() - - skewed_digest_start.clone()) - .try_into() - .ok() - .expect( - "skewed digest range is always <= max level digest range;\ - max level digest range always fits u32; qed", - ); - return Ok(( - skewed_digest_end.clone(), - skewed_digest_start, - skewed_digest_range, - None, - )) - } - } - } - - // we can't use next level digest if it touches any unknown (> max) blocks - if new_digest_end > max { - if begin < new_digest_begin { - current_begin = new_digest_begin; - } - break - } - - // we can (and will) use this digest - digest_level = new_digest_level; - digest_step = new_digest_step; - digest_interval = new_digest_interval; - current = new_current; - current_begin = new_digest_begin; - - // if current digest covers the whole range => no need to use next level digest - if current_begin <= begin && new_digest_end >= end { - break - } - } - } - - Ok((current, current_begin, digest_step, Some(digest_level))) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::changes_trie::Configuration; - - fn configuration_range<'a>( - config: &'a Configuration, - zero: u64, - ) -> ConfigurationRange<'a, u64> { - ConfigurationRange { config, zero, end: None } - } - - #[test] - fn lower_bound_max_digest_works() { - let config = Configuration { digest_interval: 4, digest_levels: 2 }; - - // when config activates at 0 - assert_eq!( - lower_bound_max_digest(configuration_range(&config, 0u64), 100_000u64, 20u64, 180u64) - .unwrap(), - (192, 176, 16, Some(2)), - ); - - // when config activates at 30 - assert_eq!( - lower_bound_max_digest(configuration_range(&config, 30u64), 100_000u64, 50u64, 210u64) - .unwrap(), - (222, 206, 16, Some(2)), - ); - } - - #[test] - fn surface_iterator_works() { - let config = Configuration { digest_interval: 4, digest_levels: 2 }; - - // when config activates at 0 - assert_eq!( - surface_iterator(configuration_range(&config, 0u64), 100_000u64, 40u64, 180u64,) - .unwrap() - .collect::>(), - vec![ - Ok((192, Some(2))), - Ok((176, Some(2))), - Ok((160, Some(2))), - Ok((144, Some(2))), - Ok((128, Some(2))), - Ok((112, Some(2))), - Ok((96, Some(2))), - Ok((80, Some(2))), - Ok((64, Some(2))), - Ok((48, Some(2))), - ], - ); - - // when config activates at 30 - assert_eq!( - surface_iterator(configuration_range(&config, 30u64), 100_000u64, 40u64, 180u64,) - .unwrap() - .collect::>(), - vec![ - Ok((190, Some(2))), - Ok((174, Some(2))), - Ok((158, Some(2))), - Ok((142, Some(2))), - Ok((126, Some(2))), - Ok((110, Some(2))), - Ok((94, Some(2))), - Ok((78, Some(2))), - Ok((62, Some(2))), - Ok((46, Some(2))), - ], - ); - - // when config activates at 0 AND max block is before next digest - assert_eq!( - surface_iterator(configuration_range(&config, 0u64), 183u64, 40u64, 183u64) - .unwrap() - .collect::>(), - vec![ - Ok((183, Some(0))), - Ok((182, Some(0))), - Ok((181, Some(0))), - Ok((180, Some(1))), - Ok((176, Some(2))), - Ok((160, Some(2))), - Ok((144, Some(2))), - Ok((128, Some(2))), - Ok((112, Some(2))), - Ok((96, Some(2))), - Ok((80, Some(2))), - Ok((64, Some(2))), - Ok((48, Some(2))), - ], - ); - } - - #[test] - fn surface_iterator_works_with_skewed_digest() { - let config = Configuration { digest_interval: 4, digest_levels: 2 }; - let mut config_range = configuration_range(&config, 0u64); - - // when config activates at 0 AND ends at 170 - config_range.end = Some(170); - assert_eq!( - surface_iterator(config_range, 100_000u64, 40u64, 170u64) - .unwrap() - .collect::>(), - vec![ - Ok((170, None)), - Ok((160, Some(2))), - Ok((144, Some(2))), - Ok((128, Some(2))), - Ok((112, Some(2))), - Ok((96, Some(2))), - Ok((80, Some(2))), - Ok((64, Some(2))), - Ok((48, Some(2))), - ], - ); - } -} diff --git a/primitives/state-machine/src/error.rs b/primitives/state-machine/src/error.rs index acc5b6080c7a..a12b3eae71bd 100644 --- a/primitives/state-machine/src/error.rs +++ b/primitives/state-machine/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index c20d8492fb1f..7b7e4b47f19b 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,12 +24,12 @@ use codec::{Decode, Encode, EncodeAppend}; use hash_db::Hasher; #[cfg(feature = "std")] use sp_core::hexdisplay::HexDisplay; -use sp_core::storage::{well_known_keys::is_child_storage_key, ChildInfo, TrackedStorageKey}; +use sp_core::storage::{ + well_known_keys::is_child_storage_key, ChildInfo, StateVersion, TrackedStorageKey, +}; use sp_externalities::{Extension, ExtensionStore, Externalities}; -use sp_trie::{empty_child_trie_root, trie_types::Layout}; +use sp_trie::{empty_child_trie_root, LayoutV1}; -#[cfg(feature = "std")] -use crate::changes_trie::State as ChangesTrieState; use crate::{log_error, trace, warn, StorageTransactionCache}; use sp_std::{ any::{Any, TypeId}, @@ -90,62 +90,52 @@ impl error::Error for Error { } /// Wraps a read-only backend, call executor, and current overlayed changes. -pub struct Ext<'a, H, N, B> +pub struct Ext<'a, H, B> where H: Hasher, B: 'a + Backend, - N: crate::changes_trie::BlockNumber, { /// The overlayed changes to write to. overlay: &'a mut OverlayedChanges, /// The storage backend to read from. backend: &'a B, /// The cache for the storage transactions. - storage_transaction_cache: &'a mut StorageTransactionCache, - /// Changes trie state to read from. - #[cfg(feature = "std")] - changes_trie_state: Option>, + storage_transaction_cache: &'a mut StorageTransactionCache, /// Pseudo-unique id used for tracing. pub id: u16, - /// Dummy usage of N arg. - _phantom: sp_std::marker::PhantomData, /// Extensions registered with this instance. #[cfg(feature = "std")] extensions: Option>, } -impl<'a, H, N, B> Ext<'a, H, N, B> +impl<'a, H, B> Ext<'a, H, B> where H: Hasher, B: Backend, - N: crate::changes_trie::BlockNumber, { /// Create a new `Ext`. #[cfg(not(feature = "std"))] pub fn new( overlay: &'a mut OverlayedChanges, - storage_transaction_cache: &'a mut StorageTransactionCache, + storage_transaction_cache: &'a mut StorageTransactionCache, backend: &'a B, ) -> Self { - Ext { overlay, backend, id: 0, storage_transaction_cache, _phantom: Default::default() } + Ext { overlay, backend, id: 0, storage_transaction_cache } } /// Create a new `Ext` from overlayed changes and read-only backend #[cfg(feature = "std")] pub fn new( overlay: &'a mut OverlayedChanges, - storage_transaction_cache: &'a mut StorageTransactionCache, + storage_transaction_cache: &'a mut StorageTransactionCache, backend: &'a B, - changes_trie_state: Option>, extensions: Option<&'a mut sp_externalities::Extensions>, ) -> Self { Self { overlay, backend, - changes_trie_state, storage_transaction_cache, id: rand::random(), - _phantom: Default::default(), extensions: extensions.map(OverlayedExtensions::new), } } @@ -159,12 +149,11 @@ where } #[cfg(test)] -impl<'a, H, N, B> Ext<'a, H, N, B> +impl<'a, H, B> Ext<'a, H, B> where H: Hasher, H::Out: Ord + 'static, B: 'a + Backend, - N: crate::changes_trie::BlockNumber, { pub fn storage_pairs(&self) -> Vec<(StorageKey, StorageValue)> { use std::collections::HashMap; @@ -181,12 +170,11 @@ where } } -impl<'a, H, N, B> Externalities for Ext<'a, H, N, B> +impl<'a, H, B> Externalities for Ext<'a, H, B> where H: Hasher, H::Out: Ord + 'static + codec::Codec, B: Backend, - N: crate::changes_trie::BlockNumber, { fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) { self.overlay.set_offchain_storage(key, value) @@ -519,7 +507,7 @@ where StorageAppend::new(current_value).append(value); } - fn storage_root(&mut self) -> Vec { + fn storage_root(&mut self, state_version: StateVersion) -> Vec { let _guard = guard(); if let Some(ref root) = self.storage_transaction_cache.transaction_storage_root { trace!( @@ -532,7 +520,9 @@ where return root.encode() } - let root = self.overlay.storage_root(self.backend, self.storage_transaction_cache); + let root = + self.overlay + .storage_root(self.backend, self.storage_transaction_cache, state_version); trace!( target: "state", method = "StorageRoot", @@ -543,7 +533,11 @@ where root.encode() } - fn child_storage_root(&mut self, child_info: &ChildInfo) -> Vec { + fn child_storage_root( + &mut self, + child_info: &ChildInfo, + state_version: StateVersion, + ) -> Vec { let _guard = guard(); let storage_key = child_info.storage_key(); let prefixed_storage_key = child_info.prefixed_storage_key(); @@ -551,7 +545,8 @@ where let root = self .storage(prefixed_storage_key.as_slice()) .and_then(|k| Decode::decode(&mut &k[..]).ok()) - .unwrap_or_else(|| empty_child_trie_root::>()); + // V1 is equivalent to V0 on empty root. + .unwrap_or_else(|| empty_child_trie_root::>()); trace!( target: "state", method = "ChildStorageRoot", @@ -564,7 +559,7 @@ where } else { let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); - Some(self.backend.child_storage_root(info, delta)) + Some(self.backend.child_storage_root(info, delta, state_version)) } else { None }; @@ -586,7 +581,7 @@ where target: "state", method = "ChildStorageRoot", ext_id = %HexDisplay::from(&self.id.to_le_bytes()), - child_info = %HexDisplay::from(&storage_key.as_ref()), + child_info = %HexDisplay::from(&storage_key), storage_root = %HexDisplay::from(&root.as_ref()), cached = false, ); @@ -597,13 +592,14 @@ where let root = self .storage(prefixed_storage_key.as_slice()) .and_then(|k| Decode::decode(&mut &k[..]).ok()) - .unwrap_or_else(|| empty_child_trie_root::>()); + // V1 is equivalent to V0 on empty root. + .unwrap_or_else(|| empty_child_trie_root::>()); trace!( target: "state", method = "ChildStorageRoot", ext_id = %HexDisplay::from(&self.id.to_le_bytes()), - child_info = %HexDisplay::from(&storage_key.as_ref()), + child_info = %HexDisplay::from(&storage_key), storage_root = %HexDisplay::from(&root.as_ref()), cached = false, ); @@ -644,54 +640,6 @@ where .add_transaction_index(IndexOperation::Renew { extrinsic: index, hash: hash.to_vec() }); } - #[cfg(not(feature = "std"))] - fn storage_changes_root(&mut self, _parent_hash: &[u8]) -> Result>, ()> { - Ok(None) - } - - #[cfg(feature = "std")] - fn storage_changes_root(&mut self, mut parent_hash: &[u8]) -> Result>, ()> { - let _guard = guard(); - if let Some(ref root) = self.storage_transaction_cache.changes_trie_transaction_storage_root - { - trace!( - target: "state", - method = "ChangesRoot", - ext_id = %HexDisplay::from(&self.id.to_le_bytes()), - parent_hash = %HexDisplay::from(&parent_hash), - ?root, - cached = true, - ); - - Ok(Some(root.encode())) - } else { - let root = self.overlay.changes_trie_root( - self.backend, - self.changes_trie_state.as_ref(), - Decode::decode(&mut parent_hash).map_err(|e| { - trace!( - target: "state", - error = %e, - "Failed to decode changes root parent hash", - ) - })?, - true, - self.storage_transaction_cache, - ); - - trace!( - target: "state", - method = "ChangesRoot", - ext_id = %HexDisplay::from(&self.id.to_le_bytes()), - parent_hash = %HexDisplay::from(&parent_hash), - ?root, - cached = false, - ); - - root.map(|r| r.map(|o| o.encode())) - } - } - fn storage_start_transaction(&mut self) { self.overlay.start_transaction() } @@ -712,10 +660,8 @@ where self.overlay .drain_storage_changes( self.backend, - #[cfg(feature = "std")] - None, - Default::default(), self.storage_transaction_cache, + Default::default(), // using any state ) .expect(EXT_NOT_ALLOWED_TO_FAIL); self.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL); @@ -726,18 +672,14 @@ where } fn commit(&mut self) { + // Bench always use latest state. + let state_version = StateVersion::default(); for _ in 0..self.overlay.transaction_depth() { self.overlay.commit_transaction().expect(BENCHMARKING_FN); } let changes = self .overlay - .drain_storage_changes( - self.backend, - #[cfg(feature = "std")] - None, - Default::default(), - self.storage_transaction_cache, - ) + .drain_storage_changes(self.backend, self.storage_transaction_cache, state_version) .expect(EXT_NOT_ALLOWED_TO_FAIL); self.backend .commit( @@ -778,12 +720,11 @@ where } } -impl<'a, H, N, B> Ext<'a, H, N, B> +impl<'a, H, B> Ext<'a, H, B> where H: Hasher, H::Out: Ord + 'static + codec::Codec, B: Backend, - N: crate::changes_trie::BlockNumber, { fn limit_remove_from_backend( &mut self, @@ -869,12 +810,11 @@ impl<'a> StorageAppend<'a> { } #[cfg(not(feature = "std"))] -impl<'a, H, N, B> ExtensionStore for Ext<'a, H, N, B> +impl<'a, H, B> ExtensionStore for Ext<'a, H, B> where H: Hasher, H::Out: Ord + 'static + codec::Codec, B: Backend, - N: crate::changes_trie::BlockNumber, { fn extension_by_type_id(&mut self, _type_id: TypeId) -> Option<&mut dyn Any> { None @@ -897,11 +837,10 @@ where } #[cfg(feature = "std")] -impl<'a, H, N, B> ExtensionStore for Ext<'a, H, N, B> +impl<'a, H, B> ExtensionStore for Ext<'a, H, B> where H: Hasher, B: 'a + Backend, - N: crate::changes_trie::BlockNumber, { fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> { self.extensions.as_mut().and_then(|exts| exts.get_mut(type_id)) @@ -938,86 +877,16 @@ where #[cfg(test)] mod tests { use super::*; - use crate::{ - changes_trie::{ - Configuration as ChangesTrieConfiguration, InMemoryStorage as TestChangesTrieStorage, - }, - InMemoryBackend, - }; + use crate::InMemoryBackend; use codec::Encode; - use hex_literal::hex; - use num_traits::Zero; use sp_core::{ map, - storage::{well_known_keys::EXTRINSIC_INDEX, Storage, StorageChild}, - Blake2Hasher, H256, + storage::{Storage, StorageChild}, + Blake2Hasher, }; type TestBackend = InMemoryBackend; - type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend>; - - fn prepare_overlay_with_changes() -> OverlayedChanges { - let mut changes = OverlayedChanges::default(); - changes.set_collect_extrinsics(true); - changes.set_extrinsic_index(1); - changes.set_storage(vec![1], Some(vec![100])); - changes.set_storage(EXTRINSIC_INDEX.to_vec(), Some(3u32.encode())); - changes.set_offchain_storage(b"k1", Some(b"v1")); - changes.set_offchain_storage(b"k2", Some(b"v2")); - changes - } - - fn changes_trie_config() -> ChangesTrieConfiguration { - ChangesTrieConfiguration { digest_interval: 0, digest_levels: 0 } - } - - #[test] - fn storage_changes_root_is_none_when_storage_is_not_provided() { - let mut overlay = prepare_overlay_with_changes(); - let mut cache = StorageTransactionCache::default(); - let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); - assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None); - } - - #[test] - fn storage_changes_root_is_none_when_state_is_not_provided() { - let mut overlay = prepare_overlay_with_changes(); - let mut cache = StorageTransactionCache::default(); - let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); - assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None); - } - - #[test] - fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() { - let mut overlay = prepare_overlay_with_changes(); - let mut cache = StorageTransactionCache::default(); - let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); - let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); - let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None); - assert_eq!( - ext.storage_changes_root(&H256::default().encode()).unwrap(), - Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").to_vec()), - ); - } - - #[test] - fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() { - let mut overlay = prepare_overlay_with_changes(); - let mut cache = StorageTransactionCache::default(); - overlay.set_collect_extrinsics(false); - overlay.set_storage(vec![1], None); - let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); - let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); - let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None); - assert_eq!( - ext.storage_changes_root(&H256::default().encode()).unwrap(), - Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").to_vec()), - ); - } + type TestExt<'a> = Ext<'a, Blake2Hasher, TestBackend>; #[test] fn next_storage_key_works() { @@ -1025,17 +894,20 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_storage(vec![20], None); overlay.set_storage(vec![30], Some(vec![31])); - let backend = Storage { - top: map![ - vec![10] => vec![10], - vec![20] => vec![20], - vec![40] => vec![40] - ], - children_default: map![], - } - .into(); + let backend = ( + Storage { + top: map![ + vec![10] => vec![10], + vec![20] => vec![20], + vec![40] => vec![40] + ], + children_default: map![], + }, + StateVersion::default(), + ) + .into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None); // next_backend < next_overlay assert_eq!(ext.next_storage_key(&[5]), Some(vec![10])); @@ -1051,7 +923,7 @@ mod tests { drop(ext); overlay.set_storage(vec![50], Some(vec![50])); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None); // next_overlay exist but next_backend doesn't exist assert_eq!(ext.next_storage_key(&[40]), Some(vec![50])); @@ -1071,15 +943,18 @@ mod tests { overlay.set_storage(vec![27], None); overlay.set_storage(vec![28], None); overlay.set_storage(vec![29], None); - let backend = Storage { - top: map![ - vec![30] => vec![30] - ], - children_default: map![], - } - .into(); + let backend = ( + Storage { + top: map![ + vec![30] => vec![30] + ], + children_default: map![], + }, + StateVersion::default(), + ) + .into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None); assert_eq!(ext.next_storage_key(&[5]), Some(vec![30])); @@ -1095,22 +970,25 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(child_info, vec![20], None); overlay.set_child_storage(child_info, vec![30], Some(vec![31])); - let backend = Storage { - top: map![], - children_default: map![ - child_info.storage_key().to_vec() => StorageChild { - data: map![ - vec![10] => vec![10], - vec![20] => vec![20], - vec![40] => vec![40] - ], - child_info: child_info.to_owned(), - } - ], - } - .into(); + let backend = ( + Storage { + top: map![], + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { + data: map![ + vec![10] => vec![10], + vec![20] => vec![20], + vec![40] => vec![40] + ], + child_info: child_info.to_owned(), + } + ], + }, + StateVersion::default(), + ) + .into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None); // next_backend < next_overlay assert_eq!(ext.next_child_storage_key(child_info, &[5]), Some(vec![10])); @@ -1126,7 +1004,7 @@ mod tests { drop(ext); overlay.set_child_storage(child_info, vec![50], Some(vec![50])); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None); // next_overlay exist but next_backend doesn't exist assert_eq!(ext.next_child_storage_key(child_info, &[40]), Some(vec![50])); @@ -1140,22 +1018,25 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(child_info, vec![20], None); overlay.set_child_storage(child_info, vec![30], Some(vec![31])); - let backend = Storage { - top: map![], - children_default: map![ - child_info.storage_key().to_vec() => StorageChild { - data: map![ - vec![10] => vec![10], - vec![20] => vec![20], - vec![30] => vec![40] - ], - child_info: child_info.to_owned(), - } - ], - } - .into(); + let backend = ( + Storage { + top: map![], + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { + data: map![ + vec![10] => vec![10], + vec![20] => vec![20], + vec![30] => vec![40] + ], + child_info: child_info.to_owned(), + } + ], + }, + StateVersion::default(), + ) + .into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None); assert_eq!(ext.child_storage(child_info, &[10]), Some(vec![10])); assert_eq!( @@ -1179,20 +1060,23 @@ mod tests { let child_info = &child_info; let mut cache = StorageTransactionCache::default(); let mut overlay = OverlayedChanges::default(); - let backend = Storage { - top: map![], - children_default: map![ - child_info.storage_key().to_vec() => StorageChild { - data: map![ - vec![30] => vec![40] - ], - child_info: child_info.to_owned(), - } - ], - } - .into(); + let backend = ( + Storage { + top: map![], + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { + data: map![ + vec![30] => vec![40] + ], + child_info: child_info.to_owned(), + } + ], + }, + StateVersion::default(), + ) + .into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None); use sp_core::storage::well_known_keys; let mut ext = ext; diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index f9f94c0c50d6..4605d07b2ab6 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,8 +22,8 @@ use crate::{ }; use codec::Codec; use hash_db::Hasher; -use sp_core::storage::{ChildInfo, Storage}; -use sp_trie::{empty_trie_root, Layout, MemoryDB}; +use sp_core::storage::{ChildInfo, StateVersion, Storage}; +use sp_trie::{empty_trie_root, LayoutV1, MemoryDB}; use std::collections::{BTreeMap, HashMap}; /// Create a new empty instance of in-memory backend. @@ -32,7 +32,8 @@ where H::Out: Codec + Ord, { let db = MemoryDB::default(); - TrieBackend::new(db, empty_trie_root::>()) + // V1 is same as V0 for an empty trie. + TrieBackend::new(db, empty_trie_root::>()) } impl TrieBackend, H> @@ -43,9 +44,10 @@ where pub fn update, StorageCollection)>>( &self, changes: T, + state_version: StateVersion, ) -> Self { let mut clone = self.clone(); - clone.insert(changes); + clone.insert(changes, state_version); clone } @@ -53,6 +55,7 @@ where pub fn insert, StorageCollection)>>( &mut self, changes: T, + state_version: StateVersion, ) { let (top, child) = changes.into_iter().partition::, _>(|v| v.0.is_none()); let (root, transaction) = self.full_storage_root( @@ -60,6 +63,7 @@ where child.iter().filter_map(|v| { v.0.as_ref().map(|c| (c, v.1.iter().map(|(k, v)| (&k[..], v.as_deref())))) }), + state_version, ); self.apply_transaction(root, transaction); @@ -103,53 +107,63 @@ where } } -impl From, BTreeMap>> +impl From<(HashMap, BTreeMap>, StateVersion)> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(inner: HashMap, BTreeMap>) -> Self { + fn from( + (inner, state_version): ( + HashMap, BTreeMap>, + StateVersion, + ), + ) -> Self { let mut backend = new_in_mem(); backend.insert( inner .into_iter() .map(|(k, m)| (k, m.into_iter().map(|(k, v)| (k, Some(v))).collect())), + state_version, ); backend } } -impl From for TrieBackend, H> +impl From<(Storage, StateVersion)> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(inners: Storage) -> Self { + fn from((inners, state_version): (Storage, StateVersion)) -> Self { let mut inner: HashMap, BTreeMap> = inners .children_default .into_iter() .map(|(_k, c)| (Some(c.child_info), c.data)) .collect(); inner.insert(None, inners.top); - inner.into() + (inner, state_version).into() } } -impl From> for TrieBackend, H> +impl From<(BTreeMap, StateVersion)> + for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(inner: BTreeMap) -> Self { + fn from((inner, state_version): (BTreeMap, StateVersion)) -> Self { let mut expanded = HashMap::new(); expanded.insert(None, inner); - expanded.into() + (expanded, state_version).into() } } -impl From, StorageCollection)>> for TrieBackend, H> +impl From<(Vec<(Option, StorageCollection)>, StateVersion)> + for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(inner: Vec<(Option, StorageCollection)>) -> Self { + fn from( + (inner, state_version): (Vec<(Option, StorageCollection)>, StateVersion), + ) -> Self { let mut expanded: HashMap, BTreeMap> = HashMap::new(); for (child_info, key_values) in inner { @@ -160,7 +174,7 @@ where } } } - expanded.into() + (expanded, state_version).into() } } @@ -168,16 +182,20 @@ where mod tests { use super::*; use crate::backend::Backend; + use sp_core::storage::StateVersion; use sp_runtime::traits::BlakeTwo256; /// Assert in memory backend with only child trie keys works as trie backend. #[test] fn in_memory_with_child_trie_only() { + let state_version = StateVersion::default(); let storage = new_in_mem::(); let child_info = ChildInfo::new_default(b"1"); let child_info = &child_info; - let storage = storage - .update(vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])]); + let storage = storage.update( + vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])], + state_version, + ); let trie_backend = storage.as_trie_backend().unwrap(); assert_eq!(trie_backend.child_storage(child_info, b"2").unwrap(), Some(b"3".to_vec())); let storage_key = child_info.prefixed_storage_key(); @@ -186,13 +204,18 @@ mod tests { #[test] fn insert_multiple_times_child_data_works() { + let state_version = StateVersion::default(); let mut storage = new_in_mem::(); let child_info = ChildInfo::new_default(b"1"); - storage - .insert(vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])]); - storage - .insert(vec![(Some(child_info.clone()), vec![(b"1".to_vec(), Some(b"3".to_vec()))])]); + storage.insert( + vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])], + state_version, + ); + storage.insert( + vec![(Some(child_info.clone()), vec![(b"1".to_vec(), Some(b"3".to_vec()))])], + state_version, + ); assert_eq!(storage.child_storage(&child_info, &b"2"[..]), Ok(Some(b"3".to_vec()))); assert_eq!(storage.child_storage(&child_info, &b"1"[..]), Ok(Some(b"3".to_vec()))); diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 7bd0c645f3c0..1a69d51d58ab 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,8 +23,6 @@ pub mod backend; #[cfg(feature = "std")] mod basic; -#[cfg(feature = "std")] -mod changes_trie; mod error; mod ext; #[cfg(feature = "std")] @@ -128,6 +126,7 @@ impl sp_std::fmt::Display for DefaultError { pub use crate::{ backend::Backend, + error::{Error, ExecutionError}, ext::Ext, overlayed_changes::{ ChildStorageCollection, IndexOperation, OffchainChangesCollection, @@ -138,30 +137,11 @@ pub use crate::{ trie_backend::TrieBackend, trie_backend_essence::{Storage, TrieBackendStorage}, }; -pub use error::{Error, ExecutionError}; - -#[cfg(not(feature = "std"))] -mod changes_trie { - /// Stub for change trie block number until - /// change trie move to no_std. - pub trait BlockNumber {} - - impl BlockNumber for N {} -} #[cfg(feature = "std")] mod std_reexport { pub use crate::{ basic::BasicExternalities, - changes_trie::{ - disabled_state as disabled_changes_trie_state, key_changes, key_changes_proof, - key_changes_proof_check, key_changes_proof_check_with_db, prune as prune_changes_tries, - AnchorBlockId as ChangesTrieAnchorBlockId, BlockNumber as ChangesTrieBlockNumber, - BuildCache as ChangesTrieBuildCache, CacheAction as ChangesTrieCacheAction, - ConfigurationRange as ChangesTrieConfigurationRange, - InMemoryStorage as InMemoryChangesTrieStorage, RootsStorage as ChangesTrieRootsStorage, - State as ChangesTrieState, Storage as ChangesTrieStorage, - }, error::{Error, ExecutionError}, in_memory_backend::new_in_mem, proving_backend::{ @@ -171,8 +151,8 @@ mod std_reexport { testing::TestExternalities, }; pub use sp_trie::{ - trie_types::{Layout, TrieDBMut}, - DBValue, MemoryDB, StorageProof, TrieMut, + trie_types::{TrieDBMutV0, TrieDBMutV1}, + CompactProof, DBValue, LayoutV0, LayoutV1, MemoryDB, StorageProof, TrieMut, }; } @@ -181,15 +161,20 @@ mod execution { use super::*; use codec::{Codec, Decode, Encode}; use hash_db::Hasher; + use smallvec::SmallVec; use sp_core::{ hexdisplay::HexDisplay, - storage::ChildInfo, + storage::{ChildInfo, ChildType, PrefixedStorageKey}, traits::{CodeExecutor, ReadRuntimeVersionExt, RuntimeCode, SpawnNamed}, NativeOrEncoded, NeverNativeValue, }; use sp_externalities::Extensions; - use std::{collections::HashMap, fmt, panic::UnwindSafe, result}; - use tracing::{trace, warn}; + use std::{ + collections::{HashMap, HashSet}, + fmt, + panic::UnwindSafe, + result, + }; const PROOF_CLOSE_TRANSACTION: &str = "\ Closing a transaction that was started in this function. Client initiated transactions @@ -200,13 +185,12 @@ mod execution { /// Default handler of the execution manager. pub type DefaultHandler = fn(CallResult, CallResult) -> CallResult; - /// Type of changes trie transaction. - pub type ChangesTrieTransaction = - (MemoryDB, ChangesTrieCacheAction<::Out, N>); - /// Trie backend with in-memory storage. pub type InMemoryBackend = TrieBackend, H>; + /// Proving Trie backend with in-memory storage. + pub type InMemoryProvingBackend<'a, H> = ProvingBackend<'a, MemoryDB, H>; + /// Strategy for executing a call into the runtime. #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum ExecutionStrategy { @@ -303,11 +287,10 @@ mod execution { } /// The substrate state machine. - pub struct StateMachine<'a, B, H, N, Exec> + pub struct StateMachine<'a, B, H, Exec> where H: Hasher, B: Backend, - N: ChangesTrieBlockNumber, { backend: &'a B, exec: &'a Exec, @@ -315,8 +298,7 @@ mod execution { call_data: &'a [u8], overlay: &'a mut OverlayedChanges, extensions: Extensions, - changes_trie_state: Option>, - storage_transaction_cache: Option<&'a mut StorageTransactionCache>, + storage_transaction_cache: Option<&'a mut StorageTransactionCache>, runtime_code: &'a RuntimeCode<'a>, stats: StateMachineStats, /// The hash of the block the state machine will be executed on. @@ -325,29 +307,26 @@ mod execution { parent_hash: Option, } - impl<'a, B, H, N, Exec> Drop for StateMachine<'a, B, H, N, Exec> + impl<'a, B, H, Exec> Drop for StateMachine<'a, B, H, Exec> where H: Hasher, B: Backend, - N: ChangesTrieBlockNumber, { fn drop(&mut self) { self.backend.register_overlay_stats(&self.stats); } } - impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> + impl<'a, B, H, Exec> StateMachine<'a, B, H, Exec> where H: Hasher, H::Out: Ord + 'static + codec::Codec, Exec: CodeExecutor + Clone + 'static, B: Backend, - N: crate::changes_trie::BlockNumber, { /// Creates new substrate state machine. pub fn new( backend: &'a B, - changes_trie_state: Option>, overlay: &'a mut OverlayedChanges, exec: &'a Exec, method: &'a str, @@ -366,7 +345,6 @@ mod execution { call_data, extensions, overlay, - changes_trie_state, storage_transaction_cache: None, runtime_code, stats: StateMachineStats::default(), @@ -381,7 +359,7 @@ mod execution { /// build that will be cached. pub fn with_storage_transaction_cache( mut self, - cache: Option<&'a mut StorageTransactionCache>, + cache: Option<&'a mut StorageTransactionCache>, ) -> Self { self.storage_transaction_cache = cache; self @@ -434,13 +412,7 @@ mod execution { .enter_runtime() .expect("StateMachine is never called from the runtime; qed"); - let mut ext = Ext::new( - self.overlay, - cache, - self.backend, - self.changes_trie_state.clone(), - Some(&mut self.extensions), - ); + let mut ext = Ext::new(self.overlay, cache, self.backend, Some(&mut self.extensions)); let ext_id = ext.id; @@ -557,9 +529,6 @@ mod execution { CallResult, ) -> CallResult, { - let changes_tries_enabled = self.changes_trie_state.is_some(); - self.overlay.set_collect_extrinsics(changes_tries_enabled); - let result = { match manager { ExecutionManager::Both(on_consensus_failure) => self @@ -583,7 +552,7 @@ mod execution { } /// Prove execution using the given state backend, overlayed changes, and call executor. - pub fn prove_execution( + pub fn prove_execution( backend: &mut B, overlay: &mut OverlayedChanges, exec: &Exec, @@ -597,13 +566,12 @@ mod execution { H: Hasher, H::Out: Ord + 'static + codec::Codec, Exec: CodeExecutor + Clone + 'static, - N: crate::changes_trie::BlockNumber, Spawn: SpawnNamed + Send + 'static, { let trie_backend = backend .as_trie_backend() .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; - prove_execution_on_trie_backend::<_, _, N, _, _>( + prove_execution_on_trie_backend::<_, _, _, _>( trie_backend, overlay, exec, @@ -623,7 +591,7 @@ mod execution { /// /// Note: changes to code will be in place if this call is made again. For running partial /// blocks (e.g. a transaction at a time), ensure a different method is used. - pub fn prove_execution_on_trie_backend( + pub fn prove_execution_on_trie_backend( trie_backend: &TrieBackend, overlay: &mut OverlayedChanges, exec: &Exec, @@ -637,13 +605,11 @@ mod execution { H: Hasher, H::Out: Ord + 'static + codec::Codec, Exec: CodeExecutor + 'static + Clone, - N: crate::changes_trie::BlockNumber, Spawn: SpawnNamed + Send + 'static, { let proving_backend = proving_backend::ProvingBackend::new(trie_backend); - let mut sm = StateMachine::<_, H, N, Exec>::new( + let mut sm = StateMachine::<_, H, Exec>::new( &proving_backend, - None, overlay, exec, method, @@ -662,7 +628,7 @@ mod execution { } /// Check execution proof, generated by `prove_execution` call. - pub fn execution_proof_check( + pub fn execution_proof_check( root: H::Out, proof: StorageProof, overlay: &mut OverlayedChanges, @@ -676,11 +642,10 @@ mod execution { H: Hasher, Exec: CodeExecutor + Clone + 'static, H::Out: Ord + 'static + codec::Codec, - N: crate::changes_trie::BlockNumber, Spawn: SpawnNamed + Send + 'static, { let trie_backend = create_proof_check_backend::(root.into(), proof)?; - execution_proof_check_on_trie_backend::<_, N, _, _>( + execution_proof_check_on_trie_backend::<_, _, _>( &trie_backend, overlay, exec, @@ -692,7 +657,7 @@ mod execution { } /// Check execution proof on proving backend, generated by `prove_execution` call. - pub fn execution_proof_check_on_trie_backend( + pub fn execution_proof_check_on_trie_backend( trie_backend: &TrieBackend, H>, overlay: &mut OverlayedChanges, exec: &Exec, @@ -705,12 +670,10 @@ mod execution { H: Hasher, H::Out: Ord + 'static + codec::Codec, Exec: CodeExecutor + Clone + 'static, - N: crate::changes_trie::BlockNumber, Spawn: SpawnNamed + Send + 'static, { - let mut sm = StateMachine::<_, H, N, Exec>::new( + let mut sm = StateMachine::<_, H, Exec>::new( trie_backend, - None, overlay, exec, method, @@ -742,6 +705,254 @@ mod execution { prove_read_on_trie_backend(trie_backend, keys) } + /// State machine only allows a single level + /// of child trie. + pub const MAX_NESTED_TRIE_DEPTH: usize = 2; + + /// Multiple key value state. + /// States are ordered by root storage key. + #[derive(PartialEq, Eq, Clone)] + pub struct KeyValueStates(pub Vec); + + /// A key value state at any storage level. + #[derive(PartialEq, Eq, Clone)] + pub struct KeyValueStorageLevel { + /// State root of the level, for + /// top trie it is as an empty byte array. + pub state_root: Vec, + /// Storage of parents, empty for top root or + /// when exporting (building proof). + pub parent_storage_keys: Vec>, + /// Pair of key and values from this state. + pub key_values: Vec<(Vec, Vec)>, + } + + impl From for KeyValueStates + where + I: IntoIterator, (Vec<(Vec, Vec)>, Vec>))>, + { + fn from(b: I) -> Self { + let mut result = Vec::new(); + for (state_root, (key_values, storage_paths)) in b.into_iter() { + result.push(KeyValueStorageLevel { + state_root, + key_values, + parent_storage_keys: storage_paths, + }) + } + KeyValueStates(result) + } + } + + impl KeyValueStates { + /// Return total number of key values in states. + pub fn len(&self) -> usize { + self.0.iter().fold(0, |nb, state| nb + state.key_values.len()) + } + + /// Update last keys accessed from this state. + pub fn update_last_key( + &self, + stopped_at: usize, + last: &mut SmallVec<[Vec; 2]>, + ) -> bool { + if stopped_at == 0 || stopped_at > MAX_NESTED_TRIE_DEPTH { + return false + } + match stopped_at { + 1 => { + let top_last = + self.0.get(0).and_then(|s| s.key_values.last().map(|kv| kv.0.clone())); + if let Some(top_last) = top_last { + match last.len() { + 0 => { + last.push(top_last); + return true + }, + 2 => { + last.pop(); + }, + _ => (), + } + // update top trie access. + last[0] = top_last; + return true + } else { + // No change in top trie accesses. + // Indicates end of reading of a child trie. + last.truncate(1); + return true + } + }, + 2 => { + let top_last = + self.0.get(0).and_then(|s| s.key_values.last().map(|kv| kv.0.clone())); + let child_last = + self.0.last().and_then(|s| s.key_values.last().map(|kv| kv.0.clone())); + + if let Some(child_last) = child_last { + if last.len() == 0 { + if let Some(top_last) = top_last { + last.push(top_last) + } else { + return false + } + } else if let Some(top_last) = top_last { + last[0] = top_last; + } + if last.len() == 2 { + last.pop(); + } + last.push(child_last); + return true + } else { + // stopped at level 2 so child last is define. + return false + } + }, + _ => (), + } + false + } + } + + /// Generate range storage read proof, with child tries + /// content. + /// A size limit is applied to the proof with the + /// exception that `start_at` and its following element + /// are always part of the proof. + /// If a key different than `start_at` is a child trie root, + /// the child trie content will be included in the proof. + pub fn prove_range_read_with_child_with_size( + backend: B, + size_limit: usize, + start_at: &[Vec], + ) -> Result<(StorageProof, u32), Box> + where + B: Backend, + H: Hasher, + H::Out: Ord + Codec, + { + let trie_backend = backend + .as_trie_backend() + .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; + prove_range_read_with_child_with_size_on_trie_backend(trie_backend, size_limit, start_at) + } + + /// Generate range storage read proof, with child tries + /// content. + /// See `prove_range_read_with_child_with_size`. + pub fn prove_range_read_with_child_with_size_on_trie_backend( + trie_backend: &TrieBackend, + size_limit: usize, + start_at: &[Vec], + ) -> Result<(StorageProof, u32), Box> + where + S: trie_backend_essence::TrieBackendStorage, + H: Hasher, + H::Out: Ord + Codec, + { + if start_at.len() > MAX_NESTED_TRIE_DEPTH { + return Err(Box::new("Invalid start of range.")) + } + + let proving_backend = proving_backend::ProvingBackend::::new(trie_backend); + let mut count = 0; + + let mut child_roots = HashSet::new(); + let (mut child_key, mut start_at) = if start_at.len() == 2 { + let storage_key = start_at.get(0).expect("Checked length.").clone(); + if let Some(state_root) = proving_backend + .storage(&storage_key) + .map_err(|e| Box::new(e) as Box)? + { + child_roots.insert(state_root.clone()); + } else { + return Err(Box::new("Invalid range start child trie key.")) + } + + (Some(storage_key), start_at.get(1).cloned()) + } else { + (None, start_at.get(0).cloned()) + }; + + loop { + let (child_info, depth) = if let Some(storage_key) = child_key.as_ref() { + let storage_key = PrefixedStorageKey::new_ref(storage_key); + ( + Some(match ChildType::from_prefixed_key(&storage_key) { + Some((ChildType::ParentKeyId, storage_key)) => + ChildInfo::new_default(storage_key), + None => return Err(Box::new("Invalid range start child trie key.")), + }), + 2, + ) + } else { + (None, 1) + }; + + let start_at_ref = start_at.as_ref().map(AsRef::as_ref); + let mut switch_child_key = None; + let mut first = start_at.is_some(); + let completed = proving_backend + .apply_to_key_values_while( + child_info.as_ref(), + None, + start_at_ref, + |key, value| { + if first { + if start_at_ref + .as_ref() + .map(|start| &key.as_slice() > start) + .unwrap_or(true) + { + first = false; + } + } + if first { + true + } else if depth < MAX_NESTED_TRIE_DEPTH && + sp_core::storage::well_known_keys::is_child_storage_key( + key.as_slice(), + ) { + count += 1; + if !child_roots.contains(value.as_slice()) { + child_roots.insert(value); + switch_child_key = Some(key); + false + } else { + // do not add two child trie with same root + true + } + } else if proving_backend.estimate_encoded_size() <= size_limit { + count += 1; + true + } else { + false + } + }, + false, + ) + .map_err(|e| Box::new(e) as Box)?; + + if switch_child_key.is_none() { + if depth == 1 { + break + } else { + if completed { + start_at = child_key.take(); + } else { + break + } + } + } else { + child_key = switch_child_key; + start_at = None; + } + } + Ok((proving_backend.extract_proof(), count)) + } + /// Generate range storage read proof. pub fn prove_range_read_with_size( backend: B, @@ -884,7 +1095,25 @@ mod execution { Ok(result) } - /// Check child storage range proof, generated by `prove_range_read` call. + /// Check storage range proof with child trie included, generated by + /// `prove_range_read_with_child_with_size` call. + /// + /// Returns key values contents and the depth of the pending state iteration + /// (0 if completed). + pub fn read_range_proof_check_with_child( + root: H::Out, + proof: StorageProof, + start_at: &[Vec], + ) -> Result<(KeyValueStates, usize), Box> + where + H: Hasher, + H::Out: Ord + Codec, + { + let proving_backend = create_proof_check_backend::(root, proof)?; + read_range_proof_check_with_child_on_proving_backend(&proving_backend, start_at) + } + + /// Check child storage range proof, generated by `prove_range_read_with_size` call. pub fn read_range_proof_check( root: H::Out, proof: StorageProof, @@ -991,16 +1220,140 @@ mod execution { Err(e) => Err(Box::new(e) as Box), } } + + /// Check storage range proof on pre-created proving backend. + /// + /// See `read_range_proof_check_with_child`. + pub fn read_range_proof_check_with_child_on_proving_backend( + proving_backend: &TrieBackend, H>, + start_at: &[Vec], + ) -> Result<(KeyValueStates, usize), Box> + where + H: Hasher, + H::Out: Ord + Codec, + { + let mut result = vec![KeyValueStorageLevel { + state_root: Default::default(), + key_values: Default::default(), + parent_storage_keys: Default::default(), + }]; + if start_at.len() > MAX_NESTED_TRIE_DEPTH { + return Err(Box::new("Invalid start of range.")) + } + + let mut child_roots = HashSet::new(); + let (mut child_key, mut start_at) = if start_at.len() == 2 { + let storage_key = start_at.get(0).expect("Checked length.").clone(); + let child_key = if let Some(state_root) = proving_backend + .storage(&storage_key) + .map_err(|e| Box::new(e) as Box)? + { + child_roots.insert(state_root.clone()); + Some((storage_key, state_root)) + } else { + return Err(Box::new("Invalid range start child trie key.")) + }; + + (child_key, start_at.get(1).cloned()) + } else { + (None, start_at.get(0).cloned()) + }; + + let completed = loop { + let (child_info, depth) = if let Some((storage_key, state_root)) = child_key.as_ref() { + result.push(KeyValueStorageLevel { + state_root: state_root.clone(), + key_values: Default::default(), + parent_storage_keys: Default::default(), + }); + + let storage_key = PrefixedStorageKey::new_ref(storage_key); + ( + Some(match ChildType::from_prefixed_key(&storage_key) { + Some((ChildType::ParentKeyId, storage_key)) => + ChildInfo::new_default(storage_key), + None => return Err(Box::new("Invalid range start child trie key.")), + }), + 2, + ) + } else { + (None, 1) + }; + + let values = if child_info.is_some() { + &mut result.last_mut().expect("Added above").key_values + } else { + &mut result[0].key_values + }; + let start_at_ref = start_at.as_ref().map(AsRef::as_ref); + let mut switch_child_key = None; + let mut first = start_at.is_some(); + let completed = proving_backend + .apply_to_key_values_while( + child_info.as_ref(), + None, + start_at_ref, + |key, value| { + if first { + if start_at_ref + .as_ref() + .map(|start| &key.as_slice() > start) + .unwrap_or(true) + { + first = false; + } + } + if !first { + values.push((key.to_vec(), value.to_vec())); + } + if first { + true + } else if depth < MAX_NESTED_TRIE_DEPTH && + sp_core::storage::well_known_keys::is_child_storage_key( + key.as_slice(), + ) { + if child_roots.contains(value.as_slice()) { + // Do not add two chid trie with same root. + true + } else { + child_roots.insert(value.clone()); + switch_child_key = Some((key, value)); + false + } + } else { + true + } + }, + true, + ) + .map_err(|e| Box::new(e) as Box)?; + + if switch_child_key.is_none() { + if !completed { + break depth + } + if depth == 1 { + break 0 + } else { + start_at = child_key.take().map(|entry| entry.0); + } + } else { + child_key = switch_child_key; + start_at = None; + } + }; + Ok((KeyValueStates(result), completed)) + } } #[cfg(test)] mod tests { - use super::{changes_trie::Configuration as ChangesTrieConfig, ext::Ext, *}; + use super::{ext::Ext, *}; use crate::execution::CallResult; use codec::{Decode, Encode}; use sp_core::{ map, - storage::ChildInfo, + storage::{ChildInfo, StateVersion}, testing::TaskExecutor, traits::{CodeExecutor, Externalities, RuntimeCode}, NativeOrEncoded, NeverNativeValue, @@ -1014,7 +1367,6 @@ mod tests { #[derive(Clone)] struct DummyCodeExecutor { - change_changes_trie_config: bool, native_available: bool, native_succeeds: bool, fallback_succeeds: bool, @@ -1035,13 +1387,6 @@ mod tests { use_native: bool, native_call: Option, ) -> (CallResult, bool) { - if self.change_changes_trie_config { - ext.place_storage( - sp_core::storage::well_known_keys::CHANGES_TRIE_CONFIG.to_vec(), - Some(ChangesTrieConfig { digest_interval: 777, digest_levels: 333 }.encode()), - ); - } - let using_native = use_native && self.native_available; match (using_native, self.native_succeeds, self.fallback_succeeds, native_call) { (true, true, _, Some(call)) => { @@ -1071,16 +1416,18 @@ mod tests { #[test] fn execute_works() { - let backend = trie_backend::tests::test_trie(); + execute_works_inner(StateVersion::V0); + execute_works_inner(StateVersion::V1); + } + fn execute_works_inner(state_version: StateVersion) { + let backend = trie_backend::tests::test_trie(state_version); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, - changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, &DummyCodeExecutor { - change_changes_trie_config: false, native_available: true, native_succeeds: true, fallback_succeeds: true, @@ -1097,16 +1444,18 @@ mod tests { #[test] fn execute_works_with_native_else_wasm() { - let backend = trie_backend::tests::test_trie(); + execute_works_with_native_else_wasm_inner(StateVersion::V0); + execute_works_with_native_else_wasm_inner(StateVersion::V1); + } + fn execute_works_with_native_else_wasm_inner(state_version: StateVersion) { + let backend = trie_backend::tests::test_trie(state_version); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, - changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, &DummyCodeExecutor { - change_changes_trie_config: false, native_available: true, native_succeeds: true, fallback_succeeds: true, @@ -1123,17 +1472,19 @@ mod tests { #[test] fn dual_execution_strategy_detects_consensus_failure() { + dual_execution_strategy_detects_consensus_failure_inner(StateVersion::V0); + dual_execution_strategy_detects_consensus_failure_inner(StateVersion::V1); + } + fn dual_execution_strategy_detects_consensus_failure_inner(state_version: StateVersion) { let mut consensus_failed = false; - let backend = trie_backend::tests::test_trie(); + let backend = trie_backend::tests::test_trie(state_version); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, - changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, &DummyCodeExecutor { - change_changes_trie_config: false, native_available: true, native_succeeds: true, fallback_succeeds: false, @@ -1159,17 +1510,20 @@ mod tests { #[test] fn prove_execution_and_proof_check_works() { + prove_execution_and_proof_check_works_inner(StateVersion::V0); + prove_execution_and_proof_check_works_inner(StateVersion::V1); + } + fn prove_execution_and_proof_check_works_inner(state_version: StateVersion) { let executor = DummyCodeExecutor { - change_changes_trie_config: false, native_available: true, native_succeeds: true, fallback_succeeds: true, }; // fetch execution proof from 'remote' full node - let mut remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(std::iter::empty()).0; - let (remote_result, remote_proof) = prove_execution::<_, _, u64, _, _>( + let mut remote_backend = trie_backend::tests::test_trie(state_version); + let remote_root = remote_backend.storage_root(std::iter::empty(), state_version).0; + let (remote_result, remote_proof) = prove_execution( &mut remote_backend, &mut Default::default(), &executor, @@ -1181,7 +1535,7 @@ mod tests { .unwrap(); // check proof locally - let local_result = execution_proof_check::( + let local_result = execution_proof_check::( remote_root, remote_proof, &mut Default::default(), @@ -1206,7 +1560,7 @@ mod tests { b"abc".to_vec() => b"2".to_vec(), b"bbb".to_vec() => b"3".to_vec() ]; - let state = InMemoryBackend::::from(initial); + let state = InMemoryBackend::::from((initial, StateVersion::default())); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); @@ -1219,13 +1573,7 @@ mod tests { let overlay_limit = overlay.clone(); { let mut cache = StorageTransactionCache::default(); - let mut ext = Ext::new( - &mut overlay, - &mut cache, - backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, backend, None); ext.clear_prefix(b"ab", None); } overlay.commit_transaction().unwrap(); @@ -1249,13 +1597,7 @@ mod tests { let mut overlay = overlay_limit; { let mut cache = StorageTransactionCache::default(); - let mut ext = Ext::new( - &mut overlay, - &mut cache, - backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, backend, None); assert_eq!((false, 1), ext.clear_prefix(b"ab", Some(1))); } overlay.commit_transaction().unwrap(); @@ -1287,7 +1629,7 @@ mod tests { b"d".to_vec() => b"3".to_vec() ], ]; - let backend = InMemoryBackend::::from(initial); + let backend = InMemoryBackend::::from((initial, StateVersion::default())); let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(&child_info, b"1".to_vec(), Some(b"1312".to_vec())); @@ -1297,13 +1639,7 @@ mod tests { { let mut cache = StorageTransactionCache::default(); - let mut ext = Ext::new( - &mut overlay, - &mut cache, - &backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, &backend, None); assert_eq!(ext.kill_child_storage(&child_info, Some(2)), (false, 2)); } @@ -1335,16 +1671,10 @@ mod tests { b"d".to_vec() => b"3".to_vec() ], ]; - let backend = InMemoryBackend::::from(initial); + let backend = InMemoryBackend::::from((initial, StateVersion::default())); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); - let mut ext = Ext::new( - &mut overlay, - &mut cache, - &backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, &backend, None); assert_eq!(ext.kill_child_storage(&child_info, Some(0)), (false, 0)); assert_eq!(ext.kill_child_storage(&child_info, Some(1)), (false, 1)); assert_eq!(ext.kill_child_storage(&child_info, Some(2)), (false, 2)); @@ -1363,13 +1693,7 @@ mod tests { let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); - let mut ext = Ext::new( - &mut overlay, - &mut cache, - backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, backend, None); ext.set_child_storage(child_info, b"abc".to_vec(), b"def".to_vec()); assert_eq!(ext.child_storage(child_info, b"abc"), Some(b"def".to_vec())); @@ -1386,26 +1710,14 @@ mod tests { let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); { - let mut ext = Ext::new( - &mut overlay, - &mut cache, - backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, backend, None); ext.storage_append(key.clone(), reference_data[0].encode()); assert_eq!(ext.storage(key.as_slice()), Some(vec![reference_data[0].clone()].encode())); } overlay.start_transaction(); { - let mut ext = Ext::new( - &mut overlay, - &mut cache, - backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, backend, None); for i in reference_data.iter().skip(1) { ext.storage_append(key.clone(), i.encode()); @@ -1414,13 +1726,7 @@ mod tests { } overlay.rollback_transaction().unwrap(); { - let ext = Ext::new( - &mut overlay, - &mut cache, - backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let ext = Ext::new(&mut overlay, &mut cache, backend, None); assert_eq!(ext.storage(key.as_slice()), Some(vec![reference_data[0].clone()].encode())); } } @@ -1442,13 +1748,7 @@ mod tests { // For example, block initialization with event. { - let mut ext = Ext::new( - &mut overlay, - &mut cache, - backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, backend, None); ext.clear_storage(key.as_slice()); ext.storage_append(key.clone(), Item::InitializationItem.encode()); } @@ -1456,13 +1756,7 @@ mod tests { // For example, first transaction resulted in panic during block building { - let mut ext = Ext::new( - &mut overlay, - &mut cache, - backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, backend, None); assert_eq!(ext.storage(key.as_slice()), Some(vec![Item::InitializationItem].encode())); @@ -1477,13 +1771,7 @@ mod tests { // Then we apply next transaction which is valid this time. { - let mut ext = Ext::new( - &mut overlay, - &mut cache, - backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, backend, None); assert_eq!(ext.storage(key.as_slice()), Some(vec![Item::InitializationItem].encode())); @@ -1496,15 +1784,9 @@ mod tests { } overlay.start_transaction(); - // Then only initlaization item and second (commited) item should persist. + // Then only initlaization item and second (committed) item should persist. { - let ext = Ext::new( - &mut overlay, - &mut cache, - backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let ext = Ext::new(&mut overlay, &mut cache, backend, None); assert_eq!( ext.storage(key.as_slice()), Some(vec![Item::InitializationItem, Item::CommitedItem].encode()), @@ -1523,13 +1805,17 @@ mod tests { #[test] fn prove_read_and_proof_check_works() { + prove_read_and_proof_check_works_inner(StateVersion::V0); + prove_read_and_proof_check_works_inner(StateVersion::V1); + } + fn prove_read_and_proof_check_works_inner(state_version: StateVersion) { let child_info = ChildInfo::new_default(b"sub1"); let missing_child_info = ChildInfo::new_default(b"sub1sub2"); // key will include other child root to proof. let child_info = &child_info; let missing_child_info = &missing_child_info; // fetch read proof from 'remote' full node - let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(std::iter::empty()).0; + let remote_backend = trie_backend::tests::test_trie(state_version); + let remote_root = remote_backend.storage_root(std::iter::empty(), state_version).0; let remote_proof = prove_read(remote_backend, &[b"value2"]).unwrap(); let remote_proof = test_compact(remote_proof, &remote_root); // check proof locally @@ -1546,8 +1832,8 @@ mod tests { ); assert_eq!(local_result2, false); // on child trie - let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(std::iter::empty()).0; + let remote_backend = trie_backend::tests::test_trie(state_version); + let remote_root = remote_backend.storage_root(std::iter::empty(), state_version).0; let remote_proof = prove_child_read(remote_backend, child_info, &[b"value3"]).unwrap(); let remote_proof = test_compact(remote_proof, &remote_root); let local_result1 = read_child_proof_check::( @@ -1574,7 +1860,7 @@ mod tests { assert_eq!( local_result1.into_iter().collect::>(), - vec![(b"value3".to_vec(), Some(vec![142]))], + vec![(b"value3".to_vec(), Some(vec![142; 33]))], ); assert_eq!(local_result2.into_iter().collect::>(), vec![(b"value2".to_vec(), None)]); assert_eq!(local_result3.into_iter().collect::>(), vec![(b"dummy".to_vec(), None)]); @@ -1611,7 +1897,8 @@ mod tests { storage.insert(Some(child_info), items); } - let trie: InMemoryBackend = storage.clone().into(); + let trie: InMemoryBackend = + (storage.clone(), StateVersion::default()).into(); let trie_root = trie.root().clone(); let backend = crate::ProvingBackend::new(&trie); let mut queries = Vec::new(); @@ -1674,15 +1961,16 @@ mod tests { #[test] fn prove_read_with_size_limit_works() { - let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(::std::iter::empty()).0; + let state_version = StateVersion::V0; + let remote_backend = trie_backend::tests::test_trie(state_version); + let remote_root = remote_backend.storage_root(::std::iter::empty(), state_version).0; let (proof, count) = prove_range_read_with_size(remote_backend, None, None, 0, None).unwrap(); - // Alwasys contains at least some nodes. + // Always contains at least some nodes. assert_eq!(proof.into_memory_db::().drain().len(), 3); assert_eq!(count, 1); - let remote_backend = trie_backend::tests::test_trie(); + let remote_backend = trie_backend::tests::test_trie(state_version); let (proof, count) = prove_range_read_with_size(remote_backend, None, None, 800, Some(&[])).unwrap(); assert_eq!(proof.clone().into_memory_db::().drain().len(), 9); @@ -1705,7 +1993,7 @@ mod tests { assert_eq!(results.len() as u32, 101); assert_eq!(completed, false); - let remote_backend = trie_backend::tests::test_trie(); + let remote_backend = trie_backend::tests::test_trie(state_version); let (proof, count) = prove_range_read_with_size(remote_backend, None, None, 50000, Some(&[])).unwrap(); assert_eq!(proof.clone().into_memory_db::().drain().len(), 11); @@ -1723,22 +2011,131 @@ mod tests { assert_eq!(completed, true); } + #[test] + fn inner_state_versioning_switch_proofs() { + let mut state_version = StateVersion::V0; + let (mut mdb, mut root) = trie_backend::tests::test_db(state_version); + { + let mut trie = TrieDBMutV0::from_existing(&mut mdb, &mut root).unwrap(); + trie.insert(b"foo", vec![1u8; 1_000].as_slice()) // big inner hash + .expect("insert failed"); + trie.insert(b"foo2", vec![3u8; 16].as_slice()) // no inner hash + .expect("insert failed"); + trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash + .expect("insert failed"); + } + + let check_proof = |mdb, root, state_version| -> StorageProof { + let remote_backend = TrieBackend::new(mdb, root); + let remote_root = remote_backend.storage_root(std::iter::empty(), state_version).0; + let remote_proof = prove_read(remote_backend, &[b"foo222"]).unwrap(); + // check proof locally + let local_result1 = + read_proof_check::(remote_root, remote_proof.clone(), &[b"foo222"]) + .unwrap(); + // check that results are correct + assert_eq!( + local_result1.into_iter().collect::>(), + vec![(b"foo222".to_vec(), Some(vec![5u8; 100]))], + ); + remote_proof + }; + + let remote_proof = check_proof(mdb.clone(), root.clone(), state_version); + // check full values in proof + assert!(remote_proof.encode().len() > 1_100); + assert!(remote_proof.encoded_size() > 1_100); + let root1 = root.clone(); + + // do switch + state_version = StateVersion::V1; + { + let mut trie = TrieDBMutV1::from_existing(&mut mdb, &mut root).unwrap(); + trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash + .expect("insert failed"); + // update with same value do change + trie.insert(b"foo", vec![1u8; 1000].as_slice()) // inner hash + .expect("insert failed"); + } + let root3 = root.clone(); + assert!(root1 != root3); + let remote_proof = check_proof(mdb.clone(), root.clone(), state_version); + // nodes foo is replaced by its hashed value form. + assert!(remote_proof.encode().len() < 1000); + assert!(remote_proof.encoded_size() < 1000); + assert_eq!(remote_proof.encode().len(), remote_proof.encoded_size()); + } + + #[test] + fn prove_range_with_child_works() { + let state_version = StateVersion::V0; + let remote_backend = trie_backend::tests::test_trie(state_version); + let remote_root = remote_backend.storage_root(std::iter::empty(), state_version).0; + let mut start_at = smallvec::SmallVec::<[Vec; 2]>::new(); + let trie_backend = remote_backend.as_trie_backend().unwrap(); + let max_iter = 1000; + let mut nb_loop = 0; + loop { + nb_loop += 1; + if max_iter == nb_loop { + panic!("Too many loop in prove range"); + } + let (proof, count) = prove_range_read_with_child_with_size_on_trie_backend( + trie_backend, + 1, + start_at.as_slice(), + ) + .unwrap(); + // Always contains at least some nodes. + assert!(proof.clone().into_memory_db::().drain().len() > 0); + assert!(count < 3); // when doing child we include parent and first child key. + + let (result, completed_depth) = read_range_proof_check_with_child::( + remote_root, + proof.clone(), + start_at.as_slice(), + ) + .unwrap(); + + if completed_depth == 0 { + break + } + assert!(result.update_last_key(completed_depth, &mut start_at)); + } + + assert_eq!(nb_loop, 10); + } + #[test] fn compact_multiple_child_trie() { + let size_no_inner_hash = compact_multiple_child_trie_inner(StateVersion::V0); + let size_inner_hash = compact_multiple_child_trie_inner(StateVersion::V1); + assert!(size_inner_hash < size_no_inner_hash); + } + fn compact_multiple_child_trie_inner(state_version: StateVersion) -> usize { // this root will be queried let child_info1 = ChildInfo::new_default(b"sub1"); // this root will not be include in proof let child_info2 = ChildInfo::new_default(b"sub2"); // this root will be include in proof let child_info3 = ChildInfo::new_default(b"sub"); - let remote_backend = trie_backend::tests::test_trie(); + let remote_backend = trie_backend::tests::test_trie(state_version); + let long_vec: Vec = (0..1024usize).map(|_| 8u8).collect(); let (remote_root, transaction) = remote_backend.full_storage_root( std::iter::empty(), vec![ ( &child_info1, - vec![(&b"key1"[..], Some(&b"val2"[..])), (&b"key2"[..], Some(&b"val3"[..]))] - .into_iter(), + vec![ + // a inner hashable node + (&b"k"[..], Some(&long_vec[..])), + // need to ensure this is not an inline node + // otherwhise we do not know what is accessed when + // storing proof. + (&b"key1"[..], Some(&vec![5u8; 32][..])), + (&b"key2"[..], Some(&b"val3"[..])), + ] + .into_iter(), ), ( &child_info2, @@ -1752,11 +2149,13 @@ mod tests { ), ] .into_iter(), + state_version, ); let mut remote_storage = remote_backend.into_storage(); remote_storage.consolidate(transaction); let remote_backend = TrieBackend::new(remote_storage, remote_root); let remote_proof = prove_child_read(remote_backend, &child_info1, &[b"key1"]).unwrap(); + let size = remote_proof.encoded_size(); let remote_proof = test_compact(remote_proof, &remote_root); let local_result1 = read_child_proof_check::( remote_root, @@ -1766,11 +2165,13 @@ mod tests { ) .unwrap(); assert_eq!(local_result1.len(), 1); - assert_eq!(local_result1.get(&b"key1"[..]), Some(&Some(b"val2".to_vec()))); + assert_eq!(local_result1.get(&b"key1"[..]), Some(&Some(vec![5u8; 32]))); + size } #[test] fn child_storage_uuid() { + let state_version = StateVersion::V0; let child_info_1 = ChildInfo::new_default(b"sub_test1"); let child_info_2 = ChildInfo::new_default(b"sub_test2"); @@ -1778,18 +2179,12 @@ mod tests { let mut overlay = OverlayedChanges::default(); let mut transaction = { - let backend = test_trie(); + let backend = test_trie(state_version); let mut cache = StorageTransactionCache::default(); - let mut ext = Ext::new( - &mut overlay, - &mut cache, - &backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, &backend, None); ext.set_child_storage(&child_info_1, b"abc".to_vec(), b"def".to_vec()); ext.set_child_storage(&child_info_2, b"abc".to_vec(), b"def".to_vec()); - ext.storage_root(); + ext.storage_root(state_version); cache.transaction.unwrap() }; let mut duplicate = false; @@ -1809,7 +2204,7 @@ mod tests { b"aaa".to_vec() => b"0".to_vec(), b"bbb".to_vec() => b"".to_vec() ]; - let state = InMemoryBackend::::from(initial); + let state = InMemoryBackend::::from((initial, StateVersion::default())); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); @@ -1823,13 +2218,7 @@ mod tests { { let mut cache = StorageTransactionCache::default(); - let mut ext = Ext::new( - &mut overlay, - &mut cache, - backend, - changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, backend, None); assert_eq!(ext.storage(b"bbb"), Some(vec![])); assert_eq!(ext.storage(b"ccc"), Some(vec![])); ext.clear_storage(b"ccc"); @@ -1841,21 +2230,20 @@ mod tests { #[test] fn runtime_registered_extensions_are_removed_after_execution() { + let state_version = StateVersion::default(); use sp_externalities::ExternalitiesExt; sp_externalities::decl_extension! { struct DummyExt(u32); } - let backend = trie_backend::tests::test_trie(); + let backend = trie_backend::tests::test_trie(state_version); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, - changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, &DummyCodeExecutor { - change_changes_trie_config: false, native_available: true, native_succeeds: true, fallback_succeeds: false, @@ -1867,7 +2255,7 @@ mod tests { TaskExecutor::new(), ); - let run_state_machine = |state_machine: &mut StateMachine<_, _, _, _>| { + let run_state_machine = |state_machine: &mut StateMachine<_, _, _>| { state_machine .execute_using_consensus_failure_handler:: _, _, _>( ExecutionManager::NativeWhenPossible, diff --git a/primitives/state-machine/src/overlayed_changes/changeset.rs b/primitives/state-machine/src/overlayed_changes/changeset.rs index 1ffd569e2828..9e7f6ffeddfd 100644 --- a/primitives/state-machine/src/overlayed_changes/changeset.rs +++ b/primitives/state-machine/src/overlayed_changes/changeset.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -69,7 +69,6 @@ struct InnerValue { /// Current value. None if value has been deleted. value: V, /// The set of extrinsic indices where the values has been changed. - /// Is filled only if runtime has announced changes trie support. extrinsics: Extrinsics, } diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index cf7af1c9a6f3..59f3e1cffa5f 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,18 +21,13 @@ mod changeset; mod offchain; use self::changeset::OverlayedChangeSet; -use crate::{backend::Backend, changes_trie::BlockNumber, stats::StateMachineStats, DefaultError}; -#[cfg(feature = "std")] -use crate::{ - changes_trie::{build_changes_trie, State as ChangesTrieState}, - ChangesTrieTransaction, -}; +use crate::{backend::Backend, stats::StateMachineStats, DefaultError}; use codec::{Decode, Encode}; use hash_db::Hasher; pub use offchain::OffchainOverlayedChanges; use sp_core::{ offchain::OffchainOverlayedChange, - storage::{well_known_keys::EXTRINSIC_INDEX, ChildInfo}, + storage::{well_known_keys::EXTRINSIC_INDEX, ChildInfo, StateVersion}, }; #[cfg(feature = "std")] use sp_externalities::{Extension, Extensions}; @@ -109,7 +104,7 @@ pub struct OverlayedChanges { stats: StateMachineStats, } -/// Transcation index operation. +/// Transaction index operation. #[derive(Debug, Clone)] pub enum IndexOperation { /// Insert transaction into index. @@ -134,7 +129,7 @@ pub enum IndexOperation { /// /// This contains all the changes to the storage and transactions to apply theses changes to the /// backend. -pub struct StorageChanges { +pub struct StorageChanges { /// All changes to the main storage. /// /// A value of `None` means that it was deleted. @@ -150,22 +145,13 @@ pub struct StorageChanges { pub transaction: Transaction, /// The storage root after applying the transaction. pub transaction_storage_root: H::Out, - /// Contains the transaction for the backend for the changes trie. - /// - /// If changes trie is disabled the value is set to `None`. - #[cfg(feature = "std")] - pub changes_trie_transaction: Option>, - /// Phantom data for block number until change trie support no_std. - #[cfg(not(feature = "std"))] - pub _ph: sp_std::marker::PhantomData, - /// Changes to the transaction index, #[cfg(feature = "std")] pub transaction_index_changes: Vec, } #[cfg(feature = "std")] -impl StorageChanges { +impl StorageChanges { /// Deconstruct into the inner values pub fn into_inner( self, @@ -175,7 +161,6 @@ impl StorageChanges { OffchainChangesCollection, Transaction, H::Out, - Option>, Vec, ) { ( @@ -184,58 +169,35 @@ impl StorageChanges { self.offchain_storage_changes, self.transaction, self.transaction_storage_root, - self.changes_trie_transaction, self.transaction_index_changes, ) } } -/// The storage transaction are calculated as part of the `storage_root` and -/// `changes_trie_storage_root`. These transactions can be reused for importing the block into the +/// Storage transactions are calculated as part of the `storage_root`. +/// These transactions can be reused for importing the block into the /// storage. So, we cache them to not require a recomputation of those transactions. -pub struct StorageTransactionCache { +pub struct StorageTransactionCache { /// Contains the changes for the main and the child storages as one transaction. pub(crate) transaction: Option, /// The storage root after applying the transaction. pub(crate) transaction_storage_root: Option, - /// Contains the changes trie transaction. - #[cfg(feature = "std")] - pub(crate) changes_trie_transaction: Option>>, - /// The storage root after applying the changes trie transaction. - #[cfg(feature = "std")] - pub(crate) changes_trie_transaction_storage_root: Option>, - /// Phantom data for block number until change trie support no_std. - #[cfg(not(feature = "std"))] - pub(crate) _ph: sp_std::marker::PhantomData, } -impl StorageTransactionCache { +impl StorageTransactionCache { /// Reset the cached transactions. pub fn reset(&mut self) { *self = Self::default(); } } -impl Default - for StorageTransactionCache -{ +impl Default for StorageTransactionCache { fn default() -> Self { - Self { - transaction: None, - transaction_storage_root: None, - #[cfg(feature = "std")] - changes_trie_transaction: None, - #[cfg(feature = "std")] - changes_trie_transaction_storage_root: None, - #[cfg(not(feature = "std"))] - _ph: Default::default(), - } + Self { transaction: None, transaction_storage_root: None } } } -impl Default - for StorageChanges -{ +impl Default for StorageChanges { fn default() -> Self { Self { main_storage_changes: Default::default(), @@ -244,10 +206,6 @@ impl Default transaction: Default::default(), transaction_storage_root: Default::default(), #[cfg(feature = "std")] - changes_trie_transaction: None, - #[cfg(not(feature = "std"))] - _ph: Default::default(), - #[cfg(feature = "std")] transaction_index_changes: Default::default(), } } @@ -539,33 +497,31 @@ impl OverlayedChanges { /// Convert this instance with all changes into a [`StorageChanges`] instance. #[cfg(feature = "std")] - pub fn into_storage_changes, H: Hasher, N: BlockNumber>( + pub fn into_storage_changes, H: Hasher>( mut self, backend: &B, - changes_trie_state: Option<&ChangesTrieState>, - parent_hash: H::Out, - mut cache: StorageTransactionCache, - ) -> Result, DefaultError> + mut cache: StorageTransactionCache, + state_version: StateVersion, + ) -> Result, DefaultError> where H::Out: Ord + Encode + 'static, { - self.drain_storage_changes(backend, changes_trie_state, parent_hash, &mut cache) + self.drain_storage_changes(backend, &mut cache, state_version) } /// Drain all changes into a [`StorageChanges`] instance. Leave empty overlay in place. - pub fn drain_storage_changes, H: Hasher, N: BlockNumber>( + pub fn drain_storage_changes, H: Hasher>( &mut self, backend: &B, - #[cfg(feature = "std")] changes_trie_state: Option<&ChangesTrieState>, - parent_hash: H::Out, - mut cache: &mut StorageTransactionCache, - ) -> Result, DefaultError> + mut cache: &mut StorageTransactionCache, + state_version: StateVersion, + ) -> Result, DefaultError> where H::Out: Ord + Encode + 'static, { // If the transaction does not exist, we generate it. if cache.transaction.is_none() { - self.storage_root(backend, &mut cache); + self.storage_root(backend, &mut cache, state_version); } let (transaction, transaction_storage_root) = cache @@ -574,21 +530,6 @@ impl OverlayedChanges { .and_then(|t| cache.transaction_storage_root.take().map(|tr| (t, tr))) .expect("Transaction was be generated as part of `storage_root`; qed"); - // If the transaction does not exist, we generate it. - #[cfg(feature = "std")] - if cache.changes_trie_transaction.is_none() { - self.changes_trie_root(backend, changes_trie_state, parent_hash, false, &mut cache) - .map_err(|_| "Failed to generate changes trie transaction")?; - } - #[cfg(not(feature = "std"))] - let _ = parent_hash; - - #[cfg(feature = "std")] - let changes_trie_transaction = cache - .changes_trie_transaction - .take() - .expect("Changes trie transaction was generated by `changes_trie_root`; qed"); - let (main_storage_changes, child_storage_changes) = self.drain_committed(); let offchain_storage_changes = self.offchain_drain_committed().collect(); @@ -604,11 +545,7 @@ impl OverlayedChanges { transaction, transaction_storage_root, #[cfg(feature = "std")] - changes_trie_transaction, - #[cfg(feature = "std")] transaction_index_changes, - #[cfg(not(feature = "std"))] - _ph: Default::default(), }) } @@ -639,10 +576,11 @@ impl OverlayedChanges { /// as seen by the current transaction. /// /// Returns the storage root and caches storage transaction in the given `cache`. - pub fn storage_root>( + pub fn storage_root>( &self, backend: &B, - cache: &mut StorageTransactionCache, + cache: &mut StorageTransactionCache, + state_version: StateVersion, ) -> H::Out where H::Out: Ord + Encode, @@ -652,7 +590,7 @@ impl OverlayedChanges { (info, changes.map(|(k, v)| (&k[..], v.value().map(|v| &v[..])))) }); - let (root, transaction) = backend.full_storage_root(delta, child_delta); + let (root, transaction) = backend.full_storage_root(delta, child_delta, state_version); cache.transaction = Some(transaction); cache.transaction_storage_root = Some(root); @@ -660,40 +598,6 @@ impl OverlayedChanges { root } - /// Generate the changes trie root. - /// - /// Returns the changes trie root and caches the storage transaction into the given `cache`. - /// - /// # Panics - /// - /// Panics on storage error, when `panic_on_storage_error` is set. - #[cfg(feature = "std")] - pub fn changes_trie_root<'a, H: Hasher, N: BlockNumber, B: Backend>( - &self, - backend: &B, - changes_trie_state: Option<&'a ChangesTrieState<'a, H, N>>, - parent_hash: H::Out, - panic_on_storage_error: bool, - cache: &mut StorageTransactionCache, - ) -> Result, ()> - where - H::Out: Ord + Encode + 'static, - { - build_changes_trie::<_, H, N>( - backend, - changes_trie_state, - self, - parent_hash, - panic_on_storage_error, - ) - .map(|r| { - let root = r.as_ref().map(|r| r.1).clone(); - cache.changes_trie_transaction = Some(r.map(|(db, _, cache)| (db, cache))); - cache.changes_trie_transaction_storage_root = Some(root); - root - }) - } - /// Returns an iterator over the keys (in lexicographic order) following `key` (excluding `key`) /// alongside its value. pub fn iter_after(&self, key: &[u8]) -> impl Iterator { @@ -927,6 +831,7 @@ mod tests { #[test] fn overlayed_storage_root_works() { + let state_version = StateVersion::default(); let initial: BTreeMap<_, _> = vec![ (b"doe".to_vec(), b"reindeer".to_vec()), (b"dog".to_vec(), b"puppyXXX".to_vec()), @@ -935,9 +840,8 @@ mod tests { ] .into_iter() .collect(); - let backend = InMemoryBackend::::from(initial); + let backend = InMemoryBackend::::from((initial, state_version)); let mut overlay = OverlayedChanges::default(); - overlay.set_collect_extrinsics(false); overlay.start_transaction(); overlay.set_storage(b"dog".to_vec(), Some(b"puppy".to_vec())); @@ -950,17 +854,11 @@ mod tests { overlay.set_storage(b"doug".to_vec(), None); let mut cache = StorageTransactionCache::default(); - let mut ext = Ext::new( - &mut overlay, - &mut cache, - &backend, - crate::changes_trie::disabled_state::<_, u64>(), - None, - ); + let mut ext = Ext::new(&mut overlay, &mut cache, &backend, None); const ROOT: [u8; 32] = hex!("39245109cef3758c2eed2ccba8d9b370a917850af3824bc8348d505df2c298fa"); - assert_eq!(&ext.storage_root()[..], &ROOT); + assert_eq!(&ext.storage_root(state_version)[..], &ROOT); } #[test] diff --git a/primitives/state-machine/src/overlayed_changes/offchain.rs b/primitives/state-machine/src/overlayed_changes/offchain.rs index ac67ca330300..98457700013a 100644 --- a/primitives/state-machine/src/overlayed_changes/offchain.rs +++ b/primitives/state-machine/src/overlayed_changes/offchain.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 690266dab1e7..eeffcc8e4705 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,14 +26,11 @@ use codec::{Codec, Decode, Encode}; use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; use log::debug; use parking_lot::RwLock; -use sp_core::storage::ChildInfo; +use sp_core::storage::{ChildInfo, StateVersion}; +pub use sp_trie::trie_types::TrieError; use sp_trie::{ empty_child_trie_root, read_child_trie_value_with, read_trie_value_with, record_all_keys, - MemoryDB, StorageProof, -}; -pub use sp_trie::{ - trie_types::{Layout, TrieError}, - Recorder, + LayoutV1, MemoryDB, Recorder, StorageProof, }; use std::{ collections::{hash_map::Entry, HashMap}, @@ -59,7 +56,8 @@ where let map_e = |e| format!("Trie lookup error: {}", e); - read_trie_value_with::, _, Ephemeral>( + // V1 is equivalent to V0 on read. + read_trie_value_with::, _, Ephemeral>( &eph, self.backend.root(), key, @@ -78,14 +76,16 @@ where let root = self .storage(storage_key)? .and_then(|r| Decode::decode(&mut &r[..]).ok()) - .unwrap_or_else(|| empty_child_trie_root::>()); + // V1 is equivalent to V0 on empty trie + .unwrap_or_else(|| empty_child_trie_root::>()); let mut read_overlay = S::Overlay::default(); let eph = Ephemeral::new(self.backend.backend_storage(), &mut read_overlay); let map_e = |e| format!("Trie lookup error: {}", e); - read_child_trie_value_with::, _, _>( + // V1 is equivalent to V0 on read + read_child_trie_value_with::, _, _>( child_info.keyspace(), &eph, &root.as_ref(), @@ -102,7 +102,8 @@ where let mut iter = move || -> Result<(), Box>> { let root = self.backend.root(); - record_all_keys::, _>(&eph, root, &mut *self.proof_recorder) + // V1 and V is equivalent to V0 on read and recorder is key read. + record_all_keys::, _>(&eph, root, &mut *self.proof_recorder) }; if let Err(e) = iter() { @@ -157,15 +158,13 @@ impl ProofRecorder { /// Convert into a [`StorageProof`]. pub fn to_storage_proof(&self) -> StorageProof { - let trie_nodes = self - .inner - .read() - .records - .iter() - .filter_map(|(_k, v)| v.as_ref().map(|v| v.to_vec())) - .collect(); - - StorageProof::new(trie_nodes) + StorageProof::new( + self.inner + .read() + .records + .iter() + .filter_map(|(_k, v)| v.as_ref().map(|v| v.to_vec())), + ) } /// Reset the internal state. @@ -221,6 +220,11 @@ where pub fn estimate_encoded_size(&self) -> usize { self.0.essence().backend_storage().proof_recorder.estimate_encoded_size() } + + /// Clear the proof recorded data. + pub fn clear_recorder(&self) { + self.0.essence().backend_storage().proof_recorder.reset() + } } impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> TrieBackendStorage @@ -333,22 +337,24 @@ where fn storage_root<'b>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (H::Out, Self::Transaction) where H::Out: Ord, { - self.0.storage_root(delta) + self.0.storage_root(delta, state_version) } fn child_storage_root<'b>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord, { - self.0.child_storage_root(child_info, delta) + self.0.child_storage_root(child_info, delta, state_version) } fn register_overlay_stats(&self, _stats: &crate::stats::StateMachineStats) {} @@ -358,7 +364,9 @@ where } } -/// Create proof check backend. +/// Create a backend used for checking the proof., using `H` as hasher. +/// +/// `proof` and `root` must match, i.e. `root` must be the correct root of `proof` nodes. pub fn create_proof_check_backend( root: H::Out, proof: StorageProof, @@ -383,6 +391,7 @@ mod tests { proving_backend::create_proof_check_backend, trie_backend::tests::test_trie, InMemoryBackend, }; + use sp_core::H256; use sp_runtime::traits::BlakeTwo256; use sp_trie::PrefixedMemoryDB; @@ -394,13 +403,21 @@ mod tests { #[test] fn proof_is_empty_until_value_is_read() { - let trie_backend = test_trie(); + proof_is_empty_until_value_is_read_inner(StateVersion::V0); + proof_is_empty_until_value_is_read_inner(StateVersion::V1); + } + fn proof_is_empty_until_value_is_read_inner(test_hash: StateVersion) { + let trie_backend = test_trie(test_hash); assert!(test_proving(&trie_backend).extract_proof().is_empty()); } #[test] fn proof_is_non_empty_after_value_is_read() { - let trie_backend = test_trie(); + proof_is_non_empty_after_value_is_read_inner(StateVersion::V0); + proof_is_non_empty_after_value_is_read_inner(StateVersion::V1); + } + fn proof_is_non_empty_after_value_is_read_inner(test_hash: StateVersion) { + let trie_backend = test_trie(test_hash); let backend = test_proving(&trie_backend); assert_eq!(backend.storage(b"key").unwrap(), Some(b"value".to_vec())); assert!(!backend.extract_proof().is_empty()); @@ -408,7 +425,6 @@ mod tests { #[test] fn proof_is_invalid_when_does_not_contains_root() { - use sp_core::H256; let result = create_proof_check_backend::( H256::from_low_u64_be(1), StorageProof::empty(), @@ -418,58 +434,82 @@ mod tests { #[test] fn passes_through_backend_calls() { - let trie_backend = test_trie(); + passes_through_backend_calls_inner(StateVersion::V0); + passes_through_backend_calls_inner(StateVersion::V1); + } + fn passes_through_backend_calls_inner(state_version: StateVersion) { + let trie_backend = test_trie(state_version); let proving_backend = test_proving(&trie_backend); assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap()); assert_eq!(trie_backend.pairs(), proving_backend.pairs()); - let (trie_root, mut trie_mdb) = trie_backend.storage_root(std::iter::empty()); - let (proving_root, mut proving_mdb) = proving_backend.storage_root(std::iter::empty()); + let (trie_root, mut trie_mdb) = + trie_backend.storage_root(std::iter::empty(), state_version); + let (proving_root, mut proving_mdb) = + proving_backend.storage_root(std::iter::empty(), state_version); assert_eq!(trie_root, proving_root); assert_eq!(trie_mdb.drain(), proving_mdb.drain()); } #[test] - fn proof_recorded_and_checked() { - let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>(); + fn proof_recorded_and_checked_top() { + proof_recorded_and_checked_inner(StateVersion::V0); + proof_recorded_and_checked_inner(StateVersion::V1); + } + fn proof_recorded_and_checked_inner(state_version: StateVersion) { + let size_content = 34; // above hashable value treshold. + let value_range = 0..64; + let contents = value_range + .clone() + .map(|i| (vec![i], Some(vec![i; size_content]))) + .collect::>(); let in_memory = InMemoryBackend::::default(); - let in_memory = in_memory.update(vec![(None, contents)]); - let in_memory_root = in_memory.storage_root(::std::iter::empty()).0; - (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); + let in_memory = in_memory.update(vec![(None, contents)], state_version); + let in_memory_root = in_memory.storage_root(std::iter::empty(), state_version).0; + value_range.clone().for_each(|i| { + assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content]) + }); let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(::std::iter::empty()).0; + let trie_root = trie.storage_root(std::iter::empty(), state_version).0; assert_eq!(in_memory_root, trie_root); - (0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i])); + value_range + .clone() + .for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); let proving = ProvingBackend::new(trie); - assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]); + assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); let proof = proving.extract_proof(); let proof_check = create_proof_check_backend::(in_memory_root.into(), proof).unwrap(); - assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]); + assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); } #[test] fn proof_recorded_and_checked_with_child() { + proof_recorded_and_checked_with_child_inner(StateVersion::V0); + proof_recorded_and_checked_with_child_inner(StateVersion::V1); + } + fn proof_recorded_and_checked_with_child_inner(state_version: StateVersion) { let child_info_1 = ChildInfo::new_default(b"sub1"); let child_info_2 = ChildInfo::new_default(b"sub2"); let child_info_1 = &child_info_1; let child_info_2 = &child_info_2; let contents = vec![ - (None, (0..64).map(|i| (vec![i], Some(vec![i]))).collect()), + (None, (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>()), (Some(child_info_1.clone()), (28..65).map(|i| (vec![i], Some(vec![i]))).collect()), (Some(child_info_2.clone()), (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; let in_memory = InMemoryBackend::::default(); - let in_memory = in_memory.update(contents); + let in_memory = in_memory.update(contents, state_version); let child_storage_keys = vec![child_info_1.to_owned(), child_info_2.to_owned()]; let in_memory_root = in_memory .full_storage_root( std::iter::empty(), child_storage_keys.iter().map(|k| (k, std::iter::empty())), + state_version, ) .0; (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); @@ -481,7 +521,7 @@ mod tests { }); let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(std::iter::empty()).0; + let trie_root = trie.storage_root(std::iter::empty(), state_version).0; assert_eq!(in_memory_root, trie_root); (0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i])); @@ -509,7 +549,11 @@ mod tests { #[test] fn storage_proof_encoded_size_estimation_works() { - let trie_backend = test_trie(); + storage_proof_encoded_size_estimation_works_inner(StateVersion::V0); + storage_proof_encoded_size_estimation_works_inner(StateVersion::V1); + } + fn storage_proof_encoded_size_estimation_works_inner(state_version: StateVersion) { + let trie_backend = test_trie(state_version); let backend = test_proving(&trie_backend); let check_estimation = @@ -536,4 +580,35 @@ mod tests { assert!(backend.storage(b"doesnotexist2").unwrap().is_none()); check_estimation(&backend); } + + #[test] + fn proof_recorded_for_same_execution_should_be_deterministic() { + let storage_changes = vec![ + (H256::random(), Some(b"value1".to_vec())), + (H256::random(), Some(b"value2".to_vec())), + (H256::random(), Some(b"value3".to_vec())), + (H256::random(), Some(b"value4".to_vec())), + (H256::random(), Some(b"value5".to_vec())), + (H256::random(), Some(b"value6".to_vec())), + (H256::random(), Some(b"value7".to_vec())), + (H256::random(), Some(b"value8".to_vec())), + ]; + + let proof_recorder = + ProofRecorder:: { inner: Arc::new(RwLock::new(ProofRecorderInner::default())) }; + storage_changes + .clone() + .into_iter() + .for_each(|(key, val)| proof_recorder.record(key, val)); + let proof1 = proof_recorder.to_storage_proof(); + + let proof_recorder = + ProofRecorder:: { inner: Arc::new(RwLock::new(ProofRecorderInner::default())) }; + storage_changes + .into_iter() + .for_each(|(key, val)| proof_recorder.record(key, val)); + let proof2 = proof_recorder.to_storage_proof(); + + assert_eq!(proof1, proof2); + } } diff --git a/primitives/state-machine/src/read_only.rs b/primitives/state-machine/src/read_only.rs index 5b7d568b0311..3c3e779c32f3 100644 --- a/primitives/state-machine/src/read_only.rs +++ b/primitives/state-machine/src/read_only.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ use crate::{Backend, StorageKey, StorageValue}; use codec::Encode; use hash_db::Hasher; use sp_core::{ - storage::{ChildInfo, TrackedStorageKey}, + storage::{ChildInfo, StateVersion, TrackedStorageKey}, traits::Externalities, Blake2Hasher, }; @@ -145,18 +145,18 @@ impl<'a, H: Hasher, B: 'a + Backend> Externalities for ReadOnlyExternalities< unimplemented!("storage_append is not supported in ReadOnlyExternalities") } - fn storage_root(&mut self) -> Vec { + fn storage_root(&mut self, _state_version: StateVersion) -> Vec { unimplemented!("storage_root is not supported in ReadOnlyExternalities") } - fn child_storage_root(&mut self, _child_info: &ChildInfo) -> Vec { + fn child_storage_root( + &mut self, + _child_info: &ChildInfo, + _state_version: StateVersion, + ) -> Vec { unimplemented!("child_storage_root is not supported in ReadOnlyExternalities") } - fn storage_changes_root(&mut self, _parent: &[u8]) -> Result>, ()> { - unimplemented!("storage_changes_root is not supported in ReadOnlyExternalities") - } - fn storage_start_transaction(&mut self) { unimplemented!("Transactions are not supported by ReadOnlyExternalities"); } diff --git a/primitives/state-machine/src/stats.rs b/primitives/state-machine/src/stats.rs index affd71f9d2e5..863ea5cfed7e 100644 --- a/primitives/state-machine/src/stats.rs +++ b/primitives/state-machine/src/stats.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 23f66ee14d87..bc462ae01ab2 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,30 +23,25 @@ use std::{ }; use crate::{ - backend::Backend, - changes_trie::{ - BlockNumber as ChangesTrieBlockNumber, Configuration as ChangesTrieConfiguration, - InMemoryStorage as ChangesTrieInMemoryStorage, State as ChangesTrieState, - }, - ext::Ext, - InMemoryBackend, OverlayedChanges, StorageKey, StorageTransactionCache, StorageValue, + backend::Backend, ext::Ext, InMemoryBackend, OverlayedChanges, StorageKey, + StorageTransactionCache, StorageValue, }; -use codec::Decode; use hash_db::Hasher; use sp_core::{ offchain::testing::TestPersistentOffchainDB, storage::{ - well_known_keys::{is_child_storage_key, CHANGES_TRIE_CONFIG, CODE}, - Storage, + well_known_keys::{is_child_storage_key, CODE}, + StateVersion, Storage, }, testing::TaskExecutor, traits::TaskExecutorExt, }; use sp_externalities::{Extension, ExtensionStore, Extensions}; +use sp_trie::StorageProof; /// Simple HashMap-based Externalities impl. -pub struct TestExternalities +pub struct TestExternalities where H::Out: codec::Codec + Ord, { @@ -54,58 +49,57 @@ where overlay: OverlayedChanges, offchain_db: TestPersistentOffchainDB, storage_transaction_cache: - StorageTransactionCache< as Backend>::Transaction, H, N>, + StorageTransactionCache< as Backend>::Transaction, H>, /// Storage backend. pub backend: InMemoryBackend, - changes_trie_config: Option, - changes_trie_storage: ChangesTrieInMemoryStorage, /// Extensions. pub extensions: Extensions, + /// State version to use during tests. + pub state_version: StateVersion, } -impl TestExternalities +impl TestExternalities where H::Out: Ord + 'static + codec::Codec, { /// Get externalities implementation. - pub fn ext(&mut self) -> Ext> { + pub fn ext(&mut self) -> Ext> { Ext::new( &mut self.overlay, &mut self.storage_transaction_cache, &self.backend, - match self.changes_trie_config.clone() { - Some(config) => Some(ChangesTrieState { - config, - zero: 0.into(), - storage: &self.changes_trie_storage, - }), - None => None, - }, Some(&mut self.extensions), ) } /// Create a new instance of `TestExternalities` with storage. pub fn new(storage: Storage) -> Self { - Self::new_with_code(&[], storage) + Self::new_with_code_and_state(&[], storage, Default::default()) + } + + /// Create a new instance of `TestExternalities` with storage for a given state version. + pub fn new_with_state_version(storage: Storage, state_version: StateVersion) -> Self { + Self::new_with_code_and_state(&[], storage, state_version) } /// New empty test externalities. pub fn new_empty() -> Self { - Self::new_with_code(&[], Storage::default()) + Self::new_with_code_and_state(&[], Storage::default(), Default::default()) } /// Create a new instance of `TestExternalities` with code and storage. - pub fn new_with_code(code: &[u8], mut storage: Storage) -> Self { - let mut overlay = OverlayedChanges::default(); - let changes_trie_config = storage - .top - .get(CHANGES_TRIE_CONFIG) - .and_then(|v| Decode::decode(&mut &v[..]).ok()); - overlay.set_collect_extrinsics(changes_trie_config.is_some()); + pub fn new_with_code(code: &[u8], storage: Storage) -> Self { + Self::new_with_code_and_state(code, storage, Default::default()) + } + /// Create a new instance of `TestExternalities` with code and storage for a given state + /// version. + pub fn new_with_code_and_state( + code: &[u8], + mut storage: Storage, + state_version: StateVersion, + ) -> Self { assert!(storage.top.keys().all(|key| !is_child_storage_key(key))); - assert!(storage.children_default.keys().all(|key| is_child_storage_key(key))); storage.top.insert(CODE.to_vec(), code.to_vec()); @@ -114,14 +108,15 @@ where let offchain_db = TestPersistentOffchainDB::new(); + let backend = (storage, state_version).into(); + TestExternalities { - overlay, + overlay: OverlayedChanges::default(), offchain_db, - changes_trie_config, extensions, - changes_trie_storage: ChangesTrieInMemoryStorage::new(), - backend: storage.into(), + backend, storage_transaction_cache: Default::default(), + state_version, } } @@ -142,7 +137,14 @@ where /// Insert key/value into backend pub fn insert(&mut self, k: StorageKey, v: StorageValue) { - self.backend.insert(vec![(None, vec![(k, Some(v))])]); + self.backend.insert(vec![(None, vec![(k, Some(v))])], self.state_version); + } + + /// Insert key/value into backend. + /// + /// This only supports inserting keys in child tries. + pub fn insert_child(&mut self, c: sp_core::storage::ChildInfo, k: StorageKey, v: StorageValue) { + self.backend.insert(vec![(Some(c), vec![(k, Some(v))])], self.state_version); } /// Registers the given extension for this instance. @@ -150,11 +152,6 @@ where self.extensions.register(ext); } - /// Get mutable reference to changes trie storage. - pub fn changes_trie_storage(&mut self) -> &mut ChangesTrieInMemoryStorage { - &mut self.changes_trie_storage - } - /// Return a new backend with all pending changes. /// /// In contrast to [`commit_all`](Self::commit_all) this will not panic if there are open @@ -171,7 +168,7 @@ where )) } - self.backend.update(transaction) + self.backend.update(transaction, self.state_version) } /// Commit all pending changes to the underlying backend. @@ -180,11 +177,10 @@ where /// /// This will panic if there are still open transactions. pub fn commit_all(&mut self) -> Result<(), String> { - let changes = self.overlay.drain_storage_changes::<_, _, N>( + let changes = self.overlay.drain_storage_changes::<_, _>( &self.backend, - None, - Default::default(), &mut Default::default(), + self.state_version, )?; self.backend @@ -200,9 +196,29 @@ where sp_externalities::set_and_run_with_externalities(&mut ext, execute) } + /// Execute the given closure while `self`, with `proving_backend` as backend, is set as + /// externalities. + /// + /// This implementation will wipe the proof recorded in between calls. Consecutive calls will + /// get their own proof from scratch. + pub fn execute_and_prove(&mut self, execute: impl FnOnce() -> R) -> (R, StorageProof) { + let proving_backend = crate::InMemoryProvingBackend::new(&self.backend); + let mut proving_ext = Ext::new( + &mut self.overlay, + &mut self.storage_transaction_cache, + &proving_backend, + Some(&mut self.extensions), + ); + + let outcome = sp_externalities::set_and_run_with_externalities(&mut proving_ext, execute); + let proof = proving_backend.extract_proof(); + + (outcome, proof) + } + /// Execute the given closure while `self` is set as externalities. /// - /// Returns the result of the given closure, if no panics occured. + /// Returns the result of the given closure, if no panics occurred. /// Otherwise, returns `Err`. pub fn execute_with_safe( &mut self, @@ -216,7 +232,7 @@ where } } -impl std::fmt::Debug for TestExternalities +impl std::fmt::Debug for TestExternalities where H::Out: Ord + codec::Codec, { @@ -225,40 +241,49 @@ where } } -impl PartialEq for TestExternalities +impl PartialEq for TestExternalities where H::Out: Ord + 'static + codec::Codec, { /// This doesn't test if they are in the same state, only if they contains the /// same data at this state - fn eq(&self, other: &TestExternalities) -> bool { + fn eq(&self, other: &TestExternalities) -> bool { self.as_backend().eq(&other.as_backend()) } } -impl Default for TestExternalities +impl Default for TestExternalities where H::Out: Ord + 'static + codec::Codec, { fn default() -> Self { - Self::new(Default::default()) + // default to default version. + Self::new_with_state_version(Storage::default(), Default::default()) } } -impl From for TestExternalities +impl From for TestExternalities where H::Out: Ord + 'static + codec::Codec, { fn from(storage: Storage) -> Self { - Self::new(storage) + Self::new_with_state_version(storage, Default::default()) + } +} + +impl From<(Storage, StateVersion)> for TestExternalities +where + H::Out: Ord + 'static + codec::Codec, +{ + fn from((storage, state_version): (Storage, StateVersion)) -> Self { + Self::new_with_state_version(storage, state_version) } } -impl sp_externalities::ExtensionStore for TestExternalities +impl sp_externalities::ExtensionStore for TestExternalities where H: Hasher, H::Out: Ord + codec::Codec, - N: ChangesTrieBlockNumber, { fn extension_by_type_id(&mut self, type_id: TypeId) -> Option<&mut dyn Any> { self.extensions.get_mut(type_id) @@ -284,11 +309,10 @@ where } } -impl sp_externalities::ExternalitiesExt for TestExternalities +impl sp_externalities::ExternalitiesExt for TestExternalities where H: Hasher, H::Out: Ord + codec::Codec, - N: ChangesTrieBlockNumber, { fn extension(&mut self) -> Option<&mut T> { self.extension_by_type_id(TypeId::of::()).and_then(::downcast_mut) @@ -312,19 +336,20 @@ mod tests { #[test] fn commit_should_work() { - let mut ext = TestExternalities::::default(); + let storage = Storage::default(); // avoid adding the trie threshold. + let mut ext = TestExternalities::::from((storage, Default::default())); let mut ext = ext.ext(); ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec()); let root = H256::from(hex!("ed4d8c799d996add422395a6abd7545491d40bd838d738afafa1b8a4de625489")); - assert_eq!(H256::from_slice(ext.storage_root().as_slice()), root); + assert_eq!(H256::from_slice(ext.storage_root(Default::default()).as_slice()), root); } #[test] fn set_and_retrieve_code() { - let mut ext = TestExternalities::::default(); + let mut ext = TestExternalities::::default(); let mut ext = ext.ext(); let code = vec![1, 2, 3]; @@ -336,12 +361,12 @@ mod tests { #[test] fn check_send() { fn assert_send() {} - assert_send::>(); + assert_send::>(); } #[test] fn commit_all_and_kill_child_storage() { - let mut ext = TestExternalities::::default(); + let mut ext = TestExternalities::::default(); let child_info = ChildInfo::new_default(&b"test_child"[..]); { @@ -366,7 +391,7 @@ mod tests { #[test] fn as_backend_generates_same_backend_as_commit_all() { - let mut ext = TestExternalities::::default(); + let mut ext = TestExternalities::::default(); { let mut ext = ext.ext(); ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 7cb725a80503..3b985ec2b99f 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,19 +18,13 @@ //! Trie-based state machine backend. use crate::{ - debug, - trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}, - warn, Backend, StorageKey, StorageValue, + trie_backend_essence::{TrieBackendEssence, TrieBackendStorage}, + Backend, StorageKey, StorageValue, }; -use codec::{Codec, Decode}; +use codec::Codec; use hash_db::Hasher; -use sp_core::storage::{ChildInfo, ChildType}; -use sp_std::{boxed::Box, vec::Vec}; -use sp_trie::{ - child_delta_trie_root, delta_trie_root, empty_child_trie_root, - trie_types::{Layout, TrieDB, TrieError}, - Trie, -}; +use sp_core::storage::{ChildInfo, StateVersion}; +use sp_std::vec::Vec; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. pub struct TrieBackend, H: Hasher> { @@ -144,108 +138,34 @@ where } fn pairs(&self) -> Vec<(StorageKey, StorageValue)> { - let collect_all = || -> Result<_, Box>> { - let trie = TrieDB::::new(self.essence(), self.essence.root())?; - let mut v = Vec::new(); - for x in trie.iter()? { - let (key, value) = x?; - v.push((key.to_vec(), value.to_vec())); - } - - Ok(v) - }; - - match collect_all() { - Ok(v) => v, - Err(e) => { - debug!(target: "trie", "Error extracting trie values: {}", e); - Vec::new() - }, - } + self.essence.pairs() } fn keys(&self, prefix: &[u8]) -> Vec { - let collect_all = || -> Result<_, Box>> { - let trie = TrieDB::::new(self.essence(), self.essence.root())?; - let mut v = Vec::new(); - for x in trie.iter()? { - let (key, _) = x?; - if key.starts_with(prefix) { - v.push(key.to_vec()); - } - } - - Ok(v) - }; - - collect_all() - .map_err(|e| debug!(target: "trie", "Error extracting trie keys: {}", e)) - .unwrap_or_default() + self.essence.keys(prefix) } fn storage_root<'a>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (H::Out, Self::Transaction) where H::Out: Ord, { - let mut write_overlay = S::Overlay::default(); - let mut root = *self.essence.root(); - - { - let mut eph = Ephemeral::new(self.essence.backend_storage(), &mut write_overlay); - - match delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta) { - Ok(ret) => root = ret, - Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), - } - } - - (root, write_overlay) + self.essence.storage_root(delta, state_version) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord, { - let default_root = match child_info.child_type() { - ChildType::ParentKeyId => empty_child_trie_root::>(), - }; - - let mut write_overlay = S::Overlay::default(); - let prefixed_storage_key = child_info.prefixed_storage_key(); - let mut root = match self.storage(prefixed_storage_key.as_slice()) { - Ok(value) => value - .and_then(|r| Decode::decode(&mut &r[..]).ok()) - .unwrap_or_else(|| default_root.clone()), - Err(e) => { - warn!(target: "trie", "Failed to read child storage root: {}", e); - default_root.clone() - }, - }; - - { - let mut eph = Ephemeral::new(self.essence.backend_storage(), &mut write_overlay); - - match child_delta_trie_root::, _, _, _, _, _, _>( - child_info.keyspace(), - &mut eph, - root, - delta, - ) { - Ok(ret) => root = ret, - Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), - } - } - - let is_default = root == default_root; - - (root, is_default, write_overlay) + self.essence.child_storage_root(child_info, delta, state_version) } fn as_trie_backend(&self) -> Option<&TrieBackend> { @@ -269,57 +189,96 @@ pub mod tests { use codec::Encode; use sp_core::H256; use sp_runtime::traits::BlakeTwo256; - use sp_trie::{trie_types::TrieDBMut, KeySpacedDBMut, PrefixedMemoryDB, TrieMut}; + use sp_trie::{ + trie_types::{TrieDBMutV0, TrieDBMutV1}, + KeySpacedDBMut, PrefixedMemoryDB, TrieMut, + }; use std::{collections::HashSet, iter}; const CHILD_KEY_1: &[u8] = b"sub1"; - fn test_db() -> (PrefixedMemoryDB, H256) { + pub(crate) fn test_db(state_version: StateVersion) -> (PrefixedMemoryDB, H256) { let child_info = ChildInfo::new_default(CHILD_KEY_1); let mut root = H256::default(); let mut mdb = PrefixedMemoryDB::::default(); { let mut mdb = KeySpacedDBMut::new(&mut mdb, child_info.keyspace()); - let mut trie = TrieDBMut::new(&mut mdb, &mut root); - trie.insert(b"value3", &[142]).expect("insert failed"); - trie.insert(b"value4", &[124]).expect("insert failed"); + match state_version { + StateVersion::V0 => { + let mut trie = TrieDBMutV0::new(&mut mdb, &mut root); + trie.insert(b"value3", &[142; 33]).expect("insert failed"); + trie.insert(b"value4", &[124; 33]).expect("insert failed"); + }, + StateVersion::V1 => { + let mut trie = TrieDBMutV1::new(&mut mdb, &mut root); + trie.insert(b"value3", &[142; 33]).expect("insert failed"); + trie.insert(b"value4", &[124; 33]).expect("insert failed"); + }, + }; }; { let mut sub_root = Vec::new(); root.encode_to(&mut sub_root); - let mut trie = TrieDBMut::new(&mut mdb, &mut root); - trie.insert(child_info.prefixed_storage_key().as_slice(), &sub_root[..]) - .expect("insert failed"); - trie.insert(b"key", b"value").expect("insert failed"); - trie.insert(b"value1", &[42]).expect("insert failed"); - trie.insert(b"value2", &[24]).expect("insert failed"); - trie.insert(b":code", b"return 42").expect("insert failed"); - for i in 128u8..255u8 { - trie.insert(&[i], &[i]).unwrap(); + + fn build( + mut trie: sp_trie::TrieDBMut, + child_info: &ChildInfo, + sub_root: &[u8], + ) { + trie.insert(child_info.prefixed_storage_key().as_slice(), sub_root) + .expect("insert failed"); + trie.insert(b"key", b"value").expect("insert failed"); + trie.insert(b"value1", &[42]).expect("insert failed"); + trie.insert(b"value2", &[24]).expect("insert failed"); + trie.insert(b":code", b"return 42").expect("insert failed"); + for i in 128u8..255u8 { + trie.insert(&[i], &[i]).unwrap(); + } } + + match state_version { + StateVersion::V0 => { + let trie = TrieDBMutV0::new(&mut mdb, &mut root); + build(trie, &child_info, &sub_root[..]) + }, + StateVersion::V1 => { + let trie = TrieDBMutV1::new(&mut mdb, &mut root); + build(trie, &child_info, &sub_root[..]) + }, + }; } (mdb, root) } - pub(crate) fn test_trie() -> TrieBackend, BlakeTwo256> { - let (mdb, root) = test_db(); + pub(crate) fn test_trie( + hashed_value: StateVersion, + ) -> TrieBackend, BlakeTwo256> { + let (mdb, root) = test_db(hashed_value); TrieBackend::new(mdb, root) } #[test] fn read_from_storage_returns_some() { - assert_eq!(test_trie().storage(b"key").unwrap(), Some(b"value".to_vec())); + read_from_storage_returns_some_inner(StateVersion::V0); + read_from_storage_returns_some_inner(StateVersion::V1); + } + fn read_from_storage_returns_some_inner(state_version: StateVersion) { + assert_eq!(test_trie(state_version).storage(b"key").unwrap(), Some(b"value".to_vec())); } #[test] fn read_from_child_storage_returns_some() { - let test_trie = test_trie(); + read_from_child_storage_returns_some_inner(StateVersion::V0); + read_from_child_storage_returns_some_inner(StateVersion::V1); + } + fn read_from_child_storage_returns_some_inner(state_version: StateVersion) { + let test_trie = test_trie(state_version); assert_eq!( test_trie .child_storage(&ChildInfo::new_default(CHILD_KEY_1), b"value3") .unwrap(), - Some(vec![142u8]), + Some(vec![142u8; 33]), ); // Change cache entry to check that caching is active. test_trie @@ -341,12 +300,20 @@ pub mod tests { #[test] fn read_from_storage_returns_none() { - assert_eq!(test_trie().storage(b"non-existing-key").unwrap(), None); + read_from_storage_returns_none_inner(StateVersion::V0); + read_from_storage_returns_none_inner(StateVersion::V1); + } + fn read_from_storage_returns_none_inner(state_version: StateVersion) { + assert_eq!(test_trie(state_version).storage(b"non-existing-key").unwrap(), None); } #[test] fn pairs_are_not_empty_on_non_empty_storage() { - assert!(!test_trie().pairs().is_empty()); + pairs_are_not_empty_on_non_empty_storage_inner(StateVersion::V0); + pairs_are_not_empty_on_non_empty_storage_inner(StateVersion::V1); + } + fn pairs_are_not_empty_on_non_empty_storage_inner(state_version: StateVersion) { + assert!(!test_trie(state_version).pairs().is_empty()); } #[test] @@ -361,25 +328,35 @@ pub mod tests { #[test] fn storage_root_is_non_default() { - assert!(test_trie().storage_root(iter::empty()).0 != H256::repeat_byte(0)); + storage_root_is_non_default_inner(StateVersion::V0); + storage_root_is_non_default_inner(StateVersion::V1); } - - #[test] - fn storage_root_transaction_is_empty() { - assert!(test_trie().storage_root(iter::empty()).1.drain().is_empty()); + fn storage_root_is_non_default_inner(state_version: StateVersion) { + assert!( + test_trie(state_version).storage_root(iter::empty(), state_version).0 != + H256::repeat_byte(0) + ); } #[test] fn storage_root_transaction_is_non_empty() { - let (new_root, mut tx) = - test_trie().storage_root(iter::once((&b"new-key"[..], Some(&b"new-value"[..])))); + storage_root_transaction_is_non_empty_inner(StateVersion::V0); + storage_root_transaction_is_non_empty_inner(StateVersion::V1); + } + fn storage_root_transaction_is_non_empty_inner(state_version: StateVersion) { + let (new_root, mut tx) = test_trie(state_version) + .storage_root(iter::once((&b"new-key"[..], Some(&b"new-value"[..]))), state_version); assert!(!tx.drain().is_empty()); - assert!(new_root != test_trie().storage_root(iter::empty()).0); + assert!(new_root != test_trie(state_version).storage_root(iter::empty(), state_version).0); } #[test] fn prefix_walking_works() { - let trie = test_trie(); + prefix_walking_works_inner(StateVersion::V0); + prefix_walking_works_inner(StateVersion::V1); + } + fn prefix_walking_works_inner(state_version: StateVersion) { + let trie = test_trie(state_version); let mut seen = HashSet::new(); trie.for_keys_with_prefix(b"value", |key| { diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 6c575f0d76bc..8531e4907d6a 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,15 +20,17 @@ use crate::{backend::Consolidate, debug, warn, StorageKey, StorageValue}; use codec::Encode; -use hash_db::{self, Hasher, Prefix}; +use hash_db::{self, AsHashDB, HashDB, HashDBRef, Hasher, Prefix}; #[cfg(feature = "std")] use parking_lot::RwLock; -use sp_core::storage::ChildInfo; +use sp_core::storage::{ChildInfo, ChildType, StateVersion}; use sp_std::{boxed::Box, vec::Vec}; use sp_trie::{ - empty_child_trie_root, read_child_trie_value, read_trie_value, - trie_types::{Layout, TrieDB, TrieError}, - DBValue, KeySpacedDB, MemoryDB, PrefixedMemoryDB, Trie, TrieDBIterator, + child_delta_trie_root, delta_trie_root, empty_child_trie_root, read_child_trie_value, + read_trie_value, + trie_types::{TrieDB, TrieError}, + DBValue, KeySpacedDB, LayoutV1 as Layout, PrefixedMemoryDB, Trie, TrieDBIterator, + TrieDBKeyIterator, }; #[cfg(feature = "std")] use std::collections::HashMap; @@ -55,12 +57,12 @@ pub trait Storage: Send + Sync { /// Local cache for child root. #[cfg(feature = "std")] -pub(crate) struct Cache { - pub child_root: HashMap, Option>>, +pub(crate) struct Cache { + pub child_root: HashMap, Option>, } #[cfg(feature = "std")] -impl Cache { +impl Cache { fn new() -> Self { Cache { child_root: HashMap::new() } } @@ -72,7 +74,7 @@ pub struct TrieBackendEssence, H: Hasher> { root: H::Out, empty: H::Out, #[cfg(feature = "std")] - pub(crate) cache: Arc>, + pub(crate) cache: Arc>>, } impl, H: Hasher> TrieBackendEssence @@ -127,22 +129,26 @@ where } /// Access the root of the child storage in its parent trie - fn child_root(&self, child_info: &ChildInfo) -> Result> { + fn child_root(&self, child_info: &ChildInfo) -> Result> { #[cfg(feature = "std")] { if let Some(result) = self.cache.read().child_root.get(child_info.storage_key()) { - return Ok(result.clone()) + return Ok(*result) } } - let result = self.storage(child_info.prefixed_storage_key().as_slice())?; + let result = self.storage(child_info.prefixed_storage_key().as_slice())?.map(|r| { + let mut hash = H::Out::default(); + + // root is fetched from DB, not writable by runtime, so it's always valid. + hash.as_mut().copy_from_slice(&r[..]); + + hash + }); #[cfg(feature = "std")] { - self.cache - .write() - .child_root - .insert(child_info.storage_key().to_vec(), result.clone()); + self.cache.write().child_root.insert(child_info.storage_key().to_vec(), result); } Ok(result) @@ -160,15 +166,7 @@ where None => return Ok(None), }; - let mut hash = H::Out::default(); - - if child_root.len() != hash.as_ref().len() { - return Err(format!("Invalid child storage hash at {:?}", child_info.storage_key())) - } - // note: child_root and hash must be same size, panics otherwise. - hash.as_mut().copy_from_slice(&child_root[..]); - - self.next_storage_key_from_root(&hash, Some(child_info), key) + self.next_storage_key_from_root(&child_root, Some(child_info), key) } /// Return next key from main trie or child trie by providing corresponding root. @@ -178,7 +176,7 @@ where child_info: Option<&ChildInfo>, key: &[u8], ) -> Result> { - let dyn_eph: &dyn hash_db::HashDBRef<_, _>; + let dyn_eph: &dyn HashDBRef<_, _>; let keyspace_eph; if let Some(child_info) = child_info.as_ref() { keyspace_eph = KeySpacedDB::new(self, child_info.keyspace()); @@ -189,7 +187,7 @@ where let trie = TrieDB::::new(dyn_eph, root).map_err(|e| format!("TrieDB creation error: {}", e))?; - let mut iter = trie.iter().map_err(|e| format!("TrieDB iteration error: {}", e))?; + let mut iter = trie.key_iter().map_err(|e| format!("TrieDB iteration error: {}", e))?; // The key just after the one given in input, basically `key++0`. // Note: We are sure this is the next key if: @@ -205,7 +203,7 @@ where let next_element = iter.next(); let next_key = if let Some(next_element) = next_element { - let (next_key, _) = + let next_key = next_element.map_err(|e| format!("TrieDB iterator next error: {}", e))?; Some(next_key) } else { @@ -228,9 +226,10 @@ where child_info: &ChildInfo, key: &[u8], ) -> Result> { - let root = self - .child_root(child_info)? - .unwrap_or_else(|| empty_child_trie_root::>().encode()); + let root = match self.child_root(child_info)? { + Some(root) => root, + None => return Ok(None), + }; let map_e = |e| format!("Trie lookup error: {}", e); @@ -250,19 +249,13 @@ where f: impl FnMut(Vec, Vec) -> bool, allow_missing_nodes: bool, ) -> Result { - let mut child_root; let root = if let Some(child_info) = child_info.as_ref() { - if let Some(fetched_child_root) = self.child_root(child_info)? { - child_root = H::Out::default(); - // root is fetched from DB, not writable by runtime, so it's always valid. - child_root.as_mut().copy_from_slice(fetched_child_root.as_slice()); - - &child_root - } else { - return Ok(true) + match self.child_root(child_info)? { + Some(child_root) => child_root, + None => return Ok(true), } } else { - &self.root + self.root }; self.trie_iter_inner(&root, prefix, f, child_info, start_at, allow_missing_nodes) @@ -276,32 +269,21 @@ where prefix: Option<&[u8]>, mut f: F, ) { - let mut child_root = H::Out::default(); let root = if let Some(child_info) = child_info.as_ref() { - let root_vec = match self.child_root(child_info) { - Ok(v) => v.unwrap_or_else(|| empty_child_trie_root::>().encode()), + match self.child_root(child_info) { + Ok(Some(v)) => v, + // If the child trie doesn't exist, there is no need to continue. + Ok(None) => return, Err(e) => { debug!(target: "trie", "Error while iterating child storage: {}", e); return }, - }; - child_root.as_mut().copy_from_slice(&root_vec); - &child_root + } } else { - &self.root + self.root }; - let _ = self.trie_iter_inner( - root, - prefix, - |k, _v| { - f(&k); - true - }, - child_info, - None, - false, - ); + self.trie_iter_key_inner(&root, prefix, |k| f(k), child_info) } /// Execute given closure for all keys starting with prefix. @@ -311,41 +293,80 @@ where prefix: &[u8], mut f: impl FnMut(&[u8]), ) { - let root_vec = match self.child_root(child_info) { - Ok(v) => v.unwrap_or_else(|| empty_child_trie_root::>().encode()), + let root = match self.child_root(child_info) { + Ok(Some(v)) => v, + // If the child trie doesn't exist, there is no need to continue. + Ok(None) => return, Err(e) => { debug!(target: "trie", "Error while iterating child storage: {}", e); return }, }; - let mut root = H::Out::default(); - root.as_mut().copy_from_slice(&root_vec); - let _ = self.trie_iter_inner( + + self.trie_iter_key_inner( &root, Some(prefix), - |k, _v| { - f(&k); + |k| { + f(k); true }, Some(child_info), - None, - false, - ); + ) } /// Execute given closure for all keys starting with prefix. pub fn for_keys_with_prefix(&self, prefix: &[u8], mut f: F) { - let _ = self.trie_iter_inner( + self.trie_iter_key_inner( &self.root, Some(prefix), - |k, _v| { - f(&k); + |k| { + f(k); true }, None, - None, - false, - ); + ) + } + + fn trie_iter_key_inner bool>( + &self, + root: &H::Out, + prefix: Option<&[u8]>, + mut f: F, + child_info: Option<&ChildInfo>, + ) { + let mut iter = move |db| -> sp_std::result::Result<(), Box>> { + let trie = TrieDB::::new(db, root)?; + let iter = if let Some(prefix) = prefix.as_ref() { + TrieDBKeyIterator::new_prefixed(&trie, prefix)? + } else { + TrieDBKeyIterator::new(&trie)? + }; + + for x in iter { + let key = x?; + + debug_assert!(prefix + .as_ref() + .map(|prefix| key.starts_with(prefix)) + .unwrap_or(true)); + + if !f(&key) { + break + } + } + + Ok(()) + }; + + let result = if let Some(child_info) = child_info { + let db = KeySpacedDB::new(self, child_info.keyspace()); + iter(&db) + } else { + iter(self) + }; + if let Err(e) = result { + debug!(target: "trie", "Error while iterating by prefix: {}", e); + } } fn trie_iter_inner, Vec) -> bool>( @@ -407,6 +428,130 @@ where false, ); } + + /// Returns all `(key, value)` pairs in the trie. + pub fn pairs(&self) -> Vec<(StorageKey, StorageValue)> { + let collect_all = || -> sp_std::result::Result<_, Box>> { + let trie = TrieDB::::new(self, &self.root)?; + let mut v = Vec::new(); + for x in trie.iter()? { + let (key, value) = x?; + v.push((key.to_vec(), value.to_vec())); + } + + Ok(v) + }; + + match collect_all() { + Ok(v) => v, + Err(e) => { + debug!(target: "trie", "Error extracting trie values: {}", e); + Vec::new() + }, + } + } + + /// Returns all keys that start with the given `prefix`. + pub fn keys(&self, prefix: &[u8]) -> Vec { + let collect_all = || -> sp_std::result::Result<_, Box>> { + let trie = TrieDB::::new(self, &self.root)?; + let mut v = Vec::new(); + for x in trie.iter()? { + let (key, _) = x?; + if key.starts_with(prefix) { + v.push(key.to_vec()); + } + } + + Ok(v) + }; + + collect_all() + .map_err(|e| debug!(target: "trie", "Error extracting trie keys: {}", e)) + .unwrap_or_default() + } + + /// Return the storage root after applying the given `delta`. + pub fn storage_root<'a>( + &self, + delta: impl Iterator)>, + state_version: StateVersion, + ) -> (H::Out, S::Overlay) + where + H::Out: Ord, + { + let mut write_overlay = S::Overlay::default(); + let mut root = self.root; + + { + let mut eph = Ephemeral::new(self.backend_storage(), &mut write_overlay); + let res = match state_version { + StateVersion::V0 => + delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta), + StateVersion::V1 => + delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta), + }; + + match res { + Ok(ret) => root = ret, + Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), + } + } + + (root, write_overlay) + } + + /// Returns the child storage root for the child trie `child_info` after applying the given + /// `delta`. + pub fn child_storage_root<'a>( + &self, + child_info: &ChildInfo, + delta: impl Iterator)>, + state_version: StateVersion, + ) -> (H::Out, bool, S::Overlay) + where + H::Out: Ord, + { + let default_root = match child_info.child_type() { + ChildType::ParentKeyId => empty_child_trie_root::>(), + }; + let mut write_overlay = S::Overlay::default(); + let mut root = match self.child_root(child_info) { + Ok(Some(hash)) => hash, + Ok(None) => default_root, + Err(e) => { + warn!(target: "trie", "Failed to read child storage root: {}", e); + default_root.clone() + }, + }; + + { + let mut eph = Ephemeral::new(self.backend_storage(), &mut write_overlay); + match match state_version { + StateVersion::V0 => + child_delta_trie_root::, _, _, _, _, _, _>( + child_info.keyspace(), + &mut eph, + root, + delta, + ), + StateVersion::V1 => + child_delta_trie_root::, _, _, _, _, _, _>( + child_info.keyspace(), + &mut eph, + root, + delta, + ), + } { + Ok(ret) => root = ret, + Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), + } + } + + let is_default = root == default_root; + + (root, is_default, write_overlay) + } } pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { @@ -414,13 +559,13 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { overlay: &'a mut S::Overlay, } -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> hash_db::AsHashDB +impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> AsHashDB for Ephemeral<'a, S, H> { - fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { + fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { self } } @@ -435,7 +580,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - if let Some(val) = hash_db::HashDB::get(self.overlay, key, prefix) { + if let Some(val) = HashDB::get(self.overlay, key, prefix) { Some(val) } else { match self.storage.get(&key, prefix) { @@ -449,38 +594,37 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB } fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - hash_db::HashDB::get(self, key, prefix).is_some() + HashDB::get(self, key, prefix).is_some() } fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { - hash_db::HashDB::insert(self.overlay, prefix, value) + HashDB::insert(self.overlay, prefix, value) } fn emplace(&mut self, key: H::Out, prefix: Prefix, value: DBValue) { - hash_db::HashDB::emplace(self.overlay, key, prefix, value) + HashDB::emplace(self.overlay, key, prefix, value) } fn remove(&mut self, key: &H::Out, prefix: Prefix) { - hash_db::HashDB::remove(self.overlay, key, prefix) + HashDB::remove(self.overlay, key, prefix) } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDBRef - for Ephemeral<'a, S, H> -{ +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - hash_db::HashDB::get(self, key, prefix) + HashDB::get(self, key, prefix) } fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - hash_db::HashDB::contains(self, key, prefix) + HashDB::contains(self, key, prefix) } } /// Key-value pairs storage that is used by trie backend essence. pub trait TrieBackendStorage: Send + Sync { /// Type of in-memory overlay. - type Overlay: hash_db::HashDB + Default + Consolidate; + type Overlay: HashDB + Default + Consolidate; + /// Get the value stored at key. fn get(&self, key: &H::Out, prefix: Prefix) -> Result>; } @@ -495,35 +639,28 @@ impl TrieBackendStorage for Arc> { } } -// This implementation is used by test storage trie clients. -impl TrieBackendStorage for PrefixedMemoryDB { - type Overlay = PrefixedMemoryDB; - - fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { - Ok(hash_db::HashDB::get(self, key, prefix)) - } -} - -impl TrieBackendStorage for MemoryDB { - type Overlay = MemoryDB; +impl TrieBackendStorage for sp_trie::GenericMemoryDB +where + H: Hasher, + KF: sp_trie::KeyFunction + Send + Sync, +{ + type Overlay = Self; fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { Ok(hash_db::HashDB::get(self, key, prefix)) } } -impl, H: Hasher> hash_db::AsHashDB - for TrieBackendEssence -{ - fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { +impl, H: Hasher> AsHashDB for TrieBackendEssence { + fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { self } } -impl, H: Hasher> hash_db::HashDB for TrieBackendEssence { +impl, H: Hasher> HashDB for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { if *key == self.empty { return Some([0u8].to_vec()) @@ -538,7 +675,7 @@ impl, H: Hasher> hash_db::HashDB for TrieBa } fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - hash_db::HashDB::get(self, key, prefix).is_some() + HashDB::get(self, key, prefix).is_some() } fn insert(&mut self, _prefix: Prefix, _value: &[u8]) -> H::Out { @@ -554,15 +691,13 @@ impl, H: Hasher> hash_db::HashDB for TrieBa } } -impl, H: Hasher> hash_db::HashDBRef - for TrieBackendEssence -{ +impl, H: Hasher> HashDBRef for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - hash_db::HashDB::get(self, key, prefix) + HashDB::get(self, key, prefix) } fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - hash_db::HashDB::contains(self, key, prefix) + HashDB::contains(self, key, prefix) } } @@ -570,7 +705,9 @@ impl, H: Hasher> hash_db::HashDBRef mod test { use super::*; use sp_core::{Blake2Hasher, H256}; - use sp_trie::{trie_types::TrieDBMut, KeySpacedDBMut, PrefixedMemoryDB, TrieMut}; + use sp_trie::{ + trie_types::TrieDBMutV1 as TrieDBMut, KeySpacedDBMut, PrefixedMemoryDB, TrieMut, + }; #[test] fn next_storage_key_and_next_child_storage_key_work() { diff --git a/primitives/std/Cargo.toml b/primitives/std/Cargo.toml index bf815c1c80c5..4c6aa8f40b49 100644 --- a/primitives/std/Cargo.toml +++ b/primitives/std/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-std" -version = "4.0.0-dev" +version = "4.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Lowest-abstraction level for the Substrate runtime: just exports useful primitives from std or client/alloc to be used with any code that depends on the runtime." diff --git a/primitives/std/src/lib.rs b/primitives/std/src/lib.rs index 3af4d07ac629..6653c3d7eade 100644 --- a/primitives/std/src/lib.rs +++ b/primitives/std/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,9 +28,13 @@ doc = "Substrate's runtime standard library as compiled without Rust's standard library." )] +/// Initialize a key-value collection from array. +/// +/// Creates a vector of given pairs and calls `collect` on the iterator from it. +/// Can be used to create a `HashMap`. #[macro_export] macro_rules! map { - ($( $name:expr => $value:expr ),*) => ( + ($( $name:expr => $value:expr ),* $(,)? ) => ( vec![ $( ( $name, $value ) ),* ].into_iter().collect() ) } @@ -95,9 +99,11 @@ impl Writer { /// This should include only things which are in the normal std prelude. pub mod prelude { pub use crate::{ + borrow::ToOwned, boxed::Box, clone::Clone, cmp::{Eq, PartialEq, Reverse}, + iter::IntoIterator, vec::Vec, }; diff --git a/primitives/std/with_std.rs b/primitives/std/with_std.rs index 8a283e8fe333..b5fa3f85ed70 100644 --- a/primitives/std/with_std.rs +++ b/primitives/std/with_std.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/std/without_std.rs b/primitives/std/without_std.rs index 38c3a8421dac..7a6d5851dfac 100755 --- a/primitives/std/without_std.rs +++ b/primitives/std/without_std.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/storage/Cargo.toml b/primitives/storage/Cargo.toml index 1a05fb996919..a304bd660cc1 100644 --- a/primitives/storage/Cargo.toml +++ b/primitives/storage/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "sp-storage" -version = "4.0.0-dev" +version = "6.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" description = "Storage related primitives" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" documentation = "https://docs.rs/sp-storage/" readme = "README.md" @@ -14,13 +14,13 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -serde = { version = "1.0.126", optional = true, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +serde = { version = "1.0.136", optional = true, features = ["derive"] } impl-serde = { version = "0.3.1", optional = true } ref-cast = "1.0.0" -sp-debug-derive = { version = "3.0.0", path = "../debug-derive" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +sp-debug-derive = { version = "4.0.0", default-features = false, path = "../debug-derive" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } [features] default = [ "std" ] -std = [ "sp-std/std", "serde", "impl-serde", "codec/std" ] +std = [ "sp-std/std", "serde", "impl-serde", "codec/std", "sp-debug-derive/std" ] diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index 45474a44693a..fecd2b24dbb0 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -119,8 +119,7 @@ impl DerefMut for PrefixedStorageKey { } impl PrefixedStorageKey { - /// Create a prefixed storage key from its byte array - /// representation. + /// Create a prefixed storage key from its byte array representation. pub fn new(inner: Vec) -> Self { PrefixedStorageKey(inner) } @@ -130,9 +129,7 @@ impl PrefixedStorageKey { PrefixedStorageKey::ref_cast(inner) } - /// Get inner key, this should - /// only be needed when writing - /// into parent trie to avoid an + /// Get inner key, this should only be needed when writing into parent trie to avoid an /// allocation. pub fn into_inner(self) -> Vec { self.0 @@ -171,10 +168,8 @@ pub struct StorageChild { pub struct Storage { /// Top trie storage data. pub top: StorageMap, - /// Children trie storage data. - /// The key does not including prefix, for the `default` - /// trie kind, so this is exclusively for the `ChildType::ParentKeyId` - /// tries. + /// Children trie storage data. Key does not include prefix, only for the `default` trie kind, + /// of `ChildType::ParentKeyId` type. pub children_default: std::collections::HashMap, StorageChild>, } @@ -204,15 +199,20 @@ pub mod well_known_keys { /// Current extrinsic index (u32) is stored under this key. pub const EXTRINSIC_INDEX: &'static [u8] = b":extrinsic_index"; - /// Changes trie configuration is stored under this key. - pub const CHANGES_TRIE_CONFIG: &'static [u8] = b":changes_trie"; - /// Prefix of child storage keys. pub const CHILD_STORAGE_KEY_PREFIX: &'static [u8] = b":child_storage:"; /// Prefix of the default child storage keys in the top trie. pub const DEFAULT_CHILD_STORAGE_KEY_PREFIX: &'static [u8] = b":child_storage:default:"; + /// Whether a key is a default child storage key. + /// + /// This is convenience function which basically checks if the given `key` starts + /// with `DEFAULT_CHILD_STORAGE_KEY_PREFIX` and doesn't do anything apart from that. + pub fn is_default_child_storage_key(key: &[u8]) -> bool { + key.starts_with(DEFAULT_CHILD_STORAGE_KEY_PREFIX) + } + /// Whether a key is a child storage key. /// /// This is convenience function which basically checks if the given `key` starts @@ -232,9 +232,12 @@ pub mod well_known_keys { } } +/// Threshold size to start using trie value nodes in state. +pub const TRIE_VALUE_NODE_THRESHOLD: u32 = 33; + /// Information related to a child state. #[derive(Debug, Clone)] -#[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] +#[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord, Encode, Decode))] pub enum ChildInfo { /// This is the one used by default. ParentKeyId(ChildTrieParentKeyId), @@ -373,16 +376,14 @@ impl ChildType { } /// A child trie of default type. -/// It uses the same default implementation as the top trie, -/// top trie being a child trie with no keyspace and no storage key. -/// Its keyspace is the variable (unprefixed) part of its storage key. -/// It shares its trie nodes backend storage with every other -/// child trie, so its storage key needs to be a unique id -/// that will be use only once. -/// Those unique id also required to be long enough to avoid any -/// unique id to be prefixed by an other unique id. +/// +/// It uses the same default implementation as the top trie, top trie being a child trie with no +/// keyspace and no storage key. Its keyspace is the variable (unprefixed) part of its storage key. +/// It shares its trie nodes backend storage with every other child trie, so its storage key needs +/// to be a unique id that will be use only once. Those unique id also required to be long enough to +/// avoid any unique id to be prefixed by an other unique id. #[derive(Debug, Clone)] -#[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord))] +#[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord, Encode, Decode))] pub struct ChildTrieParentKeyId { /// Data is the storage key without prefix. data: Vec, @@ -398,6 +399,54 @@ impl ChildTrieParentKeyId { } } +/// Different possible state version. +/// +/// V0 and V1 uses a same trie implementation, but V1 will write external value node in the trie for +/// value with size at least `TRIE_VALUE_NODE_THRESHOLD`. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum StateVersion { + /// Old state version, no value nodes. + V0 = 0, + /// New state version can use value nodes. + V1 = 1, +} + +impl Default for StateVersion { + fn default() -> Self { + StateVersion::V1 + } +} + +impl From for u8 { + fn from(version: StateVersion) -> u8 { + version as u8 + } +} + +impl TryFrom for StateVersion { + type Error = (); + fn try_from(val: u8) -> sp_std::result::Result { + match val { + 0 => Ok(StateVersion::V0), + 1 => Ok(StateVersion::V1), + _ => Err(()), + } + } +} + +impl StateVersion { + /// If defined, values in state of size bigger or equal + /// to this threshold will use a separate trie node. + /// Otherwhise, value will be inlined in branch or leaf + /// node. + pub fn state_value_threshold(&self) -> Option { + match self { + StateVersion::V0 => None, + StateVersion::V1 => Some(TRIE_VALUE_NODE_THRESHOLD), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/primitives/tasks/Cargo.toml b/primitives/tasks/Cargo.toml index ee503ae9b855..7f05afeaf0ce 100644 --- a/primitives/tasks/Cargo.toml +++ b/primitives/tasks/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-tasks" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Runtime asynchronous, pure computational tasks" documentation = "https://docs.rs/sp-tasks" @@ -15,14 +15,14 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] log = { version = "0.4.8", optional = true } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } -sp-externalities = { version = "0.10.0-dev", optional = true, path = "../externalities" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../io" } -sp-runtime-interface = { version = "4.0.0-dev", default-features = false, path = "../runtime-interface" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +sp-externalities = { version = "0.12.0", optional = true, path = "../externalities" } +sp-io = { version = "6.0.0", default-features = false, path = "../io" } +sp-runtime-interface = { version = "6.0.0", default-features = false, path = "../runtime-interface" } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } [dev-dependencies] -codec = { package = "parity-scale-codec", default-features = false, version = "2.0.0" } +codec = { package = "parity-scale-codec", default-features = false, version = "3.0.0" } [features] default = ["std"] diff --git a/primitives/tasks/src/async_externalities.rs b/primitives/tasks/src/async_externalities.rs index 975a81af4f53..70e3437538e8 100644 --- a/primitives/tasks/src/async_externalities.rs +++ b/primitives/tasks/src/async_externalities.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify @@ -19,7 +19,7 @@ //! Async externalities. use sp_core::{ - storage::{ChildInfo, TrackedStorageKey}, + storage::{ChildInfo, StateVersion, TrackedStorageKey}, traits::{Externalities, RuntimeSpawn, RuntimeSpawnExt, SpawnNamed, TaskExecutorExt}, }; use sp_externalities::{Extensions, ExternalitiesExt as _}; @@ -126,18 +126,18 @@ impl Externalities for AsyncExternalities { panic!("`storage_append`: should not be used in async externalities!") } - fn storage_root(&mut self) -> Vec { + fn storage_root(&mut self, _state_version: StateVersion) -> Vec { panic!("`storage_root`: should not be used in async externalities!") } - fn child_storage_root(&mut self, _child_info: &ChildInfo) -> Vec { + fn child_storage_root( + &mut self, + _child_info: &ChildInfo, + _state_version: StateVersion, + ) -> Vec { panic!("`child_storage_root`: should not be used in async externalities!") } - fn storage_changes_root(&mut self, _parent: &[u8]) -> Result>, ()> { - panic!("`storage_changes_root`: should not be used in async externalities!") - } - fn storage_start_transaction(&mut self) { unimplemented!("Transactions are not supported by AsyncExternalities"); } diff --git a/primitives/tasks/src/lib.rs b/primitives/tasks/src/lib.rs index e9c80ae5ff4c..3711fa71a2fa 100644 --- a/primitives/tasks/src/lib.rs +++ b/primitives/tasks/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -95,6 +95,7 @@ mod inner { let extra_scheduler = scheduler.clone(); scheduler.spawn( "parallel-runtime-spawn", + Some("substrate-runtime"), Box::pin(async move { let result = match crate::new_async_externalities(extra_scheduler) { Ok(mut ext) => { diff --git a/primitives/test-primitives/Cargo.toml b/primitives/test-primitives/Cargo.toml index 5aed5d679dd4..0d491ea217c6 100644 --- a/primitives/test-primitives/Cargo.toml +++ b/primitives/test-primitives/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-test-primitives" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" publish = false @@ -12,12 +12,12 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../application-crypto" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } -serde = { version = "1.0.126", optional = true, features = ["derive"] } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../runtime" } -parity-util-mem = { version = "0.10.0", default-features = false, features = ["primitive-types"] } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../application-crypto" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +serde = { version = "1.0.136", optional = true, features = ["derive"] } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } +parity-util-mem = { version = "0.11.0", default-features = false, features = ["primitive-types"] } [features] default = [ diff --git a/primitives/test-primitives/src/lib.rs b/primitives/test-primitives/src/lib.rs index d988160b1dc7..976bb9ddd9cd 100644 --- a/primitives/test-primitives/src/lib.rs +++ b/primitives/test-primitives/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -73,15 +73,10 @@ pub type BlockNumber = u64; /// Index of a transaction. pub type Index = u64; /// The item of a block digest. -pub type DigestItem = sp_runtime::generic::DigestItem; +pub type DigestItem = sp_runtime::generic::DigestItem; /// The digest of a block. -pub type Digest = sp_runtime::generic::Digest; +pub type Digest = sp_runtime::generic::Digest; /// A test block. pub type Block = sp_runtime::generic::Block; /// A test block's header. pub type Header = sp_runtime::generic::Header; - -/// Changes trie configuration (optionally) used in tests. -pub fn changes_trie_config() -> sp_core::ChangesTrieConfiguration { - sp_core::ChangesTrieConfiguration { digest_interval: 4, digest_levels: 2 } -} diff --git a/primitives/timestamp/Cargo.toml b/primitives/timestamp/Cargo.toml index 60daf9642df6..176a4db5cb45 100644 --- a/primitives/timestamp/Cargo.toml +++ b/primitives/timestamp/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-timestamp" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate core types and inherents for timestamps." readme = "README.md" @@ -14,11 +14,11 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sp-api = { version = "4.0.0-dev", default-features = false, path = "../api" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../runtime" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../inherents" } -thiserror = { version = "1.0.21", optional = true } +thiserror = { version = "1.0.30", optional = true } log = { version = "0.4.8", optional = true } futures-timer = { version = "3.0.2", optional = true } async-trait = { version = "0.1.50", optional = true } diff --git a/primitives/timestamp/src/lib.rs b/primitives/timestamp/src/lib.rs index 02a579497b52..b98a87c37f69 100644 --- a/primitives/timestamp/src/lib.rs +++ b/primitives/timestamp/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -42,10 +42,16 @@ impl Timestamp { } /// Returns `self` as [`Duration`]. - pub fn as_duration(self) -> Duration { + pub const fn as_duration(self) -> Duration { Duration::from_millis(self.0) } + /// Returns `self` as a `u64` representing the elapsed time since the UNIX_EPOCH in + /// milliseconds. + pub const fn as_millis(&self) -> u64 { + self.0 + } + /// Checked subtraction that returns `None` on an underflow. pub fn checked_sub(self, other: Self) -> Option { self.0.checked_sub(other.0).map(Self) @@ -138,9 +144,9 @@ impl IsFatalError for InherentError { impl InherentError { /// Try to create an instance ouf of the given identifier and data. #[cfg(feature = "std")] - pub fn try_from(id: &InherentIdentifier, data: &[u8]) -> Option { + pub fn try_from(id: &InherentIdentifier, mut data: &[u8]) -> Option { if id == &INHERENT_IDENTIFIER { - ::decode(&mut &data[..]).ok() + ::decode(&mut data).ok() } else { None } @@ -227,7 +233,7 @@ impl sp_inherents::InherentDataProvider for InherentDataProvider { &self, inherent_data: &mut InherentData, ) -> Result<(), sp_inherents::Error> { - inherent_data.put_data(INHERENT_IDENTIFIER, &InherentType::from(self.timestamp)) + inherent_data.put_data(INHERENT_IDENTIFIER, &self.timestamp) } async fn try_handle_error( diff --git a/primitives/tracing/Cargo.toml b/primitives/tracing/Cargo.toml index 3be09dcd576d..3f53cc2e6c5c 100644 --- a/primitives/tracing/Cargo.toml +++ b/primitives/tracing/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-tracing" -version = "4.0.0-dev" +version = "5.0.0" license = "Apache-2.0" authors = ["Parity Technologies "] -edition = "2018" -homepage = "https://substrate.dev" +edition = "2021" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Instrumentation primitives and macros for Substrate." readme = "README.md" @@ -18,21 +18,15 @@ features = ["with-tracing"] targets = ["x86_64-unknown-linux-gnu", "wasm32-unknown-unknown"] [dependencies] -sp-std = { version = "4.0.0-dev", path = "../std", default-features = false } -codec = { version = "2.0.0", package = "parity-scale-codec", default-features = false, features = [ +sp-std = { version = "4.0.0", path = "../std", default-features = false } +codec = { version = "3.0.0", package = "parity-scale-codec", default-features = false, features = [ "derive", ] } -tracing = { version = "0.1.25", default-features = false } -tracing-core = { version = "0.1.17", default-features = false } -log = { version = "0.4.8", optional = true } -tracing-subscriber = { version = "0.2.19", optional = true, features = [ +tracing = { version = "0.1.29", default-features = false } +tracing-core = { version = "0.1.21", default-features = false } +tracing-subscriber = { version = "0.2.25", optional = true, features = [ "tracing-log", ] } -parking_lot = { version = "0.10.0", optional = true } -erased-serde = { version = "0.3.9", optional = true } -serde = { version = "1.0.126", optional = true } -serde_json = { version = "1.0.68", optional = true } -slog = { version = "2.5.2", features = ["nested-values"], optional = true } [features] default = ["std"] @@ -43,11 +37,5 @@ std = [ "tracing-core/std", "codec/std", "sp-std/std", - "log", "tracing-subscriber", - "parking_lot", - "erased-serde", - "serde", - "serde_json", - "slog", ] diff --git a/primitives/tracing/src/lib.rs b/primitives/tracing/src/lib.rs index 9522e6df633a..1efae226a546 100644 --- a/primitives/tracing/src/lib.rs +++ b/primitives/tracing/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -108,6 +108,7 @@ pub use crate::types::{WASM_NAME_KEY, WASM_TARGET_KEY, WASM_TRACE_IDENTIFIER}; mod types; /// Try to init a simple tracing subscriber with log compatibility layer. +/// /// Ignores any error. Useful for testing. #[cfg(feature = "std")] pub fn try_init_simple() { @@ -117,6 +118,22 @@ pub fn try_init_simple() { .try_init(); } +/// Init a tracing subscriber for logging in tests. +/// +/// Be aware that this enables `TRACE` by default. It also ignores any error +/// while setting up the logger. +/// +/// The logs are not shown by default, logs are only shown when the test fails +/// or if [`nocapture`](https://doc.rust-lang.org/cargo/commands/cargo-test.html#display-options) +/// is being used. +#[cfg(feature = "std")] +pub fn init_for_tests() { + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::TRACE) + .with_test_writer() + .try_init(); +} + /// Runs given code within a tracing span, measuring it's execution time. /// /// If tracing is not enabled, the code is still executed. Pass in level and name or @@ -126,20 +143,20 @@ pub fn try_init_simple() { /// /// ``` /// sp_tracing::within_span! { -/// sp_tracing::Level::TRACE, +/// sp_tracing::Level::TRACE, /// "test-span"; /// 1 + 1; /// // some other complex code /// } /// /// sp_tracing::within_span! { -/// sp_tracing::span!(sp_tracing::Level::WARN, "warn-span", you_can_pass="any params"); +/// sp_tracing::span!(sp_tracing::Level::WARN, "warn-span", you_can_pass="any params"); /// 1 + 1; /// // some other complex code /// } /// /// sp_tracing::within_span! { -/// sp_tracing::debug_span!("debug-span", you_can_pass="any params"); +/// sp_tracing::debug_span!("debug-span", you_can_pass="any params"); /// 1 + 1; /// // some other complex code /// } diff --git a/primitives/tracing/src/types.rs b/primitives/tracing/src/types.rs index 377bd0f42c6e..d175e1f8f17e 100644 --- a/primitives/tracing/src/types.rs +++ b/primitives/tracing/src/types.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/transaction-pool/Cargo.toml b/primitives/transaction-pool/Cargo.toml index 3f77014ac53b..544b149ce3a4 100644 --- a/primitives/transaction-pool/Cargo.toml +++ b/primitives/transaction-pool/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-transaction-pool" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Transaction pool runtime facing API." documentation = "https://docs.rs/sp-transaction-pool" @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sp-api = { version = "4.0.0-dev", default-features = false, path = "../api" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } [features] default = [ "std" ] diff --git a/primitives/transaction-pool/src/lib.rs b/primitives/transaction-pool/src/lib.rs index 3c71149255ce..143958f06d16 100644 --- a/primitives/transaction-pool/src/lib.rs +++ b/primitives/transaction-pool/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/transaction-pool/src/runtime_api.rs b/primitives/transaction-pool/src/runtime_api.rs index be631ee03b9d..87a0c82e9133 100644 --- a/primitives/transaction-pool/src/runtime_api.rs +++ b/primitives/transaction-pool/src/runtime_api.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/transaction-storage-proof/Cargo.toml b/primitives/transaction-storage-proof/Cargo.toml index 8a41105b20b7..966142a7e14f 100644 --- a/primitives/transaction-storage-proof/Cargo.toml +++ b/primitives/transaction-storage-proof/Cargo.toml @@ -3,9 +3,9 @@ name = "sp-transaction-storage-proof" version = "4.0.0-dev" authors = ["Parity Technologies "] description = "Transaction storage proof primitives" -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -14,12 +14,12 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../inherents" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../runtime" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -sp-trie = { version = "4.0.0-dev", optional = true, path = "../trie" } -sp-core = { version = "4.0.0-dev", path = "../core", optional = true } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +sp-trie = { version = "6.0.0", optional = true, path = "../trie" } +sp-core = { version = "6.0.0", path = "../core", optional = true } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } log = { version = "0.4.8", optional = true } async-trait = { version = "0.1.50", optional = true } diff --git a/primitives/transaction-storage-proof/src/lib.rs b/primitives/transaction-storage-proof/src/lib.rs index 4b01a8d45d45..2e5aa3b2b9c7 100644 --- a/primitives/transaction-storage-proof/src/lib.rs +++ b/primitives/transaction-storage-proof/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -67,7 +67,7 @@ pub trait TransactionStorageProofInherentData { impl TransactionStorageProofInherentData for InherentData { fn storage_proof(&self) -> Result, Error> { - Ok(self.get_data(&INHERENT_IDENTIFIER)?) + self.get_data(&INHERENT_IDENTIFIER) } } @@ -98,13 +98,13 @@ impl sp_inherents::InherentDataProvider for InherentDataProvider { async fn try_handle_error( &self, identifier: &InherentIdentifier, - error: &[u8], + mut error: &[u8], ) -> Option> { if *identifier != INHERENT_IDENTIFIER { return None } - let error = InherentError::decode(&mut &error[..]).ok()?; + let error = InherentError::decode(&mut error).ok()?; Some(Err(Error::Application(Box::from(format!("{:?}", error))))) } @@ -143,7 +143,7 @@ pub mod registration { use sp_trie::TrieMut; type Hasher = sp_core::Blake2Hasher; - type TrieLayout = sp_trie::Layout; + type TrieLayout = sp_trie::LayoutV1; /// Create a new inherent data provider instance for a given parent block hash. pub fn new_data_provider( diff --git a/primitives/trie/Cargo.toml b/primitives/trie/Cargo.toml index 5a2de4f16f9a..f434a15b8964 100644 --- a/primitives/trie/Cargo.toml +++ b/primitives/trie/Cargo.toml @@ -1,12 +1,12 @@ [package] name = "sp-trie" -version = "4.0.0-dev" +version = "6.0.0" authors = ["Parity Technologies "] description = "Patricia trie stuff using a parity-scale-codec node format" repository = "https://github.com/paritytech/substrate/" license = "Apache-2.0" -edition = "2018" -homepage = "https://substrate.dev" +edition = "2021" +homepage = "https://substrate.io" documentation = "https://docs.rs/sp-trie" readme = "README.md" @@ -18,21 +18,22 @@ name = "bench" harness = false [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } hash-db = { version = "0.15.2", default-features = false } -trie-db = { version = "0.22.6", default-features = false } -trie-root = { version = "0.16.0", default-features = false } -memory-db = { version = "0.27.0", default-features = false } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../core" } +trie-db = { version = "0.23.1", default-features = false } +trie-root = { version = "0.17.0", default-features = false } +memory-db = { version = "0.29.0", default-features = false } +sp-core = { version = "6.0.0", default-features = false, path = "../core" } +thiserror = { version = "1.0.30", optional = true } [dev-dependencies] -trie-bench = "0.28.0" +trie-bench = "0.30.0" trie-standardmap = "0.15.2" criterion = "0.3.3" -hex-literal = "0.3.1" -sp-runtime = { version = "4.0.0-dev", path = "../runtime" } +hex-literal = "0.3.4" +sp-runtime = { version = "6.0.0", path = "../runtime" } [features] default = ["std"] @@ -45,5 +46,6 @@ std = [ "trie-db/std", "trie-root/std", "sp-core/std", + "thiserror", ] memory-tracker = [] diff --git a/primitives/trie/benches/bench.rs b/primitives/trie/benches/bench.rs index 8c84c6354f2c..f1670b0c2fa4 100644 --- a/primitives/trie/benches/bench.rs +++ b/primitives/trie/benches/bench.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2015-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2015-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,11 +21,11 @@ criterion_main!(benches); fn benchmark(c: &mut Criterion) { trie_bench::standard_benchmark::< - sp_trie::Layout, + sp_trie::LayoutV1, sp_trie::TrieStream, >(c, "substrate-blake2"); trie_bench::standard_benchmark::< - sp_trie::Layout, + sp_trie::LayoutV1, sp_trie::TrieStream, >(c, "substrate-keccak"); } diff --git a/primitives/trie/src/error.rs b/primitives/trie/src/error.rs index 30a164c61475..e0b3642b6db7 100644 --- a/primitives/trie/src/error.rs +++ b/primitives/trie/src/error.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2015-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2015-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,18 +15,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "std")] -use std::error::Error as StdError; -#[cfg(feature = "std")] -use std::fmt; - -#[derive(Debug, PartialEq, Eq, Clone)] /// Error for trie node decoding. +#[derive(Debug, PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(thiserror::Error))] pub enum Error { - /// Bad format. + #[cfg_attr(feature = "std", error("Bad format"))] BadFormat, - /// Decoding error. - Decode(codec::Error), + #[cfg_attr(feature = "std", error("Decoding failed: {0}"))] + Decode(#[cfg_attr(feature = "std", source)] codec::Error), } impl From for Error { @@ -34,23 +30,3 @@ impl From for Error { Error::Decode(x) } } - -#[cfg(feature = "std")] -impl StdError for Error { - fn description(&self) -> &str { - match self { - Error::BadFormat => "Bad format error", - Error::Decode(_) => "Decoding error", - } - } -} - -#[cfg(feature = "std")] -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Error::Decode(e) => write!(f, "Decode error: {}", e), - Error::BadFormat => write!(f, "Bad format"), - } - } -} diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 8ba13284d379..c4d4c7210bd4 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2015-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2015-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -45,31 +45,85 @@ pub use trie_db::proof::VerifyError; use trie_db::proof::{generate_proof, verify_proof}; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ - nibble_ops, CError, DBValue, Query, Recorder, Trie, TrieConfiguration, TrieDBIterator, + nibble_ops, + node::{NodePlan, ValuePlan}, + CError, DBValue, Query, Recorder, Trie, TrieConfiguration, TrieDBIterator, TrieDBKeyIterator, TrieLayout, TrieMut, }; /// The Substrate format implementation of `TrieStream`. pub use trie_stream::TrieStream; -#[derive(Default)] /// substrate trie layout -pub struct Layout(sp_std::marker::PhantomData); +pub struct LayoutV0(sp_std::marker::PhantomData); -impl TrieLayout for Layout { +/// substrate trie layout, with external value nodes. +pub struct LayoutV1(sp_std::marker::PhantomData); + +impl TrieLayout for LayoutV0 +where + H: Hasher, +{ const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; + const MAX_INLINE_VALUE: Option = None; + type Hash = H; type Codec = NodeCodec; } -impl TrieConfiguration for Layout { +impl TrieConfiguration for LayoutV0 +where + H: Hasher, +{ + fn trie_root(input: I) -> ::Out + where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + { + trie_root::trie_root_no_extension::(input, Self::MAX_INLINE_VALUE) + } + + fn trie_root_unhashed(input: I) -> Vec + where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + { + trie_root::unhashed_trie_no_extension::( + input, + Self::MAX_INLINE_VALUE, + ) + } + + fn encode_index(input: u32) -> Vec { + codec::Encode::encode(&codec::Compact(input)) + } +} + +impl TrieLayout for LayoutV1 +where + H: Hasher, +{ + const USE_EXTENSION: bool = false; + const ALLOW_EMPTY: bool = true; + const MAX_INLINE_VALUE: Option = Some(sp_core::storage::TRIE_VALUE_NODE_THRESHOLD); + + type Hash = H; + type Codec = NodeCodec; +} + +impl TrieConfiguration for LayoutV1 +where + H: Hasher, +{ fn trie_root(input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::trie_root_no_extension::(input) + trie_root::trie_root_no_extension::(input, Self::MAX_INLINE_VALUE) } fn trie_root_unhashed(input: I) -> Vec @@ -78,7 +132,10 @@ impl TrieConfiguration for Layout { A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::unhashed_trie_no_extension::(input) + trie_root::unhashed_trie_no_extension::( + input, + Self::MAX_INLINE_VALUE, + ) } fn encode_index(input: u32) -> Vec { @@ -118,17 +175,22 @@ pub type TrieDBMut<'a, L> = trie_db::TrieDBMut<'a, L>; pub type Lookup<'a, L, Q> = trie_db::Lookup<'a, L, Q>; /// Hash type for a trie layout. pub type TrieHash = <::Hash as Hasher>::Out; - /// This module is for non generic definition of trie type. /// Only the `Hasher` trait is generic in this case. pub mod trie_types { - pub type Layout = super::Layout; + use super::*; + /// Persistent trie database read-access interface for the a given hasher. - pub type TrieDB<'a, H> = super::TrieDB<'a, Layout>; + /// Read only V1 and V0 are compatible, thus we always use V1. + pub type TrieDB<'a, H> = super::TrieDB<'a, LayoutV1>; /// Persistent trie database write-access interface for the a given hasher. - pub type TrieDBMut<'a, H> = super::TrieDBMut<'a, Layout>; + pub type TrieDBMutV0<'a, H> = super::TrieDBMut<'a, LayoutV0>; + /// Persistent trie database write-access interface for the a given hasher. + pub type TrieDBMutV1<'a, H> = super::TrieDBMut<'a, LayoutV1>; + /// Querying interface, as in `trie_db` but less generic. + pub type LookupV0<'a, H, Q> = trie_db::Lookup<'a, LayoutV0, Q>; /// Querying interface, as in `trie_db` but less generic. - pub type Lookup<'a, H, Q> = trie_db::Lookup<'a, Layout, Q>; + pub type LookupV1<'a, H, Q> = trie_db::Lookup<'a, LayoutV1, Q>; /// As in `trie_db`, but less generic, error type for the crate. pub type TrieError = trie_db::TrieError; } @@ -141,16 +203,18 @@ pub mod trie_types { /// For a key `K` that is included in the `db` a proof of inclusion is generated. /// For a key `K` that is not included in the `db` a proof of non-inclusion is generated. /// These can be later checked in `verify_trie_proof`. -pub fn generate_trie_proof<'a, L: TrieConfiguration, I, K, DB>( +pub fn generate_trie_proof<'a, L, I, K, DB>( db: &DB, root: TrieHash, keys: I, ) -> Result>, Box>> where + L: TrieConfiguration, I: IntoIterator, K: 'a + AsRef<[u8]>, DB: hash_db::HashDBRef, { + // Can use default layout (read only). let trie = TrieDB::::new(db, &root)?; generate_proof(&trie, keys) } @@ -163,17 +227,18 @@ where /// checked for inclusion in the proof. /// If the value is omitted (`(key, None)`), this key will be checked for non-inclusion in the /// proof. -pub fn verify_trie_proof<'a, L: TrieConfiguration, I, K, V>( +pub fn verify_trie_proof<'a, L, I, K, V>( root: &TrieHash, proof: &[Vec], items: I, -) -> Result<(), VerifyError, error::Error>> +) -> Result<(), VerifyError, CError>> where + L: TrieConfiguration, I: IntoIterator)>, K: 'a + AsRef<[u8]>, V: 'a + AsRef<[u8]>, { - verify_proof::, _, _, _>(root, proof, items) + verify_proof::(root, proof, items) } /// Determine a trie root given a hash DB and delta values. @@ -207,25 +272,30 @@ where } /// Read a value from the trie. -pub fn read_trie_value>( +pub fn read_trie_value( db: &DB, root: &TrieHash, key: &[u8], -) -> Result>, Box>> { +) -> Result>, Box>> +where + L: TrieConfiguration, + DB: hash_db::HashDBRef, +{ TrieDB::::new(&*db, root)?.get(key).map(|x| x.map(|val| val.to_vec())) } /// Read a value from the trie with given Query. -pub fn read_trie_value_with< - L: TrieConfiguration, - Q: Query, - DB: hash_db::HashDBRef, ->( +pub fn read_trie_value_with( db: &DB, root: &TrieHash, key: &[u8], query: Q, -) -> Result>, Box>> { +) -> Result>, Box>> +where + L: TrieConfiguration, + Q: Query, + DB: hash_db::HashDBRef, +{ TrieDB::::new(&*db, root)? .get_with(key, query) .map(|x| x.map(|val| val.to_vec())) @@ -304,22 +374,18 @@ where pub fn read_child_trie_value( keyspace: &[u8], db: &DB, - root_slice: &[u8], + root: &TrieHash, key: &[u8], ) -> Result>, Box>> where DB: hash_db::HashDBRef, { - let mut root = TrieHash::::default(); - // root is fetched from DB, not writable by runtime, so it's always valid. - root.as_mut().copy_from_slice(root_slice); - let db = KeySpacedDB::new(&*db, keyspace); - TrieDB::::new(&db, &root)?.get(key).map(|x| x.map(|val| val.to_vec())) + TrieDB::::new(&db, root)?.get(key).map(|x| x.map(|val| val.to_vec())) } /// Read a value from the child trie with given query. -pub fn read_child_trie_value_with, DB>( +pub fn read_child_trie_value_with( keyspace: &[u8], db: &DB, root_slice: &[u8], @@ -327,6 +393,8 @@ pub fn read_child_trie_value_with Result>, Box>> where + L: TrieConfiguration, + Q: Query, DB: hash_db::HashDBRef, { let mut root = TrieHash::::default(); @@ -444,11 +512,15 @@ where /// Constants used into trie simplification codec. mod trie_constants { - pub const EMPTY_TRIE: u8 = 0; - pub const NIBBLE_SIZE_BOUND: usize = u16::MAX as usize; + const FIRST_PREFIX: u8 = 0b_00 << 6; + pub const NIBBLE_SIZE_BOUND: usize = u16::max_value() as usize; pub const LEAF_PREFIX_MASK: u8 = 0b_01 << 6; pub const BRANCH_WITHOUT_MASK: u8 = 0b_10 << 6; pub const BRANCH_WITH_MASK: u8 = 0b_11 << 6; + pub const EMPTY_TRIE: u8 = FIRST_PREFIX | (0b_00 << 4); + pub const ALT_HASHING_LEAF_PREFIX_MASK: u8 = FIRST_PREFIX | (0b_1 << 5); + pub const ALT_HASHING_BRANCH_WITH_MASK: u8 = FIRST_PREFIX | (0b_01 << 4); + pub const ESCAPE_COMPACT_HEADER: u8 = EMPTY_TRIE | 0b_00_01; } #[cfg(test)] @@ -461,7 +533,11 @@ mod tests { use trie_db::{DBValue, NodeCodec as NodeCodecT, Trie, TrieMut}; use trie_standardmap::{Alphabet, StandardMap, ValueMode}; - type Layout = super::Layout; + type LayoutV0 = super::LayoutV0; + type LayoutV1 = super::LayoutV1; + + type MemoryDBMeta = + memory_db::MemoryDB, trie_db::DBValue, MemTracker>; fn hashed_null_node() -> TrieHash { ::hashed_null_node() @@ -473,7 +549,7 @@ mod tests { let d = T::trie_root_unhashed(input.clone()); println!("Data: {:#x?}, {:#x?}", d, Blake2Hasher::hash(&d[..])); let persistent = { - let mut memdb = MemoryDB::default(); + let mut memdb = MemoryDBMeta::default(); let mut root = Default::default(); let mut t = TrieDBMut::::new(&mut memdb, &mut root); for (x, y) in input.iter().rev() { @@ -486,7 +562,7 @@ mod tests { } fn check_iteration(input: &Vec<(&[u8], &[u8])>) { - let mut memdb = MemoryDB::default(); + let mut memdb = MemoryDBMeta::default(); let mut root = Default::default(); { let mut t = TrieDBMut::::new(&mut memdb, &mut root); @@ -506,14 +582,21 @@ mod tests { } } + fn check_input(input: &Vec<(&[u8], &[u8])>) { + check_equivalent::(input); + check_iteration::(input); + check_equivalent::(input); + check_iteration::(input); + } + #[test] fn default_trie_root() { let mut db = MemoryDB::default(); - let mut root = TrieHash::::default(); - let mut empty = TrieDBMut::::new(&mut db, &mut root); + let mut root = TrieHash::::default(); + let mut empty = TrieDBMut::::new(&mut db, &mut root); empty.commit(); let root1 = empty.root().as_ref().to_vec(); - let root2: Vec = Layout::trie_root::<_, Vec, Vec>(std::iter::empty()) + let root2: Vec = LayoutV1::trie_root::<_, Vec, Vec>(std::iter::empty()) .as_ref() .iter() .cloned() @@ -525,31 +608,27 @@ mod tests { #[test] fn empty_is_equivalent() { let input: Vec<(&[u8], &[u8])> = vec![]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] fn leaf_is_equivalent() { let input: Vec<(&[u8], &[u8])> = vec![(&[0xaa][..], &[0xbb][..])]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] fn branch_is_equivalent() { let input: Vec<(&[u8], &[u8])> = vec![(&[0xaa][..], &[0x10][..]), (&[0xba][..], &[0x11][..])]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] fn extension_and_branch_is_equivalent() { let input: Vec<(&[u8], &[u8])> = vec![(&[0xaa][..], &[0x10][..]), (&[0xab][..], &[0x11][..])]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -564,8 +643,7 @@ mod tests { let mut d = st.make(); d.sort_by(|&(ref a, _), &(ref b, _)| a.cmp(b)); let dr = d.iter().map(|v| (&v.0[..], &v.1[..])).collect(); - check_equivalent::(&dr); - check_iteration::(&dr); + check_input(&dr); } #[test] @@ -575,8 +653,7 @@ mod tests { (&[0xaa, 0xaa][..], &[0xaa][..]), (&[0xaa, 0xbb][..], &[0xab][..]), ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -589,8 +666,7 @@ mod tests { (&[0xbb, 0xbb][..], &[0xbb][..]), (&[0xbb, 0xcc][..], &[0xbc][..]), ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -602,8 +678,7 @@ mod tests { ), (&[0xba][..], &[0x11][..]), ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -618,15 +693,17 @@ mod tests { &b"ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC"[..], ), ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } - fn populate_trie<'db, T: TrieConfiguration>( + fn populate_trie<'db, T>( db: &'db mut dyn HashDB, root: &'db mut TrieHash, v: &[(Vec, Vec)], - ) -> TrieDBMut<'db, T> { + ) -> TrieDBMut<'db, T> + where + T: TrieConfiguration, + { let mut t = TrieDBMut::::new(db, root); for i in 0..v.len() { let key: &[u8] = &v[i].0; @@ -648,8 +725,12 @@ mod tests { #[test] fn random_should_work() { + random_should_work_inner::(); + random_should_work_inner::(); + } + fn random_should_work_inner() { let mut seed = ::Out::zero(); - for test_i in 0..10000 { + for test_i in 0..10_000 { if test_i % 50 == 0 { println!("{:?} of 10000 stress tests done", test_i); } @@ -662,10 +743,11 @@ mod tests { } .make_with(seed.as_fixed_bytes_mut()); - let real = Layout::trie_root(x.clone()); + let real = L::trie_root(x.clone()); let mut memdb = MemoryDB::default(); let mut root = Default::default(); - let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); + + let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); memtrie.commit(); if *memtrie.root() != real { @@ -677,9 +759,9 @@ mod tests { } } assert_eq!(*memtrie.root(), real); - unpopulate_trie::(&mut memtrie, &x); + unpopulate_trie::(&mut memtrie, &x); memtrie.commit(); - let hashed_null_node = hashed_null_node::(); + let hashed_null_node = hashed_null_node::(); if *memtrie.root() != hashed_null_node { println!("- TRIE MISMATCH"); println!(""); @@ -699,7 +781,7 @@ mod tests { #[test] fn codec_trie_empty() { let input: Vec<(&[u8], &[u8])> = vec![]; - let trie = Layout::trie_root_unhashed::<_, _, _>(input); + let trie = LayoutV1::trie_root_unhashed(input); println!("trie: {:#x?}", trie); assert_eq!(trie, vec![0x0]); } @@ -707,7 +789,7 @@ mod tests { #[test] fn codec_trie_single_tuple() { let input = vec![(vec![0xaa], vec![0xbb])]; - let trie = Layout::trie_root_unhashed::<_, _, _>(input); + let trie = LayoutV1::trie_root_unhashed(input); println!("trie: {:#x?}", trie); assert_eq!( trie, @@ -723,7 +805,7 @@ mod tests { #[test] fn codec_trie_two_tuples_disjoint_keys() { let input = vec![(&[0x48, 0x19], &[0xfe]), (&[0x13, 0x14], &[0xff])]; - let trie = Layout::trie_root_unhashed::<_, _, _>(input); + let trie = LayoutV1::trie_root_unhashed(input); println!("trie: {:#x?}", trie); let mut ex = Vec::::new(); ex.push(0x80); // branch, no value (0b_10..) no nibble @@ -747,6 +829,10 @@ mod tests { #[test] fn iterator_works() { + iterator_works_inner::(); + iterator_works_inner::(); + } + fn iterator_works_inner() { let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("0400000000").to_vec()), (hex!("0103000000000000000469").to_vec(), hex!("0401000000").to_vec()), @@ -777,15 +863,15 @@ mod tests { let mut memdb = MemoryDB::default(); let mut root = Default::default(); - populate_trie::(&mut memdb, &mut root, &pairs); + populate_trie::(&mut memdb, &mut root, &pairs); let non_included_key: Vec = hex!("0909").to_vec(); let proof = - generate_trie_proof::(&memdb, root, &[non_included_key.clone()]) + generate_trie_proof::(&memdb, root, &[non_included_key.clone()]) .unwrap(); // Verifying that the K was not included into the trie should work. - assert!(verify_trie_proof::>( + assert!(verify_trie_proof::>( &root, &proof, &[(non_included_key.clone(), None)], @@ -793,7 +879,7 @@ mod tests { .is_ok()); // Verifying that the K was included into the trie should fail. - assert!(verify_trie_proof::>( + assert!(verify_trie_proof::>( &root, &proof, &[(non_included_key, Some(hex!("1010").to_vec()))], @@ -810,13 +896,13 @@ mod tests { let mut memdb = MemoryDB::default(); let mut root = Default::default(); - populate_trie::(&mut memdb, &mut root, &pairs); + populate_trie::(&mut memdb, &mut root, &pairs); let proof = - generate_trie_proof::(&memdb, root, &[pairs[0].0.clone()]).unwrap(); + generate_trie_proof::(&memdb, root, &[pairs[0].0.clone()]).unwrap(); // Check that a K, V included into the proof are verified. - assert!(verify_trie_proof::( + assert!(verify_trie_proof::( &root, &proof, &[(pairs[0].0.clone(), Some(pairs[0].1.clone()))] @@ -824,7 +910,7 @@ mod tests { .is_ok()); // Absence of the V is not verified with the proof that has K, V included. - assert!(verify_trie_proof::>( + assert!(verify_trie_proof::>( &root, &proof, &[(pairs[0].0.clone(), None)] @@ -832,7 +918,7 @@ mod tests { .is_err()); // K not included into the trie is not verified. - assert!(verify_trie_proof::( + assert!(verify_trie_proof::( &root, &proof, &[(hex!("4242").to_vec(), Some(pairs[0].1.clone()))] @@ -840,7 +926,7 @@ mod tests { .is_err()); // K included into the trie but not included into the proof is not verified. - assert!(verify_trie_proof::( + assert!(verify_trie_proof::( &root, &proof, &[(pairs[1].0.clone(), Some(pairs[1].1.clone()))] @@ -865,13 +951,13 @@ mod tests { .unwrap(); let proof_db = proof.into_memory_db::(); - let first_storage_root = delta_trie_root::( + let first_storage_root = delta_trie_root::( &mut proof_db.clone(), storage_root, valid_delta, ) .unwrap(); - let second_storage_root = delta_trie_root::( + let second_storage_root = delta_trie_root::( &mut proof_db.clone(), storage_root, invalid_delta, diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index d5ffb3219cf6..bd0ba27483e6 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2015-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2015-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,8 +23,8 @@ use codec::{Compact, Decode, Encode, Input}; use hash_db::Hasher; use sp_std::{borrow::Borrow, marker::PhantomData, ops::Range, vec::Vec}; use trie_db::{ - self, nibble_ops, - node::{NibbleSlicePlan, NodeHandlePlan, NodePlan}, + nibble_ops, + node::{NibbleSlicePlan, NodeHandlePlan, NodePlan, Value, ValuePlan}, ChildReference, NodeCodec as NodeCodecT, Partial, }; @@ -54,9 +54,7 @@ impl<'a> ByteSliceInput<'a> { impl<'a> Input for ByteSliceInput<'a> { fn remaining_len(&mut self) -> Result, codec::Error> { - let remaining = - if self.offset <= self.data.len() { Some(self.data.len() - self.offset) } else { None }; - Ok(remaining) + Ok(Some(self.data.len().saturating_sub(self.offset))) } fn read(&mut self, into: &mut [u8]) -> Result<(), codec::Error> { @@ -76,11 +74,17 @@ impl<'a> Input for ByteSliceInput<'a> { } } -/// Concrete implementation of a `NodeCodec` with Parity Codec encoding, generic over the `Hasher` +/// Concrete implementation of a [`NodeCodecT`] with SCALE encoding. +/// +/// It is generic over `H` the [`Hasher`]. #[derive(Default, Clone)] pub struct NodeCodec(PhantomData); -impl NodeCodecT for NodeCodec { +impl NodeCodecT for NodeCodec +where + H: Hasher, +{ + const ESCAPE_HEADER: Option = Some(trie_constants::ESCAPE_COMPACT_HEADER); type Error = Error; type HashOut = H::Out; @@ -88,11 +92,22 @@ impl NodeCodecT for NodeCodec { H::hash(::empty_node()) } - fn decode_plan(data: &[u8]) -> sp_std::result::Result { + fn decode_plan(data: &[u8]) -> Result { let mut input = ByteSliceInput::new(data); - match NodeHeader::decode(&mut input)? { + + let header = NodeHeader::decode(&mut input)?; + let contains_hash = header.contains_hash_of_value(); + + let branch_has_value = if let NodeHeader::Branch(has_value, _) = &header { + *has_value + } else { + // hashed_value_branch + true + }; + + match header { NodeHeader::Null => Ok(NodePlan::Empty), - NodeHeader::Branch(has_value, nibble_count) => { + NodeHeader::HashedValueBranch(nibble_count) | NodeHeader::Branch(_, nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -105,9 +120,13 @@ impl NodeCodecT for NodeCodec { let partial_padding = nibble_ops::number_padding(nibble_count); let bitmap_range = input.take(BITMAP_LENGTH)?; let bitmap = Bitmap::decode(&data[bitmap_range])?; - let value = if has_value { - let count = >::decode(&mut input)?.0 as usize; - Some(input.take(count)?) + let value = if branch_has_value { + Some(if contains_hash { + ValuePlan::Node(input.take(H::LENGTH)?) + } else { + let count = >::decode(&mut input)?.0 as usize; + ValuePlan::Inline(input.take(count)?) + }) } else { None }; @@ -132,7 +151,7 @@ impl NodeCodecT for NodeCodec { children, }) }, - NodeHeader::Leaf(nibble_count) => { + NodeHeader::HashedValueLeaf(nibble_count) | NodeHeader::Leaf(nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -143,10 +162,16 @@ impl NodeCodecT for NodeCodec { nibble_ops::NIBBLE_PER_BYTE, )?; let partial_padding = nibble_ops::number_padding(nibble_count); - let count = >::decode(&mut input)?.0 as usize; + let value = if contains_hash { + ValuePlan::Node(input.take(H::LENGTH)?) + } else { + let count = >::decode(&mut input)?.0 as usize; + ValuePlan::Inline(input.take(count)?) + }; + Ok(NodePlan::Leaf { partial: NibbleSlicePlan::new(partial, partial_padding), - value: input.take(count)?, + value, }) }, } @@ -160,9 +185,23 @@ impl NodeCodecT for NodeCodec { &[trie_constants::EMPTY_TRIE] } - fn leaf_node(partial: Partial, value: &[u8]) -> Vec { - let mut output = partial_encode(partial, NodeKind::Leaf); - value.encode_to(&mut output); + fn leaf_node(partial: Partial, value: Value) -> Vec { + let contains_hash = matches!(&value, Value::Node(..)); + let mut output = if contains_hash { + partial_encode(partial, NodeKind::HashedValueLeaf) + } else { + partial_encode(partial, NodeKind::Leaf) + }; + match value { + Value::Inline(value) => { + Compact(value.len() as u32).encode_to(&mut output); + output.extend_from_slice(value); + }, + Value::Node(hash, _) => { + debug_assert!(hash.len() == H::LENGTH); + output.extend_from_slice(hash); + }, + } output } @@ -171,33 +210,46 @@ impl NodeCodecT for NodeCodec { _nbnibble: usize, _child: ChildReference<::Out>, ) -> Vec { - unreachable!() + unreachable!("No extension codec.") } fn branch_node( _children: impl Iterator::Out>>>>, - _maybe_value: Option<&[u8]>, + _maybe_value: Option, ) -> Vec { - unreachable!() + unreachable!("No extension codec.") } fn branch_node_nibbled( partial: impl Iterator, number_nibble: usize, children: impl Iterator::Out>>>>, - maybe_value: Option<&[u8]>, + value: Option, ) -> Vec { - let mut output = if maybe_value.is_some() { - partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue) - } else { - partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) + let contains_hash = matches!(&value, Some(Value::Node(..))); + let mut output = match (&value, contains_hash) { + (&None, _) => + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue), + (_, false) => + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue), + (_, true) => + partial_from_iterator_encode(partial, number_nibble, NodeKind::HashedValueBranch), }; + let bitmap_index = output.len(); let mut bitmap: [u8; BITMAP_LENGTH] = [0; BITMAP_LENGTH]; (0..BITMAP_LENGTH).for_each(|_| output.push(0)); - if let Some(value) = maybe_value { - value.encode_to(&mut output); - }; + match value { + Some(Value::Inline(value)) => { + Compact(value.len() as u32).encode_to(&mut output); + output.extend_from_slice(value); + }, + Some(Value::Node(hash, _)) => { + debug_assert!(hash.len() == H::LENGTH); + output.extend_from_slice(hash); + }, + None => (), + } Bitmap::encode( children.map(|maybe_child| match maybe_child.borrow() { Some(ChildReference::Hash(h)) => { @@ -229,11 +281,15 @@ fn partial_from_iterator_encode>( ) -> Vec { let nibble_count = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, nibble_count); - let mut output = Vec::with_capacity(3 + (nibble_count / nibble_ops::NIBBLE_PER_BYTE)); + let mut output = Vec::with_capacity(4 + (nibble_count / nibble_ops::NIBBLE_PER_BYTE)); match node_kind { NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), + NodeKind::HashedValueLeaf => + NodeHeader::HashedValueLeaf(nibble_count).encode_to(&mut output), + NodeKind::HashedValueBranch => + NodeHeader::HashedValueBranch(nibble_count).encode_to(&mut output), }; output.extend(partial); output @@ -247,11 +303,15 @@ fn partial_encode(partial: Partial, node_kind: NodeKind) -> Vec { let nibble_count = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, nibble_count); - let mut output = Vec::with_capacity(3 + partial.1.len()); + let mut output = Vec::with_capacity(4 + partial.1.len()); match node_kind { NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), + NodeKind::HashedValueLeaf => + NodeHeader::HashedValueLeaf(nibble_count).encode_to(&mut output), + NodeKind::HashedValueBranch => + NodeHeader::HashedValueBranch(nibble_count).encode_to(&mut output), }; if number_nibble_encoded > 0 { output.push(nibble_ops::pad_right((partial.0).1)); diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index 9f05113a3593..c2c9510c5ac4 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2015-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2015-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,8 +25,20 @@ use sp_std::iter::once; #[derive(Copy, Clone, PartialEq, Eq, sp_core::RuntimeDebug)] pub(crate) enum NodeHeader { Null, + // contains wether there is a value and nibble count Branch(bool, usize), + // contains nibble count Leaf(usize), + // contains nibble count. + HashedValueBranch(usize), + // contains nibble count. + HashedValueLeaf(usize), +} + +impl NodeHeader { + pub(crate) fn contains_hash_of_value(&self) -> bool { + matches!(self, NodeHeader::HashedValueBranch(_) | NodeHeader::HashedValueLeaf(_)) + } } /// NodeHeader without content @@ -34,6 +46,8 @@ pub(crate) enum NodeKind { Leaf, BranchNoValue, BranchWithValue, + HashedValueLeaf, + HashedValueBranch, } impl Encode for NodeHeader { @@ -41,11 +55,27 @@ impl Encode for NodeHeader { match self { NodeHeader::Null => output.push_byte(trie_constants::EMPTY_TRIE), NodeHeader::Branch(true, nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITH_MASK, output), - NodeHeader::Branch(false, nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITHOUT_MASK, output), + encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITH_MASK, 2, output), + NodeHeader::Branch(false, nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::BRANCH_WITHOUT_MASK, + 2, + output, + ), NodeHeader::Leaf(nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, output), + encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, 2, output), + NodeHeader::HashedValueBranch(nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::ALT_HASHING_BRANCH_WITH_MASK, + 4, + output, + ), + NodeHeader::HashedValueLeaf(nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, + 3, + output, + ), } } } @@ -59,13 +89,22 @@ impl Decode for NodeHeader { return Ok(NodeHeader::Null) } match i & (0b11 << 6) { - trie_constants::LEAF_PREFIX_MASK => Ok(NodeHeader::Leaf(decode_size(i, input)?)), - trie_constants::BRANCH_WITHOUT_MASK => - Ok(NodeHeader::Branch(false, decode_size(i, input)?)), + trie_constants::LEAF_PREFIX_MASK => Ok(NodeHeader::Leaf(decode_size(i, input, 2)?)), trie_constants::BRANCH_WITH_MASK => - Ok(NodeHeader::Branch(true, decode_size(i, input)?)), - // do not allow any special encoding - _ => Err("Unallowed encoding".into()), + Ok(NodeHeader::Branch(true, decode_size(i, input, 2)?)), + trie_constants::BRANCH_WITHOUT_MASK => + Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), + trie_constants::EMPTY_TRIE => { + if i & (0b111 << 5) == trie_constants::ALT_HASHING_LEAF_PREFIX_MASK { + Ok(NodeHeader::HashedValueLeaf(decode_size(i, input, 3)?)) + } else if i & (0b1111 << 4) == trie_constants::ALT_HASHING_BRANCH_WITH_MASK { + Ok(NodeHeader::HashedValueBranch(decode_size(i, input, 4)?)) + } else { + // do not allow any special encoding + Err("Unallowed encoding".into()) + } + }, + _ => unreachable!(), } } } @@ -73,12 +112,20 @@ impl Decode for NodeHeader { /// Returns an iterator over encoded bytes for node header and size. /// Size encoding allows unlimited, length inefficient, representation, but /// is bounded to 16 bit maximum value to avoid possible DOS. -pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8) -> impl Iterator { +pub(crate) fn size_and_prefix_iterator( + size: usize, + prefix: u8, + prefix_mask: usize, +) -> impl Iterator { let size = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, size); - let l1 = sp_std::cmp::min(62, size); - let (first_byte, mut rem) = - if size == l1 { (once(prefix + l1 as u8), 0) } else { (once(prefix + 63), size - l1) }; + let max_value = 255u8 >> prefix_mask; + let l1 = sp_std::cmp::min((max_value as usize).saturating_sub(1), size); + let (first_byte, mut rem) = if size == l1 { + (once(prefix + l1 as u8), 0) + } else { + (once(prefix + max_value as u8), size - l1) + }; let next_bytes = move || { if rem > 0 { if rem < 256 { @@ -97,16 +144,24 @@ pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8) -> impl Iterator } /// Encodes size and prefix to a stream output. -fn encode_size_and_prefix(size: usize, prefix: u8, out: &mut W) { - for b in size_and_prefix_iterator(size, prefix) { +fn encode_size_and_prefix(size: usize, prefix: u8, prefix_mask: usize, out: &mut W) +where + W: Output + ?Sized, +{ + for b in size_and_prefix_iterator(size, prefix, prefix_mask) { out.push_byte(b) } } /// Decode size only from stream input and header byte. -fn decode_size(first: u8, input: &mut impl Input) -> Result { - let mut result = (first & 255u8 >> 2) as usize; - if result < 63 { +fn decode_size( + first: u8, + input: &mut impl Input, + prefix_mask: usize, +) -> Result { + let max_value = 255u8 >> prefix_mask; + let mut result = (first & max_value) as usize; + if result < max_value as usize { return Ok(result) } result -= 1; diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index cfdb8566ea75..f6139584dbba 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +18,10 @@ use codec::{Decode, Encode}; use hash_db::{HashDB, Hasher}; use scale_info::TypeInfo; -use sp_std::vec::Vec; +use sp_std::{collections::btree_set::BTreeSet, iter::IntoIterator, vec::Vec}; +// Note that `LayoutV1` usage here (proof compaction) is compatible +// with `LayoutV0`. +use crate::LayoutV1 as Layout; /// A proof that some set of key-value pairs are included in the storage trie. The proof contains /// the storage values so that the partial storage backend can be reconstructed by a verifier that @@ -29,19 +32,13 @@ use sp_std::vec::Vec; /// the serialized nodes and performing the key lookups. #[derive(Debug, PartialEq, Eq, Clone, Encode, Decode, TypeInfo)] pub struct StorageProof { - trie_nodes: Vec>, -} - -/// Storage proof in compact form. -#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode, TypeInfo)] -pub struct CompactProof { - pub encoded_nodes: Vec>, + trie_nodes: BTreeSet>, } impl StorageProof { /// Constructs a storage proof from a subset of encoded trie nodes in a storage backend. - pub fn new(trie_nodes: Vec>) -> Self { - StorageProof { trie_nodes } + pub fn new(trie_nodes: impl IntoIterator>) -> Self { + StorageProof { trie_nodes: BTreeSet::from_iter(trie_nodes) } } /// Returns a new empty proof. @@ -49,7 +46,7 @@ impl StorageProof { /// An empty proof is capable of only proving trivial statements (ie. that an empty set of /// key-value pairs exist in storage). pub fn empty() -> Self { - StorageProof { trie_nodes: Vec::new() } + StorageProof { trie_nodes: BTreeSet::new() } } /// Returns whether this is an empty proof. @@ -57,17 +54,18 @@ impl StorageProof { self.trie_nodes.is_empty() } - /// Create an iterator over trie nodes constructed from the proof. The nodes are not guaranteed - /// to be traversed in any particular order. + /// Create an iterator over encoded trie nodes in lexicographical order constructed + /// from the proof. pub fn iter_nodes(self) -> StorageProofNodeIterator { StorageProofNodeIterator::new(self) } /// Convert into plain node vector. - pub fn into_nodes(self) -> Vec> { + pub fn into_nodes(self) -> BTreeSet> { self.trie_nodes } - /// Creates a `MemoryDB` from `Self`. + + /// Creates a [`MemoryDB`](crate::MemoryDB) from `Self`. pub fn into_memory_db(self) -> crate::MemoryDB { self.into() } @@ -75,10 +73,7 @@ impl StorageProof { /// Merges multiple storage proofs covering potentially different sets of keys into one proof /// covering all keys. The merged proof output may be smaller than the aggregate size of the /// input proofs due to deduplication of trie nodes. - pub fn merge(proofs: I) -> Self - where - I: IntoIterator, - { + pub fn merge(proofs: impl IntoIterator) -> Self { let trie_nodes = proofs .into_iter() .flat_map(|proof| proof.iter_nodes()) @@ -89,19 +84,19 @@ impl StorageProof { Self { trie_nodes } } - /// Encode as a compact proof with default - /// trie layout. + /// Encode as a compact proof with default trie layout. pub fn into_compact_proof( self, root: H::Out, - ) -> Result>> { - crate::encode_compact::>(self, root) + ) -> Result> { + crate::encode_compact::>(self, root) } /// Returns the estimated encoded size of the compact proof. /// - /// Runing this operation is a slow operation (build the whole compact proof) and should only be - /// in non sensitive path. + /// Running this operation is a slow operation (build the whole compact proof) and should only + /// be in non sensitive path. + /// /// Return `None` on error. pub fn encoded_compact_size(self, root: H::Out) -> Option { let compact_proof = self.into_compact_proof::(root); @@ -109,6 +104,22 @@ impl StorageProof { } } +impl From for crate::MemoryDB { + fn from(proof: StorageProof) -> Self { + let mut db = crate::MemoryDB::default(); + proof.iter_nodes().for_each(|n| { + db.insert(crate::EMPTY_PREFIX, &n); + }); + db + } +} + +/// Storage proof in compact form. +#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode, TypeInfo)] +pub struct CompactProof { + pub encoded_nodes: Vec>, +} + impl CompactProof { /// Return an iterator on the compact encoded nodes. pub fn iter_compact_encoded_nodes(&self) -> impl Iterator { @@ -116,35 +127,52 @@ impl CompactProof { } /// Decode to a full storage_proof. - /// - /// Method use a temporary `HashDB`, and `sp_trie::decode_compact` - /// is often better. pub fn to_storage_proof( &self, expected_root: Option<&H::Out>, - ) -> Result<(StorageProof, H::Out), crate::CompactProofError>> { + ) -> Result<(StorageProof, H::Out), crate::CompactProofError> { let mut db = crate::MemoryDB::::new(&[]); - let root = crate::decode_compact::, _, _>( + let root = crate::decode_compact::, _, _>( &mut db, self.iter_compact_encoded_nodes(), expected_root, )?; Ok(( - StorageProof::new( - db.drain() - .into_iter() - .filter_map(|kv| if (kv.1).1 > 0 { Some((kv.1).0) } else { None }) - .collect(), - ), + StorageProof::new(db.drain().into_iter().filter_map(|kv| { + if (kv.1).1 > 0 { + Some((kv.1).0) + } else { + None + } + })), root, )) } + + /// Convert self into a [`MemoryDB`](crate::MemoryDB). + /// + /// `expected_root` is the expected root of this compact proof. + /// + /// Returns the memory db and the root of the trie. + pub fn to_memory_db( + &self, + expected_root: Option<&H::Out>, + ) -> Result<(crate::MemoryDB, H::Out), crate::CompactProofError> { + let mut db = crate::MemoryDB::::new(&[]); + let root = crate::decode_compact::, _, _>( + &mut db, + self.iter_compact_encoded_nodes(), + expected_root, + )?; + + Ok((db, root)) + } } /// An iterator over trie nodes constructed from a storage proof. The nodes are not guaranteed to /// be traversed in any particular order. pub struct StorageProofNodeIterator { - inner: > as IntoIterator>::IntoIter, + inner: > as IntoIterator>::IntoIter, } impl StorageProofNodeIterator { @@ -160,13 +188,3 @@ impl Iterator for StorageProofNodeIterator { self.inner.next() } } - -impl From for crate::MemoryDB { - fn from(proof: StorageProof) -> Self { - let mut db = crate::MemoryDB::default(); - for item in proof.iter_nodes() { - db.insert(crate::EMPTY_PREFIX, &item); - } - db - } -} diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index 1596229f2b5d..a7f292271565 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,80 +20,34 @@ //! This uses compact proof from trie crate and extends //! it to substrate specific layout and child trie system. -use crate::{ - CompactProof, HashDBT, StorageProof, TrieConfiguration, TrieError, TrieHash, EMPTY_PREFIX, -}; +use crate::{CompactProof, HashDBT, StorageProof, TrieConfiguration, TrieHash, EMPTY_PREFIX}; use sp_std::{boxed::Box, vec::Vec}; -#[cfg(feature = "std")] -use std::error::Error as StdError; -#[cfg(feature = "std")] -use std::fmt; -use trie_db::Trie; +use trie_db::{CError, Trie}; /// Error for trie node decoding. -pub enum Error { - /// Verification failed due to root mismatch. - RootMismatch(TrieHash, TrieHash), - /// Missing nodes in proof. +#[derive(Debug)] +#[cfg_attr(feature = "std", derive(thiserror::Error))] +pub enum Error { + #[cfg_attr(feature = "std", error("Invalid root {0:x?}, expected {1:x?}"))] + RootMismatch(H, H), + #[cfg_attr(feature = "std", error("Missing nodes in the proof"))] IncompleteProof, - /// Compact node is not needed. + #[cfg_attr(feature = "std", error("Child node content with no root in proof"))] ExtraneousChildNode, - /// Child content with root not in proof. - ExtraneousChildProof(TrieHash), - /// Bad child trie root. + #[cfg_attr(feature = "std", error("Proof of child trie {0:x?} not in parent proof"))] + ExtraneousChildProof(H), + #[cfg_attr(feature = "std", error("Invalid root {0:x?}, expected {1:x?}"))] InvalidChildRoot(Vec, Vec), - /// Errors from trie crate. - TrieError(Box>), + #[cfg_attr(feature = "std", error("Trie error: {0:?}"))] + TrieError(Box>), } -impl From>> for Error { - fn from(error: Box>) -> Self { +impl From>> for Error { + fn from(error: Box>) -> Self { Error::TrieError(error) } } -#[cfg(feature = "std")] -impl StdError for Error { - fn description(&self) -> &str { - match self { - Error::InvalidChildRoot(..) => "Invalid child root error", - Error::TrieError(..) => "Trie db error", - Error::RootMismatch(..) => "Trie db error", - Error::IncompleteProof => "Incomplete proof", - Error::ExtraneousChildNode => "Extraneous child node", - Error::ExtraneousChildProof(..) => "Extraneous child proof", - } - } -} - -#[cfg(feature = "std")] -impl fmt::Debug for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - ::fmt(&self, f) - } -} - -#[cfg(feature = "std")] -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Error::InvalidChildRoot(k, v) => write!(f, "InvalidChildRoot at {:x?}: {:x?}", k, v), - Error::TrieError(e) => write!(f, "Trie error: {}", e), - Error::IncompleteProof => write!(f, "Incomplete proof"), - Error::ExtraneousChildNode => write!(f, "Child node content with no root in proof"), - Error::ExtraneousChildProof(root) => { - write!(f, "Proof of child trie {:x?} not in parent proof", root.as_ref()) - }, - Error::RootMismatch(root, expected) => write!( - f, - "Verification error, root is {:x?}, expected: {:x?}", - root.as_ref(), - expected.as_ref(), - ), - } - } -} - /// Decode a compact proof. /// /// Takes as input a destination `db` for decoded node and `encoded` @@ -105,15 +59,14 @@ pub fn decode_compact<'a, L, DB, I>( db: &mut DB, encoded: I, expected_root: Option<&TrieHash>, -) -> Result, Error> +) -> Result, Error, CError>> where L: TrieConfiguration, DB: HashDBT + hash_db::HashDBRef, I: IntoIterator, { let mut nodes_iter = encoded.into_iter(); - let (top_root, _nb_used) = - trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; + let (top_root, _nb_used) = trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; // Only check root if expected root is passed as argument. if let Some(expected_root) = expected_root { @@ -164,8 +117,7 @@ where let mut nodes_iter = nodes_iter.peekable(); for child_root in child_tries.into_iter() { if previous_extracted_child_trie.is_none() && nodes_iter.peek().is_some() { - let (top_root, _) = - trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; + let (top_root, _) = trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; previous_extracted_child_trie = Some(top_root); } @@ -197,7 +149,10 @@ where /// Then parse all child trie root and compress main trie content first /// then all child trie contents. /// Child trie are ordered by the order of their roots in the top trie. -pub fn encode_compact(proof: StorageProof, root: TrieHash) -> Result> +pub fn encode_compact( + proof: StorageProof, + root: TrieHash, +) -> Result, CError>> where L: TrieConfiguration, { diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index e0e26fea67c2..a17d7c25e1b8 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2015-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2015-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,21 +18,18 @@ //! `TrieStream` implementation for Substrate's trie format. use crate::{ - node_codec::Bitmap, node_header::{size_and_prefix_iterator, NodeKind}, trie_constants, }; -use codec::Encode; +use codec::{Compact, Encode}; use hash_db::Hasher; use sp_std::vec::Vec; use trie_root; -const BRANCH_NODE_NO_VALUE: u8 = 254; -const BRANCH_NODE_WITH_VALUE: u8 = 255; - -#[derive(Default, Clone)] /// Codec-flavored TrieStream. +#[derive(Default, Clone)] pub struct TrieStream { + /// Current node buffer. buffer: Vec, } @@ -60,51 +57,76 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK), + NodeKind::Leaf => size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK, 2), NodeKind::BranchNoValue => - size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK), + size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK, 2), NodeKind::BranchWithValue => - size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK), + size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK, 2), + NodeKind::HashedValueLeaf => + size_and_prefix_iterator(size, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3), + NodeKind::HashedValueBranch => + size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4), }; iter_start .chain(if nibbles.len() % 2 == 1 { Some(nibbles[0]) } else { None }) .chain(nibbles[nibbles.len() % 2..].chunks(2).map(|ch| ch[0] << 4 | ch[1])) } +use trie_root::Value as TrieStreamValue; impl trie_root::TrieStream for TrieStream { fn new() -> Self { - TrieStream { buffer: Vec::new() } + Self { buffer: Vec::new() } } fn append_empty_data(&mut self) { self.buffer.push(trie_constants::EMPTY_TRIE); } - fn append_leaf(&mut self, key: &[u8], value: &[u8]) { - self.buffer.extend(fuse_nibbles_node(key, NodeKind::Leaf)); - value.encode_to(&mut self.buffer); + fn append_leaf(&mut self, key: &[u8], value: TrieStreamValue) { + let kind = match &value { + TrieStreamValue::Inline(..) => NodeKind::Leaf, + TrieStreamValue::Node(..) => NodeKind::HashedValueLeaf, + }; + self.buffer.extend(fuse_nibbles_node(key, kind)); + match &value { + TrieStreamValue::Inline(value) => { + Compact(value.len() as u32).encode_to(&mut self.buffer); + self.buffer.extend_from_slice(value); + }, + TrieStreamValue::Node(hash) => { + self.buffer.extend_from_slice(hash.as_slice()); + }, + }; } fn begin_branch( &mut self, maybe_partial: Option<&[u8]>, - maybe_value: Option<&[u8]>, + maybe_value: Option, has_children: impl Iterator, ) { if let Some(partial) = maybe_partial { - if maybe_value.is_some() { - self.buffer.extend(fuse_nibbles_node(partial, NodeKind::BranchWithValue)); - } else { - self.buffer.extend(fuse_nibbles_node(partial, NodeKind::BranchNoValue)); - } + let kind = match &maybe_value { + None => NodeKind::BranchNoValue, + Some(TrieStreamValue::Inline(..)) => NodeKind::BranchWithValue, + Some(TrieStreamValue::Node(..)) => NodeKind::HashedValueBranch, + }; + + self.buffer.extend(fuse_nibbles_node(partial, kind)); let bm = branch_node_bit_mask(has_children); self.buffer.extend([bm.0, bm.1].iter()); } else { - debug_assert!(false, "trie stream codec only for no extension trie"); - self.buffer.extend(&branch_node(maybe_value.is_some(), has_children)); + unreachable!("trie stream codec only for no extension trie"); } - if let Some(value) = maybe_value { - value.encode_to(&mut self.buffer); + match maybe_value { + None => (), + Some(TrieStreamValue::Inline(value)) => { + Compact(value.len() as u32).encode_to(&mut self.buffer); + self.buffer.extend_from_slice(value); + }, + Some(TrieStreamValue::Node(hash)) => { + self.buffer.extend_from_slice(hash.as_slice()); + }, } } @@ -124,18 +146,3 @@ impl trie_root::TrieStream for TrieStream { self.buffer } } - -fn branch_node(has_value: bool, has_children: impl Iterator) -> [u8; 3] { - let mut result = [0, 0, 0]; - branch_node_buffered(has_value, has_children, &mut result[..]); - result -} - -fn branch_node_buffered(has_value: bool, has_children: I, output: &mut [u8]) -where - I: Iterator, -{ - let first = if has_value { BRANCH_NODE_WITH_VALUE } else { BRANCH_NODE_NO_VALUE }; - output[0] = first; - Bitmap::encode(has_children, &mut output[1..]); -} diff --git a/primitives/version/Cargo.toml b/primitives/version/Cargo.toml index fcab1eeabcaf..7488d924070c 100644 --- a/primitives/version/Cargo.toml +++ b/primitives/version/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-version" -version = "4.0.0-dev" +version = "5.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Version module for the Substrate runtime; Provides a function that returns the runtime version." documentation = "https://docs.rs/sp-version" @@ -13,17 +13,17 @@ readme = "README.md" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] - [dependencies] impl-serde = { version = "0.3.1", optional = true } -serde = { version = "1.0.126", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../std" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../runtime" } +serde = { version = "1.0.136", optional = true, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +sp-std = { version = "4.0.0", default-features = false, path = "../std" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" } sp-version-proc-macro = { version = "4.0.0-dev", default-features = false, path = "proc-macro" } parity-wasm = { version = "0.42.2", optional = true } -thiserror = { version = "1.0.21", optional = true } +sp-core-hashing-proc-macro = { version = "5.0.0", path = "../core/hashing/proc-macro" } +thiserror = { version = "1.0.30", optional = true } [features] default = ["std"] diff --git a/primitives/version/proc-macro/Cargo.toml b/primitives/version/proc-macro/Cargo.toml index c3c801431434..e54012a516f2 100644 --- a/primitives/version/proc-macro/Cargo.toml +++ b/primitives/version/proc-macro/Cargo.toml @@ -2,9 +2,9 @@ name = "sp-version-proc-macro" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Macro for defining a runtime version." documentation = "https://docs.rs/sp-api-proc-macro" @@ -16,10 +16,10 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -quote = "1.0.3" -syn = { version = "1.0.58", features = ["full", "fold", "extra-traits", "visit"] } -proc-macro2 = "1.0.29" -codec = { package = "parity-scale-codec", version = "2.0.0", features = [ "derive" ] } +quote = "1.0.10" +syn = { version = "1.0.82", features = ["full", "fold", "extra-traits", "visit"] } +proc-macro2 = "1.0.36" +codec = { package = "parity-scale-codec", version = "3.0.0", features = [ "derive" ] } [dev-dependencies] -sp-version = { version = "4.0.0-dev", path = ".." } +sp-version = { version = "5.0.0", path = ".." } diff --git a/primitives/version/proc-macro/src/decl_runtime_version.rs b/primitives/version/proc-macro/src/decl_runtime_version.rs index eef6314be4c8..9ca1a67cc7fd 100644 --- a/primitives/version/proc-macro/src/decl_runtime_version.rs +++ b/primitives/version/proc-macro/src/decl_runtime_version.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -63,6 +63,7 @@ struct RuntimeVersion { impl_version: u32, apis: u8, transaction_version: u32, + state_version: u8, } #[derive(Default, Debug)] @@ -73,6 +74,7 @@ struct ParseRuntimeVersion { spec_version: Option, impl_version: Option, transaction_version: Option, + state_version: Option, } impl ParseRuntimeVersion { @@ -122,6 +124,8 @@ impl ParseRuntimeVersion { parse_once(&mut self.impl_version, field_value, Self::parse_num_literal)?; } else if field_name == "transaction_version" { parse_once(&mut self.transaction_version, field_value, Self::parse_num_literal)?; + } else if field_name == "state_version" { + parse_once(&mut self.state_version, field_value, Self::parse_num_literal_u8)?; } else if field_name == "apis" { // Intentionally ignored // @@ -147,6 +151,18 @@ impl ParseRuntimeVersion { lit.base10_parse::() } + fn parse_num_literal_u8(expr: &Expr) -> Result { + let lit = match *expr { + Expr::Lit(ExprLit { lit: Lit::Int(ref lit), .. }) => lit, + _ => + return Err(Error::new( + expr.span(), + "only numeric literals (e.g. `10`) are supported here", + )), + }; + lit.base10_parse::() + } + fn parse_str_literal(expr: &Expr) -> Result { let mac = match *expr { Expr::Macro(syn::ExprMacro { ref mac, .. }) => mac, @@ -182,6 +198,7 @@ impl ParseRuntimeVersion { spec_version, impl_version, transaction_version, + state_version, } = self; Ok(RuntimeVersion { @@ -191,6 +208,7 @@ impl ParseRuntimeVersion { spec_version: required!(spec_version), impl_version: required!(impl_version), transaction_version: required!(transaction_version), + state_version: required!(state_version), apis: 0, }) } @@ -210,7 +228,6 @@ fn generate_emit_link_section_decl(contents: &[u8], section_name: &str) -> Token #[cfg(test)] mod tests { use super::*; - use codec::DecodeAll; use std::borrow::Cow; #[test] @@ -223,11 +240,13 @@ mod tests { impl_version: 1, apis: 0, transaction_version: 2, + state_version: 1, } .encode(); assert_eq!( - sp_version::RuntimeVersion::decode_all(&mut &version_bytes[..]).unwrap(), + sp_version::RuntimeVersion::decode_with_version_hint(&mut &version_bytes[..], Some(4)) + .unwrap(), sp_version::RuntimeVersion { spec_name: "hello".into(), impl_name: "world".into(), @@ -236,6 +255,7 @@ mod tests { impl_version: 1, apis: Cow::Owned(vec![]), transaction_version: 2, + state_version: 1, }, ); } diff --git a/primitives/version/proc-macro/src/lib.rs b/primitives/version/proc-macro/src/lib.rs index 9a6d4d60bbf9..8be18b15868f 100644 --- a/primitives/version/proc-macro/src/lib.rs +++ b/primitives/version/proc-macro/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/version/src/embed.rs b/primitives/version/src/embed.rs index 452762dcf687..e6b468e2e58c 100644 --- a/primitives/version/src/embed.rs +++ b/primitives/version/src/embed.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/primitives/version/src/lib.rs b/primitives/version/src/lib.rs index 58216bc494dd..46f8af9e22c5 100644 --- a/primitives/version/src/lib.rs +++ b/primitives/version/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +15,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Version module for the Substrate runtime; Provides a function that returns the runtime version. +//! Substrate runtime version +//! +//! Each runtime that should be executed by a Substrate based node needs to have a runtime version. +//! The runtime version is defined by [`RuntimeVersion`]. The runtime version is used to +//! distinguish different runtimes. The most important field is the +//! [`spec_version`](RuntimeVersion::spec_version). The `spec_version` should be increased in a +//! runtime when a new runtime build includes breaking changes that would make other runtimes unable +//! to import blocks built by this runtime or vice-versa, where the new runtime could not import +//! blocks built by the old runtime. The runtime version also carries other version information +//! about the runtime, see [`RuntimeVersion`] for more information on this. +//! +//! Substrate will fetch the runtime version from a `wasm` blob by first checking the +//! `runtime_version` link section or calling the `Core::version` runtime api. The link section can +//! be generated in the runtime using the [`runtime_version`] attribute. The `Core` runtime api also +//! needs to be implemented for the runtime using `impl_runtime_apis!`. #![cfg_attr(not(feature = "std"), no_std)] @@ -26,10 +40,10 @@ use std::collections::HashSet; #[cfg(feature = "std")] use std::fmt; -use codec::{Decode, Encode}; +use codec::{Decode, Encode, Input}; use scale_info::TypeInfo; -pub use sp_runtime::create_runtime_str; use sp_runtime::RuntimeString; +pub use sp_runtime::{create_runtime_str, StateVersion}; #[doc(hidden)] pub use sp_std; @@ -65,6 +79,7 @@ pub mod embed; /// impl_version: 1, /// apis: RUNTIME_API_VERSIONS, /// transaction_version: 2, +/// state_version: 1, /// }; /// /// # const RUNTIME_API_VERSIONS: sp_version::ApisVec = sp_version::create_apis_vec!([]); @@ -104,7 +119,23 @@ pub use sp_version_proc_macro::runtime_version; /// The id is generated by hashing the name of the runtime api with BLAKE2 using a hash size /// of 8 bytes. /// -/// The name of the runtime api is the name of the trait when using `decl_runtime_apis!` macro. +/// The name of the runtime api is the name of the trait when using `decl_runtime_apis!` macro. So, +/// in the following runtime api declaration: +/// +/// ```nocompile +/// decl_runtime_apis! { +/// trait TestApi { +/// fn do_test(); +/// } +/// } +/// ``` +/// +/// The name of the trait would be `TestApi` and would be taken as input to the BLAKE2 hash +/// function. +/// +/// As Rust supports renaming of traits, the name of a runtime api given to `impl_runtime_apis!` +/// doesn't need to be the same as in `decl_runtime_apis!`, but only the name in +/// `decl_runtime_apis!` is the important one! pub type ApiId = [u8; 8]; /// A vector of pairs of `ApiId` and a `u32` for version. @@ -124,7 +155,7 @@ macro_rules! create_apis_vec { /// In particular: bug fixes should result in an increment of `spec_version` and possibly /// `authoring_version`, absolutely not `impl_version` since they change the semantics of the /// runtime. -#[derive(Clone, PartialEq, Eq, Encode, Decode, Default, sp_runtime::RuntimeDebug, TypeInfo)] +#[derive(Clone, PartialEq, Eq, Encode, Default, sp_runtime::RuntimeDebug, TypeInfo)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] #[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] pub struct RuntimeVersion { @@ -177,6 +208,53 @@ pub struct RuntimeVersion { /// /// It need *not* change when a new module is added or when a dispatchable is added. pub transaction_version: u32, + + /// Version of the state implementation used by this runtime. + /// Use of an incorrect version is consensus breaking. + pub state_version: u8, +} + +impl RuntimeVersion { + /// `Decode` while giving a "version hint" + /// + /// There exists multiple versions of [`RuntimeVersion`] and they are versioned using the `Core` + /// runtime api: + /// - `Core` version < 3 is a runtime version without a transaction version and state version. + /// - `Core` version 3 is a runtime version without a state version. + /// - `Core` version 4 is the latest runtime version. + pub fn decode_with_version_hint( + input: &mut I, + core_version: Option, + ) -> Result { + let spec_name = Decode::decode(input)?; + let impl_name = Decode::decode(input)?; + let authoring_version = Decode::decode(input)?; + let spec_version = Decode::decode(input)?; + let impl_version = Decode::decode(input)?; + let apis = Decode::decode(input)?; + let core_version = + if core_version.is_some() { core_version } else { core_version_from_apis(&apis) }; + let transaction_version = + if core_version.map(|v| v >= 3).unwrap_or(false) { Decode::decode(input)? } else { 1 }; + let state_version = + if core_version.map(|v| v >= 4).unwrap_or(false) { Decode::decode(input)? } else { 0 }; + Ok(RuntimeVersion { + spec_name, + impl_name, + authoring_version, + spec_version, + impl_version, + apis, + transaction_version, + state_version, + }) + } +} + +impl Decode for RuntimeVersion { + fn decode(input: &mut I) -> Result { + Self::decode_with_version_hint(input, None) + } } #[cfg(feature = "std")] @@ -195,6 +273,16 @@ impl fmt::Display for RuntimeVersion { } } +fn has_api_with bool>(apis: &ApisVec, id: &ApiId, predicate: P) -> bool { + apis.iter().any(|(s, v)| s == id && predicate(*v)) +} + +/// Returns the version of the `Core` runtime api. +pub fn core_version_from_apis(apis: &ApisVec) -> Option { + let id = sp_core_hashing_proc_macro::blake2b_64!(b"Core"); + apis.iter().find(|(s, _v)| s == &id).map(|(_s, v)| *v) +} + #[cfg(feature = "std")] impl RuntimeVersion { /// Check if this version matches other version for calling into runtime. @@ -207,7 +295,7 @@ impl RuntimeVersion { /// Check if the given api with `api_id` is implemented and the version passes the given /// `predicate`. pub fn has_api_with bool>(&self, id: &ApiId, predicate: P) -> bool { - self.apis.iter().any(|(s, v)| s == id && predicate(*v)) + has_api_with(&self.apis, id, predicate) } /// Returns the api version found for api with `id`. @@ -216,12 +304,28 @@ impl RuntimeVersion { } } -#[cfg(feature = "std")] +impl RuntimeVersion { + /// Returns state version to use for update. + /// + /// For runtime with core api version less than 4, + /// V0 trie version will be applied to state. + /// Otherwhise, V1 trie version will be use. + pub fn state_version(&self) -> StateVersion { + // If version > than 1, keep using latest version. + self.state_version.try_into().unwrap_or(StateVersion::V1) + } +} + +/// The version of the native runtime. +/// +/// In contrast to the bare [`RuntimeVersion`] this also carries a list of `spec_version`s of +/// runtimes this native runtime can be used to author blocks for. #[derive(Debug)] +#[cfg(feature = "std")] pub struct NativeVersion { /// Basic runtime version info. pub runtime_version: RuntimeVersion, - /// Authoring runtimes that this native runtime supports. + /// Authoring runtimes (`spec_version`s) that this native runtime supports. pub can_author_with: HashSet, } diff --git a/primitives/wasm-interface/Cargo.toml b/primitives/wasm-interface/Cargo.toml index ba8a7b4e4b46..5fa0627490a2 100644 --- a/primitives/wasm-interface/Cargo.toml +++ b/primitives/wasm-interface/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "sp-wasm-interface" -version = "4.0.0-dev" +version = "6.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Types and traits for interfacing between the host and the wasm runtime." documentation = "https://docs.rs/sp-wasm-interface" @@ -14,11 +14,13 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -wasmi = { version = "0.9.0", optional = true } -impl-trait-for-tuples = "0.2.1" -sp-std = { version = "4.0.0-dev", path = "../std", default-features = false } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +wasmi = { version = "0.9.1", optional = true } +wasmtime = { version = "0.33.0", optional = true, default-features = false } +log = { version = "0.4.14", optional = true } +impl-trait-for-tuples = "0.2.2" +sp-std = { version = "4.0.0", path = "../std", default-features = false } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } [features] default = [ "std" ] -std = [ "wasmi", "sp-std/std", "codec/std" ] +std = [ "wasmi", "sp-std/std", "codec/std", "log" ] diff --git a/primitives/wasm-interface/src/lib.rs b/primitives/wasm-interface/src/lib.rs index e1903ef425ae..6dfc3116ddc4 100644 --- a/primitives/wasm-interface/src/lib.rs +++ b/primitives/wasm-interface/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,6 +24,25 @@ use sp_std::{borrow::Cow, iter::Iterator, marker::PhantomData, mem, result, vec, #[cfg(feature = "std")] mod wasmi_impl; +#[cfg(not(all(feature = "std", feature = "wasmtime")))] +#[macro_export] +macro_rules! if_wasmtime_is_enabled { + ($($token:tt)*) => {}; +} + +#[cfg(all(feature = "std", feature = "wasmtime"))] +#[macro_export] +macro_rules! if_wasmtime_is_enabled { + ($($token:tt)*) => { + $($token)* + } +} + +if_wasmtime_is_enabled! { + // Reexport wasmtime so that its types are accessible from the procedural macro. + pub use wasmtime; +} + /// Result type used by traits in this crate. #[cfg(feature = "std")] pub type Result = result::Result; @@ -54,7 +73,7 @@ impl From for u8 { } } -impl sp_std::convert::TryFrom for ValueType { +impl TryFrom for ValueType { type Error = (); fn try_from(val: u8) -> sp_std::result::Result { @@ -105,7 +124,8 @@ impl Value { } } -/// Provides `Sealed` trait to prevent implementing trait `PointerType` outside of this crate. +/// Provides `Sealed` trait to prevent implementing trait `PointerType` and `WasmTy` outside of this +/// crate. mod private { pub trait Sealed {} @@ -113,6 +133,9 @@ mod private { impl Sealed for u16 {} impl Sealed for u32 {} impl Sealed for u64 {} + + impl Sealed for i32 {} + impl Sealed for i64 {} } /// Something that can be wrapped in a wasm `Pointer`. @@ -282,6 +305,29 @@ pub trait FunctionContext { fn deallocate_memory(&mut self, ptr: Pointer) -> Result<()>; /// Provides access to the sandbox. fn sandbox(&mut self) -> &mut dyn Sandbox; + + /// Registers a panic error message within the executor. + /// + /// This is meant to be used in situations where the runtime + /// encounters an unrecoverable error and intends to panic. + /// + /// Panicking in WASM is done through the [`unreachable`](https://webassembly.github.io/spec/core/syntax/instructions.html#syntax-instr-control) + /// instruction which causes an unconditional trap and immediately aborts + /// the execution. It does not however allow for any diagnostics to be + /// passed through to the host, so while we do know that *something* went + /// wrong we don't have any direct indication of what *exactly* went wrong. + /// + /// As a workaround we use this method right before the execution is + /// actually aborted to pass an error message to the host so that it + /// can associate it with the next trap, and return that to the caller. + /// + /// A WASM trap should be triggered immediately after calling this method; + /// otherwise the error message might be associated with a completely + /// unrelated trap. + /// + /// It should only be called once, however calling it more than once + /// is harmless and will overwrite the previously set error message. + fn register_panic_error_message(&mut self, message: &str); } /// Sandbox memory identifier. @@ -338,10 +384,48 @@ pub trait Sandbox { fn get_global_val(&self, instance_idx: u32, name: &str) -> Result>; } +if_wasmtime_is_enabled! { + /// A trait used to statically register host callbacks with the WASM executor, + /// so that they call be called from within the runtime with minimal overhead. + /// + /// This is used internally to interface the wasmtime-based executor with the + /// host functions' definitions generated through the runtime interface macro, + /// and is not meant to be used directly. + pub trait HostFunctionRegistry { + type State; + type Error; + type FunctionContext: FunctionContext; + + /// Wraps the given `caller` in a type which implements `FunctionContext` + /// and calls the given `callback`. + fn with_function_context( + caller: wasmtime::Caller, + callback: impl FnOnce(&mut dyn FunctionContext) -> R, + ) -> R; + + /// Registers a given host function with the WASM executor. + /// + /// The function has to be statically callable, and all of its arguments + /// and its return value have to be compatible with WASM FFI. + fn register_static( + &mut self, + fn_name: &str, + func: impl wasmtime::IntoFunc + 'static, + ) -> core::result::Result<(), Self::Error>; + } +} + /// Something that provides implementations for host functions. -pub trait HostFunctions: 'static { +pub trait HostFunctions: 'static + Send + Sync { /// Returns the host functions `Self` provides. fn host_functions() -> Vec<&'static dyn Function>; + + if_wasmtime_is_enabled! { + /// Statically registers the host functions. + fn register_static(registry: &mut T) -> core::result::Result<(), T::Error> + where + T: HostFunctionRegistry; + } } #[impl_trait_for_tuples::impl_for_tuples(30)] @@ -353,8 +437,146 @@ impl HostFunctions for Tuple { host_functions } + + #[cfg(all(feature = "std", feature = "wasmtime"))] + fn register_static(registry: &mut T) -> core::result::Result<(), T::Error> + where + T: HostFunctionRegistry, + { + for_tuples!( + #( Tuple::register_static(registry)?; )* + ); + + Ok(()) + } } +/// A wrapper which merges two sets of host functions, and allows the second set to override +/// the host functions from the first set. +pub struct ExtendedHostFunctions { + phantom: PhantomData<(Base, Overlay)>, +} + +impl HostFunctions for ExtendedHostFunctions +where + Base: HostFunctions, + Overlay: HostFunctions, +{ + fn host_functions() -> Vec<&'static dyn Function> { + let mut base = Base::host_functions(); + let overlay = Overlay::host_functions(); + base.retain(|host_fn| { + !overlay.iter().any(|ext_host_fn| host_fn.name() == ext_host_fn.name()) + }); + base.extend(overlay); + base + } + + if_wasmtime_is_enabled! { + fn register_static(registry: &mut T) -> core::result::Result<(), T::Error> + where + T: HostFunctionRegistry, + { + struct Proxy<'a, T> { + registry: &'a mut T, + seen_overlay: std::collections::HashSet, + seen_base: std::collections::HashSet, + overlay_registered: bool, + } + + impl<'a, T> HostFunctionRegistry for Proxy<'a, T> + where + T: HostFunctionRegistry, + { + type State = T::State; + type Error = T::Error; + type FunctionContext = T::FunctionContext; + + fn with_function_context( + caller: wasmtime::Caller, + callback: impl FnOnce(&mut dyn FunctionContext) -> R, + ) -> R { + T::with_function_context(caller, callback) + } + + fn register_static( + &mut self, + fn_name: &str, + func: impl wasmtime::IntoFunc + 'static, + ) -> core::result::Result<(), Self::Error> { + if self.overlay_registered { + if !self.seen_base.insert(fn_name.to_owned()) { + log::warn!( + target: "extended_host_functions", + "Duplicate base host function: '{}'", + fn_name, + ); + + // TODO: Return an error here? + return Ok(()) + } + + if self.seen_overlay.contains(fn_name) { + // Was already registered when we went through the overlay, so just ignore it. + log::debug!( + target: "extended_host_functions", + "Overriding base host function: '{}'", + fn_name, + ); + + return Ok(()) + } + } else if !self.seen_overlay.insert(fn_name.to_owned()) { + log::warn!( + target: "extended_host_functions", + "Duplicate overlay host function: '{}'", + fn_name, + ); + + // TODO: Return an error here? + return Ok(()) + } + + self.registry.register_static(fn_name, func) + } + } + + let mut proxy = Proxy { + registry, + seen_overlay: Default::default(), + seen_base: Default::default(), + overlay_registered: false, + }; + + // The functions from the `Overlay` can override those from the `Base`, + // so `Overlay` is registered first, and then we skip those functions + // in `Base` if they were already registered from the `Overlay`. + Overlay::register_static(&mut proxy)?; + proxy.overlay_registered = true; + Base::register_static(&mut proxy)?; + + Ok(()) + } + } +} + +/// A trait for types directly usable at the WASM FFI boundary without any conversion at all. +/// +/// This trait is sealed and should not be implemented downstream. +#[cfg(all(feature = "std", feature = "wasmtime"))] +pub trait WasmTy: wasmtime::WasmTy + private::Sealed {} + +/// A trait for types directly usable at the WASM FFI boundary without any conversion at all. +/// +/// This trait is sealed and should not be implemented downstream. +#[cfg(not(all(feature = "std", feature = "wasmtime")))] +pub trait WasmTy: private::Sealed {} + +impl WasmTy for i32 {} +impl WasmTy for u32 {} +impl WasmTy for i64 {} +impl WasmTy for u64 {} + /// Something that can be converted into a wasm compatible `Value`. pub trait IntoValue { /// The type of the value in wasm. diff --git a/primitives/wasm-interface/src/wasmi_impl.rs b/primitives/wasm-interface/src/wasmi_impl.rs index f7e0ec6f16d4..39afce4df4eb 100644 --- a/primitives/wasm-interface/src/wasmi_impl.rs +++ b/primitives/wasm-interface/src/wasmi_impl.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/.maintain/common/lib.sh b/scripts/ci/common/lib.sh similarity index 100% rename from .maintain/common/lib.sh rename to scripts/ci/common/lib.sh diff --git a/.maintain/deny.toml b/scripts/ci/deny.toml similarity index 100% rename from .maintain/deny.toml rename to scripts/ci/deny.toml diff --git a/.maintain/docker/subkey.Dockerfile b/scripts/ci/docker/subkey.Dockerfile similarity index 93% rename from .maintain/docker/subkey.Dockerfile rename to scripts/ci/docker/subkey.Dockerfile index 5797295806d0..3483502845cf 100644 --- a/.maintain/docker/subkey.Dockerfile +++ b/scripts/ci/docker/subkey.Dockerfile @@ -8,7 +8,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="parity/subkey" \ io.parity.image.description="Subkey: key generating utility for Substrate." \ - io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/.maintain/docker/subkey.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/ci/docker/subkey.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/substrate/tree/${VCS_REF}/subkey" diff --git a/.maintain/docker/substrate.Dockerfile b/scripts/ci/docker/substrate.Dockerfile similarity index 96% rename from .maintain/docker/substrate.Dockerfile rename to scripts/ci/docker/substrate.Dockerfile index e13dfb426adf..b4c103ed5244 100644 --- a/.maintain/docker/substrate.Dockerfile +++ b/scripts/ci/docker/substrate.Dockerfile @@ -8,7 +8,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="parity/substrate" \ io.parity.image.description="Substrate: The platform for blockchain innovators." \ - io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/.maintain/docker/Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/ci/docker/Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://wiki.parity.io/Parity-Substrate" diff --git a/.maintain/github/check_labels.sh b/scripts/ci/github/check_labels.sh similarity index 100% rename from .maintain/github/check_labels.sh rename to scripts/ci/github/check_labels.sh diff --git a/.maintain/gitlab/generate_changelog.sh b/scripts/ci/github/generate_changelog.sh similarity index 100% rename from .maintain/gitlab/generate_changelog.sh rename to scripts/ci/github/generate_changelog.sh diff --git a/.maintain/gitlab/check_runtime.sh b/scripts/ci/gitlab/check_runtime.sh similarity index 100% rename from .maintain/gitlab/check_runtime.sh rename to scripts/ci/gitlab/check_runtime.sh diff --git a/.maintain/gitlab/check_signed.sh b/scripts/ci/gitlab/check_signed.sh similarity index 100% rename from .maintain/gitlab/check_signed.sh rename to scripts/ci/gitlab/check_signed.sh diff --git a/.maintain/ensure-deps.sh b/scripts/ci/gitlab/ensure-deps.sh similarity index 100% rename from .maintain/ensure-deps.sh rename to scripts/ci/gitlab/ensure-deps.sh diff --git a/.maintain/gitlab/publish_draft_release.sh b/scripts/ci/gitlab/publish_draft_release.sh similarity index 100% rename from .maintain/gitlab/publish_draft_release.sh rename to scripts/ci/gitlab/publish_draft_release.sh diff --git a/.maintain/gitlab/skip_if_draft.sh b/scripts/ci/gitlab/skip_if_draft.sh similarity index 100% rename from .maintain/gitlab/skip_if_draft.sh rename to scripts/ci/gitlab/skip_if_draft.sh diff --git a/.maintain/monitoring/alerting-rules/alerting-rule-tests.yaml b/scripts/ci/monitoring/alerting-rules/alerting-rule-tests.yaml similarity index 61% rename from .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml rename to scripts/ci/monitoring/alerting-rules/alerting-rule-tests.yaml index 7ad916f02215..df5e020d067e 100644 --- a/.maintain/monitoring/alerting-rules/alerting-rule-tests.yaml +++ b/scripts/ci/monitoring/alerting-rules/alerting-rule-tests.yaml @@ -6,39 +6,39 @@ evaluation_interval: 1m tests: - interval: 1m input_series: - - series: 'polkadot_sub_libp2p_peers_count{ - job="polkadot", - pod="polkadot-abcdef01234-abcdef", - instance="polkadot-abcdef01234-abcdef", + - series: 'substrate_sub_libp2p_peers_count{ + job="substrate", + pod="substrate-abcdef01234-abcdef", + instance="substrate-abcdef01234-abcdef", }' values: '3 2+0x4 1+0x9' # 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 - - series: 'polkadot_sub_txpool_validations_scheduled{ - job="polkadot", - pod="polkadot-abcdef01234-abcdef", - instance="polkadot-abcdef01234-abcdef", + - series: 'substrate_sub_txpool_validations_scheduled{ + job="substrate", + pod="substrate-abcdef01234-abcdef", + instance="substrate-abcdef01234-abcdef", }' values: '11+1x10 22+2x30 10043x5' - - series: 'polkadot_sub_txpool_validations_finished{ - job="polkadot", - pod="polkadot-abcdef01234-abcdef", - instance="polkadot-abcdef01234-abcdef", + - series: 'substrate_sub_txpool_validations_finished{ + job="substrate", + pod="substrate-abcdef01234-abcdef", + instance="substrate-abcdef01234-abcdef", }' values: '0+1x42 42x5' - - series: 'polkadot_block_height{ - status="best", job="polkadot", - pod="polkadot-abcdef01234-abcdef", - instance="polkadot-abcdef01234-abcdef", + - series: 'substrate_block_height{ + status="best", job="substrate", + pod="substrate-abcdef01234-abcdef", + instance="substrate-abcdef01234-abcdef", }' values: '1+1x3 4+0x13' # 1 2 3 4 4 4 4 4 4 4 4 4 ... - - series: 'polkadot_block_height{ + - series: 'substrate_block_height{ status="finalized", - job="polkadot", - pod="polkadot-abcdef01234-abcdef", - instance="polkadot-abcdef01234-abcdef", + job="substrate", + pod="substrate-abcdef01234-abcdef", + instance="substrate-abcdef01234-abcdef", }' values: '1+1x3 4+0x13' # 1 2 3 4 4 4 4 4 4 4 4 4 ... @@ -56,13 +56,13 @@ tests: exp_alerts: - exp_labels: severity: warning - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate status: best exp_annotations: message: "Best block on instance - polkadot-abcdef01234-abcdef increases by less than 1 per + substrate-abcdef01234-abcdef increases by less than 1 per minute for more than 3 minutes." - eval_time: 14m @@ -70,23 +70,23 @@ tests: exp_alerts: - exp_labels: severity: warning - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate status: best exp_annotations: message: "Best block on instance - polkadot-abcdef01234-abcdef increases by less than 1 per + substrate-abcdef01234-abcdef increases by less than 1 per minute for more than 3 minutes." - exp_labels: severity: critical - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate status: best exp_annotations: message: "Best block on instance - polkadot-abcdef01234-abcdef increases by less than 1 per + substrate-abcdef01234-abcdef increases by less than 1 per minute for more than 10 minutes." ###################################################################### @@ -101,13 +101,13 @@ tests: exp_alerts: - exp_labels: severity: warning - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate status: finalized exp_annotations: message: "Finalized block on instance - polkadot-abcdef01234-abcdef increases by less than 1 per + substrate-abcdef01234-abcdef increases by less than 1 per minute for more than 3 minutes." - eval_time: 14m @@ -115,23 +115,23 @@ tests: exp_alerts: - exp_labels: severity: warning - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate status: finalized exp_annotations: message: "Finalized block on instance - polkadot-abcdef01234-abcdef increases by less than 1 per + substrate-abcdef01234-abcdef increases by less than 1 per minute for more than 3 minutes." - exp_labels: severity: critical - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate status: finalized exp_annotations: message: "Finalized block on instance - polkadot-abcdef01234-abcdef increases by less than 1 per + substrate-abcdef01234-abcdef increases by less than 1 per minute for more than 10 minutes." ###################################################################### @@ -152,12 +152,12 @@ tests: exp_alerts: - exp_labels: severity: warning - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate exp_annotations: message: "The transaction pool size on node - polkadot-abcdef01234-abcdef has been monotonically + substrate-abcdef01234-abcdef has been monotonically increasing for more than 10 minutes." - eval_time: 43m alertname: TransactionQueueSizeIncreasing @@ -167,21 +167,21 @@ tests: exp_alerts: - exp_labels: severity: warning - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate exp_annotations: message: "The transaction pool size on node - polkadot-abcdef01234-abcdef has been monotonically + substrate-abcdef01234-abcdef has been monotonically increasing for more than 10 minutes." - exp_labels: severity: warning - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate exp_annotations: message: "The transaction pool size on node - polkadot-abcdef01234-abcdef has been monotonically + substrate-abcdef01234-abcdef has been monotonically increasing for more than 30 minutes." - eval_time: 49m alertname: TransactionQueueSizeHigh @@ -191,12 +191,12 @@ tests: exp_alerts: - exp_labels: severity: warning - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate exp_annotations: message: "The transaction pool size on node - polkadot-abcdef01234-abcdef has been above 10_000 for more + substrate-abcdef01234-abcdef has been above 10_000 for more than 5 minutes." ###################################################################### @@ -211,11 +211,11 @@ tests: exp_alerts: - exp_labels: severity: warning - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate exp_annotations: - message: "The node polkadot-abcdef01234-abcdef has less + message: "The node substrate-abcdef01234-abcdef has less than 3 peers for more than 3 minutes" - eval_time: 16m # Values: 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 @@ -223,17 +223,17 @@ tests: exp_alerts: - exp_labels: severity: warning - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate exp_annotations: - message: "The node polkadot-abcdef01234-abcdef has less + message: "The node substrate-abcdef01234-abcdef has less than 3 peers for more than 3 minutes" - exp_labels: severity: critical - pod: polkadot-abcdef01234-abcdef - instance: polkadot-abcdef01234-abcdef - job: polkadot + pod: substrate-abcdef01234-abcdef + instance: substrate-abcdef01234-abcdef + job: substrate exp_annotations: - message: "The node polkadot-abcdef01234-abcdef has less + message: "The node substrate-abcdef01234-abcdef has less than 3 peers for more than 15 minutes" diff --git a/.maintain/monitoring/alerting-rules/alerting-rules.yaml b/scripts/ci/monitoring/alerting-rules/alerting-rules.yaml similarity index 70% rename from .maintain/monitoring/alerting-rules/alerting-rules.yaml rename to scripts/ci/monitoring/alerting-rules/alerting-rules.yaml index 7a69cba66c3f..4171f92f68fe 100644 --- a/.maintain/monitoring/alerting-rules/alerting-rules.yaml +++ b/scripts/ci/monitoring/alerting-rules/alerting-rules.yaml @@ -1,5 +1,5 @@ groups: -- name: polkadot.rules +- name: substrate.rules rules: ############################################################################## @@ -10,7 +10,7 @@ groups: annotations: message: 'Best block on instance {{ $labels.instance }} increases by less than 1 per minute for more than 3 minutes.' - expr: increase(polkadot_block_height{status="best"}[1m]) < 1 + expr: increase(substrate_block_height{status="best"}[1m]) < 1 for: 3m labels: severity: warning @@ -18,7 +18,7 @@ groups: annotations: message: 'Best block on instance {{ $labels.instance }} increases by less than 1 per minute for more than 10 minutes.' - expr: increase(polkadot_block_height{status="best"}[1m]) < 1 + expr: increase(substrate_block_height{status="best"}[1m]) < 1 for: 10m labels: severity: critical @@ -28,7 +28,7 @@ groups: ############################################################################## - alert: BlockFinalizationSlow - expr: increase(polkadot_block_height{status="finalized"}[1m]) < 1 + expr: increase(substrate_block_height{status="finalized"}[1m]) < 1 for: 3m labels: severity: warning @@ -36,7 +36,7 @@ groups: message: 'Finalized block on instance {{ $labels.instance }} increases by less than 1 per minute for more than 3 minutes.' - alert: BlockFinalizationSlow - expr: increase(polkadot_block_height{status="finalized"}[1m]) < 1 + expr: increase(substrate_block_height{status="finalized"}[1m]) < 1 for: 10m labels: severity: critical @@ -47,8 +47,8 @@ groups: # Under the assumption of an average block production of 6 seconds, # "best" and "finalized" being more than 10 blocks apart would imply # more than a 1 minute delay between block production and finalization. - expr: '(polkadot_block_height{status="best"} - ignoring(status) - polkadot_block_height{status="finalized"}) > 10' + expr: '(substrate_block_height{status="best"} - ignoring(status) + substrate_block_height{status="finalized"}) > 10' for: 8m labels: severity: critical @@ -61,8 +61,8 @@ groups: ############################################################################## - alert: TransactionQueueSizeIncreasing - expr: 'increase(polkadot_sub_txpool_validations_scheduled[5m]) - - increase(polkadot_sub_txpool_validations_finished[5m]) > 0' + expr: 'increase(substrate_sub_txpool_validations_scheduled[5m]) - + increase(substrate_sub_txpool_validations_finished[5m]) > 0' for: 10m labels: severity: warning @@ -70,8 +70,8 @@ groups: message: 'The transaction pool size on node {{ $labels.instance }} has been monotonically increasing for more than 10 minutes.' - alert: TransactionQueueSizeIncreasing - expr: 'increase(polkadot_sub_txpool_validations_scheduled[5m]) - - increase(polkadot_sub_txpool_validations_finished[5m]) > 0' + expr: 'increase(substrate_sub_txpool_validations_scheduled[5m]) - + increase(substrate_sub_txpool_validations_finished[5m]) > 0' for: 30m labels: severity: warning @@ -79,8 +79,8 @@ groups: message: 'The transaction pool size on node {{ $labels.instance }} has been monotonically increasing for more than 30 minutes.' - alert: TransactionQueueSizeHigh - expr: 'polkadot_sub_txpool_validations_scheduled - - polkadot_sub_txpool_validations_finished > 10000' + expr: 'substrate_sub_txpool_validations_scheduled - + substrate_sub_txpool_validations_finished > 10000' for: 5m labels: severity: warning @@ -93,7 +93,7 @@ groups: ############################################################################## - alert: NumberOfPeersLow - expr: polkadot_sub_libp2p_peers_count < 3 + expr: substrate_sub_libp2p_peers_count < 3 for: 3m labels: severity: warning @@ -101,7 +101,7 @@ groups: message: 'The node {{ $labels.instance }} has less than 3 peers for more than 3 minutes' - alert: NumberOfPeersLow - expr: polkadot_sub_libp2p_peers_count < 3 + expr: substrate_sub_libp2p_peers_count < 3 for: 15m labels: severity: critical @@ -109,7 +109,7 @@ groups: message: 'The node {{ $labels.instance }} has less than 3 peers for more than 15 minutes' - alert: NoIncomingConnection - expr: increase(polkadot_sub_libp2p_incoming_connections_total[20m]) == 0 + expr: increase(substrate_sub_libp2p_incoming_connections_total[20m]) == 0 labels: severity: warning annotations: @@ -121,7 +121,7 @@ groups: ############################################################################## - alert: NumberOfFileDescriptorsHigh - expr: 'node_filefd_allocated{domain=~"kusama|polkadot"} > 10000' + expr: 'node_filefd_allocated{chain!=""} > 10000' for: 3m labels: severity: warning @@ -133,20 +133,10 @@ groups: # Others ############################################################################## - - alert: ContinuousTaskEnded - expr: '(polkadot_tasks_spawned_total{task_name != "basic-authorship-proposer", task_name != "substrate-rpc-subscription"} == 1) - - on(instance, task_name) group_left() (polkadot_tasks_ended_total == 1)' - for: 5m - labels: - severity: warning - annotations: - message: 'Continuous task {{ $labels.task_name }} on node - {{ $labels.instance }} ended unexpectedly.' - - alert: AuthorityDiscoveryDiscoveryFailureHigh - expr: 'polkadot_authority_discovery_handle_value_found_event_failure / + expr: 'substrate_authority_discovery_handle_value_found_event_failure / ignoring(name) - polkadot_authority_discovery_dht_event_received{name="value_found"} > 0.5' + substrate_authority_discovery_dht_event_received{name="value_found"} > 0.5' for: 2h labels: severity: warning @@ -157,9 +147,9 @@ groups: - alert: UnboundedChannelPersistentlyLarge expr: '( - (polkadot_unbounded_channel_len{action = "send"} - - ignoring(action) polkadot_unbounded_channel_len{action = "received"}) - or on(instance) polkadot_unbounded_channel_len{action = "send"} + (substrate_unbounded_channel_len{action = "send"} - + ignoring(action) substrate_unbounded_channel_len{action = "received"}) + or on(instance) substrate_unbounded_channel_len{action = "send"} ) >= 200' for: 5m labels: @@ -170,9 +160,9 @@ groups: - alert: UnboundedChannelVeryLarge expr: '( - (polkadot_unbounded_channel_len{action = "send"} - - ignoring(action) polkadot_unbounded_channel_len{action = "received"}) - or on(instance) polkadot_unbounded_channel_len{action = "send"} + (substrate_unbounded_channel_len{action = "send"} - + ignoring(action) substrate_unbounded_channel_len{action = "received"}) + or on(instance) substrate_unbounded_channel_len{action = "send"} ) > 15000' labels: severity: warning diff --git a/.maintain/monitoring/grafana-dashboards/README_dashboard.md b/scripts/ci/monitoring/grafana-dashboards/README_dashboard.md similarity index 73% rename from .maintain/monitoring/grafana-dashboards/README_dashboard.md rename to scripts/ci/monitoring/grafana-dashboards/README_dashboard.md index e00b89449cfa..50f54a107e93 100644 --- a/.maintain/monitoring/grafana-dashboards/README_dashboard.md +++ b/scripts/ci/monitoring/grafana-dashboards/README_dashboard.md @@ -4,4 +4,4 @@ Shared templated Grafana dashboards. To import the dashboards follow the [Grafana documentation](https://grafana.com/docs/grafana/latest/reference/export_import/). -You can see an example setup [here](../../../.maintain/sentry-node). +You can see an example setup [here](./substrate-networking.json). diff --git a/.maintain/monitoring/grafana-dashboards/substrate-networking.json b/scripts/ci/monitoring/grafana-dashboards/substrate-networking.json similarity index 99% rename from .maintain/monitoring/grafana-dashboards/substrate-networking.json rename to scripts/ci/monitoring/grafana-dashboards/substrate-networking.json index 46942cf582fc..abd675ed13ec 100644 --- a/.maintain/monitoring/grafana-dashboards/substrate-networking.json +++ b/scripts/ci/monitoring/grafana-dashboards/substrate-networking.json @@ -4,7 +4,7 @@ "name": "VAR_METRIC_NAMESPACE", "type": "constant", "label": "Prefix of the metrics", - "value": "polkadot", + "value": "substrate", "description": "" } ], diff --git a/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json b/scripts/ci/monitoring/grafana-dashboards/substrate-service-tasks.json similarity index 99% rename from .maintain/monitoring/grafana-dashboards/substrate-service-tasks.json rename to scripts/ci/monitoring/grafana-dashboards/substrate-service-tasks.json index 2f08ac7bb34c..ce7e9f78cd8a 100644 --- a/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json +++ b/scripts/ci/monitoring/grafana-dashboards/substrate-service-tasks.json @@ -4,7 +4,7 @@ "name": "VAR_METRIC_NAMESPACE", "type": "constant", "label": "Prefix of the metrics", - "value": "polkadot", + "value": "substrate", "description": "" } ], diff --git a/.maintain/node-template-release.sh b/scripts/ci/node-template-release.sh similarity index 84% rename from .maintain/node-template-release.sh rename to scripts/ci/node-template-release.sh index cb5e72e7fa98..09ef98e04627 100755 --- a/.maintain/node-template-release.sh +++ b/scripts/ci/node-template-release.sh @@ -11,6 +11,6 @@ if [ "$#" -ne 1 ]; then fi PATH_TO_ARCHIVE=$1 -cd $PROJECT_ROOT/.maintain/node-template-release +cd $PROJECT_ROOT/scripts/ci/node-template-release cargo run $PROJECT_ROOT/bin/node-template $PROJECT_ROOT/$PATH_TO_ARCHIVE diff --git a/.maintain/node-template-release/Cargo.toml b/scripts/ci/node-template-release/Cargo.toml similarity index 82% rename from .maintain/node-template-release/Cargo.toml rename to scripts/ci/node-template-release/Cargo.toml index c1d9f2da7fae..667281f6dcad 100644 --- a/.maintain/node-template-release/Cargo.toml +++ b/scripts/ci/node-template-release/Cargo.toml @@ -2,14 +2,14 @@ name = "node-template-release" version = "3.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "GPL-3.0" [dependencies] toml = "0.4" tar = "0.4" glob = "0.2" -structopt = "0.3" +clap = { version = "3.0", features = ["derive"] } tempfile = "3" fs_extra = "1" git2 = "0.8" diff --git a/.maintain/node-template-release/src/main.rs b/scripts/ci/node-template-release/src/main.rs similarity index 97% rename from .maintain/node-template-release/src/main.rs rename to scripts/ci/node-template-release/src/main.rs index 7dcb1f0f4d81..62e9b6671576 100644 --- a/.maintain/node-template-release/src/main.rs +++ b/scripts/ci/node-template-release/src/main.rs @@ -1,4 +1,4 @@ -use structopt::StructOpt; +use clap::Parser; use std::{ collections::HashMap, @@ -26,13 +26,13 @@ const SUBSTRATE_GIT_URL: &str = "https://github.com/paritytech/substrate.git"; type CargoToml = HashMap; -#[derive(StructOpt)] +#[derive(Parser)] struct Options { /// The path to the `node-template` source. - #[structopt(parse(from_os_str))] + #[clap(parse(from_os_str))] node_template: PathBuf, /// The path where to output the generated `tar.gz` file. - #[structopt(parse(from_os_str))] + #[clap(parse(from_os_str))] output: PathBuf, } @@ -209,7 +209,7 @@ fn build_and_test(path: &Path, cargo_tomls: &[PathBuf]) { } fn main() { - let options = Options::from_args(); + let options = Options::parse(); let build_dir = tempfile::tempdir().expect("Creates temp build dir"); @@ -261,8 +261,7 @@ fn main() { // adding root rustfmt to node template build path let node_template_rustfmt_toml_path = node_template_path.join("rustfmt.toml"); - let root_rustfmt_toml = - &options.node_template.join("../../rustfmt.toml"); + let root_rustfmt_toml = &options.node_template.join("../../rustfmt.toml"); if root_rustfmt_toml.exists() { fs::copy(&root_rustfmt_toml, &node_template_rustfmt_toml_path) .expect("Copying rustfmt.toml."); diff --git a/shell.nix b/shell.nix index a86af005383f..023946ce16de 100644 --- a/shell.nix +++ b/shell.nix @@ -2,10 +2,10 @@ let mozillaOverlay = import (builtins.fetchGit { url = "https://github.com/mozilla/nixpkgs-mozilla.git"; - rev = "4a07484cf0e49047f82d83fd119acffbad3b235f"; + rev = "15b7a05f20aab51c4ffbefddb1b448e862dccb7d"; }); nixpkgs = import { overlays = [ mozillaOverlay ]; }; - rust-nightly = with nixpkgs; ((rustChannelOf { date = "2021-09-10"; channel = "nightly"; }).rust.override { + rust-nightly = with nixpkgs; ((rustChannelOf { date = "2022-02-10"; channel = "nightly"; }).rust.override { extensions = [ "rust-src" ]; targets = [ "wasm32-unknown-unknown" ]; }); diff --git a/simnet_tests/README.md b/simnet_tests/README.md deleted file mode 100644 index cb1b13ae9850..000000000000 --- a/simnet_tests/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Simulation tests, or high level integration tests. - - -_The content of this directory is meant to be used by Parity's private CI/CD -infrastructure with private tools. At the moment those tools are still early -stage of development and we don't when if / when they will available for -public use._ - - -## Content of this dir. - -`configs` dir contains config files in toml format that describe how to -configure the simulation network that you want to launch. - -`tests` dir contains [cucumber](https://cucumber.io/) files. Those are -Behavior-Driven Development test files that describe tests in plain English. -Under the hood there are assertions that specific metrics should have specific -values. - -At the moment we have 2 tests: `tests/quick/001-smoketest.feature` and -`tests/long/002-loadtest.feature` -The load test uses a JS script that we added to simnet image and it's launched -by this step in the cucumber file: -`Then launch 'node' with parameters '/usr/local/bin/sub-flood --finalization --url ws://localhost:11222'` - -`run_test.sh` is a script meant to ease up launching a test. -In order to use this script locally, you need to install -[gurke](https://github.com/paritytech/gurke) -This script also helps preparing the test environment. Once you have access to -a kubernetes cluster (meaning you can do `kubectl get pods`) you can run this -script with no arguments, like `./run_test.sh` and tests should run. -Kubernetes cluster can be local, spawned with -[kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) -or an instance living in the -[cloud](https://github.com/paritytech/gurke/blob/main/docs/How-to-setup-access-to-gke-k8s-cluster.md) - - -### [Here is link to barcamp presenation of simnet](https://www.crowdcast.io/e/ph49xu01) -### [Here is link to the simnet repo, hosted on private gitlab](https://gitlab.parity.io/parity/simnet/-/tree/master) diff --git a/simnet_tests/configs/default_local_testnet.toml b/simnet_tests/configs/default_local_testnet.toml deleted file mode 100644 index 066bd4c9e332..000000000000 --- a/simnet_tests/configs/default_local_testnet.toml +++ /dev/null @@ -1,14 +0,0 @@ -[settings] -bootnode-domain-name = "bootnode.{{get_env(name="NAMESPACE")}}.svc.cluster.local" - - -[settings.setup] -timeout = 300 - -[settings.defaults] -timeout = 300 - -[nodes] -alice = { extra-args = ["--alice"], validator = true } -bob = { extra-args = ["--bob"], validator = true } -charlie = { extra-args = ["--charlie"], validator = true } diff --git a/simnet_tests/run_tests.sh b/simnet_tests/run_tests.sh deleted file mode 100755 index 3b8ac8a71dad..000000000000 --- a/simnet_tests/run_tests.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash - -### ARGS FOR THIS SCRIPT ### -# ./${SCRIPT_NAME} NAMESPACE IMAGE LOG_PATH FEATURES -# NAMESPACE the kubernetes namespace where the test will run -# IMAGE Substrate image used to spawn network -# LOG_PATH path to dir where to save logs from external JS script that is run as part -# of step in features file -# FEATURES directory containing cucumber files or single cucumber file that describes -# what to test. -# -# All args have default values, specify args to override -# e.g: ./${SCRIPT_NAME} test-name parity/substrate:latest logs quick - -set -eou pipefail -SCRIPT_NAME="$0" -SCRIPT_PATH=$(dirname "${SCRIPT_NAME}") # relative -SCRIPT_PATH=$(cd "${SCRIPT_PATH}" && pwd) # absolutized and normalized - -function random_string { - head -1 <(fold -w 30 <(tr -dc 'a-z0-9' < /dev/urandom)) - } - -# -### Script args -# - -NAMESPACE=${1:-gurke-"$(random_string)"-runtest} -IMAGE=${2:-"parity/substrate:latest"} -LOG_PATH=${3:-"${SCRIPT_PATH}/logs"} -FEATURES=${4:-"ALL"} - -mkdir -p "${SCRIPT_PATH}"/logs - -echo "Running tests in namespace: ${NAMESPACE}" -echo "Testing image: ${IMAGE}" -echo "Storing scripts logs to: ${LOG_PATH}" -echo "Using features files from: ${FEATURES}" - -# -### Script logic -# - -function forward_port { - # RUN_IN_CONTAINER is env var that is set in the dockerfile - # use the -v operator to explicitly test if a variable is set - if [[ ! -v RUN_IN_CONTAINER ]] ; then - if is_port_forward_running ; then - kill_previous_job - fi - fi - start_forwading_job -} - -FORWARD_GREP_FILTER='kubectl.*[p]ort-forward.*svc/rpc.*11222' - -function is_port_forward_running { - # shellcheck disable=SC2009 - ps aux | grep -qE "${FORWARD_GREP_FILTER}" -} - -function kill_previous_job { - # shellcheck disable=SC2009 - job_pid=$(ps aux | grep -E "${FORWARD_GREP_FILTER}" | awk '{ print $2 }') - echo "INFO Killed forwading port 9944 into bootnode" - kill "${job_pid}" -} - -function start_forwading_job { - kubectl -n "${NAMESPACE}" \ - expose pod bootnode \ - --name=rpc \ - --type=NodePort \ - --target-port=9944 \ - --port=9944 - kubectl -n "${NAMESPACE}" \ - port-forward svc/rpc 11222:9944 &> "${LOG_PATH}/forward-${NAMESPACE}.log" & - sleep 2 - echo "INFO Started forwading port 9944 into bootnode" -} - -function update_api { - echo "INFO: Updating Polkadot JS API" - pwd - cd "${SCRIPT_PATH}"/../../sub-flood/ - npm run build - cd - -} - -function run_test { - case "${FEATURES}" in - quick) - gurke test "${NAMESPACE}" "${SCRIPT_PATH}"/tests/quick --log-path "${LOG_PATH}" - ;; - long) - gurke test "${NAMESPACE}" "${SCRIPT_PATH}"/tests/long --log-path "${LOG_PATH}" - ;; - ALL ) - gurke test "${NAMESPACE}" "${SCRIPT_PATH}"/tests --log-path "${LOG_PATH}" - ;; - ??* ) - gurke test \ - "${NAMESPACE}" \ - "${SCRIPT_PATH}"/"${FEATURES}" \ - --log-path "${LOG_PATH}" - ;; - esac -} - - -export NAMESPACE="${NAMESPACE}" - -set -x # echo the commands to stdout -gurke spawn --config "${SCRIPT_PATH}"/configs/default_local_testnet.toml \ - -n "${NAMESPACE}" \ - --image "${IMAGE}" - -echo "INFO: Checking if pods launched correctly" -kubectl -n "${NAMESPACE}" get pods -o wide - -update_api - -forward_port -run_test - - diff --git a/simnet_tests/tests/long/002-loadtest.feature b/simnet_tests/tests/long/002-loadtest.feature deleted file mode 100644 index 67d108ea5541..000000000000 --- a/simnet_tests/tests/long/002-loadtest.feature +++ /dev/null @@ -1,5 +0,0 @@ -Feature: LoadTesting - - Scenario: spawn 50k transactions and wait their finalization - Given a test network - Then launch 'node' with parameters '/usr/local/bin/sub-flood --finalization --url ws://localhost:11222' diff --git a/simnet_tests/tests/quick/001-smoketest.feature b/simnet_tests/tests/quick/001-smoketest.feature deleted file mode 100644 index a07041e4ea62..000000000000 --- a/simnet_tests/tests/quick/001-smoketest.feature +++ /dev/null @@ -1,16 +0,0 @@ -Feature: Smoketest - - Scenario: Minimal Example - Given a test network - Then alice is up - And alice reports substrate_node_roles is 4 - And alice reports substrate_sub_libp2p_is_major_syncing is 0 - When alice's best block should be above 30 - Then alice reports block height is greater than 30 - And alice reports peers count is at least 2 - Then bob is up - And bob reports block height is greater than 30 - And bob reports peers count is at least 2 - Then charlie is up - And charlie reports block height is greater than 30 - And charlie reports peers count is at least 2 diff --git a/ss58-registry.json b/ss58-registry.json deleted file mode 100644 index 7c95f421586a..000000000000 --- a/ss58-registry.json +++ /dev/null @@ -1,653 +0,0 @@ -{ - "specification": "https://github.com/paritytech/substrate/wiki/External-Address-Format-(SS58)", - "schema": { - "prefix": "The address prefix. Must be an integer and unique.", - "network": "Unique identifier for the network that will use this prefix, string, no spaces. To integrate with CLI tools, e.g. `--network polkadot`.", - "displayName": "The name of the network that will use this prefix, in a format friendly for display.", - "symbols": "Array of symbols of any tokens the chain uses, usually 2-5 characters. Most chains will only have one. Chains that have multiple instances of the Balances pallet should order the array by instance.", - "decimals": "Array of integers representing the number of decimals that represent a single unit to the end user. Must be same length as `symbols` to represent each token's denomination.", - "standardAccount": "Signing curve for standard account. Substrate supports ed25519, sr25519, and secp256k1.", - "website": "A website or Github repo associated with the network." - }, - "registry": [ - { - "prefix": 0, - "network": "polkadot", - "displayName": "Polkadot Relay Chain", - "symbols": ["DOT"], - "decimals": [10], - "standardAccount": "*25519", - "website": "https://polkadot.network" - }, - { - "prefix": 1, - "network": null, - "displayName": "Bare 32-bit Schnorr/Ristretto (S/R 25519) public key.", - "symbols": null, - "decimals": null, - "standardAccount": null, - "website": null - }, - { - "prefix": 2, - "network": "kusama", - "displayName": "Kusama Relay Chain", - "symbols": ["KSM"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://kusama.network" - }, - { - "prefix": 3, - "network": null, - "displayName": "Bare 32-bit Ed25519 public key.", - "symbols": null, - "decimals": null, - "standardAccount": null, - "website": null - }, - { - "prefix": 4, - "network": "katalchain", - "displayName": "Katal Chain", - "symbols": null, - "decimals": null, - "standardAccount": "*25519", - "website": null - }, - { - "prefix": 5, - "network": "plasm", - "displayName": "Plasm Network", - "symbols": ["PLM"], - "decimals": [15], - "standardAccount": "*25519", - "website": "https://plasmnet.io" - }, - { - "prefix": 6, - "network": "bifrost", - "displayName": "Bifrost", - "symbols": ["BNC"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://bifrost.finance/" - }, - { - "prefix": 7, - "network": "edgeware", - "displayName": "Edgeware", - "symbols": ["EDG"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://edgewa.re" - }, - { - "prefix": 8, - "network": "karura", - "displayName": "Karura", - "symbols": ["KAR"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://karura.network/" - }, - { - "prefix": 9, - "network": "reynolds", - "displayName": "Laminar Reynolds Canary", - "symbols": ["REY"], - "decimals": [18], - "standardAccount": "*25519", - "website": "http://laminar.network/" - }, - { - "prefix": 10, - "network": "acala", - "displayName": "Acala", - "symbols": ["ACA"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://acala.network/" - }, - { - "prefix": 11, - "network": "laminar", - "displayName": "Laminar", - "symbols": ["LAMI"], - "decimals": [18], - "standardAccount": "*25519", - "website": "http://laminar.network/" - }, - { - "prefix": 12, - "network": "polymesh", - "displayName": "Polymesh", - "symbols": ["POLYX"], - "decimals": [6], - "standardAccount": "*25519", - "website": "https://polymath.network/" - }, - { - "prefix": 13, - "network": "integritee", - "displayName": "Integritee", - "symbols": ["TEER"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://integritee.network" - }, - { - "prefix": 14, - "network": "totem", - "displayName": "Totem", - "symbols": ["XTX"], - "decimals": [0], - "standardAccount": "*25519", - "website": "https://totemaccounting.com" - }, - { - "prefix": 15, - "network": "synesthesia", - "displayName": "Synesthesia", - "symbols": ["SYN"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://synesthesia.network/" - }, - { - "prefix": 16, - "network": "kulupu", - "displayName": "Kulupu", - "symbols": ["KLP"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://kulupu.network/" - }, - { - "prefix": 17, - "network": "dark", - "displayName": "Dark Mainnet", - "symbols": null, - "decimals": null, - "standardAccount": "*25519", - "website": null - }, - { - "prefix": 18, - "network": "darwinia", - "displayName": "Darwinia Network", - "symbols": ["RING", "KTON"], - "decimals": [9, 9], - "standardAccount": "*25519", - "website": "https://darwinia.network/" - }, - { - "prefix": 19, - "network": "geek", - "displayName": "GeekCash", - "symbols": ["GEEK"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://geekcash.org" - }, - { - "prefix": 20, - "network": "stafi", - "displayName": "Stafi", - "symbols": ["FIS"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://stafi.io" - }, - { - "prefix": 21, - "network": "dock-testnet", - "displayName": "Dock Testnet", - "symbols": ["DCK"], - "decimals": [6], - "standardAccount": "*25519", - "website": "https://dock.io" - }, - { - "prefix": 22, - "network": "dock-mainnet", - "displayName": "Dock Mainnet", - "symbols": ["DCK"], - "decimals": [6], - "standardAccount": "*25519", - "website": "https://dock.io" - }, - { - "prefix": 23, - "network": "shift", - "displayName": "ShiftNrg", - "symbols": null, - "decimals": null, - "standardAccount": "*25519", - "website": null - }, - { - "prefix": 24, - "network": "zero", - "displayName": "ZERO", - "symbols": ["PLAY"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://zero.io" - }, - { - "prefix": 25, - "network": "zero-alphaville", - "displayName": "ZERO Alphaville", - "symbols": ["PLAY"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://zero.io" - }, - { - "prefix": 26, - "network": "jupiter", - "displayName": "Jupiter", - "symbols": ["jDOT"], - "decimals": [10], - "standardAccount": "*25519", - "website": "https://jupiter.patract.io" - }, - { - "prefix": 28, - "network": "subsocial", - "displayName": "Subsocial", - "symbols": null, - "decimals": null, - "standardAccount": "*25519", - "website": null - }, - { - "prefix": 29, - "network": "cord", - "displayName": "Dhiway CORD Network", - "symbols": ["DCU"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://dhiway.com/" - }, - { - "prefix": 30, - "network": "phala", - "displayName": "Phala Network", - "symbols": ["PHA"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://phala.network" - }, - { - "prefix": 31, - "network": "litentry", - "displayName": "Litentry Network", - "symbols": ["LIT"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://litentry.com/" - }, - { - "prefix": 32, - "network": "robonomics", - "displayName": "Robonomics", - "symbols": ["XRT"], - "decimals": [9], - "standardAccount": "*25519", - "website": "https://robonomics.network" - }, - { - "prefix": 33, - "network": "datahighway", - "displayName": "DataHighway", - "symbols": null, - "decimals": null, - "standardAccount": "*25519", - "website": null - }, - { - "prefix": 34, - "network": "ares", - "displayName": "Ares Protocol", - "symbols": ["ARES"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://www.aresprotocol.com/" - }, - { - "prefix": 35, - "network": "vln", - "displayName": "Valiu Liquidity Network", - "symbols": ["USDv"], - "decimals": [15], - "standardAccount": "*25519", - "website": "https://valiu.com/" - }, - { - "prefix": 36, - "network": "centrifuge", - "displayName": "Centrifuge Chain", - "symbols": ["CFG"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://centrifuge.io/" - }, - { - "prefix": 37, - "network": "nodle", - "displayName": "Nodle Chain", - "symbols": ["NODL"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://nodle.io/" - }, - { - "prefix": 38, - "network": "kilt", - "displayName": "KILT Chain", - "symbols": ["KILT"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://kilt.io/" - }, - { - "prefix": 39, - "network": "mathchain", - "displayName": "MathChain mainnet", - "symbols": ["MATH"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://mathwallet.org" - }, - { - "prefix": 40, - "network": "mathchain-testnet", - "displayName": "MathChain testnet", - "symbols": ["MATH"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://mathwallet.org" - }, - { - "prefix": 41, - "network": "poli", - "displayName": "Polimec Chain", - "symbols": null, - "decimals": null, - "standardAccount": "*25519", - "website": "https://polimec.io/" - }, - { - "prefix": 42, - "network": "substrate", - "displayName": "Substrate", - "symbols": null, - "decimals": null, - "standardAccount": "*25519", - "website": "https://substrate.dev/" - }, - { - "prefix": 43, - "network": null, - "displayName": "Bare 32-bit ECDSA SECP-256k1 public key.", - "symbols": null, - "decimals": null, - "standardAccount": null, - "website": null - }, - { - "prefix": 44, - "network": "chainx", - "displayName": "ChainX", - "symbols": ["PCX"], - "decimals": [8], - "standardAccount": "*25519", - "website": "https://chainx.org/" - }, - { - "prefix": 45, - "network": "uniarts", - "displayName": "UniArts Network", - "symbols": ["UART", "UINK"], - "decimals": [12, 12], - "standardAccount": "*25519", - "website": "https://uniarts.me" - }, - { - "prefix": 46, - "network": "reserved46", - "displayName": "This prefix is reserved.", - "symbols": null, - "decimals": null, - "standardAccount": null, - "website": null - }, - { - "prefix": 47, - "network": "reserved47", - "displayName": "This prefix is reserved.", - "symbols": null, - "decimals": null, - "standardAccount": null, - "website": null - }, - { - "prefix": 48, - "network": "neatcoin", - "displayName": "Neatcoin Mainnet", - "symbols": ["NEAT"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://neatcoin.org" - }, - { - "prefix": 49, - "network": "picasso", - "displayName": "Picasso", - "symbols": ["PICA"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://picasso.composable.finance" - }, - { - "prefix": 50, - "network": "composable", - "displayName": "Composable", - "symbols": ["LAYR"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://composable.finance" - }, - { - "prefix": 63, - "network": "hydradx", - "displayName": "HydraDX", - "symbols": ["HDX"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://hydradx.io" - }, - { - "prefix": 65, - "network": "aventus", - "displayName": "AvN Mainnet", - "symbols": ["AVT"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://aventus.io" - }, - { - "prefix": 66, - "network": "crust", - "displayName": "Crust Network", - "symbols": ["CRU"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://crust.network" - }, - { - "prefix": 67, - "network": "equilibrium", - "displayName": "Equilibrium Network", - "symbols": ["Unknown", "USD", "EQ", "ETH", "BTC", "EOS", "DOT", "CRV"], - "decimals": [0,9,9,9,9,9,9,9], - "standardAccount": "*25519", - "website": "https://equilibrium.io" - }, - { - "prefix": 69, - "network": "sora", - "displayName": "SORA Network", - "symbols": ["XOR"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://sora.org" - }, - { - "prefix": 73, - "network": "zeitgeist", - "displayName": "Zeitgeist", - "symbols": ["ZTG"], - "decimals": [10], - "standardAccount": "*25519", - "website": "https://zeitgeist.pm" - }, - { - "prefix": 77, - "network": "manta", - "displayName": "Manta network", - "symbols": ["MA"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://manta.network" - }, - { - "prefix": 78, - "network": "calamari", - "displayName": "Calamari: Manta Canary Network", - "symbols": ["KMA"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://manta.network" - }, - { - "prefix": 88, - "network": "polkadex", - "displayName": "Polkadex Mainnet", - "symbols": ["PDEX"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://polkadex.trade" - }, - { - "prefix": 98, - "network": "polkasmith", - "displayName": "PolkaSmith Canary Network", - "symbols": ["PKS"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://polkafoundry.com" - }, - { - "prefix": 99, - "network": "polkafoundry", - "displayName": "PolkaFoundry Network", - "symbols": ["PKF"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://polkafoundry.com" - }, - { - "prefix": 101, - "network": "origintrail-parachain", - "displayName": "OriginTrail Parachain", - "symbols": ["TRAC"], - "decimals": [18], - "standardAccount": "secp256k1", - "website": "https://origintrail.io" - }, - { - "prefix": 110, - "network": "heiko", - "displayName": "Heiko", - "symbols": ["HKO"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://parallel.fi/" - }, - { - "prefix": 113, - "network": "integritee-incognito", - "displayName": "Integritee Incognito", - "symbols": null, - "decimals": null, - "standardAccount": "*25519", - "website": "https://integritee.network" - }, - { - "prefix": 128, - "network": "clover", - "displayName": "Clover Finance", - "symbols": ["CLV"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://clover.finance" - }, - { - "prefix": 136, - "network": "altair", - "displayName": "Altair", - "symbols": ["AIR"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://centrifuge.io/" - }, - { - "prefix": 172, - "network": "parallel", - "displayName": "Parallel", - "symbols": ["PARA"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://parallel.fi/" - }, - { - "prefix": 252, - "network": "social-network", - "displayName": "Social Network", - "symbols": ["NET"], - "decimals": [18], - "standardAccount": "*25519", - "website": "https://social.network" - }, - { - "prefix": 1284, - "network": "moonbeam", - "displayName": "Moonbeam", - "symbols": ["GLMR"], - "decimals": [18], - "standardAccount": "secp256k1", - "website": "https://moonbeam.network" - }, - { - "prefix": 1285, - "network": "moonriver", - "displayName": "Moonriver", - "symbols": ["MOVR"], - "decimals": [18], - "standardAccount": "secp256k1", - "website": "https://moonbeam.network" - }, - { - "prefix": 10041, - "network": "basilisk", - "displayName": "Basilisk", - "symbols": ["BSX"], - "decimals": [12], - "standardAccount": "*25519", - "website": "https://bsx.fi" - } - ] -} diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index 4eed6e5e2913..9f66fa812bb4 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -2,9 +2,9 @@ name = "substrate-test-utils" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate test utilities" @@ -14,8 +14,8 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] futures = "0.3.16" substrate-test-utils-derive = { version = "0.10.0-dev", path = "./derive" } -tokio = { version = "1.10", features = ["macros", "time"] } +tokio = { version = "1.17.0", features = ["macros", "time"] } [dev-dependencies] sc-service = { version = "0.10.0-dev", path = "../client/service" } -trybuild = { version = "1.0.43", features = [ "diff" ] } +trybuild = { version = "1.0.53", features = [ "diff" ] } diff --git a/test-utils/client/Cargo.toml b/test-utils/client/Cargo.toml index 34238872cad8..f99300f27eac 100644 --- a/test-utils/client/Cargo.toml +++ b/test-utils/client/Cargo.toml @@ -2,9 +2,9 @@ name = "substrate-test-client" version = "2.0.1" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" publish = false @@ -12,27 +12,26 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0" } -futures = "0.3.16" +codec = { package = "parity-scale-codec", version = "3.0.0" } +futures = "0.3.21" hex = "0.4" -serde = "1.0.126" -serde_json = "1.0.68" +serde = "1.0.136" +serde_json = "1.0.79" sc-client-api = { version = "4.0.0-dev", path = "../../client/api" } sc-client-db = { version = "0.10.0-dev", features = [ "test-helpers", ], path = "../../client/db" } sc-consensus = { version = "0.10.0-dev", path = "../../client/consensus/common" } sc-executor = { version = "0.10.0-dev", path = "../../client/executor" } -sc-light = { version = "4.0.0-dev", path = "../../client/light" } sc-offchain = { version = "4.0.0-dev", path = "../../client/offchain" } sc-service = { version = "0.10.0-dev", default-features = false, features = [ "test-helpers", ], path = "../../client/service" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore" } -sp-keyring = { version = "4.0.0-dev", path = "../../primitives/keyring" } -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sp-state-machine = { version = "0.10.0-dev", path = "../../primitives/state-machine" } +sp-core = { version = "6.0.0", path = "../../primitives/core" } +sp-keystore = { version = "0.12.0", path = "../../primitives/keystore" } +sp-keyring = { version = "6.0.0", path = "../../primitives/keyring" } +sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" } +sp-state-machine = { version = "0.12.0", path = "../../primitives/state-machine" } async-trait = "0.1.50" diff --git a/test-utils/client/src/client_ext.rs b/test-utils/client/src/client_ext.rs index bf1c9898972c..f2b99a5b355f 100644 --- a/test-utils/client/src/client_ext.rs +++ b/test-utils/client/src/client_ext.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -69,7 +69,7 @@ pub trait ClientBlockImportExt: Sized { impl ClientExt for Client where B: sc_client_api::backend::Backend, - E: sc_client_api::CallExecutor + 'static, + E: sc_client_api::CallExecutor + sc_executor::RuntimeVersionOf + 'static, Self: BlockImport, Block: BlockT, { diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 9bc411af5d3e..94a350b5f5df 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -45,21 +45,13 @@ use sc_client_api::BlockchainEvents; use sc_service::client::{ClientConfig, LocalCallExecutor}; use serde::Deserialize; use sp_core::storage::ChildInfo; -use sp_runtime::{ - codec::Encode, - traits::{BlakeTwo256, Block as BlockT}, - OpaqueExtrinsic, -}; +use sp_runtime::{codec::Encode, traits::Block as BlockT, OpaqueExtrinsic}; use std::{ collections::{HashMap, HashSet}, pin::Pin, sync::Arc, }; -/// Test client light database backend. -pub type LightBackend = - sc_light::Backend, BlakeTwo256>; - /// A genesis storage initialization trait. pub trait GenesisInit: Default { /// Construct genesis storage. @@ -113,11 +105,7 @@ impl /// Create new `TestClientBuilder` with default backend and storage chain mode pub fn with_tx_storage(keep_blocks: u32) -> Self { - let backend = Arc::new(Backend::new_test_with_tx_storage( - keep_blocks, - 0, - sc_client_db::TransactionStorageMode::StorageChain, - )); + let backend = Arc::new(Backend::new_test_with_tx_storage(keep_blocks, 0)); Self::with_backend(backend) } } @@ -216,13 +204,13 @@ impl sc_consensus::LongestChain, ) where - ExecutorDispatch: sc_client_api::CallExecutor + 'static, + ExecutorDispatch: + sc_client_api::CallExecutor + sc_executor::RuntimeVersionOf + 'static, Backend: sc_client_api::backend::Backend, >::OffchainStorage: 'static, { let storage = { let mut storage = self.genesis_init.genesis_storage(); - // Add some child storage keys. for (key, child_content) in self.child_storage_extension { storage.children_default.insert( @@ -270,7 +258,8 @@ impl client::LocalCallExecutor>, Backend, G, - > + > where + D: sc_executor::NativeExecutionDispatch, { /// Build the test client with the given native executor. pub fn build_with_native_executor( @@ -291,7 +280,7 @@ impl Backend: sc_client_api::backend::Backend + 'static, { let executor = executor.into().unwrap_or_else(|| { - NativeElseWasmExecutor::new(WasmExecutionMethod::Interpreted, None, 8) + NativeElseWasmExecutor::new(WasmExecutionMethod::Interpreted, None, 8, 2) }); let executor = LocalCallExecutor::new( self.backend.clone(), diff --git a/test-utils/derive/Cargo.toml b/test-utils/derive/Cargo.toml index 545e8cf33261..16e0b9822f7b 100644 --- a/test-utils/derive/Cargo.toml +++ b/test-utils/derive/Cargo.toml @@ -2,17 +2,17 @@ name = "substrate-test-utils-derive" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate test utilities macros" [dependencies] -quote = "1.0.6" -syn = { version = "1.0.58", features = ["full"] } -proc-macro-crate = "1.0.0" -proc-macro2 = "1.0.29" +quote = "1.0.10" +syn = { version = "1.0.82", features = ["full"] } +proc-macro-crate = "1.1.3" +proc-macro2 = "1.0.36" [lib] proc-macro = true diff --git a/test-utils/derive/src/lib.rs b/test-utils/derive/src/lib.rs index 3f14f67477fa..06b7d2463cbd 100644 --- a/test-utils/derive/src/lib.rs +++ b/test-utils/derive/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/test-utils/runtime/Cargo.toml b/test-utils/runtime/Cargo.toml index 24f4d404c18b..cbefed27ca9b 100644 --- a/test-utils/runtime/Cargo.toml +++ b/test-utils/runtime/Cargo.toml @@ -2,10 +2,10 @@ name = "substrate-test-runtime" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" build = "build.rs" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" publish = false @@ -13,49 +13,50 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } +beefy-primitives = { version = "4.0.0-dev", default-features = false, path = "../../primitives/beefy" } +sp-application-crypto = { version = "6.0.0", default-features = false, path = "../../primitives/application-crypto" } sp-consensus-aura = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/aura" } sp-consensus-babe = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/babe" } sp-block-builder = { version = "4.0.0-dev", default-features = false, path = "../../primitives/block-builder" } -codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "1.0", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../primitives/inherents" } -sp-keyring = { version = "4.0.0-dev", optional = true, path = "../../primitives/keyring" } -memory-db = { version = "0.27.0", default-features = false } +sp-keyring = { version = "6.0.0", optional = true, path = "../../primitives/keyring" } +memory-db = { version = "0.29.0", default-features = false } sp-offchain = { version = "4.0.0-dev", default-features = false, path = "../../primitives/offchain" } -sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } -sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } -sp-runtime-interface = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime-interface" } -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" } +sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" } +sp-runtime-interface = { version = "6.0.0", default-features = false, path = "../../primitives/runtime-interface" } +sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../../frame/support" } -sp-version = { version = "4.0.0-dev", default-features = false, path = "../../primitives/version" } +sp-version = { version = "5.0.0", default-features = false, path = "../../primitives/version" } sp-session = { version = "4.0.0-dev", default-features = false, path = "../../primitives/session" } sp-api = { version = "4.0.0-dev", default-features = false, path = "../../primitives/api" } -sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } +sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" } pallet-babe = { version = "4.0.0-dev", default-features = false, path = "../../frame/babe" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../../frame/system" } frame-system-rpc-runtime-api = { version = "4.0.0-dev", default-features = false, path = "../../frame/system/rpc/runtime-api" } pallet-timestamp = { version = "4.0.0-dev", default-features = false, path = "../../frame/timestamp" } sp-finality-grandpa = { version = "4.0.0-dev", default-features = false, path = "../../primitives/finality-grandpa" } -sp-trie = { version = "4.0.0-dev", default-features = false, path = "../../primitives/trie" } +sp-trie = { version = "6.0.0", default-features = false, path = "../../primitives/trie" } sp-transaction-pool = { version = "4.0.0-dev", default-features = false, path = "../../primitives/transaction-pool" } -trie-db = { version = "0.22.6", default-features = false } -parity-util-mem = { version = "0.10.0", default-features = false, features = ["primitive-types"] } +trie-db = { version = "0.23.1", default-features = false } +parity-util-mem = { version = "0.11.0", default-features = false, features = ["primitive-types"] } sc-service = { version = "0.10.0-dev", default-features = false, optional = true, features = ["test-helpers"], path = "../../client/service" } -sp-state-machine = { version = "0.10.0-dev", default-features = false, path = "../../primitives/state-machine" } -sp-externalities = { version = "0.10.0-dev", default-features = false, path = "../../primitives/externalities" } +sp-state-machine = { version = "0.12.0", default-features = false, path = "../../primitives/state-machine" } +sp-externalities = { version = "0.12.0", default-features = false, path = "../../primitives/externalities" } # 3rd party cfg-if = "1.0" log = { version = "0.4.14", default-features = false } -serde = { version = "1.0.126", optional = true, features = ["derive"] } +serde = { version = "1.0.136", optional = true, features = ["derive"] } [dev-dependencies] sc-block-builder = { version = "0.10.0-dev", path = "../../client/block-builder" } sc-executor = { version = "0.10.0-dev", path = "../../client/executor" } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } substrate-test-runtime-client = { version = "2.0.0", path = "./client" } -futures = "0.3.9" +futures = "0.3.21" [build-dependencies] substrate-wasm-builder = { version = "5.0.0-dev", path = "../../utils/wasm-builder" } @@ -65,6 +66,7 @@ default = [ "std", ] std = [ + "beefy-primitives/std", "sp-application-crypto/std", "sp-consensus-aura/std", "sp-consensus-babe/std", diff --git a/test-utils/runtime/build.rs b/test-utils/runtime/build.rs index 50c455b4ad83..5a7b518d0bd7 100644 --- a/test-utils/runtime/build.rs +++ b/test-utils/runtime/build.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/test-utils/runtime/client/Cargo.toml b/test-utils/runtime/client/Cargo.toml index 3561697042f2..a22d9f302a9e 100644 --- a/test-utils/runtime/client/Cargo.toml +++ b/test-utils/runtime/client/Cargo.toml @@ -2,9 +2,9 @@ name = "substrate-test-runtime-client" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" publish = false @@ -12,16 +12,15 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -sc-light = { version = "4.0.0-dev", path = "../../../client/light" } sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } sc-consensus = { version = "0.10.0-dev", path = "../../../client/consensus/common" } sc-block-builder = { version = "0.10.0-dev", path = "../../../client/block-builder" } substrate-test-client = { version = "2.0.0", path = "../../client" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } substrate-test-runtime = { version = "2.0.0", path = "../../runtime" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -codec = { package = "parity-scale-codec", version = "2.0.0" } +codec = { package = "parity-scale-codec", version = "3.0.0" } sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" } -futures = "0.3.9" +futures = "0.3.21" diff --git a/test-utils/runtime/client/src/block_builder_ext.rs b/test-utils/runtime/client/src/block_builder_ext.rs index e8c1d2ac5cd4..66b0b4c3b881 100644 --- a/test-utils/runtime/client/src/block_builder_ext.rs +++ b/test-utils/runtime/client/src/block_builder_ext.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,6 @@ use sc_client_api::backend; use sp_api::{ApiExt, ProvideRuntimeApi}; -use sp_core::ChangesTrieConfiguration; use sc_block_builder::BlockBuilderApi; @@ -36,11 +35,6 @@ pub trait BlockBuilderExt { key: Vec, value: Option>, ) -> Result<(), sp_blockchain::Error>; - /// Add changes trie configuration update extrinsic to the block. - fn push_changes_trie_configuration_update( - &mut self, - new_config: Option, - ) -> Result<(), sp_blockchain::Error>; } impl<'a, A, B> BlockBuilderExt @@ -68,11 +62,4 @@ where ) -> Result<(), sp_blockchain::Error> { self.push(substrate_test_runtime::Extrinsic::StorageChange(key, value)) } - - fn push_changes_trie_configuration_update( - &mut self, - new_config: Option, - ) -> Result<(), sp_blockchain::Error> { - self.push(substrate_test_runtime::Extrinsic::ChangesTrieConfigUpdate(new_config)) - } } diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index dc5ccadc4574..fe0fef351667 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,22 +24,18 @@ pub mod trait_tests; mod block_builder_ext; pub use sc_consensus::LongestChain; -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; pub use substrate_test_client::*; pub use substrate_test_runtime as runtime; pub use self::block_builder_ext::BlockBuilderExt; -use sc_client_api::light::{ - Fetcher, RemoteBodyRequest, RemoteCallRequest, RemoteChangesRequest, RemoteHeaderRequest, - RemoteReadChildRequest, RemoteReadRequest, -}; use sp_core::{ sr25519, storage::{ChildInfo, Storage, StorageChild}, - ChangesTrieConfiguration, + Pair, }; -use sp_runtime::traits::{Block as BlockT, Hash as HashT, HashFor, Header as HeaderT, NumberFor}; +use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header as HeaderT}; use substrate_test_runtime::genesismap::{additional_storage_with_genesis, GenesisConfig}; /// A prelude to import in tests. @@ -51,8 +47,8 @@ pub mod prelude { }; // Client structs pub use super::{ - Backend, ExecutorDispatch, LightBackend, LightExecutor, LocalExecutorDispatch, - NativeElseWasmExecutor, TestClient, TestClientBuilder, WasmExecutionMethod, + Backend, ExecutorDispatch, LocalExecutorDispatch, NativeElseWasmExecutor, TestClient, + TestClientBuilder, WasmExecutionMethod, }; // Keyring pub use super::{AccountKeyring, Sr25519Keyring}; @@ -84,26 +80,9 @@ pub type ExecutorDispatch = client::LocalCallExecutor< NativeElseWasmExecutor, >; -/// Test client light database backend. -pub type LightBackend = substrate_test_client::LightBackend; - -/// Test client light executor. -pub type LightExecutor = sc_light::GenesisCallExecutor< - LightBackend, - client::LocalCallExecutor< - substrate_test_runtime::Block, - sc_light::Backend< - sc_client_db::light::LightStorage, - HashFor, - >, - NativeElseWasmExecutor, - >, ->; - /// Parameters of test-client builder with test-runtime. #[derive(Default)] pub struct GenesisParameters { - changes_trie_config: Option, heap_pages_override: Option, extra_storage: Storage, wasm_code: Option>, @@ -112,17 +91,20 @@ pub struct GenesisParameters { impl GenesisParameters { fn genesis_config(&self) -> GenesisConfig { GenesisConfig::new( - self.changes_trie_config.clone(), vec![ sr25519::Public::from(Sr25519Keyring::Alice).into(), sr25519::Public::from(Sr25519Keyring::Bob).into(), sr25519::Public::from(Sr25519Keyring::Charlie).into(), ], - vec![ - AccountKeyring::Alice.into(), - AccountKeyring::Bob.into(), - AccountKeyring::Charlie.into(), - ], + (0..16_usize) + .into_iter() + .map(|i| AccountKeyring::numeric(i).public()) + .chain(vec![ + AccountKeyring::Alice.into(), + AccountKeyring::Bob.into(), + AccountKeyring::Charlie.into(), + ]) + .collect(), 1000, self.heap_pages_override, self.extra_storage.clone(), @@ -133,6 +115,11 @@ impl GenesisParameters { pub fn set_wasm_code(&mut self, code: Vec) { self.wasm_code = Some(code); } + + /// Access extra genesis storage. + pub fn extra_storage(&mut self) -> &mut Storage { + &mut self.extra_storage + } } impl substrate_test_client::GenesisInit for GenesisParameters { @@ -151,6 +138,7 @@ impl substrate_test_client::GenesisInit for GenesisParameters { let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( child_content.data.clone().into_iter().collect(), + sp_runtime::StateVersion::V1, ); let prefixed_storage_key = child_content.child_info.prefixed_storage_key(); (prefixed_storage_key.into_inner(), state_root.encode()) @@ -158,6 +146,7 @@ impl substrate_test_client::GenesisInit for GenesisParameters { let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( storage.top.clone().into_iter().chain(child_roots).collect(), + sp_runtime::StateVersion::V1, ); let block: runtime::Block = client::genesis::construct_genesis_block(state_root); storage.top.extend(additional_storage_with_genesis(&block)); @@ -206,12 +195,6 @@ pub trait TestClientBuilderExt: Sized { /// Returns a mutable reference to the genesis parameters. fn genesis_init_mut(&mut self) -> &mut GenesisParameters; - /// Set changes trie configuration for genesis. - fn changes_trie_config(mut self, config: Option) -> Self { - self.genesis_init_mut().changes_trie_config = config; - self - } - /// Override the default value for Wasm heap pages. fn set_heap_pages(mut self, heap_pages: u64) -> Self { self.genesis_init_mut().heap_pages_override = Some(heap_pages); @@ -299,143 +282,17 @@ impl TestClientBuilderExt } } -/// Type of optional fetch callback. -type MaybeFetcherCallback = - Option Result + Send + Sync>>; - -/// Type of fetcher future result. -type FetcherFutureResult = futures::future::Ready>; - -/// Implementation of light client fetcher used in tests. -#[derive(Default)] -pub struct LightFetcher { - call: MaybeFetcherCallback, Vec>, - body: MaybeFetcherCallback< - RemoteBodyRequest, - Vec, - >, -} - -impl LightFetcher { - /// Sets remote call callback. - pub fn with_remote_call( - self, - call: MaybeFetcherCallback, Vec>, - ) -> Self { - LightFetcher { call, body: self.body } - } - - /// Sets remote body callback. - pub fn with_remote_body( - self, - body: MaybeFetcherCallback< - RemoteBodyRequest, - Vec, - >, - ) -> Self { - LightFetcher { call: self.call, body } - } -} - -impl Fetcher for LightFetcher { - type RemoteHeaderResult = FetcherFutureResult; - type RemoteReadResult = FetcherFutureResult, Option>>>; - type RemoteCallResult = FetcherFutureResult>; - type RemoteChangesResult = - FetcherFutureResult, u32)>>; - type RemoteBodyResult = FetcherFutureResult>; - - fn remote_header( - &self, - _: RemoteHeaderRequest, - ) -> Self::RemoteHeaderResult { - unimplemented!() - } - - fn remote_read( - &self, - _: RemoteReadRequest, - ) -> Self::RemoteReadResult { - unimplemented!() - } - - fn remote_read_child( - &self, - _: RemoteReadChildRequest, - ) -> Self::RemoteReadResult { - unimplemented!() - } - - fn remote_call( - &self, - req: RemoteCallRequest, - ) -> Self::RemoteCallResult { - match self.call { - Some(ref call) => futures::future::ready(call(req)), - None => unimplemented!(), - } - } - - fn remote_changes( - &self, - _: RemoteChangesRequest, - ) -> Self::RemoteChangesResult { - unimplemented!() - } - - fn remote_body( - &self, - req: RemoteBodyRequest, - ) -> Self::RemoteBodyResult { - match self.body { - Some(ref body) => futures::future::ready(body(req)), - None => unimplemented!(), - } - } -} - /// Creates new client instance used for tests. pub fn new() -> Client { TestClientBuilder::new().build() } -/// Creates new light client instance used for tests. -pub fn new_light() -> ( - client::Client< - LightBackend, - LightExecutor, - substrate_test_runtime::Block, - substrate_test_runtime::RuntimeApi, - >, - Arc, -) { - let storage = sc_client_db::light::LightStorage::new_test(); - let blockchain = Arc::new(sc_light::Blockchain::new(storage)); - let backend = Arc::new(LightBackend::new(blockchain)); - let executor = new_native_executor(); - let local_call_executor = client::LocalCallExecutor::new( - backend.clone(), - executor, - Box::new(sp_core::testing::TaskExecutor::new()), - Default::default(), - ) - .expect("Creates LocalCallExecutor"); - let call_executor = LightExecutor::new(backend.clone(), local_call_executor); - - ( - TestClientBuilder::with_backend(backend.clone()) - .build_with_executor(call_executor) - .0, - backend, - ) -} - -/// Creates new light client fetcher used for tests. -pub fn new_light_fetcher() -> LightFetcher { - LightFetcher::default() -} - /// Create a new native executor. pub fn new_native_executor() -> sc_executor::NativeElseWasmExecutor { - sc_executor::NativeElseWasmExecutor::new(sc_executor::WasmExecutionMethod::Interpreted, None, 8) + sc_executor::NativeElseWasmExecutor::new( + sc_executor::WasmExecutionMethod::Interpreted, + None, + 8, + 2, + ) } diff --git a/test-utils/runtime/client/src/trait_tests.rs b/test-utils/runtime/client/src/trait_tests.rs index 938aeda36d31..65aa3e65e614 100644 --- a/test-utils/runtime/client/src/trait_tests.rs +++ b/test-utils/runtime/client/src/trait_tests.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2018-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2018-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/test-utils/runtime/src/genesismap.rs b/test-utils/runtime/src/genesismap.rs index a8801b8519df..71118b4183ef 100644 --- a/test-utils/runtime/src/genesismap.rs +++ b/test-utils/runtime/src/genesismap.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,7 +23,6 @@ use sc_service::client::genesis; use sp_core::{ map, storage::{well_known_keys, Storage}, - ChangesTrieConfiguration, }; use sp_io::hashing::{blake2_256, twox_128}; use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header as HeaderT}; @@ -31,7 +30,6 @@ use std::collections::BTreeMap; /// Configuration of a general Substrate test genesis block. pub struct GenesisConfig { - changes_trie_config: Option, authorities: Vec, balances: Vec<(AccountId, u64)>, heap_pages_override: Option, @@ -41,7 +39,6 @@ pub struct GenesisConfig { impl GenesisConfig { pub fn new( - changes_trie_config: Option, authorities: Vec, endowed_accounts: Vec, balance: u64, @@ -49,7 +46,6 @@ impl GenesisConfig { extra_storage: Storage, ) -> Self { GenesisConfig { - changes_trie_config, authorities, balances: endowed_accounts.into_iter().map(|a| (a, balance)).collect(), heap_pages_override, @@ -77,9 +73,6 @@ impl GenesisConfig { .into_iter(), ) .collect(); - if let Some(ref changes_trie_config) = self.changes_trie_config { - map.insert(well_known_keys::CHANGES_TRIE_CONFIG.to_vec(), changes_trie_config.encode()); - } map.insert(twox_128(&b"sys:auth"[..])[..].to_vec(), self.authorities.encode()); // Add the extra storage entries. map.extend(self.extra_storage.top.clone().into_iter()); @@ -102,6 +95,7 @@ pub fn insert_genesis_block(storage: &mut Storage) -> sp_core::hash::H256 { let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( child_content.data.clone().into_iter().collect(), + sp_runtime::StateVersion::V1, ); (sk.clone(), state_root.encode()) }); @@ -109,6 +103,7 @@ pub fn insert_genesis_block(storage: &mut Storage) -> sp_core::hash::H256 { storage.top.extend(child_roots); let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( storage.top.clone().into_iter().collect(), + sp_runtime::StateVersion::V1, ); let block: crate::Block = genesis::construct_genesis_block(state_root); let genesis_hash = block.header.hash(); diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 943c41c247f7..743652a0ee89 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,17 +28,14 @@ use scale_info::TypeInfo; use sp_std::{marker::PhantomData, prelude::*}; use sp_application_crypto::{ecdsa, ed25519, sr25519, RuntimeAppPublic}; -use sp_core::{offchain::KeyTypeId, ChangesTrieConfiguration, OpaqueMetadata, RuntimeDebug}; -use sp_trie::{ - trie_types::{TrieDB, TrieDBMut}, - PrefixedMemoryDB, StorageProof, -}; +use sp_core::{offchain::KeyTypeId, OpaqueMetadata, RuntimeDebug}; +use sp_trie::{trie_types::TrieDB, PrefixedMemoryDB, StorageProof}; use trie_db::{Trie, TrieMut}; use cfg_if::cfg_if; use frame_support::{ parameter_types, - traits::{CrateVersion, KeyOwnerProofSystem}, + traits::{ConstU32, ConstU64, CrateVersion, KeyOwnerProofSystem}, weights::RuntimeDbWeight, }; use frame_system::limits::{BlockLength, BlockWeights}; @@ -62,6 +59,8 @@ use sp_runtime::{ #[cfg(any(feature = "std", test))] use sp_version::NativeVersion; use sp_version::RuntimeVersion; +// bench on latest state. +use sp_trie::trie_types::TrieDBMutV1 as TrieDBMut; // Ensure Babe and Aura use the same crypto to simplify things a bit. pub use sp_consensus_babe::{AllowedSlots, AuthorityId, Slot}; @@ -105,6 +104,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { impl_version: 2, apis: RUNTIME_API_VERSIONS, transaction_version: 1, + state_version: 1, }; fn version() -> RuntimeVersion { @@ -132,8 +132,7 @@ impl Transfer { pub fn into_signed_tx(self) -> Extrinsic { let signature = sp_keyring::AccountKeyring::from_public(&self.from) .expect("Creates keyring from public key.") - .sign(&self.encode()) - .into(); + .sign(&self.encode()); Extrinsic::Transfer { transfer: self, signature, exhaust_resources_when_not_first: false } } @@ -144,8 +143,7 @@ impl Transfer { pub fn into_resources_exhausting_tx(self) -> Extrinsic { let signature = sp_keyring::AccountKeyring::from_public(&self.from) .expect("Creates keyring from public key.") - .sign(&self.encode()) - .into(); + .sign(&self.encode()); Extrinsic::Transfer { transfer: self, signature, exhaust_resources_when_not_first: true } } } @@ -161,7 +159,6 @@ pub enum Extrinsic { }, IncludeData(Vec), StorageChange(Vec, Option>), - ChangesTrieConfigUpdate(Option), OffchainIndexSet(Vec, Vec), OffchainIndexClear(Vec), Store(Vec), @@ -197,8 +194,6 @@ impl BlindCheckable for Extrinsic { }, Extrinsic::IncludeData(v) => Ok(Extrinsic::IncludeData(v)), Extrinsic::StorageChange(key, value) => Ok(Extrinsic::StorageChange(key, value)), - Extrinsic::ChangesTrieConfigUpdate(new_config) => - Ok(Extrinsic::ChangesTrieConfigUpdate(new_config)), Extrinsic::OffchainIndexSet(key, value) => Ok(Extrinsic::OffchainIndexSet(key, value)), Extrinsic::OffchainIndexClear(key) => Ok(Extrinsic::OffchainIndexClear(key)), Extrinsic::Store(data) => Ok(Extrinsic::Store(data)), @@ -265,9 +260,9 @@ pub type BlockNumber = u64; /// Index of a transaction. pub type Index = u64; /// The item of a block digest. -pub type DigestItem = sp_runtime::generic::DigestItem; +pub type DigestItem = sp_runtime::generic::DigestItem; /// The digest of a block. -pub type Digest = sp_runtime::generic::Digest; +pub type Digest = sp_runtime::generic::Digest; /// A test block. pub type Block = sp_runtime::generic::Block; /// A test block's header. @@ -280,9 +275,9 @@ pub fn run_tests(mut input: &[u8]) -> Vec { print("run_tests..."); let block = Block::decode(&mut input).unwrap(); print("deserialized block."); - let stxs = block.extrinsics.iter().map(Encode::encode).collect::>(); + let stxs = block.extrinsics.iter().map(Encode::encode); print("reserialized transactions."); - [stxs.len() as u8].encode() + [stxs.count() as u8].encode() } /// A type that can not be decoded. @@ -299,9 +294,9 @@ impl Encode for DecodeFails { impl codec::EncodeLike for DecodeFails {} -impl DecodeFails { - /// Create a new instance. - pub fn new() -> DecodeFails { +impl Default for DecodeFails { + /// Create a default instance. + fn default() -> DecodeFails { DecodeFails { _phantom: Default::default() } } } @@ -431,7 +426,7 @@ impl GetRuntimeBlockType for Runtime { type RuntimeBlock = Block; } -#[derive(Clone, RuntimeDebug)] +#[derive(Clone, RuntimeDebug, Encode, Decode, PartialEq, Eq, TypeInfo)] pub struct Origin; impl From> for Origin { @@ -439,8 +434,8 @@ impl From> for Origin { unimplemented!("Not required in tests!") } } -impl Into, Origin>> for Origin { - fn into(self) -> Result, Origin> { +impl From for Result, Origin> { + fn from(_origin: Origin) -> Result, Origin> { unimplemented!("Not required in tests!") } } @@ -559,7 +554,6 @@ impl frame_support::traits::PalletInfo for Runtime { parameter_types! { pub const BlockHashCount: BlockNumber = 2400; - pub const MinimumPeriod: u64 = 5; pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { read: 100, write: 1000, @@ -594,25 +588,24 @@ impl frame_system::Config for Runtime { type SystemWeightInfo = (); type SS58Prefix = (); type OnSetCode = (); + type MaxConsumers = ConstU32<16>; } impl pallet_timestamp::Config for Runtime { /// A timestamp: milliseconds since the unix epoch. type Moment = u64; type OnTimestampSet = (); - type MinimumPeriod = MinimumPeriod; + type MinimumPeriod = ConstU64<5>; type WeightInfo = (); } parameter_types! { pub const EpochDuration: u64 = 6; - pub const ExpectedBlockTime: u64 = 10_000; - pub const MaxAuthorities: u32 = 10; } impl pallet_babe::Config for Runtime { type EpochDuration = EpochDuration; - type ExpectedBlockTime = ExpectedBlockTime; + type ExpectedBlockTime = ConstU64<10_000>; // there is no actual runtime in this test-runtime, so testing crates // are manually adding the digests. normally in this situation you'd use // pallet_babe::SameAuthoritiesForever. @@ -632,7 +625,7 @@ impl pallet_babe::Config for Runtime { type HandleEquivocation = (); type WeightInfo = (); - type MaxAuthorities = MaxAuthorities; + type MaxAuthorities = ConstU32<10>; } /// Adds one to the given input and returns the final result. @@ -656,12 +649,9 @@ fn code_using_trie() -> u64 { let mut mdb = PrefixedMemoryDB::default(); let mut root = sp_std::default::Default::default(); let _ = { - let v = &pairs; let mut t = TrieDBMut::::new(&mut mdb, &mut root); - for i in 0..v.len() { - let key: &[u8] = &v[i].0; - let val: &[u8] = &v[i].1; - if !t.insert(key, val).is_ok() { + for (key, value) in &pairs { + if t.insert(key, value).is_err() { return 101 } } @@ -670,13 +660,7 @@ fn code_using_trie() -> u64 { if let Ok(trie) = TrieDB::::new(&mdb, &root) { if let Ok(iter) = trie.iter() { - let mut iter_pairs = Vec::new(); - for pair in iter { - if let Ok((key, value)) = pair { - iter_pairs.push((key, value.to_vec())); - } - } - iter_pairs.len() as u64 + iter.flatten().count() as u64 } else { 102 } @@ -772,7 +756,7 @@ cfg_if! { fn fail_convert_parameter(_: DecodeFails) {} fn fail_convert_return_value() -> DecodeFails { - DecodeFails::new() + DecodeFails::default() } fn function_signature_changed() -> u64 { @@ -942,6 +926,12 @@ cfg_if! { } } + impl beefy_primitives::BeefyApi for RuntimeApi { + fn validator_set() -> Option> { + None + } + } + impl frame_system_rpc_runtime_api::AccountNonceApi for Runtime { fn account_nonce(_account: AccountId) -> Index { 0 @@ -1026,7 +1016,7 @@ cfg_if! { fn fail_convert_parameter(_: DecodeFails) {} fn fail_convert_return_value() -> DecodeFails { - DecodeFails::new() + DecodeFails::default() } fn function_signature_changed() -> Vec { @@ -1264,20 +1254,18 @@ fn test_witness(proof: StorageProof, root: crate::Hash) { let db: sp_trie::MemoryDB = proof.into_memory_db(); let backend = sp_state_machine::TrieBackend::<_, crate::Hashing>::new(db, root); let mut overlay = sp_state_machine::OverlayedChanges::default(); - let mut cache = sp_state_machine::StorageTransactionCache::<_, _, BlockNumber>::default(); + let mut cache = sp_state_machine::StorageTransactionCache::<_, _>::default(); let mut ext = sp_state_machine::Ext::new( &mut overlay, &mut cache, &backend, #[cfg(feature = "std")] None, - #[cfg(feature = "std")] - None, ); assert!(ext.storage(b"value3").is_some()); - assert!(ext.storage_root().as_slice() == &root[..]); + assert!(ext.storage_root(Default::default()).as_slice() == &root[..]); ext.place_storage(vec![0], Some(vec![1])); - assert!(ext.storage_root().as_slice() != &root[..]); + assert!(ext.storage_root(Default::default()).as_slice() != &root[..]); } #[cfg(test)] @@ -1341,7 +1329,7 @@ mod tests { let mut root = crate::Hash::default(); let mut mdb = sp_trie::MemoryDB::::default(); { - let mut trie = sp_trie::trie_types::TrieDBMut::new(&mut mdb, &mut root); + let mut trie = sp_trie::trie_types::TrieDBMutV1::new(&mut mdb, &mut root); trie.insert(b"value3", &[142]).expect("insert failed"); trie.insert(b"value4", &[124]).expect("insert failed"); }; diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index 334569d055a0..6df35421d361 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,12 +24,8 @@ use crate::{ use codec::{Decode, Encode, KeyedVec}; use frame_support::{decl_module, decl_storage, storage}; use frame_system::Config; -use sp_core::{storage::well_known_keys, ChangesTrieConfiguration}; -use sp_io::{ - hashing::blake2_256, - storage::{changes_root as storage_changes_root, root as storage_root}, - trie, -}; +use sp_core::storage::well_known_keys; +use sp_io::{hashing::blake2_256, storage::root as storage_root, trie}; use sp_runtime::{ generic, traits::Header as _, @@ -54,7 +50,6 @@ decl_storage! { Number get(fn number): Option; ParentHash get(fn parent_hash): Hash; NewAuthorities get(fn new_authorities): Option>; - NewChangesTrieConfig get(fn new_changes_trie_config): Option>; StorageDigest get(fn storage_digest): Option; Authorities get(fn authorities) config(): Vec; } @@ -199,38 +194,26 @@ pub fn execute_transaction(utx: Extrinsic) -> ApplyExtrinsicResult { /// Finalize the block. pub fn finalize_block() -> Header { + use sp_core::storage::StateVersion; let extrinsic_index: u32 = storage::unhashed::take(well_known_keys::EXTRINSIC_INDEX).unwrap(); let txs: Vec<_> = (0..extrinsic_index).map(ExtrinsicData::take).collect(); - let extrinsics_root = trie::blake2_256_ordered_root(txs).into(); + let extrinsics_root = trie::blake2_256_ordered_root(txs, StateVersion::V0); let number = ::take().expect("Number is set by `initialize_block`"); let parent_hash = ::take(); let mut digest = ::take().expect("StorageDigest is set by `initialize_block`"); let o_new_authorities = ::take(); - let new_changes_trie_config = ::take(); // This MUST come after all changes to storage are done. Otherwise we will fail the // “Storage root does not match that calculated” assertion. - let storage_root = - Hash::decode(&mut &storage_root()[..]).expect("`storage_root` is a valid hash"); - let storage_changes_root = storage_changes_root(&parent_hash.encode()) - .map(|r| Hash::decode(&mut &r[..]).expect("`storage_changes_root` is a valid hash")); - - if let Some(storage_changes_root) = storage_changes_root { - digest.push(generic::DigestItem::ChangesTrieRoot(storage_changes_root)); - } + let storage_root = Hash::decode(&mut &storage_root(StateVersion::V1)[..]) + .expect("`storage_root` is a valid hash"); if let Some(new_authorities) = o_new_authorities { digest.push(generic::DigestItem::Consensus(*b"aura", new_authorities.encode())); digest.push(generic::DigestItem::Consensus(*b"babe", new_authorities.encode())); } - if let Some(new_config) = new_changes_trie_config { - digest.push(generic::DigestItem::ChangesTrieSignal( - generic::ChangesTrieSignal::NewConfiguration(new_config), - )); - } - Header { number, extrinsics_root, state_root: storage_root, parent_hash, digest } } @@ -251,14 +234,12 @@ fn execute_transaction_backend(utx: &Extrinsic, extrinsic_index: u32) -> ApplyEx Extrinsic::IncludeData(_) => Ok(Ok(())), Extrinsic::StorageChange(key, value) => execute_storage_change(key, value.as_ref().map(|v| &**v)), - Extrinsic::ChangesTrieConfigUpdate(ref new_config) => - execute_changes_trie_config_update(new_config.clone()), Extrinsic::OffchainIndexSet(key, value) => { - sp_io::offchain_index::set(&key, &value); + sp_io::offchain_index::set(key, value); Ok(Ok(())) }, Extrinsic::OffchainIndexClear(key) => { - sp_io::offchain_index::clear(&key); + sp_io::offchain_index::clear(key); Ok(Ok(())) }, Extrinsic::Store(data) => execute_store(data.clone()), @@ -269,7 +250,7 @@ fn execute_transfer_backend(tx: &Transfer) -> ApplyExtrinsicResult { // check nonce let nonce_key = tx.from.to_keyed_vec(NONCE_OF); let expected_nonce: u64 = storage::hashed::get_or(&blake2_256, &nonce_key, 0); - if !(tx.nonce == expected_nonce) { + if tx.nonce != expected_nonce { return Err(InvalidTransaction::Stale.into()) } @@ -281,7 +262,7 @@ fn execute_transfer_backend(tx: &Transfer) -> ApplyExtrinsicResult { let from_balance: u64 = storage::hashed::get_or(&blake2_256, &from_balance_key, 0); // enact transfer - if !(tx.amount <= from_balance) { + if tx.amount > from_balance { return Err(InvalidTransaction::Payment.into()) } let to_balance_key = tx.to.to_keyed_vec(BALANCE_OF); @@ -311,18 +292,6 @@ fn execute_storage_change(key: &[u8], value: Option<&[u8]>) -> ApplyExtrinsicRes Ok(Ok(())) } -fn execute_changes_trie_config_update( - new_config: Option, -) -> ApplyExtrinsicResult { - match new_config.clone() { - Some(new_config) => - storage::unhashed::put_raw(well_known_keys::CHANGES_TRIE_CONFIG, &new_config.encode()), - None => storage::unhashed::kill(well_known_keys::CHANGES_TRIE_CONFIG), - } - ::put(new_config); - Ok(Ok(())) -} - #[cfg(feature = "std")] fn info_expect_equal_hash(given: &Hash, expected: &Hash) { use sp_core::hexdisplay::HexDisplay; @@ -374,7 +343,7 @@ mod tests { } fn executor() -> NativeElseWasmExecutor { - NativeElseWasmExecutor::new(WasmExecutionMethod::Interpreted, None, 8) + NativeElseWasmExecutor::new(WasmExecutionMethod::Interpreted, None, 8, 2) } fn new_test_ext() -> TestExternalities { @@ -383,6 +352,7 @@ mod tests { Sr25519Keyring::Bob.to_raw_public(), Sr25519Keyring::Charlie.to_raw_public(), ]; + TestExternalities::new_with_code( wasm_binary_unwrap(), sp_core::storage::Storage { @@ -391,7 +361,7 @@ mod tests { twox_128(b"sys:auth").to_vec() => authorities.encode(), blake2_256(&AccountKeyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => { vec![111u8, 0, 0, 0, 0, 0, 0, 0] - } + }, ], children_default: map![], }, diff --git a/test-utils/runtime/transaction-pool/Cargo.toml b/test-utils/runtime/transaction-pool/Cargo.toml index 09839ebae6ff..59d576acdcb8 100644 --- a/test-utils/runtime/transaction-pool/Cargo.toml +++ b/test-utils/runtime/transaction-pool/Cargo.toml @@ -2,9 +2,9 @@ name = "substrate-test-runtime-transaction-pool" version = "2.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" publish = false @@ -13,11 +13,11 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] substrate-test-runtime-client = { version = "2.0.0", path = "../client" } -parking_lot = "0.11.1" -codec = { package = "parity-scale-codec", version = "2.0.0" } +parking_lot = "0.12.0" +codec = { package = "parity-scale-codec", version = "3.0.0" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -sc-transaction-pool = { version = "4.0.0-dev", path = "../../../client/transaction-pool", features = ["test-helpers"] } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } +sc-transaction-pool = { version = "4.0.0-dev", path = "../../../client/transaction-pool" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../../client/transaction-pool/api" } -futures = "0.3.16" -derive_more = "0.99.2" +futures = "0.3.21" +thiserror = "1.0" diff --git a/test-utils/runtime/transaction-pool/src/lib.rs b/test-utils/runtime/transaction-pool/src/lib.rs index d0cd50394c53..400842762349 100644 --- a/test-utils/runtime/transaction-pool/src/lib.rs +++ b/test-utils/runtime/transaction-pool/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,7 +25,9 @@ use parking_lot::RwLock; use sp_blockchain::CachedHeaderMetadata; use sp_runtime::{ generic::{self, BlockId}, - traits::{BlakeTwo256, Block as BlockT, Hash as HashT, Header as _}, + traits::{ + BlakeTwo256, Block as BlockT, Hash as HashT, Header as _, NumberFor, TrailingZeroInput, + }, transaction_validity::{ InvalidTransaction, TransactionSource, TransactionValidity, TransactionValidityError, ValidTransaction, @@ -38,8 +40,9 @@ use substrate_test_runtime_client::{ }; /// Error type used by [`TestApi`]. -#[derive(Debug, derive_more::From, derive_more::Display)] -pub struct Error(sc_transaction_pool_api::error::Error); +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +pub struct Error(#[from] sc_transaction_pool_api::error::Error); impl sc_transaction_pool_api::error::IntoPoolError for Error { fn into_pool_error(self) -> Result { @@ -47,12 +50,6 @@ impl sc_transaction_pool_api::error::IntoPoolError for Error { } } -impl std::error::Error for Error { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - Some(&self.0) - } -} - pub enum IsBestBlock { Yes, No, @@ -177,13 +174,13 @@ impl TestApi { /// Add a block to the internal state. pub fn add_block(&self, block: Block, is_best_block: bool) { let hash = block.header.hash(); - let block_number = block.header.number().clone(); + let block_number = block.header.number(); let mut chain = self.chain.write(); chain.block_by_hash.insert(hash, block.clone()); chain .block_by_number - .entry(block_number) + .entry(*block_number) .or_default() .push((block, is_best_block.into())); } @@ -227,7 +224,7 @@ impl TestApi { } } -impl sc_transaction_pool::test_helpers::ChainApi for TestApi { +impl sc_transaction_pool::ChainApi for TestApi { type Block = Block; type Error = Error; type ValidationFuture = futures::future::Ready>; @@ -237,7 +234,7 @@ impl sc_transaction_pool::test_helpers::ChainApi for TestApi { &self, at: &BlockId, _source: TransactionSource, - uxt: sc_transaction_pool::test_helpers::ExtrinsicFor, + uxt: ::Extrinsic, ) -> Self::ValidationFuture { self.validation_requests.write().push(uxt.clone()); @@ -257,15 +254,13 @@ impl sc_transaction_pool::test_helpers::ChainApi for TestApi { if !found_best { return ready(Ok(Err(TransactionValidityError::Invalid( InvalidTransaction::Custom(1), - ) - .into()))) + )))) } }, Ok(None) => return ready(Ok(Err(TransactionValidityError::Invalid( InvalidTransaction::Custom(2), - ) - .into()))), + )))), Err(e) => return ready(Err(e)), } @@ -281,9 +276,7 @@ impl sc_transaction_pool::test_helpers::ChainApi for TestApi { }; if self.chain.read().invalid_hashes.contains(&self.hash_and_length(&uxt).0) { - return ready(Ok(Err( - TransactionValidityError::Invalid(InvalidTransaction::Custom(0)).into() - ))) + return ready(Ok(Err(TransactionValidityError::Invalid(InvalidTransaction::Custom(0))))) } let mut validity = @@ -297,7 +290,7 @@ impl sc_transaction_pool::test_helpers::ChainApi for TestApi { fn block_id_to_number( &self, at: &BlockId, - ) -> Result>, Error> { + ) -> Result>, Error> { Ok(match at { generic::BlockId::Hash(x) => self.chain.read().block_by_hash.get(x).map(|b| *b.header.number()), @@ -308,9 +301,9 @@ impl sc_transaction_pool::test_helpers::ChainApi for TestApi { fn block_id_to_hash( &self, at: &BlockId, - ) -> Result>, Error> { + ) -> Result::Hash>, Error> { Ok(match at { - generic::BlockId::Hash(x) => Some(x.clone()), + generic::BlockId::Hash(x) => Some(*x), generic::BlockId::Number(num) => self.chain.read().block_by_number.get(num).and_then(|blocks| { blocks.iter().find(|b| b.1.is_best()).map(|b| b.0.header().hash()) @@ -318,10 +311,7 @@ impl sc_transaction_pool::test_helpers::ChainApi for TestApi { }) } - fn hash_and_length( - &self, - ex: &sc_transaction_pool::test_helpers::ExtrinsicFor, - ) -> (Hash, usize) { + fn hash_and_length(&self, ex: &::Extrinsic) -> (Hash, usize) { Self::hash_and_length_inner(ex) } @@ -370,7 +360,8 @@ impl sp_blockchain::HeaderMetadata for TestApi { /// /// Part of the test api. pub fn uxt(who: AccountKeyring, nonce: Index) -> Extrinsic { - let transfer = Transfer { from: who.into(), to: AccountId::default(), nonce, amount: 1 }; - let signature = transfer.using_encoded(|e| who.sign(e)).into(); + let dummy = codec::Decode::decode(&mut TrailingZeroInput::zeroes()).unwrap(); + let transfer = Transfer { from: who.into(), to: dummy, nonce, amount: 1 }; + let signature = transfer.using_encoded(|e| who.sign(e)); Extrinsic::Transfer { transfer, signature, exhaust_resources_when_not_first: false } } diff --git a/test-utils/src/lib.rs b/test-utils/src/lib.rs index b68994926533..643985940792 100644 --- a/test-utils/src/lib.rs +++ b/test-utils/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2017-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/test-utils/test-crate/Cargo.toml b/test-utils/test-crate/Cargo.toml index fff39c3964ad..f6fea8407eaa 100644 --- a/test-utils/test-crate/Cargo.toml +++ b/test-utils/test-crate/Cargo.toml @@ -2,9 +2,9 @@ name = "substrate-test-utils-test-crate" version = "0.1.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" publish = false @@ -12,6 +12,6 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dev-dependencies] -tokio = { version = "1.10", features = ["macros"] } +tokio = { version = "1.17.0", features = ["macros"] } test-utils = { version = "4.0.0-dev", path = "..", package = "substrate-test-utils" } sc-service = { version = "0.10.0-dev", path = "../../client/service" } diff --git a/test-utils/test-crate/src/main.rs b/test-utils/test-crate/src/main.rs index 554adcb88406..4696e71779c1 100644 --- a/test-utils/test-crate/src/main.rs +++ b/test-utils/test-crate/src/main.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/test-utils/test-runner/Cargo.toml b/test-utils/test-runner/Cargo.toml deleted file mode 100644 index b5b115771b53..000000000000 --- a/test-utils/test-runner/Cargo.toml +++ /dev/null @@ -1,54 +0,0 @@ -[package] -name = "test-runner" -version = "0.9.0" -authors = ["Parity Technologies "] -edition = "2018" -publish = false - -[dependencies] -# client deps -sc-executor = { path = "../../client/executor" } -sc-service = { path = "../../client/service" } -sc-informant = { path = "../../client/informant" } -sc-network = { path = "../../client/network" } -sc-cli = { path = "../../client/cli" } -sc-basic-authorship = { path = "../../client/basic-authorship" } -sc-rpc = { path = "../../client/rpc" } -sc-transaction-pool = { path = "../../client/transaction-pool" } -grandpa = { package = "sc-finality-grandpa", path = "../../client/finality-grandpa" } -sp-finality-grandpa = { path = "../../primitives/finality-grandpa" } -sp-consensus-babe = { path = "../../primitives/consensus/babe" } -sc-consensus-babe = { path = "../../client/consensus/babe" } -sc-consensus = { path = "../../client/consensus/common" } -sc-transaction-pool-api = { path = "../../client/transaction-pool/api" } -sc-client-api = { path = "../../client/api" } -sc-rpc-server = { path = "../../client/rpc-servers" } -manual-seal = { package = "sc-consensus-manual-seal", path = "../../client/consensus/manual-seal" } - -# primitive deps -sp-core = { path = "../../primitives/core" } -sp-blockchain = { path = "../../primitives/blockchain" } -sp-block-builder = { path = "../../primitives/block-builder" } -sp-api = { path = "../../primitives/api" } -sp-transaction-pool = { path = "../../primitives/transaction-pool" } -sp-consensus = { path = "../../primitives/consensus/common" } -sp-runtime = { path = "../../primitives/runtime" } -sp-session = { path = "../../primitives/session" } -sp-offchain = { path = "../../primitives/offchain" } -sp-inherents = { path = "../../primitives/inherents" } -sp-keyring = { path = "../../primitives/keyring" } - -sp-externalities = { path = "../../primitives/externalities" } -sp-state-machine = { path = "../../primitives/state-machine" } -sp-wasm-interface = { path = "../../primitives/wasm-interface" } -sp-runtime-interface = { path = "../../primitives/runtime-interface" } - -# pallets -frame-system = { path = "../../frame/system" } - -log = "0.4.8" -futures = "0.3.16" -tokio = { version = "1.10", features = ["signal"] } -# Calling RPC -jsonrpc-core = "18.0" -num-traits = "0.2.14" diff --git a/test-utils/test-runner/src/host_functions.rs b/test-utils/test-runner/src/host_functions.rs deleted file mode 100644 index 731abfbb9db0..000000000000 --- a/test-utils/test-runner/src/host_functions.rs +++ /dev/null @@ -1,90 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -/// Use this to override host functions. -/// eg -/// ```rust -/// use test_runner::override_host_functions; -/// pub struct SignatureVerificationOverride; -/// -/// impl sp_wasm_interface::HostFunctions for SignatureVerificationOverride { -/// fn host_functions() -> Vec<&'static dyn sp_wasm_interface::Function> { -/// override_host_functions!( -/// "ext_crypto_ecdsa_verify_version_1", EcdsaVerify, -/// ) -/// } -/// } -/// ``` -#[macro_export] -macro_rules! override_host_functions { - ($($fn_name:expr, $name:ident,)*) => {{ - let mut host_functions = vec![]; - $( - struct $name; - impl sp_wasm_interface::Function for $name { - fn name(&self) -> &str { - &$fn_name - } - - fn signature(&self) -> sp_wasm_interface::Signature { - sp_wasm_interface::Signature { - args: std::borrow::Cow::Owned(vec![ - sp_wasm_interface::ValueType::I32, - sp_wasm_interface::ValueType::I64, - sp_wasm_interface::ValueType::I32, - ]), - return_value: Some(sp_wasm_interface::ValueType::I32), - } - } - - fn execute( - &self, - context: &mut dyn sp_wasm_interface::FunctionContext, - _args: &mut dyn Iterator, - ) -> Result, String> { - ::into_ffi_value(true, context) - .map(sp_wasm_interface::IntoValue::into_value) - .map(Some) - } - } - host_functions.push(&$name as &'static dyn sp_wasm_interface::Function); - )* - host_functions - }}; -} - -/// Provides host functions that overrides runtime signature verification -/// to always return true. -pub struct SignatureVerificationOverride; - -impl sp_wasm_interface::HostFunctions for SignatureVerificationOverride { - fn host_functions() -> Vec<&'static dyn sp_wasm_interface::Function> { - override_host_functions!( - "ext_crypto_ecdsa_verify_version_1", - EcdsaVerify, - "ext_crypto_ecdsa_verify_version_2", - EcdsaVerifyV2, - "ext_crypto_ed25519_verify_version_1", - Ed25519Verify, - "ext_crypto_sr25519_verify_version_1", - Sr25519Verify, - "ext_crypto_sr25519_verify_version_2", - Sr25519VerifyV2, - ) - } -} diff --git a/test-utils/test-runner/src/lib.rs b/test-utils/test-runner/src/lib.rs deleted file mode 100644 index ca2c518fd692..000000000000 --- a/test-utils/test-runner/src/lib.rs +++ /dev/null @@ -1,309 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -#![deny(missing_docs, unused_extern_crates)] - -//! Test runner -//! # Substrate Test Runner -//! -//! Allows you to test -//!
-//! -//! - Migrations -//! - Runtime Upgrades -//! - Pallets and general runtime functionality. -//! -//! This works by running a full node with a Manual Seal-BABE™ hybrid consensus for block authoring. -//! -//!

Note

-//! The running node has no signature verification, which allows us author extrinsics for any -//! account on chain.
-//!
-//! -//!

How do I Use this?

-//! -//! -//! ```rust,ignore -//! use test_runner::{Node, ChainInfo, SignatureVerificationOverride, base_path, NodeConfig}; -//! use sc_finality_grandpa::GrandpaBlockImport; -//! use sc_service::{ -//! TFullBackend, TFullClient, Configuration, TaskManager, new_full_parts, BasePath, -//! DatabaseSource, KeepBlocks, TransactionStorageMode, ChainSpec, Role, -//! config::{NetworkConfiguration, KeystoreConfig}, -//! }; -//! use std::sync::Arc; -//! use sp_inherents::InherentDataProviders; -//! use sc_consensus_babe::BabeBlockImport; -//! use sp_keystore::SyncCryptoStorePtr; -//! use sp_keyring::sr25519::Keyring::{Alice, Bob}; -//! use node_cli::chain_spec::development_config; -//! use sp_consensus_babe::AuthorityId; -//! use manual_seal::{ConsensusDataProvider, consensus::babe::BabeConsensusDataProvider}; -//! use sp_runtime::{traits::IdentifyAccount, MultiSigner, generic::Era}; -//! use sc_executor::WasmExecutionMethod; -//! use sc_network::{multiaddr, config::TransportConfig}; -//! use sc_client_api::execution_extensions::ExecutionStrategies; -//! use sc_informant::OutputFormat; -//! use sp_api::TransactionFor; -//! -//! type BlockImport = BabeBlockImport>; -//! -//! pub struct ExecutorDispatch; -//! -//! impl sc_executor::NativeExecutionDispatch for ExecutorDispatch { -//! type ExtendHostFunctions = SignatureVerificationOverride; -//! -//! fn dispatch(method: &str, data: &[u8]) -> Option> { -//! node_runtime::api::dispatch(method, data) -//! } -//! -//! fn native_version() -> sc_executor::NativeVersion { -//! node_runtime::native_version() -//! } -//! } -//! -//! struct Requirements; -//! -//! impl ChainInfo for Requirements { -//! /// Provide a Block type with an OpaqueExtrinsic -//! type Block = node_primitives::Block; -//! /// Provide an ExecutorDispatch type for the runtime -//! type ExecutorDispatch = ExecutorDispatch; -//! /// Provide the runtime itself -//! type Runtime = node_runtime::Runtime; -//! /// A touch of runtime api -//! type RuntimeApi = node_runtime::RuntimeApi; -//! /// A pinch of SelectChain implementation -//! type SelectChain = sc_consensus::LongestChain, Self::Block>; -//! /// A slice of concrete BlockImport type -//! type BlockImport = BlockImport< -//! Self::Block, -//! TFullBackend, -//! TFullClient>, -//! Self::SelectChain, -//! >; -//! /// and a dash of SignedExtensions -//! type SignedExtras = node_runtime::SignedExtra; -//! -//! /// Create your signed extras here. -//! fn signed_extras( -//! from: ::AccountId, -//! ) -> Self::SignedExtension { -//! let nonce = frame_system::Pallet::::account_nonce(from); -//! -//! ( -//! frame_system::CheckSpecVersion::::new(), -//! frame_system::CheckTxVersion::::new(), -//! frame_system::CheckGenesis::::new(), -//! frame_system::CheckMortality::::from(Era::Immortal), -//! frame_system::CheckNonce::::from(nonce), -//! frame_system::CheckWeight::::new(), -//! pallet_transaction_payment::ChargeTransactionPayment::::from(0), -//! ) -//! } -//! -//! /// The function signature tells you all you need to know. ;) -//! fn create_client_parts(config: &Configuration) -> Result< -//! ( -//! Arc>>, -//! Arc>, -//! KeyStorePtr, -//! TaskManager, -//! InherentDataProviders, -//! Option>, -//! Self::Block -//! >, -//! > -//! >>, -//! Self::SelectChain, -//! Self::BlockImport -//! ), -//! sc_service::Error -//! > { -//! let ( -//! client, -//! backend, -//! keystore, -//! task_manager, -//! ) = new_full_parts::>(config)?; -//! let client = Arc::new(client); -//! -//! let inherent_providers = InherentDataProviders::new(); -//! let select_chain = sc_consensus::LongestChain::new(backend.clone()); -//! -//! let (grandpa_block_import, ..) = -//! sc_finality_grandpa::block_import(client.clone(), &(client.clone() as Arc<_>), select_chain.clone())?; -//! -//! let (block_import, babe_link) = sc_consensus_babe::block_import( -//! sc_consensus_babe::Config::get_or_compute(&*client)?, -//! grandpa_block_import, -//! client.clone(), -//! )?; -//! -//! let consensus_data_provider = BabeConsensusDataProvider::new( -//! client.clone(), -//! keystore.clone(), -//! &inherent_providers, -//! babe_link.epoch_changes().clone(), -//! vec![(AuthorityId::from(Alice.public()), 1000)] -//! ) -//! .expect("failed to create ConsensusDataProvider"); -//! -//! Ok(( -//! client, -//! backend, -//! keystore, -//! task_manager, -//! inherent_providers, -//! Some(Box::new(consensus_data_provider)), -//! select_chain, -//! block_import -//! )) -//! } -//! -//! fn dispatch_with_root(call: ::Call, node: &mut Node) { -//! let alice = MultiSigner::from(Alice.public()).into_account(); -//! // for chains that support sudo, otherwise, you'd have to use pallet-democracy here. -//! let call = pallet_sudo::Call::sudo(Box::new(call)); -//! node.submit_extrinsic(call, alice); -//! node.seal_blocks(1); -//! } -//! } -//! -//! /// And now for the most basic test -//! -//! #[test] -//! fn simple_balances_test() { -//! // given -//! let config = NodeConfig { -//! execution_strategies: ExecutionStrategies { -//! syncing: sc_client_api::ExecutionStrategy::NativeWhenPossible, -//! importing: sc_client_api::ExecutionStrategy::NativeWhenPossible, -//! block_construction: sc_client_api::ExecutionStrategy::NativeWhenPossible, -//! offchain_worker: sc_client_api::ExecutionStrategy::NativeWhenPossible, -//! other: sc_client_api::ExecutionStrategy::NativeWhenPossible, -//! }, -//! chain_spec: Box::new(development_config()), -//! log_targets: vec![], -//! }; -//! let mut node = Node::::new(config).unwrap(); -//! -//! type Balances = pallet_balances::Pallet; -//! -//! let (alice, bob) = (Alice.pair(), Bob.pair()); -//! let (alice_account_id, bob_acount_id) = ( -//! MultiSigner::from(alice.public()).into_account(), -//! MultiSigner::from(bob.public()).into_account() -//! ); -//! -//! /// the function with_state allows us to read state, pretty cool right? :D -//! let old_balance = node.with_state(|| Balances::free_balance(alice_account_id.clone())); -//! -//! // 70 dots -//! let amount = 70_000_000_000_000; -//! -//! /// Send extrinsic in action. -//! node.submit_extrinsic(BalancesCall::transfer(bob_acount_id.clone(), amount), alice_account_id.clone()); -//! -//! /// Produce blocks in action, Powered by manual-seal™. -//! node.seal_blocks(1); -//! -//! /// we can check the new state :D -//! let new_balance = node.with_state(|| Balances::free_balance(alice_account_id)); -//! -//! /// we can now make assertions on how state has changed. -//! assert_eq!(old_balance + amount, new_balance); -//! } -//! ``` - -use sc_consensus::BlockImport; -use sc_executor::{NativeElseWasmExecutor, NativeExecutionDispatch}; -use sc_service::TFullClient; -use sp_api::{ConstructRuntimeApi, TransactionFor}; -use sp_consensus::SelectChain; -use sp_inherents::InherentDataProvider; -use sp_runtime::traits::{Block as BlockT, SignedExtension}; - -mod client; -mod host_functions; -mod node; -mod utils; - -pub use client::*; -pub use host_functions::*; -pub use node::*; -pub use utils::*; - -/// Wrapper trait for concrete type required by this testing framework. -pub trait ChainInfo: Sized { - /// Opaque block type - type Block: BlockT; - - /// ExecutorDispatch dispatch type - type ExecutorDispatch: NativeExecutionDispatch + 'static; - - /// Runtime - type Runtime: frame_system::Config; - - /// RuntimeApi - type RuntimeApi: Send - + Sync - + 'static - + ConstructRuntimeApi< - Self::Block, - TFullClient< - Self::Block, - Self::RuntimeApi, - NativeElseWasmExecutor, - >, - >; - - /// select chain type. - type SelectChain: SelectChain + 'static; - - /// Block import type. - type BlockImport: Send - + Sync - + Clone - + BlockImport< - Self::Block, - Error = sp_consensus::Error, - Transaction = TransactionFor< - TFullClient< - Self::Block, - Self::RuntimeApi, - NativeElseWasmExecutor, - >, - Self::Block, - >, - > + 'static; - - /// The signed extras required by the runtime - type SignedExtras: SignedExtension; - - /// The inherent data providers. - type InherentDataProviders: InherentDataProvider + 'static; - - /// Signed extras, this function is caled in an externalities provided environment. - fn signed_extras( - from: ::AccountId, - ) -> Self::SignedExtras; -} diff --git a/test-utils/test-runner/src/node.rs b/test-utils/test-runner/src/node.rs deleted file mode 100644 index 9114013b747f..000000000000 --- a/test-utils/test-runner/src/node.rs +++ /dev/null @@ -1,288 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use std::sync::Arc; - -use crate::ChainInfo; -use futures::{ - channel::{mpsc, oneshot}, - FutureExt, SinkExt, -}; -use jsonrpc_core::MetaIoHandler; -use manual_seal::EngineCommand; -use sc_client_api::{ - backend::{self, Backend}, - CallExecutor, ExecutorProvider, -}; -use sc_executor::NativeElseWasmExecutor; -use sc_service::{TFullBackend, TFullCallExecutor, TFullClient, TaskManager}; -use sc_transaction_pool_api::TransactionPool; -use sp_api::{OverlayedChanges, StorageTransactionCache}; -use sp_blockchain::HeaderBackend; -use sp_core::ExecutionContext; -use sp_runtime::{ - generic::{BlockId, UncheckedExtrinsic}, - traits::{Block as BlockT, Extrinsic, Header, NumberFor}, - transaction_validity::TransactionSource, - MultiAddress, MultiSignature, -}; -use sp_state_machine::Ext; - -/// This holds a reference to a running node on another thread, -/// the node process is dropped when this struct is dropped -/// also holds logs from the process. -pub struct Node { - /// rpc handler for communicating with the node over rpc. - rpc_handler: Arc>, - /// handle to the running node. - task_manager: Option, - /// client instance - client: Arc>>, - /// transaction pool - pool: Arc< - dyn TransactionPool< - Block = ::Block, - Hash = <::Block as BlockT>::Hash, - Error = sc_transaction_pool::error::Error, - InPoolTransaction = sc_transaction_pool::Transaction< - <::Block as BlockT>::Hash, - <::Block as BlockT>::Extrinsic, - >, - >, - >, - /// channel to communicate with manual seal on. - manual_seal_command_sink: mpsc::Sender::Hash>>, - /// backend type. - backend: Arc>, - /// Block number at initialization of this Node. - initial_block_number: NumberFor, -} - -type EventRecord = frame_system::EventRecord< - ::Event, - ::Hash, ->; - -impl Node -where - T: ChainInfo, - <::Header as Header>::Number: From, -{ - /// Creates a new node. - pub fn new( - rpc_handler: Arc>, - task_manager: TaskManager, - client: Arc< - TFullClient>, - >, - pool: Arc< - dyn TransactionPool< - Block = ::Block, - Hash = <::Block as BlockT>::Hash, - Error = sc_transaction_pool::error::Error, - InPoolTransaction = sc_transaction_pool::Transaction< - <::Block as BlockT>::Hash, - <::Block as BlockT>::Extrinsic, - >, - >, - >, - command_sink: mpsc::Sender::Hash>>, - backend: Arc>, - ) -> Self { - Self { - rpc_handler, - task_manager: Some(task_manager), - client: client.clone(), - pool, - backend, - manual_seal_command_sink: command_sink, - initial_block_number: client.info().best_number, - } - } - - /// Returns a reference to the rpc handlers, use this to send rpc requests. - /// eg - /// ```ignore - /// let request = r#"{"jsonrpc":"2.0","method":"engine_createBlock","params": [true, true],"id":1}"#; - /// let response = node.rpc_handler() - /// .handle_request_sync(request, Default::default()); - /// ``` - pub fn rpc_handler( - &self, - ) -> Arc> { - self.rpc_handler.clone() - } - - /// Return a reference to the Client - pub fn client( - &self, - ) -> Arc>> { - self.client.clone() - } - - /// Return a reference to the pool. - pub fn pool( - &self, - ) -> Arc< - dyn TransactionPool< - Block = ::Block, - Hash = <::Block as BlockT>::Hash, - Error = sc_transaction_pool::error::Error, - InPoolTransaction = sc_transaction_pool::Transaction< - <::Block as BlockT>::Hash, - <::Block as BlockT>::Extrinsic, - >, - >, - > { - self.pool.clone() - } - - /// Executes closure in an externalities provided environment. - pub fn with_state(&self, closure: impl FnOnce() -> R) -> R - where - > as CallExecutor>::Error: - std::fmt::Debug, - { - let id = BlockId::Hash(self.client.info().best_hash); - let mut overlay = OverlayedChanges::default(); - let changes_trie = - backend::changes_tries_state_at_block(&id, self.backend.changes_trie_storage()) - .unwrap(); - let mut cache = StorageTransactionCache::< - T::Block, - as Backend>::State, - >::default(); - let mut extensions = self - .client - .execution_extensions() - .extensions(&id, ExecutionContext::BlockConstruction); - let state_backend = self - .backend - .state_at(id.clone()) - .expect(&format!("State at block {} not found", id)); - - let mut ext = Ext::new( - &mut overlay, - &mut cache, - &state_backend, - changes_trie.clone(), - Some(&mut extensions), - ); - sp_externalities::set_and_run_with_externalities(&mut ext, closure) - } - - /// submit some extrinsic to the node. if signer is None, will submit unsigned_extrinsic. - pub async fn submit_extrinsic( - &self, - call: impl Into<::Call>, - signer: Option<::AccountId>, - ) -> Result<::Hash, sc_transaction_pool::error::Error> - where - ::Extrinsic: From< - UncheckedExtrinsic< - MultiAddress< - ::AccountId, - ::Index, - >, - ::Call, - MultiSignature, - T::SignedExtras, - >, - >, - { - let signed_data = if let Some(signer) = signer { - let extra = self.with_state(|| T::signed_extras(signer.clone())); - Some((signer.into(), MultiSignature::Sr25519(Default::default()), extra)) - } else { - None - }; - let ext = UncheckedExtrinsic::< - MultiAddress< - ::AccountId, - ::Index, - >, - ::Call, - MultiSignature, - T::SignedExtras, - >::new(call.into(), signed_data) - .expect("UncheckedExtrinsic::new() always returns Some"); - let at = self.client.info().best_hash; - - self.pool - .submit_one(&BlockId::Hash(at), TransactionSource::Local, ext.into()) - .await - } - - /// Get the events of the most recently produced block - pub fn events(&self) -> Vec> { - self.with_state(|| frame_system::Pallet::::events()) - } - - /// Instructs manual seal to seal new, possibly empty blocks. - pub async fn seal_blocks(&self, num: usize) { - let mut sink = self.manual_seal_command_sink.clone(); - - for count in 0..num { - let (sender, future_block) = oneshot::channel(); - let future = sink.send(EngineCommand::SealNewBlock { - create_empty: true, - finalize: false, - parent_hash: None, - sender: Some(sender), - }); - - const ERROR: &'static str = "manual-seal authorship task is shutting down"; - future.await.expect(ERROR); - - match future_block.await.expect(ERROR) { - Ok(block) => { - log::info!("sealed {} (hash: {}) of {} blocks", count + 1, block.hash, num) - }, - Err(err) => { - log::error!("failed to seal block {} of {}, error: {:?}", count + 1, num, err) - }, - } - } - } - - /// Revert count number of blocks from the chain. - pub fn revert_blocks(&self, count: NumberFor) { - self.backend.revert(count, true).expect("Failed to revert blocks: "); - } - - /// so you've decided to run the test runner as a binary, use this to shutdown gracefully. - pub async fn until_shutdown(mut self) { - let manager = self.task_manager.take(); - if let Some(mut task_manager) = manager { - let task = task_manager.future().fuse(); - let signal = tokio::signal::ctrl_c(); - futures::pin_mut!(signal); - futures::future::select(task, signal).await; - // we don't really care whichever comes first. - task_manager.clean_shutdown().await - } - } -} - -impl Drop for Node { - fn drop(&mut self) { - // Revert all blocks added since creation of the node. - let diff = self.client.info().best_number - self.initial_block_number; - self.revert_blocks(diff); - } -} diff --git a/test-utils/test-runner/src/utils.rs b/test-utils/test-runner/src/utils.rs deleted file mode 100644 index 8e8c84e6b4f8..000000000000 --- a/test-utils/test-runner/src/utils.rs +++ /dev/null @@ -1,117 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use sc_client_api::execution_extensions::ExecutionStrategies; -use sc_executor::WasmExecutionMethod; -use sc_informant::OutputFormat; -use sc_network::{ - config::{NetworkConfiguration, Role, TransportConfig}, - multiaddr, -}; -use sc_service::{ - config::KeystoreConfig, BasePath, ChainSpec, Configuration, DatabaseSource, KeepBlocks, - TransactionStorageMode, -}; -use sp_keyring::sr25519::Keyring::Alice; -use tokio::runtime::Handle; - -pub use sc_cli::build_runtime; - -/// Base db path gotten from env -pub fn base_path() -> BasePath { - if let Some(base) = std::env::var("DB_BASE_PATH").ok() { - BasePath::new(base) - } else { - BasePath::new_temp_dir().expect("couldn't create a temp dir") - } -} - -/// Produces a default configuration object, suitable for use with most set ups. -pub fn default_config(tokio_handle: Handle, mut chain_spec: Box) -> Configuration { - let base_path = base_path(); - let root_path = base_path.path().to_path_buf().join("chains").join(chain_spec.id()); - - let storage = chain_spec - .as_storage_builder() - .build_storage() - .expect("could not build storage"); - - chain_spec.set_storage(storage); - let key_seed = Alice.to_seed(); - - let mut network_config = NetworkConfiguration::new( - format!("Test Node for: {}", key_seed), - "network/test/0.1", - Default::default(), - None, - ); - let informant_output_format = OutputFormat { enable_color: false }; - network_config.allow_non_globals_in_dht = true; - - network_config.listen_addresses.push(multiaddr::Protocol::Memory(0).into()); - - network_config.transport = TransportConfig::MemoryOnly; - - Configuration { - impl_name: "test-node".to_string(), - impl_version: "0.1".to_string(), - role: Role::Authority, - tokio_handle, - transaction_pool: Default::default(), - network: network_config, - keystore: KeystoreConfig::Path { path: root_path.join("key"), password: None }, - database: DatabaseSource::RocksDb { path: root_path.join("db"), cache_size: 128 }, - state_cache_size: 16777216, - state_cache_child_ratio: None, - chain_spec, - wasm_method: WasmExecutionMethod::Interpreted, - execution_strategies: ExecutionStrategies { - syncing: sc_client_api::ExecutionStrategy::AlwaysWasm, - importing: sc_client_api::ExecutionStrategy::AlwaysWasm, - block_construction: sc_client_api::ExecutionStrategy::AlwaysWasm, - offchain_worker: sc_client_api::ExecutionStrategy::AlwaysWasm, - other: sc_client_api::ExecutionStrategy::AlwaysWasm, - }, - rpc_http: None, - rpc_ws: None, - rpc_ipc: None, - rpc_ws_max_connections: None, - rpc_cors: None, - rpc_methods: Default::default(), - rpc_max_payload: None, - prometheus_config: None, - telemetry_endpoints: None, - default_heap_pages: None, - offchain_worker: Default::default(), - force_authoring: false, - disable_grandpa: false, - dev_key_seed: Some(key_seed), - tracing_targets: None, - tracing_receiver: Default::default(), - max_runtime_instances: 8, - announce_block: true, - base_path: Some(base_path), - wasm_runtime_overrides: None, - informant_output_format, - disable_log_reloading: false, - keystore_remote: None, - keep_blocks: KeepBlocks::All, - state_pruning: Default::default(), - transaction_storage: TransactionStorageMode::BlockBody, - } -} diff --git a/test-utils/tests/basic.rs b/test-utils/tests/basic.rs index 527ca3e365ed..4daca836c7fd 100644 --- a/test-utils/tests/basic.rs +++ b/test-utils/tests/basic.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/test-utils/tests/ui.rs b/test-utils/tests/ui.rs index 119162fdc21b..be3b9c111b3e 100644 --- a/test-utils/tests/ui.rs +++ b/test-utils/tests/ui.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/test-utils/tests/ui/too-many-func-parameters.rs b/test-utils/tests/ui/too-many-func-parameters.rs index b1789b9d3ee7..51059f37ab62 100644 --- a/test-utils/tests/ui/too-many-func-parameters.rs +++ b/test-utils/tests/ui/too-many-func-parameters.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify diff --git a/utils/build-script-utils/Cargo.toml b/utils/build-script-utils/Cargo.toml index fbef70db93bf..7150b4fa5adc 100644 --- a/utils/build-script-utils/Cargo.toml +++ b/utils/build-script-utils/Cargo.toml @@ -2,9 +2,9 @@ name = "substrate-build-script-utils" version = "3.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Crate with utility functions for `build.rs` scripts." readme = "README.md" @@ -13,4 +13,4 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -platforms = "1.1" +platforms = "2.0" diff --git a/utils/build-script-utils/src/git.rs b/utils/build-script-utils/src/git.rs index 66a15737f84c..db9a4b291ffd 100644 --- a/utils/build-script-utils/src/git.rs +++ b/utils/build-script-utils/src/git.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/utils/build-script-utils/src/lib.rs b/utils/build-script-utils/src/lib.rs index 0c45c4b34ebe..7e69f2ac85d4 100644 --- a/utils/build-script-utils/src/lib.rs +++ b/utils/build-script-utils/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/utils/build-script-utils/src/version.rs b/utils/build-script-utils/src/version.rs index 52336eb0b6a2..773949e30d8d 100644 --- a/utils/build-script-utils/src/version.rs +++ b/utils/build-script-utils/src/version.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,21 +20,23 @@ use std::{borrow::Cow, process::Command}; /// Generate the `cargo:` key output pub fn generate_cargo_keys() { - let output = Command::new("git").args(&["rev-parse", "--short", "HEAD"]).output(); - - let commit = match output { - Ok(o) if o.status.success() => { - let sha = String::from_utf8_lossy(&o.stdout).trim().to_owned(); - Cow::from(sha) - }, - Ok(o) => { - println!("cargo:warning=Git command failed with status: {}", o.status); - Cow::from("unknown") - }, - Err(err) => { - println!("cargo:warning=Failed to execute git command: {}", err); - Cow::from("unknown") - }, + let commit = if let Ok(hash) = std::env::var("SUBSTRATE_CLI_GIT_COMMIT_HASH") { + Cow::from(hash.trim().to_owned()) + } else { + match Command::new("git").args(&["rev-parse", "--short", "HEAD"]).output() { + Ok(o) if o.status.success() => { + let sha = String::from_utf8_lossy(&o.stdout).trim().to_owned(); + Cow::from(sha) + }, + Ok(o) => { + println!("cargo:warning=Git command failed with status: {}", o.status); + Cow::from("unknown") + }, + Err(err) => { + println!("cargo:warning=Failed to execute git command: {}", err); + Cow::from("unknown") + }, + } }; println!("cargo:rustc-env=SUBSTRATE_CLI_IMPL_VERSION={}", get_version(&commit)) diff --git a/utils/fork-tree/Cargo.toml b/utils/fork-tree/Cargo.toml index 11c269bc3cba..4ac176f645bd 100644 --- a/utils/fork-tree/Cargo.toml +++ b/utils/fork-tree/Cargo.toml @@ -2,9 +2,9 @@ name = "fork-tree" version = "3.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Utility library for managing tree-like ordered data with logic for pruning the tree while finalizing nodes." documentation = "https://docs.rs/fork-tree" @@ -14,4 +14,4 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", features = ["derive"] } diff --git a/utils/fork-tree/src/lib.rs b/utils/fork-tree/src/lib.rs index 9143da89a77e..c23a4f55f44a 100644 --- a/utils/fork-tree/src/lib.rs +++ b/utils/fork-tree/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -69,6 +69,17 @@ pub enum FinalizationResult { Unchanged, } +/// Filtering action. +#[derive(Debug, PartialEq)] +pub enum FilterAction { + /// Remove the node and its subtree. + Remove, + /// Maintain the node. + KeepNode, + /// Maintain the node and its subtree. + KeepTree, +} + /// A tree data structure that stores several nodes across multiple branches. /// Top-level branches are called roots. The tree has functionality for /// finalizing nodes, which means that that node is traversed, and all competing @@ -498,14 +509,14 @@ where } /// Checks if any node in the tree is finalized by either finalizing the - /// node itself or a child node that's not in the tree, guaranteeing that - /// the node being finalized isn't a descendent of any of the node's - /// children. Returns `Some(true)` if the node being finalized is a root, - /// `Some(false)` if the node being finalized is not a root, and `None` if - /// no node in the tree is finalized. The given `predicate` is checked on - /// the prospective finalized root and must pass for finalization to occur. - /// The given function `is_descendent_of` should return `true` if the second - /// hash (target) is a descendent of the first hash (base). + /// node itself or a node's descendent that's not in the tree, guaranteeing + /// that the node being finalized isn't a descendent of (or equal to) any of + /// the node's children. Returns `Some(true)` if the node being finalized is + /// a root, `Some(false)` if the node being finalized is not a root, and + /// `None` if no node in the tree is finalized. The given `predicate` is + /// checked on the prospective finalized root and must pass for finalization + /// to occur. The given function `is_descendent_of` should return `true` if + /// the second hash (target) is a descendent of the first hash (base). pub fn finalizes_any_with_descendent_if( &self, hash: &H, @@ -530,8 +541,10 @@ where for node in self.node_iter() { if predicate(&node.data) { if node.hash == *hash || is_descendent_of(&node.hash, hash)? { - for node in node.children.iter() { - if node.number <= number && is_descendent_of(&node.hash, &hash)? { + for child in node.children.iter() { + if child.number <= number && + (child.hash == *hash || is_descendent_of(&child.hash, hash)?) + { return Err(Error::UnfinalizedAncestor) } } @@ -545,12 +558,12 @@ where } /// Finalize a root in the tree by either finalizing the node itself or a - /// child node that's not in the tree, guaranteeing that the node being - /// finalized isn't a descendent of any of the root's children. The given - /// `predicate` is checked on the prospective finalized root and must pass for - /// finalization to occur. The given function `is_descendent_of` should - /// return `true` if the second hash (target) is a descendent of the first - /// hash (base). + /// node's descendent that's not in the tree, guaranteeing that the node + /// being finalized isn't a descendent of (or equal to) any of the root's + /// children. The given `predicate` is checked on the prospective finalized + /// root and must pass for finalization to occur. The given function + /// `is_descendent_of` should return `true` if the second hash (target) is a + /// descendent of the first hash (base). pub fn finalize_with_descendent_if( &mut self, hash: &H, @@ -576,8 +589,10 @@ where for (i, root) in self.roots.iter().enumerate() { if predicate(&root.data) { if root.hash == *hash || is_descendent_of(&root.hash, hash)? { - for node in root.children.iter() { - if node.number <= number && is_descendent_of(&node.hash, &hash)? { + for child in root.children.iter() { + if child.number <= number && + (child.hash == *hash || is_descendent_of(&child.hash, hash)?) + { return Err(Error::UnfinalizedAncestor) } } @@ -595,12 +610,11 @@ where node.data }); - // if the block being finalized is earlier than a given root, then it - // must be its ancestor, otherwise we can prune the root. if there's a - // root at the same height then the hashes must match. otherwise the - // node being finalized is higher than the root so it must be its - // descendent (in this case the node wasn't finalized earlier presumably - // because the predicate didn't pass). + // Retain only roots that are descendents of the finalized block (this + // happens if the node has been properly finalized) or that are + // ancestors (or equal) to the finalized block (in this case the node + // wasn't finalized earlier presumably because the predicate didn't + // pass). let mut changed = false; let roots = std::mem::take(&mut self.roots); @@ -624,6 +638,29 @@ where (None, false) => Ok(FinalizationResult::Unchanged), } } + + /// Remove from the tree some nodes (and their subtrees) using a `filter` predicate. + /// The `filter` is called over tree nodes and returns a filter action: + /// - `Remove` if the node and its subtree should be removed; + /// - `KeepNode` if we should maintain the node and keep processing the tree. + /// - `KeepTree` if we should maintain the node and its entire subtree. + /// An iterator over all the pruned nodes is returned. + pub fn drain_filter(&mut self, mut filter: F) -> impl Iterator + where + F: FnMut(&H, &N, &V) -> FilterAction, + { + let mut removed = Vec::new(); + let mut i = 0; + while i < self.roots.len() { + if self.roots[i].drain_filter(&mut filter, &mut removed) { + removed.push(self.roots.remove(i)); + } else { + i += 1; + } + } + self.rebalance(); + RemovedIterator { stack: removed } + } } // Workaround for: https://github.com/rust-lang/rust/issues/34537 @@ -849,6 +886,34 @@ mod node_implementation { }, } } + + /// Calls a `filter` predicate for the given node. + /// The `filter` is called over tree nodes and returns a filter action: + /// - `Remove` if the node and its subtree should be removed; + /// - `KeepNode` if we should maintain the node and keep processing the tree; + /// - `KeepTree` if we should maintain the node and its entire subtree. + /// Pruned subtrees are added to the `removed` list. + /// Returns a booleans indicateing if this node (and its subtree) should be removed. + pub fn drain_filter(&mut self, filter: &mut F, removed: &mut Vec>) -> bool + where + F: FnMut(&H, &N, &V) -> FilterAction, + { + match filter(&self.hash, &self.number, &self.data) { + FilterAction::KeepNode => { + let mut i = 0; + while i < self.children.len() { + if self.children[i].drain_filter(filter, removed) { + removed.push(self.children.remove(i)); + } else { + i += 1; + } + } + false + }, + FilterAction::KeepTree => false, + FilterAction::Remove => true, + } + } } } @@ -895,6 +960,8 @@ impl Iterator for RemovedIterator { #[cfg(test)] mod test { + use crate::FilterAction; + use super::{Error, FinalizationResult, ForkTree}; #[derive(Debug, PartialEq)] @@ -919,11 +986,11 @@ mod test { // / - G // / / // A - F - H - I - // \ - // - L - M \ - // - O - // \ - // — J - K + // \ \ + // \ - L - M + // \ \ + // \ - O + // - J - K // // (where N is not a part of fork tree) // @@ -1211,18 +1278,31 @@ mod test { Ok(None), ); + // finalizing "D" is not allowed since it is not a root. + assert_eq!( + tree.finalize_with_descendent_if(&"D", 10, &is_descendent_of, |c| c.effective <= 10), + Err(Error::UnfinalizedAncestor) + ); + // finalizing "D" will finalize a block from the tree, but it can't be applied yet - // since it is not a root change + // since it is not a root change. assert_eq!( tree.finalizes_any_with_descendent_if(&"D", 10, &is_descendent_of, |c| c.effective == - 10,), + 10), Ok(Some(false)), ); + // finalizing "E" is not allowed since there are not finalized anchestors. + assert_eq!( + tree.finalizes_any_with_descendent_if(&"E", 15, &is_descendent_of, |c| c.effective == + 10), + Err(Error::UnfinalizedAncestor) + ); + // finalizing "B" doesn't finalize "A0" since the predicate doesn't pass, // although it will clear out "A1" from the tree assert_eq!( - tree.finalize_with_descendent_if(&"B", 2, &is_descendent_of, |c| c.effective <= 2,), + tree.finalize_with_descendent_if(&"B", 2, &is_descendent_of, |c| c.effective <= 2), Ok(FinalizationResult::Changed(None)), ); @@ -1243,7 +1323,7 @@ mod test { ); assert_eq!( - tree.finalize_with_descendent_if(&"C", 5, &is_descendent_of, |c| c.effective <= 5,), + tree.finalize_with_descendent_if(&"C", 5, &is_descendent_of, |c| c.effective <= 5), Ok(FinalizationResult::Changed(Some(Change { effective: 5 }))), ); @@ -1262,12 +1342,12 @@ mod test { // it will work with "G" though since it is not in the same branch as "E" assert_eq!( tree.finalizes_any_with_descendent_if(&"G", 100, &is_descendent_of, |c| c.effective <= - 100,), + 100), Ok(Some(true)), ); assert_eq!( - tree.finalize_with_descendent_if(&"G", 100, &is_descendent_of, |c| c.effective <= 100,), + tree.finalize_with_descendent_if(&"G", 100, &is_descendent_of, |c| c.effective <= 100), Ok(FinalizationResult::Changed(Some(Change { effective: 10 }))), ); @@ -1458,4 +1538,28 @@ mod test { ["A", "F", "H", "L", "O", "P", "M", "I", "G", "B", "C", "D", "E", "J", "K"] ); } + + #[test] + fn tree_drain_filter() { + let (mut tree, _) = test_fork_tree(); + + let filter = |h: &&str, _: &u64, _: &()| match *h { + "A" | "B" | "F" | "G" => FilterAction::KeepNode, + "C" => FilterAction::KeepTree, + "H" | "J" => FilterAction::Remove, + _ => panic!("Unexpected filtering for node: {}", *h), + }; + + let removed = tree.drain_filter(filter); + + assert_eq!( + tree.iter().map(|(h, _, _)| *h).collect::>(), + ["A", "B", "C", "D", "E", "F", "G"] + ); + + assert_eq!( + removed.map(|(h, _, _)| h).collect::>(), + ["J", "K", "H", "L", "M", "O", "I"] + ); + } } diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index 93616b590f61..5cb81232085a 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -2,9 +2,9 @@ name = "frame-benchmarking-cli" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "CLI for benchmarking FRAME" readme = "README.md" @@ -15,24 +15,44 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] frame-benchmarking = { version = "4.0.0-dev", path = "../../../frame/benchmarking" } frame-support = { version = "4.0.0-dev", path = "../../../frame/support" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } +frame-system = { version = "4.0.0-dev", path = "../../../frame/system" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sc-block-builder = { version = "0.10.0-dev", path = "../../../client/block-builder" } sc-service = { version = "0.10.0-dev", default-features = false, path = "../../../client/service" } +sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" } sc-cli = { version = "0.10.0-dev", path = "../../../client/cli" } sc-client-db = { version = "0.10.0-dev", path = "../../../client/db" } sc-executor = { version = "0.10.0-dev", path = "../../../client/executor" } -sp-externalities = { version = "0.10.0-dev", path = "../../../primitives/externalities" } -sp-keystore = { version = "0.10.0-dev", path = "../../../primitives/keystore" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -sp-state-machine = { version = "0.10.0-dev", path = "../../../primitives/state-machine" } -codec = { version = "2.0.0", package = "parity-scale-codec" } -structopt = "0.3.8" + +sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } +sp-externalities = { version = "0.12.0", path = "../../../primitives/externalities" } +sp-database = { version = "4.0.0-dev", path = "../../../primitives/database" } +sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } +sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } +sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" } +sp-storage = { version = "6.0.0", path = "../../../primitives/storage" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } +sp-state-machine = { version = "0.12.0", path = "../../../primitives/state-machine" } +sp-trie = { version = "6.0.0", path = "../../../primitives/trie" } +codec = { version = "3.0.0", package = "parity-scale-codec" } +clap = { version = "3.1.6", features = ["derive"] } chrono = "0.4" -serde = "1.0.126" -handlebars = "3.5.0" +serde = "1.0.136" +serde_json = "1.0.79" +handlebars = "4.2.2" Inflector = "0.11.4" linked-hash-map = "0.5.4" log = "0.4.8" +itertools = "0.10.3" +serde_nanos = "0.1.2" +kvdb = "0.11.0" +hash-db = "0.15.2" +hex = "0.4.3" +memory-db = "0.29.0" +rand = { version = "0.8.4", features = ["small_rng"] } +thousands = "0.2.0" [features] -default = ["db"] +default = ["db", "sc-client-db/runtime-benchmarks"] db = ["sc-client-db/with-kvdb-rocksdb", "sc-client-db/with-parity-db"] diff --git a/utils/frame/benchmarking-cli/src/block/bench.rs b/utils/frame/benchmarking-cli/src/block/bench.rs new file mode 100644 index 000000000000..e48a7e8b3c6f --- /dev/null +++ b/utils/frame/benchmarking-cli/src/block/bench.rs @@ -0,0 +1,176 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Contains the core benchmarking logic. + +use codec::DecodeAll; +use frame_support::weights::constants::WEIGHT_PER_NANOS; +use frame_system::ConsumedWeight; +use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider}; +use sc_cli::{Error, Result}; +use sc_client_api::{Backend as ClientBackend, BlockBackend, StorageProvider, UsageProvider}; +use sp_api::{ApiExt, Core, HeaderT, ProvideRuntimeApi}; +use sp_blockchain::Error::RuntimeApiError; +use sp_runtime::{generic::BlockId, traits::Block as BlockT, DigestItem, OpaqueExtrinsic}; +use sp_storage::StorageKey; + +use clap::Args; +use log::{info, warn}; +use serde::Serialize; +use std::{fmt::Debug, marker::PhantomData, sync::Arc, time::Instant}; +use thousands::Separable; + +use crate::shared::{StatSelect, Stats}; + +/// Log target for printing block weight info. +const LOG_TARGET: &'static str = "benchmark::block::weight"; + +/// Parameters for modifying the benchmark behaviour. +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct BenchmarkParams { + /// Number of the first block to consider. + #[clap(long)] + pub from: u32, + + /// Last block number to consider. + #[clap(long)] + pub to: u32, + + /// Number of times that the benchmark should be repeated for each block. + #[clap(long, default_value = "10")] + pub repeat: u32, +} + +/// Convenience closure for the [`Benchmark::run()`] function. +pub struct Benchmark { + client: Arc, + params: BenchmarkParams, + _p: PhantomData<(Block, BA, C)>, +} + +/// Helper for nano seconds. +type NanoSeconds = u64; + +impl Benchmark +where + Block: BlockT, + BA: ClientBackend, + C: BlockBuilderProvider + + ProvideRuntimeApi + + StorageProvider + + UsageProvider + + BlockBackend, + C::Api: ApiExt + BlockBuilderApi, +{ + /// Returns a new [`Self`] from the arguments. + pub fn new(client: Arc, params: BenchmarkParams) -> Self { + Self { client, params, _p: PhantomData } + } + + /// Benchmark the execution speed of historic blocks and log the results. + pub fn run(&self) -> Result<()> { + if self.params.from == 0 { + return Err("Cannot benchmark the genesis block".into()) + } + + for i in self.params.from..=self.params.to { + let block_num = BlockId::Number(i.into()); + let parent_num = BlockId::Number(((i - 1) as u32).into()); + let consumed = self.consumed_weight(&block_num)?; + + let block = + self.client.block(&block_num)?.ok_or(format!("Block {} not found", block_num))?; + let block = self.unsealed(block.block); + let took = self.measure_block(&block, &parent_num)?; + + self.log_weight(i, block.extrinsics().len(), consumed, took); + } + + Ok(()) + } + + /// Return the average *execution* aka. *import* time of the block. + fn measure_block(&self, block: &Block, parent_num: &BlockId) -> Result { + let mut record = Vec::::default(); + // Interesting part here: + // Execute the block multiple times and collect stats about its execution time. + for _ in 0..self.params.repeat { + let block = block.clone(); + let runtime_api = self.client.runtime_api(); + let start = Instant::now(); + + runtime_api + .execute_block(&parent_num, block) + .map_err(|e| Error::Client(RuntimeApiError(e)))?; + + record.push(start.elapsed().as_nanos() as NanoSeconds); + } + + let took = Stats::new(&record)?.select(StatSelect::Average); + Ok(took) + } + + /// Returns the total nanoseconds of a [`frame_system::ConsumedWeight`] for a block number. + /// + /// This is the post-dispatch corrected weight and is only available + /// after executing the block. + fn consumed_weight(&self, block: &BlockId) -> Result { + // Hard-coded key for System::BlockWeight. It could also be passed in as argument + // for the benchmark, but I think this should work as well. + let hash = hex::decode("26aa394eea5630e07c48ae0c9558cef734abf5cb34d6244378cddbf18e849d96")?; + let key = StorageKey(hash); + + let mut raw_weight = &self + .client + .storage(&block, &key)? + .ok_or(format!("Could not find System::BlockWeight for block: {}", block))? + .0[..]; + + let weight = ConsumedWeight::decode_all(&mut raw_weight)?; + // Should be divisible, but still use floats in case we ever change that. + Ok((weight.total() as f64 / WEIGHT_PER_NANOS as f64).floor() as NanoSeconds) + } + + /// Prints the weight info of a block to the console. + fn log_weight(&self, num: u32, num_ext: usize, consumed: NanoSeconds, took: NanoSeconds) { + // The ratio of weight that the block used vs what it consumed. + // This should in general not exceed 100% (minus outliers). + let percent = (took as f64 / consumed as f64) * 100.0; + + let msg = format!( + "Block {} with {: >5} tx used {: >6.2}% of its weight ({: >14} of {: >14} ns)", + num, + num_ext, + percent, + took.separate_with_commas(), + consumed.separate_with_commas() + ); + + if took <= consumed { + info!(target: LOG_TARGET, "{}", msg); + } else { + warn!(target: LOG_TARGET, "{} - OVER WEIGHT!", msg); + } + } + + /// Removes the consensus seal from the block. + fn unsealed(&self, block: Block) -> Block { + let (mut header, exts) = block.deconstruct(); + header.digest_mut().logs.retain(|item| !matches!(item, DigestItem::Seal(_, _))); + Block::new(header, exts) + } +} diff --git a/utils/frame/benchmarking-cli/src/block/cmd.rs b/utils/frame/benchmarking-cli/src/block/cmd.rs new file mode 100644 index 000000000000..e4e1716b1c5a --- /dev/null +++ b/utils/frame/benchmarking-cli/src/block/cmd.rs @@ -0,0 +1,101 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Contains the [`BlockCmd`] as entry point for the CLI to execute +//! the *block* benchmark. + +use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider}; +use sc_cli::{CliConfiguration, ImportParams, Result, SharedParams}; +use sc_client_api::{Backend as ClientBackend, BlockBackend, StorageProvider, UsageProvider}; +use sp_api::{ApiExt, ProvideRuntimeApi}; +use sp_runtime::{traits::Block as BlockT, OpaqueExtrinsic}; + +use clap::Parser; +use std::{fmt::Debug, sync::Arc}; + +use super::bench::{Benchmark, BenchmarkParams}; + +/// Benchmark the execution time of historic blocks. +/// +/// This can be used to verify that blocks do not use more weight than they consumed +/// in their `WeightInfo`. Example: +/// +/// Let's say you are on a Substrate chain and want to verify that the first 3 blocks +/// did not use more weight than declared which would otherwise be an issue. +/// To test this with a dev node, first create one with a temp directory: +/// +/// $ substrate --dev -d /tmp/my-dev --execution wasm --wasm-execution compiled +/// +/// And wait some time to let it produce 3 blocks. Then benchmark them with: +/// +/// $ substrate benchmark-block --from 1 --to 3 --dev -d /tmp/my-dev +/// --execution wasm --wasm-execution compiled --pruning archive +/// +/// The output will be similar to this: +/// +/// Block 1 with 1 tx used 77.34% of its weight ( 5,308,964 of 6,864,645 ns) +/// Block 2 with 1 tx used 77.99% of its weight ( 5,353,992 of 6,864,645 ns) +/// Block 3 with 1 tx used 75.91% of its weight ( 5,305,938 of 6,989,645 ns) +/// +/// The percent number is important and indicates how much weight +/// was used as compared to the consumed weight. +/// This number should be below 100% for reference hardware. +#[derive(Debug, Parser)] +pub struct BlockCmd { + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub import_params: ImportParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub params: BenchmarkParams, +} + +impl BlockCmd { + /// Benchmark the execution time of historic blocks and compare it to their consumed weight. + /// + /// Output will be printed to console. + pub fn run(&self, client: Arc) -> Result<()> + where + Block: BlockT, + BA: ClientBackend, + C: BlockBuilderProvider + + BlockBackend + + ProvideRuntimeApi + + StorageProvider + + UsageProvider, + C::Api: ApiExt + BlockBuilderApi, + { + // Put everything in the benchmark type to have the generic types handy. + Benchmark::new(client, self.params.clone()).run() + } +} + +// Boilerplate +impl CliConfiguration for BlockCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } + + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) + } +} diff --git a/utils/frame/benchmarking-cli/src/block/mod.rs b/utils/frame/benchmarking-cli/src/block/mod.rs new file mode 100644 index 000000000000..97fdb6ad2c20 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/block/mod.rs @@ -0,0 +1,24 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Crate to benchmark the execution time of historic blocks +//! and compare it to their consumed weight. + +mod bench; +mod cmd; + +pub use cmd::BlockCmd; diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 316ddfb8d0c1..2543a63b2f15 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,137 +15,85 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod command; -mod writer; - -use sc_cli::{ExecutionStrategy, WasmExecutionMethod}; -use std::fmt::Debug; - -// Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be -// used like crate names with `_` -fn parse_pallet_name(pallet: &str) -> String { - pallet.replace("-", "_") +//! Contains the root [`BenchmarkCmd`] command and exports its sub-commands. + +mod block; +mod overhead; +mod pallet; +mod shared; +mod storage; + +pub use block::BlockCmd; +pub use overhead::{ExtrinsicBuilder, OverheadCmd}; +pub use pallet::PalletCmd; +pub use storage::StorageCmd; + +use sc_cli::{CliConfiguration, DatabaseParams, ImportParams, PruningParams, Result, SharedParams}; + +/// The root `benchmarking` command. +/// +/// Has no effect itself besides printing a help menu of the sub-commands. +#[derive(Debug, clap::Subcommand)] +pub enum BenchmarkCmd { + Pallet(PalletCmd), + Storage(StorageCmd), + Overhead(OverheadCmd), + Block(BlockCmd), } -/// The `benchmark` command used to benchmark FRAME Pallets. -#[derive(Debug, structopt::StructOpt)] -pub struct BenchmarkCmd { - /// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`). - #[structopt(short, long, parse(from_str = parse_pallet_name), required_unless = "list")] - pub pallet: Option, - - /// Select an extrinsic inside the pallet to benchmark, or `*` for all. - #[structopt(short, long, required_unless = "list")] - pub extrinsic: Option, - - /// Select how many samples we should take across the variable components. - #[structopt(short, long, default_value = "1")] - pub steps: u32, - - /// Indicates lowest values for each of the component ranges. - #[structopt(long = "low", use_delimiter = true)] - pub lowest_range_values: Vec, - - /// Indicates highest values for each of the component ranges. - #[structopt(long = "high", use_delimiter = true)] - pub highest_range_values: Vec, - - /// Select how many repetitions of this benchmark should run from within the wasm. - #[structopt(short, long, default_value = "1")] - pub repeat: u32, - - /// Select how many repetitions of this benchmark should run from the client. - /// - /// NOTE: Using this alone may give slower results, but will afford you maximum Wasm memory. - #[structopt(long, default_value = "1")] - pub external_repeat: u32, - - /// Print the raw results. - #[structopt(long = "raw")] - pub raw_data: bool, - - /// Don't print the median-slopes linear regression analysis. - #[structopt(long)] - pub no_median_slopes: bool, - - /// Don't print the min-squares linear regression analysis. - #[structopt(long)] - pub no_min_squares: bool, - - /// Output the benchmarks to a Rust file at the given path. - #[structopt(long)] - pub output: Option, - - /// Add a header file to your outputted benchmarks - #[structopt(long)] - pub header: Option, - - /// Path to Handlebars template file used for outputting benchmark results. (Optional) - #[structopt(long)] - pub template: Option, - - /// Which analysis function to use when outputting benchmarks: - /// * min-squares (default) - /// * median-slopes - /// * max (max of min squares and median slopes for each value) - #[structopt(long)] - pub output_analysis: Option, - - /// Set the heap pages while running benchmarks. If not set, the default value from the client - /// is used. - #[structopt(long)] - pub heap_pages: Option, - - /// Disable verification logic when running benchmarks. - #[structopt(long)] - pub no_verify: bool, - - /// Display and run extra benchmarks that would otherwise not be needed for weight - /// construction. - #[structopt(long)] - pub extra: bool, - - /// Estimate PoV size. - #[structopt(long)] - pub record_proof: bool, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: sc_cli::SharedParams, - - /// The execution strategy that should be used for benchmarks - #[structopt( - long = "execution", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - )] - pub execution: Option, - - /// Method for executing Wasm runtime code. - #[structopt( - long = "wasm-execution", - value_name = "METHOD", - possible_values = &WasmExecutionMethod::variants(), - case_insensitive = true, - default_value = "compiled" - )] - pub wasm_method: WasmExecutionMethod, - - /// Limit the memory the database cache can use. - #[structopt(long = "db-cache", value_name = "MiB", default_value = "128")] - pub database_cache_size: u32, - - /// List the benchmarks that match your query rather than running them. - /// - /// When nothing is provided, we list all benchmarks. - #[structopt(long)] - pub list: bool, +/// Unwraps a [`BenchmarkCmd`] into its concrete sub-command. +macro_rules! unwrap_cmd { + { + $self:expr, + $cmd:ident, + $code:expr + } => { + match $self { + BenchmarkCmd::Pallet($cmd) => $code, + BenchmarkCmd::Storage($cmd) => $code, + BenchmarkCmd::Overhead($cmd) => $code, + BenchmarkCmd::Block($cmd) => $code, + } + } +} - /// If enabled, the storage info is not displayed in the output next to the analysis. - /// - /// This is independent of the storage info appearing in the *output file*. Use a Handlebar - /// template for that purpose. - #[structopt(long)] - pub no_storage_info: bool, +/// Forward the [`CliConfiguration`] trait implementation. +/// +/// Each time a sub-command exposes a new config option, it must be added here. +impl CliConfiguration for BenchmarkCmd { + fn shared_params(&self) -> &SharedParams { + unwrap_cmd! { + self, cmd, cmd.shared_params() + } + } + + fn import_params(&self) -> Option<&ImportParams> { + unwrap_cmd! { + self, cmd, cmd.import_params() + } + } + + fn database_params(&self) -> Option<&DatabaseParams> { + unwrap_cmd! { + self, cmd, cmd.database_params() + } + } + + fn pruning_params(&self) -> Option<&PruningParams> { + unwrap_cmd! { + self, cmd, cmd.pruning_params() + } + } + + fn state_cache_size(&self) -> Result { + unwrap_cmd! { + self, cmd, cmd.state_cache_size() + } + } + + fn chain_id(&self, is_dev: bool) -> Result { + unwrap_cmd! { + self, cmd, cmd.chain_id(is_dev) + } + } } diff --git a/utils/frame/benchmarking-cli/src/overhead/bench.rs b/utils/frame/benchmarking-cli/src/overhead/bench.rs new file mode 100644 index 000000000000..68f3f6597b46 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/overhead/bench.rs @@ -0,0 +1,211 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Contains the core benchmarking logic. + +use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider}; +use sc_cli::{Error, Result}; +use sc_client_api::Backend as ClientBackend; +use sp_api::{ApiExt, BlockId, Core, ProvideRuntimeApi}; +use sp_blockchain::{ + ApplyExtrinsicFailed::Validity, + Error::{ApplyExtrinsicFailed, RuntimeApiError}, +}; +use sp_runtime::{ + traits::{Block as BlockT, Zero}, + transaction_validity::{InvalidTransaction, TransactionValidityError}, + OpaqueExtrinsic, +}; + +use clap::Args; +use log::info; +use serde::Serialize; +use std::{marker::PhantomData, sync::Arc, time::Instant}; + +use super::cmd::ExtrinsicBuilder; +use crate::shared::Stats; + +/// Parameters to configure an *overhead* benchmark. +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct BenchmarkParams { + /// Rounds of warmups before measuring. + #[clap(long, default_value = "100")] + pub warmup: u32, + + /// How many times the benchmark should be repeated. + #[clap(long, default_value = "1000")] + pub repeat: u32, + + /// Maximal number of extrinsics that should be put into a block. + /// + /// Only useful for debugging. + #[clap(long)] + pub max_ext_per_block: Option, +} + +/// The results of multiple runs in nano seconds. +pub(crate) type BenchRecord = Vec; + +/// Type of a benchmark. +#[derive(Serialize, Clone, PartialEq, Copy)] +pub(crate) enum BenchmarkType { + /// Measure the per-extrinsic execution overhead. + Extrinsic, + /// Measure the per-block execution overhead. + Block, +} + +/// Holds all objects needed to run the *overhead* benchmarks. +pub(crate) struct Benchmark { + client: Arc, + params: BenchmarkParams, + inherent_data: sp_inherents::InherentData, + ext_builder: Arc, + _p: PhantomData<(Block, BA)>, +} + +impl Benchmark +where + Block: BlockT, + BA: ClientBackend, + C: BlockBuilderProvider + ProvideRuntimeApi, + C::Api: ApiExt + BlockBuilderApi, +{ + /// Create a new [`Self`] from the arguments. + pub fn new( + client: Arc, + params: BenchmarkParams, + inherent_data: sp_inherents::InherentData, + ext_builder: Arc, + ) -> Self { + Self { client, params, inherent_data, ext_builder, _p: PhantomData } + } + + /// Run the specified benchmark. + pub fn bench(&self, bench_type: BenchmarkType) -> Result { + let (block, num_ext) = self.build_block(bench_type)?; + let record = self.measure_block(&block, num_ext, bench_type)?; + Stats::new(&record) + } + + /// Builds a block for the given benchmark type. + /// + /// Returns the block and the number of extrinsics in the block + /// that are not inherents. + fn build_block(&self, bench_type: BenchmarkType) -> Result<(Block, u64)> { + let mut builder = self.client.new_block(Default::default())?; + // Create and insert the inherents. + let inherents = builder.create_inherents(self.inherent_data.clone())?; + for inherent in inherents { + builder.push(inherent)?; + } + + // Return early if we just want a block with inherents and no additional extrinsics. + if bench_type == BenchmarkType::Block { + return Ok((builder.build()?.block, 0)) + } + + // Put as many extrinsics into the block as possible and count them. + info!("Building block, this takes some time..."); + let mut num_ext = 0; + for nonce in 0..self.max_ext_per_block() { + let ext = self.ext_builder.remark(nonce)?; + match builder.push(ext.clone()) { + Ok(()) => {}, + Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid( + InvalidTransaction::ExhaustsResources, + )))) => break, // Block is full + Err(e) => return Err(Error::Client(e)), + } + num_ext += 1; + } + if num_ext == 0 { + return Err("A Block must hold at least one extrinsic".into()) + } + info!("Extrinsics per block: {}", num_ext); + let block = builder.build()?.block; + + Ok((block, num_ext)) + } + + /// Measures the time that it take to execute a block or an extrinsic. + fn measure_block( + &self, + block: &Block, + num_ext: u64, + bench_type: BenchmarkType, + ) -> Result { + let mut record = BenchRecord::new(); + if bench_type == BenchmarkType::Extrinsic && num_ext == 0 { + return Err("Cannot measure the extrinsic time of an empty block".into()) + } + let genesis = BlockId::Number(Zero::zero()); + + info!("Running {} warmups...", self.params.warmup); + for _ in 0..self.params.warmup { + self.client + .runtime_api() + .execute_block(&genesis, block.clone()) + .map_err(|e| Error::Client(RuntimeApiError(e)))?; + } + + info!("Executing block {} times", self.params.repeat); + // Interesting part here: + // Execute a block multiple times and record each execution time. + for _ in 0..self.params.repeat { + let block = block.clone(); + let runtime_api = self.client.runtime_api(); + let start = Instant::now(); + + runtime_api + .execute_block(&genesis, block) + .map_err(|e| Error::Client(RuntimeApiError(e)))?; + + let elapsed = start.elapsed().as_nanos(); + if bench_type == BenchmarkType::Extrinsic { + // Checked for non-zero div above. + record.push((elapsed as f64 / num_ext as f64).ceil() as u64); + } else { + record.push(elapsed as u64); + } + } + + Ok(record) + } + + fn max_ext_per_block(&self) -> u32 { + self.params.max_ext_per_block.unwrap_or(u32::MAX) + } +} + +impl BenchmarkType { + /// Short name of the benchmark type. + pub(crate) fn short_name(&self) -> &'static str { + match self { + Self::Extrinsic => "extrinsic", + Self::Block => "block", + } + } + + /// Long name of the benchmark type. + pub(crate) fn long_name(&self) -> &'static str { + match self { + Self::Extrinsic => "ExtrinsicBase", + Self::Block => "BlockExecution", + } + } +} diff --git a/utils/frame/benchmarking-cli/src/overhead/cmd.rs b/utils/frame/benchmarking-cli/src/overhead/cmd.rs new file mode 100644 index 000000000000..3cf281986861 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/overhead/cmd.rs @@ -0,0 +1,126 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Contains the [`OverheadCmd`] as entry point for the CLI to execute +//! the *overhead* benchmarks. + +use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider}; +use sc_cli::{CliConfiguration, ImportParams, Result, SharedParams}; +use sc_client_api::Backend as ClientBackend; +use sc_service::Configuration; +use sp_api::{ApiExt, ProvideRuntimeApi}; +use sp_runtime::{traits::Block as BlockT, OpaqueExtrinsic}; + +use clap::{Args, Parser}; +use log::info; +use serde::Serialize; +use std::{fmt::Debug, sync::Arc}; + +use crate::{ + overhead::{ + bench::{Benchmark, BenchmarkParams, BenchmarkType}, + template::TemplateData, + }, + shared::WeightParams, +}; + +/// Benchmark the execution overhead per-block and per-extrinsic. +#[derive(Debug, Parser)] +pub struct OverheadCmd { + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub import_params: ImportParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub params: OverheadParams, +} + +/// Configures the benchmark, the post-processing and weight generation. +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct OverheadParams { + #[allow(missing_docs)] + #[clap(flatten)] + pub weight: WeightParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub bench: BenchmarkParams, +} + +/// Used by the benchmark to build signed extrinsics. +/// +/// The built extrinsics only need to be valid in the first block +/// who's parent block is the genesis block. +pub trait ExtrinsicBuilder { + /// Build a `System::remark` extrinsic. + fn remark(&self, nonce: u32) -> std::result::Result; +} + +impl OverheadCmd { + /// Measure the per-block and per-extrinsic execution overhead. + /// + /// Writes the results to console and into two instances of the + /// `weights.hbs` template, one for each benchmark. + pub fn run( + &self, + cfg: Configuration, + client: Arc, + inherent_data: sp_inherents::InherentData, + ext_builder: Arc, + ) -> Result<()> + where + Block: BlockT, + BA: ClientBackend, + C: BlockBuilderProvider + ProvideRuntimeApi, + C::Api: ApiExt + BlockBuilderApi, + { + let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data, ext_builder); + + // per-block execution overhead + { + let stats = bench.bench(BenchmarkType::Block)?; + info!("Per-block execution overhead [ns]:\n{:?}", stats); + let template = TemplateData::new(BenchmarkType::Block, &cfg, &self.params, &stats)?; + template.write(&self.params.weight.weight_path)?; + } + // per-extrinsic execution overhead + { + let stats = bench.bench(BenchmarkType::Extrinsic)?; + info!("Per-extrinsic execution overhead [ns]:\n{:?}", stats); + let template = TemplateData::new(BenchmarkType::Extrinsic, &cfg, &self.params, &stats)?; + template.write(&self.params.weight.weight_path)?; + } + + Ok(()) + } +} + +// Boilerplate +impl CliConfiguration for OverheadCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } + + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) + } +} diff --git a/utils/frame/benchmarking-cli/src/overhead/mod.rs b/utils/frame/benchmarking-cli/src/overhead/mod.rs new file mode 100644 index 000000000000..abdeac22b789 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/overhead/mod.rs @@ -0,0 +1,22 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod bench; +pub mod cmd; +mod template; + +pub use cmd::{ExtrinsicBuilder, OverheadCmd}; diff --git a/utils/frame/benchmarking-cli/src/overhead/template.rs b/utils/frame/benchmarking-cli/src/overhead/template.rs new file mode 100644 index 000000000000..d5f90d487386 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/overhead/template.rs @@ -0,0 +1,110 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Converts a benchmark result into [`TemplateData`] and writes +//! it into the `weights.hbs` template. + +use sc_cli::Result; +use sc_service::Configuration; + +use handlebars::Handlebars; +use log::info; +use serde::Serialize; +use std::{env, fs, path::PathBuf}; + +use crate::{ + overhead::{bench::BenchmarkType, cmd::OverheadParams}, + shared::{Stats, UnderscoreHelper}, +}; + +static VERSION: &'static str = env!("CARGO_PKG_VERSION"); +static TEMPLATE: &str = include_str!("./weights.hbs"); + +/// Data consumed by Handlebar to fill out the `weights.hbs` template. +#[derive(Serialize, Debug, Clone)] +pub(crate) struct TemplateData { + /// Short name of the benchmark. Can be "block" or "extrinsic". + long_name: String, + /// Long name of the benchmark. Can be "BlockExecution" or "ExtrinsicBase". + short_name: String, + /// Name of the runtime. Taken from the chain spec. + runtime_name: String, + /// Version of the benchmarking CLI used. + version: String, + /// Date that the template was filled out. + date: String, + /// Command line arguments that were passed to the CLI. + args: Vec, + /// Params of the executed command. + params: OverheadParams, + /// Stats about the benchmark result. + stats: Stats, + /// The resulting weight in ns. + weight: u64, +} + +impl TemplateData { + /// Returns a new [`Self`] from the given params. + pub(crate) fn new( + t: BenchmarkType, + cfg: &Configuration, + params: &OverheadParams, + stats: &Stats, + ) -> Result { + let weight = params.weight.calc_weight(stats)?; + + Ok(TemplateData { + short_name: t.short_name().into(), + long_name: t.long_name().into(), + runtime_name: cfg.chain_spec.name().into(), + version: VERSION.into(), + date: chrono::Utc::now().format("%Y-%m-%d (Y/M/D)").to_string(), + args: env::args().collect::>(), + params: params.clone(), + stats: stats.clone(), + weight, + }) + } + + /// Fill out the `weights.hbs` HBS template with its own data. + /// Writes the result to `path` which can be a directory or a file. + pub fn write(&self, path: &Option) -> Result<()> { + let mut handlebars = Handlebars::new(); + // Format large integers with underscores. + handlebars.register_helper("underscore", Box::new(UnderscoreHelper)); + // Don't HTML escape any characters. + handlebars.register_escape_fn(|s| -> String { s.to_string() }); + + let out_path = self.build_path(path)?; + let mut fd = fs::File::create(&out_path)?; + info!("Writing weights to {:?}", fs::canonicalize(&out_path)?); + handlebars + .render_template_to_write(&TEMPLATE, &self, &mut fd) + .map_err(|e| format!("HBS template write: {:?}", e).into()) + } + + /// Build a path for the weight file. + fn build_path(&self, weight_out: &Option) -> Result { + let mut path = weight_out.clone().unwrap_or(PathBuf::from(".")); + + if !path.is_dir() { + return Err("Need directory as --weight-path".into()) + } + path.push(format!("{}_weights.rs", self.short_name)); + Ok(path) + } +} diff --git a/utils/frame/benchmarking-cli/src/overhead/weights.hbs b/utils/frame/benchmarking-cli/src/overhead/weights.hbs new file mode 100644 index 000000000000..ad33f55a9f36 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/overhead/weights.hbs @@ -0,0 +1,80 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION {{version}} +//! DATE: {{date}} +//! +//! SHORT-NAME: `{{short_name}}`, LONG-NAME: `{{long_name}}`, RUNTIME: `{{runtime_name}}` +//! WARMUPS: `{{params.bench.warmup}}`, REPEAT: `{{params.bench.repeat}}` +//! WEIGHT-PATH: `{{params.weight.weight_path}}` +//! WEIGHT-METRIC: `{{params.weight.weight_metric}}`, WEIGHT-MUL: `{{params.weight.weight_mul}}`, WEIGHT-ADD: `{{params.weight.weight_add}}` + +// Executed Command: +{{#each args as |arg|}} +// {{arg}} +{{/each}} + +use frame_support::{ + parameter_types, + weights::{constants::WEIGHT_PER_NANOS, Weight}, +}; + +parameter_types! { + {{#if (eq short_name "block")}} + /// Time to execute an empty block. + {{else}} + /// Time to execute a NO-OP extrinsic, for example `System::remark`. + {{/if}} + /// Calculated by multiplying the *{{params.weight.weight_metric}}* with `{{params.weight.weight_mul}}` and adding `{{params.weight.weight_add}}`. + /// + /// Stats [NS]: + /// Min, Max: {{underscore stats.min}}, {{underscore stats.max}} + /// Average: {{underscore stats.avg}} + /// Median: {{underscore stats.median}} + /// Std-Dev: {{stats.stddev}} + /// + /// Percentiles [NS]: + /// 99th: {{underscore stats.p99}} + /// 95th: {{underscore stats.p95}} + /// 75th: {{underscore stats.p75}} + pub const {{long_name}}Weight: Weight = {{underscore weight}} * WEIGHT_PER_NANOS; +} + +#[cfg(test)] +mod test_weights { + use frame_support::weights::constants; + + /// Checks that the weight exists and is sane. + // NOTE: If this test fails but you are sure that the generated values are fine, + // you can delete it. + #[test] + fn sane() { + let w = super::{{long_name}}Weight::get(); + + {{#if (eq short_name "block")}} + // At least 100 µs. + assert!(w >= 100 * constants::WEIGHT_PER_MICROS, "Weight should be at least 100 µs."); + // At most 50 ms. + assert!(w <= 50 * constants::WEIGHT_PER_MILLIS, "Weight should be at most 50 ms."); + {{else}} + // At least 10 µs. + assert!(w >= 10 * constants::WEIGHT_PER_MICROS, "Weight should be at least 10 µs."); + // At most 1 ms. + assert!(w <= constants::WEIGHT_PER_MILLIS, "Weight should be at most 1 ms."); + {{/if}} + } +} diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/pallet/command.rs similarity index 81% rename from utils/frame/benchmarking-cli/src/command.rs rename to utils/frame/benchmarking-cli/src/pallet/command.rs index 5efa970d9358..89b8d018bcb5 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/pallet/command.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::BenchmarkCmd; +use super::{writer, PalletCmd}; use codec::{Decode, Encode}; use frame_benchmarking::{ Analysis, BenchmarkBatch, BenchmarkBatchSplitResults, BenchmarkList, BenchmarkParameter, @@ -33,9 +33,9 @@ use sp_core::offchain::{ }; use sp_externalities::Extensions; use sp_keystore::{testing::KeyStore, KeystoreExt, SyncCryptoStorePtr}; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use sp_state_machine::StateMachine; -use std::{fmt::Debug, sync::Arc, time}; +use std::{fmt::Debug, fs, sync::Arc, time}; // This takes multiple benchmark batches and combines all the results where the pallet, instance, // and benchmark are the same. @@ -87,7 +87,13 @@ fn combine_batches( .collect::>() } -impl BenchmarkCmd { +/// Explains possible reasons why the metadata for the benchmarking could not be found. +const ERROR_METADATA_NOT_FOUND: &'static str = "Did not find the benchmarking metadata. \ +This could mean that you either did not build the node correctly with the \ +`--features runtime-benchmarks` flag, or the chain spec that you are using was \ +not created by a node that was compiled with the flag"; + +impl PalletCmd { /// Runs the command and benchmarks the chain. pub fn run(&self, config: Configuration) -> Result<()> where @@ -120,7 +126,8 @@ impl BenchmarkCmd { let pallet = self.pallet.clone().unwrap_or_else(|| String::new()); let pallet = pallet.as_bytes(); let extrinsic = self.extrinsic.clone().unwrap_or_else(|| String::new()); - let extrinsic = extrinsic.as_bytes(); + let extrinsic_split: Vec<&str> = extrinsic.split(',').collect(); + let extrinsics: Vec<_> = extrinsic_split.iter().map(|x| x.trim().as_bytes()).collect(); let genesis_storage = spec.build_storage()?; let mut changes = Default::default(); @@ -137,6 +144,7 @@ impl BenchmarkCmd { wasm_method, self.heap_pages, 2, // The runtime instances cache size. + 2, // The runtime cache size ); let extensions = || -> Extensions { @@ -152,9 +160,8 @@ impl BenchmarkCmd { // Get Benchmark List let state = &state_without_tracking; - let result = StateMachine::<_, _, NumberFor, _>::new( + let result = StateMachine::new( state, - None, &mut changes, &executor, "Benchmark_benchmark_metadata", @@ -164,7 +171,7 @@ impl BenchmarkCmd { sp_core::testing::TaskExecutor::new(), ) .execute(strategy.into()) - .map_err(|e| format!("Error getting benchmark list: {:?}", e))?; + .map_err(|e| format!("{}: {}", ERROR_METADATA_NOT_FOUND, e))?; let (list, storage_info) = <(Vec, Vec) as Decode>::decode(&mut &result[..]) @@ -176,9 +183,10 @@ impl BenchmarkCmd { .filter(|item| pallet.is_empty() || pallet == &b"*"[..] || pallet == &item.pallet[..]) .for_each(|item| { for benchmark in &item.benchmarks { + let benchmark_name = &benchmark.name; if extrinsic.is_empty() || - &extrinsic[..] == &b"*"[..] || - extrinsic == benchmark.name + extrinsic.as_bytes() == &b"*"[..] || + extrinsics.contains(&&benchmark_name[..]) { benchmarks_to_run.push(( item.pallet.clone(), @@ -243,9 +251,8 @@ impl BenchmarkCmd { if !self.no_verify { // Dont use these results since verification code will add overhead let state = &state_without_tracking; - let _results = StateMachine::<_, _, NumberFor, _>::new( + let _results = StateMachine::new( state, - None, &mut changes, &executor, "Benchmark_dispatch_benchmark", @@ -264,15 +271,14 @@ impl BenchmarkCmd { ) .execute(strategy.into()) .map_err(|e| { - format!("Error executing and verifying runtime benchmark: {:?}", e) + format!("Error executing and verifying runtime benchmark: {}", e) })?; } // Do one loop of DB tracking. { let state = &state_with_tracking; - let result = StateMachine::<_, _, NumberFor, _>::new( + let result = StateMachine::new( state, // todo remove tracking - None, &mut changes, &executor, "Benchmark_dispatch_benchmark", @@ -290,7 +296,7 @@ impl BenchmarkCmd { sp_core::testing::TaskExecutor::new(), ) .execute(strategy.into()) - .map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?; + .map_err(|e| format!("Error executing runtime benchmark: {}", e))?; let batch = , String> as Decode>::decode( @@ -303,9 +309,8 @@ impl BenchmarkCmd { // Finally run a bunch of loops to get extrinsic timing information. for r in 0..self.external_repeat { let state = &state_without_tracking; - let result = StateMachine::<_, _, NumberFor, _>::new( + let result = StateMachine::new( state, // todo remove tracking - None, &mut changes, &executor, "Benchmark_dispatch_benchmark", @@ -323,7 +328,7 @@ impl BenchmarkCmd { sp_core::testing::TaskExecutor::new(), ) .execute(strategy.into()) - .map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?; + .map_err(|e| format!("Error executing runtime benchmark: {}", e))?; let batch = , String> as Decode>::decode( @@ -338,14 +343,14 @@ impl BenchmarkCmd { if elapsed >= time::Duration::from_secs(5) { timer = time::SystemTime::now(); log::info!( - "Running Benchmark:\t{}\t{}\t{}/{}\t{}/{}", + "Running Benchmark: {}.{} {}/{} {}/{}", String::from_utf8(pallet.clone()) .expect("Encoded from String; qed"), String::from_utf8(extrinsic.clone()) .expect("Encoded from String; qed"), - s, // todo show step + s + 1, // s starts at 0. todo show step self.steps, - r, + r + 1, self.external_repeat, ); } @@ -358,62 +363,64 @@ impl BenchmarkCmd { // are together. let batches: Vec = combine_batches(batches, batches_db); + // Create the weights.rs file. if let Some(output_path) = &self.output { - crate::writer::write_results(&batches, &storage_info, output_path, self)?; + writer::write_results(&batches, &storage_info, output_path, self)?; + } + + // Jsonify the result and write it to a file or stdout if desired. + if !self.jsonify(&batches)? { + // Print the summary only if `jsonify` did not write to stdout. + self.print_summary(&batches, &storage_info) + } + Ok(()) + } + + /// Jsonifies the passed batches and writes them to stdout or into a file. + /// Can be configured via `--json` and `--json-file`. + /// Returns whether it wrote to stdout. + fn jsonify(&self, batches: &Vec) -> Result { + if self.json_output || self.json_file.is_some() { + let json = serde_json::to_string_pretty(&batches) + .map_err(|e| format!("Serializing into JSON: {:?}", e))?; + + if let Some(path) = &self.json_file { + fs::write(path, json)?; + } else { + println!("{}", json); + return Ok(true) + } } + Ok(false) + } + + /// Prints the results as human-readable summary without raw timing data. + fn print_summary( + &self, + batches: &Vec, + storage_info: &Vec, + ) { for batch in batches.into_iter() { // Print benchmark metadata println!( - "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", - String::from_utf8(batch.pallet).expect("Encoded from String; qed"), - String::from_utf8(batch.benchmark).expect("Encoded from String; qed"), - self.lowest_range_values, - self.highest_range_values, - self.steps, - self.repeat, - ); + "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", + String::from_utf8(batch.pallet.clone()).expect("Encoded from String; qed"), + String::from_utf8(batch.benchmark.clone()).expect("Encoded from String; qed"), + self.lowest_range_values, + self.highest_range_values, + self.steps, + self.repeat, + ); // Skip raw data + analysis if there are no results if batch.time_results.is_empty() { continue } - if self.raw_data { - // Print the table header - batch.time_results[0] - .components - .iter() - .for_each(|param| print!("{:?},", param.0)); - - print!("extrinsic_time_ns,storage_root_time_ns,reads,repeat_reads,writes,repeat_writes,proof_size_bytes\n"); - // Print the values - batch.time_results.iter().for_each(|result| { - let parameters = &result.components; - parameters.iter().for_each(|param| print!("{:?},", param.1)); - // Print extrinsic time and storage root time - print!( - "{:?},{:?},{:?},{:?},{:?},{:?},{:?}\n", - result.extrinsic_time, - result.storage_root_time, - result.reads, - result.repeat_reads, - result.writes, - result.repeat_writes, - result.proof_size, - ); - }); - - println!(); - } - if !self.no_storage_info { let mut comments: Vec = Default::default(); - crate::writer::add_storage_comments( - &mut comments, - &batch.db_results, - &storage_info, - ); + writer::add_storage_comments(&mut comments, &batch.db_results, &storage_info); println!("Raw Storage Info\n========"); for comment in comments { println!("{}", comment); @@ -461,12 +468,10 @@ impl BenchmarkCmd { println!(""); } } - - Ok(()) } } -impl CliConfiguration for BenchmarkCmd { +impl CliConfiguration for PalletCmd { fn shared_params(&self) -> &SharedParams { &self.shared_params } diff --git a/utils/frame/benchmarking-cli/src/pallet/mod.rs b/utils/frame/benchmarking-cli/src/pallet/mod.rs new file mode 100644 index 000000000000..48ddcc7ce8ee --- /dev/null +++ b/utils/frame/benchmarking-cli/src/pallet/mod.rs @@ -0,0 +1,150 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod command; +mod writer; + +use sc_cli::{ExecutionStrategy, WasmExecutionMethod, DEFAULT_WASM_EXECUTION_METHOD}; +use std::{fmt::Debug, path::PathBuf}; + +// Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be +// used like crate names with `_` +fn parse_pallet_name(pallet: &str) -> String { + pallet.replace("-", "_") +} + +/// Benchmark the extrinsic weight of FRAME Pallets. +#[derive(Debug, clap::Parser)] +pub struct PalletCmd { + /// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`). + #[clap(short, long, parse(from_str = parse_pallet_name), required_unless_present = "list")] + pub pallet: Option, + + /// Select an extrinsic inside the pallet to benchmark, or `*` for all. + #[clap(short, long, required_unless_present = "list")] + pub extrinsic: Option, + + /// Select how many samples we should take across the variable components. + #[clap(short, long, default_value = "1")] + pub steps: u32, + + /// Indicates lowest values for each of the component ranges. + #[clap(long = "low", use_value_delimiter = true)] + pub lowest_range_values: Vec, + + /// Indicates highest values for each of the component ranges. + #[clap(long = "high", use_value_delimiter = true)] + pub highest_range_values: Vec, + + /// Select how many repetitions of this benchmark should run from within the wasm. + #[clap(short, long, default_value = "1")] + pub repeat: u32, + + /// Select how many repetitions of this benchmark should run from the client. + /// + /// NOTE: Using this alone may give slower results, but will afford you maximum Wasm memory. + #[clap(long, default_value = "1")] + pub external_repeat: u32, + + /// Print the raw results in JSON format. + #[clap(long = "json")] + pub json_output: bool, + + /// Write the raw results in JSON format into the given file. + #[clap(long, conflicts_with = "json-output")] + pub json_file: Option, + + /// Don't print the median-slopes linear regression analysis. + #[clap(long)] + pub no_median_slopes: bool, + + /// Don't print the min-squares linear regression analysis. + #[clap(long)] + pub no_min_squares: bool, + + /// Output the benchmarks to a Rust file at the given path. + #[clap(long)] + pub output: Option, + + /// Add a header file to your outputted benchmarks. + #[clap(long)] + pub header: Option, + + /// Path to Handlebars template file used for outputting benchmark results. (Optional) + #[clap(long)] + pub template: Option, + + /// Which analysis function to use when outputting benchmarks: + /// * min-squares (default) + /// * median-slopes + /// * max (max of min squares and median slopes for each value) + #[clap(long)] + pub output_analysis: Option, + + /// Set the heap pages while running benchmarks. If not set, the default value from the client + /// is used. + #[clap(long)] + pub heap_pages: Option, + + /// Disable verification logic when running benchmarks. + #[clap(long)] + pub no_verify: bool, + + /// Display and run extra benchmarks that would otherwise not be needed for weight + /// construction. + #[clap(long)] + pub extra: bool, + + /// Estimate PoV size. + #[clap(long)] + pub record_proof: bool, + + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: sc_cli::SharedParams, + + /// The execution strategy that should be used for benchmarks. + #[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)] + pub execution: Option, + + /// Method for executing Wasm runtime code. + #[clap( + long = "wasm-execution", + value_name = "METHOD", + possible_values = WasmExecutionMethod::variants(), + ignore_case = true, + default_value = DEFAULT_WASM_EXECUTION_METHOD, + )] + pub wasm_method: WasmExecutionMethod, + + /// Limit the memory the database cache can use. + #[clap(long = "db-cache", value_name = "MiB", default_value = "1024")] + pub database_cache_size: u32, + + /// List the benchmarks that match your query rather than running them. + /// + /// When nothing is provided, we list all benchmarks. + #[clap(long)] + pub list: bool, + + /// If enabled, the storage info is not displayed in the output next to the analysis. + /// + /// This is independent of the storage info appearing in the *output file*. Use a Handlebar + /// template for that purpose. + #[clap(long)] + pub no_storage_info: bool, +} diff --git a/utils/frame/benchmarking-cli/src/template.hbs b/utils/frame/benchmarking-cli/src/pallet/template.hbs similarity index 78% rename from utils/frame/benchmarking-cli/src/template.hbs rename to utils/frame/benchmarking-cli/src/pallet/template.hbs index 36abf27f59a6..ea734e165919 100644 --- a/utils/frame/benchmarking-cli/src/template.hbs +++ b/utils/frame/benchmarking-cli/src/pallet/template.hbs @@ -6,7 +6,7 @@ //! EXECUTION: {{cmd.execution}}, WASM-EXECUTION: {{cmd.wasm_execution}}, CHAIN: {{cmd.chain}}, DB CACHE: {{cmd.db_cache}} // Executed Command: -{{#each args as |arg|~}} +{{#each args as |arg|}} // {{arg}} {{/each}} @@ -20,32 +20,32 @@ use sp_std::marker::PhantomData; /// Weight functions for `{{pallet}}`. pub struct WeightInfo(PhantomData); impl {{pallet}}::WeightInfo for WeightInfo { - {{~#each benchmarks as |benchmark|}} - {{~#each benchmark.comments as |comment|}} + {{#each benchmarks as |benchmark|}} + {{#each benchmark.comments as |comment|}} // {{comment}} - {{~/each}} + {{/each}} fn {{benchmark.name~}} ( {{~#each benchmark.components as |c| ~}} {{~#if (not c.is_used)}}_{{/if}}{{c.name}}: u32, {{/each~}} ) -> Weight { ({{underscore benchmark.base_weight}} as Weight) - {{~#each benchmark.component_weight as |cw|}} + {{#each benchmark.component_weight as |cw|}} // Standard Error: {{underscore cw.error}} .saturating_add(({{underscore cw.slope}} as Weight).saturating_mul({{cw.name}} as Weight)) - {{~/each}} - {{~#if (ne benchmark.base_reads "0")}} + {{/each}} + {{#if (ne benchmark.base_reads "0")}} .saturating_add(T::DbWeight::get().reads({{benchmark.base_reads}} as Weight)) - {{~/if}} - {{~#each benchmark.component_reads as |cr|}} + {{/if}} + {{#each benchmark.component_reads as |cr|}} .saturating_add(T::DbWeight::get().reads(({{cr.slope}} as Weight).saturating_mul({{cr.name}} as Weight))) - {{~/each}} - {{~#if (ne benchmark.base_writes "0")}} + {{/each}} + {{#if (ne benchmark.base_writes "0")}} .saturating_add(T::DbWeight::get().writes({{benchmark.base_writes}} as Weight)) - {{~/if}} - {{~#each benchmark.component_writes as |cw|}} + {{/if}} + {{#each benchmark.component_writes as |cw|}} .saturating_add(T::DbWeight::get().writes(({{cw.slope}} as Weight).saturating_mul({{cw.name}} as Weight))) - {{~/each}} + {{/each}} } - {{~/each}} + {{/each}} } diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/pallet/writer.rs similarity index 93% rename from utils/frame/benchmarking-cli/src/writer.rs rename to utils/frame/benchmarking-cli/src/pallet/writer.rs index ede5b2d1355a..cd97b3efbd9d 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/pallet/writer.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +17,6 @@ // Outputs benchmark results to Rust files that can be ingested by the runtime. -use core::convert::TryInto; use std::{ collections::{HashMap, HashSet}, fs, @@ -27,7 +26,7 @@ use std::{ use inflector::Inflector; use serde::Serialize; -use crate::BenchmarkCmd; +use crate::{shared::UnderscoreHelper, PalletCmd}; use frame_benchmarking::{ Analysis, AnalysisChoice, BenchmarkBatchSplitResults, BenchmarkResult, BenchmarkSelector, RegressionModel, @@ -69,7 +68,7 @@ struct BenchmarkData { comments: Vec, } -// This forwards some specific metadata from the `BenchmarkCmd` +// This forwards some specific metadata from the `PalletCmd` #[derive(Serialize, Default, Debug, Clone)] struct CmdData { steps: u32, @@ -256,7 +255,7 @@ pub fn write_results( batches: &[BenchmarkBatchSplitResults], storage_info: &[StorageInfo], path: &PathBuf, - cmd: &BenchmarkCmd, + cmd: &PalletCmd, ) -> Result<(), std::io::Error> { // Use custom template if provided. let template: String = match &cmd.template { @@ -417,44 +416,6 @@ pub(crate) fn add_storage_comments( } } -// Add an underscore after every 3rd character, i.e. a separator for large numbers. -fn underscore(i: Number) -> String -where - Number: std::string::ToString, -{ - let mut s = String::new(); - let i_str = i.to_string(); - let a = i_str.chars().rev().enumerate(); - for (idx, val) in a { - if idx != 0 && idx % 3 == 0 { - s.insert(0, '_'); - } - s.insert(0, val); - } - s -} - -// A Handlebars helper to add an underscore after every 3rd character, -// i.e. a separator for large numbers. -#[derive(Clone, Copy)] -struct UnderscoreHelper; -impl handlebars::HelperDef for UnderscoreHelper { - fn call<'reg: 'rc, 'rc>( - &self, - h: &handlebars::Helper, - _: &handlebars::Handlebars, - _: &handlebars::Context, - _rc: &mut handlebars::RenderContext, - out: &mut dyn handlebars::Output, - ) -> handlebars::HelperResult { - use handlebars::JsonRender; - let param = h.param(0).unwrap(); - let underscore_param = underscore(param.value().render()); - out.write(&underscore_param)?; - Ok(()) - } -} - // A helper to join a string of vectors. #[derive(Clone, Copy)] struct JoinHelper; diff --git a/utils/frame/benchmarking-cli/src/shared/mod.rs b/utils/frame/benchmarking-cli/src/shared/mod.rs new file mode 100644 index 000000000000..f08d79b9aafc --- /dev/null +++ b/utils/frame/benchmarking-cli/src/shared/mod.rs @@ -0,0 +1,65 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Code that is shared among all benchmarking sub-commands. + +pub mod record; +pub mod stats; +pub mod weight_params; + +pub use record::BenchRecord; +pub use stats::{StatSelect, Stats}; +pub use weight_params::WeightParams; + +/// A Handlebars helper to add an underscore after every 3rd character, +/// i.e. a separator for large numbers. +#[derive(Clone, Copy)] +pub struct UnderscoreHelper; + +impl handlebars::HelperDef for UnderscoreHelper { + fn call<'reg: 'rc, 'rc>( + &self, + h: &handlebars::Helper, + _: &handlebars::Handlebars, + _: &handlebars::Context, + _rc: &mut handlebars::RenderContext, + out: &mut dyn handlebars::Output, + ) -> handlebars::HelperResult { + use handlebars::JsonRender; + let param = h.param(0).unwrap(); + let underscore_param = underscore(param.value().render()); + out.write(&underscore_param)?; + Ok(()) + } +} + +/// Add an underscore after every 3rd character, i.e. a separator for large numbers. +fn underscore(i: Number) -> String +where + Number: std::string::ToString, +{ + let mut s = String::new(); + let i_str = i.to_string(); + let a = i_str.chars().rev().enumerate(); + for (idx, val) in a { + if idx != 0 && idx % 3 == 0 { + s.insert(0, '_'); + } + s.insert(0, val); + } + s +} diff --git a/utils/frame/benchmarking-cli/src/shared/record.rs b/utils/frame/benchmarking-cli/src/shared/record.rs new file mode 100644 index 000000000000..79ab37f65152 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/shared/record.rs @@ -0,0 +1,72 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Defines the [`BenchRecord`] and its facilities for computing [`super::Stats`]. + +use sc_cli::Result; +use sc_service::Configuration; + +use log::info; +use serde::Serialize; +use std::{fs, path::PathBuf, time::Duration}; + +use super::Stats; + +/// Raw output of a Storage benchmark. +#[derive(Debug, Default, Clone, Serialize)] +pub struct BenchRecord { + /// Multi-Map of value sizes and the time that it took to access them. + ns_per_size: Vec<(u64, u64)>, +} + +impl BenchRecord { + /// Appends a new record. Uses safe casts. + pub fn append(&mut self, size: usize, d: Duration) -> Result<()> { + let size: u64 = size.try_into().map_err(|e| format!("Size overflow u64: {}", e))?; + let ns: u64 = d + .as_nanos() + .try_into() + .map_err(|e| format!("Nanoseconds overflow u64: {}", e))?; + self.ns_per_size.push((size, ns)); + Ok(()) + } + + /// Returns the statistics for *time* and *value size*. + pub fn calculate_stats(self) -> Result<(Stats, Stats)> { + let (size, time): (Vec<_>, Vec<_>) = self.ns_per_size.into_iter().unzip(); + let size = Stats::new(&size)?; + let time = Stats::new(&time)?; + Ok((time, size)) // The swap of time/size here is intentional. + } + + /// Unless a path is specified, saves the raw results in a json file in the current directory. + /// Prefixes it with the DB name and suffixed with `path_suffix`. + pub fn save_json(&self, cfg: &Configuration, out_path: &PathBuf, suffix: &str) -> Result<()> { + let mut path = PathBuf::from(out_path); + if path.is_dir() || path.as_os_str().is_empty() { + path.push(&format!("{}_{}", cfg.database, suffix).to_lowercase()); + path.set_extension("json"); + } + + let json = serde_json::to_string_pretty(&self) + .map_err(|e| format!("Serializing as JSON: {:?}", e))?; + + fs::write(&path, json)?; + info!("Raw data written to {:?}", fs::canonicalize(&path)?); + Ok(()) + } +} diff --git a/utils/frame/benchmarking-cli/src/shared/stats.rs b/utils/frame/benchmarking-cli/src/shared/stats.rs new file mode 100644 index 000000000000..7785965fed4a --- /dev/null +++ b/utils/frame/benchmarking-cli/src/shared/stats.rs @@ -0,0 +1,188 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Handles statistics that were generated from benchmarking results and +//! that can be used to fill out weight templates. + +use sc_cli::Result; + +use serde::Serialize; +use std::{fmt, result, str::FromStr}; + +/// Various statistics that help to gauge the quality of the produced weights. +/// Will be written to the weight file and printed to console. +#[derive(Serialize, Default, Clone)] +pub struct Stats { + /// Sum of all values. + pub sum: u64, + /// Minimal observed value. + pub min: u64, + /// Maximal observed value. + pub max: u64, + + /// Average of all values. + pub avg: u64, + /// Median of all values. + pub median: u64, + /// Standard derivation of all values. + pub stddev: f64, + + /// 99th percentile. At least 99% of all values are below this threshold. + pub p99: u64, + /// 95th percentile. At least 95% of all values are below this threshold. + pub p95: u64, + /// 75th percentile. At least 75% of all values are below this threshold. + pub p75: u64, +} + +/// Selects a specific field from a [`Stats`] object. +/// Not all fields are available. +#[derive(Debug, Clone, Copy, Serialize, PartialEq)] +pub enum StatSelect { + /// Select the maximum. + Maximum, + /// Select the average. + Average, + /// Select the median. + Median, + /// Select the 99th percentile. + P99Percentile, + /// Select the 95th percentile. + P95Percentile, + /// Select the 75th percentile. + P75Percentile, +} + +impl Stats { + /// Calculates statistics and returns them. + pub fn new(xs: &Vec) -> Result { + if xs.is_empty() { + return Err("Empty input is invalid".into()) + } + let (avg, stddev) = Self::avg_and_stddev(&xs); + + Ok(Self { + sum: xs.iter().sum(), + min: *xs.iter().min().expect("Checked for non-empty above"), + max: *xs.iter().max().expect("Checked for non-empty above"), + + avg: avg as u64, + median: Self::percentile(xs.clone(), 0.50), + stddev: (stddev * 100.0).round() / 100.0, // round to 1/100 + + p99: Self::percentile(xs.clone(), 0.99), + p95: Self::percentile(xs.clone(), 0.95), + p75: Self::percentile(xs.clone(), 0.75), + }) + } + + /// Returns the selected stat. + pub fn select(&self, s: StatSelect) -> u64 { + match s { + StatSelect::Maximum => self.max, + StatSelect::Average => self.avg, + StatSelect::Median => self.median, + StatSelect::P99Percentile => self.p99, + StatSelect::P95Percentile => self.p95, + StatSelect::P75Percentile => self.p75, + } + } + + /// Returns the *average* and the *standard derivation*. + fn avg_and_stddev(xs: &Vec) -> (f64, f64) { + let avg = xs.iter().map(|x| *x as f64).sum::() / xs.len() as f64; + let variance = xs.iter().map(|x| (*x as f64 - avg).powi(2)).sum::() / xs.len() as f64; + (avg, variance.sqrt()) + } + + /// Returns the specified percentile for the given data. + /// This is best effort since it ignores the interpolation case. + fn percentile(mut xs: Vec, p: f64) -> u64 { + xs.sort(); + let index = (xs.len() as f64 * p).ceil() as usize - 1; + xs[index.clamp(0, xs.len() - 1)] + } +} + +impl fmt::Debug for Stats { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Total: {}\n", self.sum)?; + write!(f, "Min: {}, Max: {}\n", self.min, self.max)?; + write!(f, "Average: {}, Median: {}, Stddev: {}\n", self.avg, self.median, self.stddev)?; + write!(f, "Percentiles 99th, 95th, 75th: {}, {}, {}", self.p99, self.p95, self.p75) + } +} + +impl Default for StatSelect { + /// Returns the `Average` selector. + fn default() -> Self { + Self::Average + } +} + +impl FromStr for StatSelect { + type Err = &'static str; + + fn from_str(day: &str) -> result::Result { + match day.to_lowercase().as_str() { + "max" => Ok(Self::Maximum), + "average" => Ok(Self::Average), + "median" => Ok(Self::Median), + "p99" => Ok(Self::P99Percentile), + "p95" => Ok(Self::P95Percentile), + "p75" => Ok(Self::P75Percentile), + _ => Err("String was not a StatSelect"), + } + } +} + +#[cfg(test)] +mod test_stats { + use super::Stats; + use rand::{seq::SliceRandom, thread_rng}; + + #[test] + fn stats_correct() { + let mut data: Vec = (1..=100).collect(); + data.shuffle(&mut thread_rng()); + let stats = Stats::new(&data).unwrap(); + + assert_eq!(stats.sum, 5050); + assert_eq!(stats.min, 1); + assert_eq!(stats.max, 100); + + assert_eq!(stats.avg, 50); + assert_eq!(stats.median, 50); // 50.5 to be exact. + assert_eq!(stats.stddev, 28.87); // Rounded with 1/100 precision. + + assert_eq!(stats.p99, 99); + assert_eq!(stats.p95, 95); + assert_eq!(stats.p75, 75); + } + + #[test] + fn no_panic_short_lengths() { + // Empty input does error. + assert!(Stats::new(&vec![]).is_err()); + + // Different small input lengths are fine. + for l in 1..10 { + let data = (0..=l).collect(); + assert!(Stats::new(&data).is_ok()); + } + } +} diff --git a/utils/frame/benchmarking-cli/src/shared/weight_params.rs b/utils/frame/benchmarking-cli/src/shared/weight_params.rs new file mode 100644 index 000000000000..4dd80cd41ff3 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/shared/weight_params.rs @@ -0,0 +1,95 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Calculates a weight from the [`super::Stats`] of a benchmark result. + +use sc_cli::Result; + +use clap::Args; +use serde::Serialize; +use std::path::PathBuf; + +use super::{StatSelect, Stats}; + +/// Configures the weight generation. +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct WeightParams { + /// File or directory to write the *weight* files to. + /// + /// For Substrate this should be `frame/support/src/weights`. + #[clap(long)] + pub weight_path: Option, + + /// Select a specific metric to calculate the final weight output. + #[clap(long = "metric", default_value = "average")] + pub weight_metric: StatSelect, + + /// Multiply the resulting weight with the given factor. Must be positive. + /// + /// Is applied before `weight_add`. + #[clap(long = "mul", default_value = "1")] + pub weight_mul: f64, + + /// Add the given offset to the resulting weight. + /// + /// Is applied after `weight_mul`. + #[clap(long = "add", default_value = "0")] + pub weight_add: u64, +} + +/// Calculates the final weight by multiplying the selected metric with +/// `weight_mul` and adding `weight_add`. +/// Does not use safe casts and can overflow. +impl WeightParams { + pub fn calc_weight(&self, stat: &Stats) -> Result { + if self.weight_mul.is_sign_negative() || !self.weight_mul.is_normal() { + return Err("invalid floating number for `weight_mul`".into()) + } + let s = stat.select(self.weight_metric) as f64; + let w = s.mul_add(self.weight_mul, self.weight_add as f64).ceil(); + Ok(w as u64) // No safe cast here since there is no `From` for `u64`. + } +} + +#[cfg(test)] +mod test_weight_params { + use super::WeightParams; + use crate::shared::{StatSelect, Stats}; + + #[test] + fn calc_weight_works() { + let stats = Stats { avg: 113, ..Default::default() }; + let params = WeightParams { + weight_metric: StatSelect::Average, + weight_mul: 0.75, + weight_add: 3, + ..Default::default() + }; + + let want = (113.0f64 * 0.75 + 3.0).ceil() as u64; // Ceil for overestimation. + let got = params.calc_weight(&stats).unwrap(); + assert_eq!(want, got); + } + + #[test] + fn calc_weight_detects_negative_mul() { + let stats = Stats::default(); + let params = WeightParams { weight_mul: -0.75, ..Default::default() }; + + assert!(params.calc_weight(&stats).is_err()); + } +} diff --git a/utils/frame/benchmarking-cli/src/storage/cmd.rs b/utils/frame/benchmarking-cli/src/storage/cmd.rs new file mode 100644 index 000000000000..c2cc219ef152 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/cmd.rs @@ -0,0 +1,206 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sc_cli::{CliConfiguration, DatabaseParams, PruningParams, Result, SharedParams}; +use sc_client_api::{Backend as ClientBackend, StorageProvider, UsageProvider}; +use sc_client_db::DbHash; +use sc_service::Configuration; +use sp_blockchain::HeaderBackend; +use sp_core::storage::StorageKey; +use sp_database::{ColumnId, Database}; +use sp_runtime::traits::{Block as BlockT, HashFor}; +use sp_state_machine::Storage; +use sp_storage::StateVersion; + +use clap::{Args, Parser}; +use log::info; +use rand::prelude::*; +use serde::Serialize; +use sp_runtime::generic::BlockId; +use std::{fmt::Debug, path::PathBuf, sync::Arc}; + +use super::template::TemplateData; +use crate::shared::WeightParams; + +/// Benchmark the storage speed of a chain snapshot. +#[derive(Debug, Parser)] +pub struct StorageCmd { + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub database_params: DatabaseParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub pruning_params: PruningParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub params: StorageParams, +} + +/// Parameters for modifying the benchmark behaviour and the post processing of the results. +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct StorageParams { + #[allow(missing_docs)] + #[clap(flatten)] + pub weight_params: WeightParams, + + /// Skip the `read` benchmark. + #[clap(long)] + pub skip_read: bool, + + /// Skip the `write` benchmark. + #[clap(long)] + pub skip_write: bool, + + /// Specify the Handlebars template to use for outputting benchmark results. + #[clap(long)] + pub template_path: Option, + + /// Path to write the raw 'read' results in JSON format to. Can be a file or directory. + #[clap(long)] + pub json_read_path: Option, + + /// Path to write the raw 'write' results in JSON format to. Can be a file or directory. + #[clap(long)] + pub json_write_path: Option, + + /// Rounds of warmups before measuring. + #[clap(long, default_value = "1")] + pub warmups: u32, + + /// The `StateVersion` to use. Substrate `--dev` should use `V1` and Polkadot `V0`. + /// Selecting the wrong version can corrupt the DB. + #[clap(long, possible_values = ["0", "1"])] + pub state_version: u8, + + /// State cache size. + #[clap(long, default_value = "0")] + pub state_cache_size: usize, +} + +impl StorageCmd { + /// Calls into the Read and Write benchmarking functions. + /// Processes the output and writes it into files and stdout. + pub fn run( + &self, + cfg: Configuration, + client: Arc, + db: (Arc>, ColumnId), + storage: Arc>>, + ) -> Result<()> + where + BA: ClientBackend, + Block: BlockT, + C: UsageProvider + StorageProvider + HeaderBackend, + { + let mut template = TemplateData::new(&cfg, &self.params); + + let block_id = BlockId::::Number(client.usage_info().chain.best_number); + template.set_block_number(block_id.to_string()); + + if !self.params.skip_read { + self.bench_warmup(&client)?; + let record = self.bench_read(client.clone())?; + if let Some(path) = &self.params.json_read_path { + record.save_json(&cfg, path, "read")?; + } + let stats = record.calculate_stats()?; + info!("Time summary [ns]:\n{:?}\nValue size summary:\n{:?}", stats.0, stats.1); + template.set_stats(Some(stats), None)?; + } + + if !self.params.skip_write { + self.bench_warmup(&client)?; + let record = self.bench_write(client, db, storage)?; + if let Some(path) = &self.params.json_write_path { + record.save_json(&cfg, path, "write")?; + } + let stats = record.calculate_stats()?; + info!("Time summary [ns]:\n{:?}\nValue size summary:\n{:?}", stats.0, stats.1); + template.set_stats(None, Some(stats))?; + } + + template.write(&self.params.weight_params.weight_path, &self.params.template_path) + } + + /// Returns the specified state version. + pub(crate) fn state_version(&self) -> StateVersion { + match self.params.state_version { + 0 => StateVersion::V0, + 1 => StateVersion::V1, + _ => unreachable!("Clap set to only allow 0 and 1"), + } + } + + /// Creates an rng from a random seed. + pub(crate) fn setup_rng() -> impl rand::Rng { + let seed = rand::thread_rng().gen::(); + info!("Using seed {}", seed); + StdRng::seed_from_u64(seed) + } + + /// Run some rounds of the (read) benchmark as warmup. + /// See `frame_benchmarking_cli::storage::read::bench_read` for detailed comments. + fn bench_warmup(&self, client: &Arc) -> Result<()> + where + C: UsageProvider + StorageProvider, + B: BlockT + Debug, + BA: ClientBackend, + { + let block = BlockId::Number(client.usage_info().chain.best_number); + let empty_prefix = StorageKey(Vec::new()); + let mut keys = client.storage_keys(&block, &empty_prefix)?; + let mut rng = Self::setup_rng(); + keys.shuffle(&mut rng); + + for i in 0..self.params.warmups { + info!("Warmup round {}/{}", i + 1, self.params.warmups); + for key in keys.clone() { + let _ = client + .storage(&block, &key) + .expect("Checked above to exist") + .ok_or("Value unexpectedly empty"); + } + } + + Ok(()) + } +} + +// Boilerplate +impl CliConfiguration for StorageCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } + + fn database_params(&self) -> Option<&DatabaseParams> { + Some(&self.database_params) + } + + fn pruning_params(&self) -> Option<&PruningParams> { + Some(&self.pruning_params) + } + + fn state_cache_size(&self) -> Result { + Ok(self.params.state_cache_size) + } +} diff --git a/utils/frame/benchmarking-cli/src/storage/mod.rs b/utils/frame/benchmarking-cli/src/storage/mod.rs new file mode 100644 index 000000000000..0c722fdd4702 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/mod.rs @@ -0,0 +1,23 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod cmd; +pub mod read; +pub mod template; +pub mod write; + +pub use cmd::StorageCmd; diff --git a/utils/frame/benchmarking-cli/src/storage/read.rs b/utils/frame/benchmarking-cli/src/storage/read.rs new file mode 100644 index 000000000000..f58f3c3de0c1 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/read.rs @@ -0,0 +1,66 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sc_cli::Result; +use sc_client_api::{Backend as ClientBackend, StorageProvider, UsageProvider}; +use sp_core::storage::StorageKey; +use sp_runtime::{ + generic::BlockId, + traits::{Block as BlockT, Header as HeaderT}, +}; + +use log::info; +use rand::prelude::*; +use std::{fmt::Debug, sync::Arc, time::Instant}; + +use super::cmd::StorageCmd; +use crate::shared::BenchRecord; + +impl StorageCmd { + /// Benchmarks the time it takes to read a single Storage item. + /// Uses the latest state that is available for the given client. + pub(crate) fn bench_read(&self, client: Arc) -> Result + where + C: UsageProvider + StorageProvider, + B: BlockT + Debug, + BA: ClientBackend, + <::Header as HeaderT>::Number: From, + { + let mut record = BenchRecord::default(); + let block = BlockId::Number(client.usage_info().chain.best_number); + + info!("Preparing keys from block {}", block); + // Load all keys and randomly shuffle them. + let empty_prefix = StorageKey(Vec::new()); + let mut keys = client.storage_keys(&block, &empty_prefix)?; + let mut rng = Self::setup_rng(); + keys.shuffle(&mut rng); + + // Interesting part here: + // Read all the keys in the database and measure the time it takes to access each. + info!("Reading {} keys", keys.len()); + for key in keys.clone() { + let start = Instant::now(); + let v = client + .storage(&block, &key) + .expect("Checked above to exist") + .ok_or("Value unexpectedly empty")?; + record.append(v.0.len(), start.elapsed())?; + } + Ok(record) + } +} diff --git a/utils/frame/benchmarking-cli/src/storage/template.rs b/utils/frame/benchmarking-cli/src/storage/template.rs new file mode 100644 index 000000000000..26aa8a962301 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/template.rs @@ -0,0 +1,133 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sc_cli::Result; +use sc_service::Configuration; + +use log::info; +use serde::Serialize; +use std::{env, fs, path::PathBuf}; + +use super::cmd::StorageParams; +use crate::shared::{Stats, UnderscoreHelper}; + +static VERSION: &'static str = env!("CARGO_PKG_VERSION"); +static TEMPLATE: &str = include_str!("./weights.hbs"); + +/// Data consumed by Handlebar to fill out the `weights.hbs` template. +#[derive(Serialize, Default, Debug, Clone)] +pub(crate) struct TemplateData { + /// Name of the database used. + db_name: String, + /// Block number that was used. + block_number: String, + /// Name of the runtime. Taken from the chain spec. + runtime_name: String, + /// Version of the benchmarking CLI used. + version: String, + /// Date that the template was filled out. + date: String, + /// Command line arguments that were passed to the CLI. + args: Vec, + /// Storage params of the executed command. + params: StorageParams, + /// The weight for one `read`. + read_weight: u64, + /// The weight for one `write`. + write_weight: u64, + /// Stats about a `read` benchmark. Contains *time* and *value size* stats. + /// The *value size* stats are currently not used in the template. + read: Option<(Stats, Stats)>, + /// Stats about a `write` benchmark. Contains *time* and *value size* stats. + /// The *value size* stats are currently not used in the template. + write: Option<(Stats, Stats)>, +} + +impl TemplateData { + /// Returns a new [`Self`] from the given configuration. + pub fn new(cfg: &Configuration, params: &StorageParams) -> Self { + TemplateData { + db_name: format!("{}", cfg.database), + runtime_name: cfg.chain_spec.name().into(), + version: VERSION.into(), + date: chrono::Utc::now().format("%Y-%m-%d (Y/M/D)").to_string(), + args: env::args().collect::>(), + params: params.clone(), + ..Default::default() + } + } + + /// Sets the stats and calculates the final weights. + pub fn set_stats( + &mut self, + read: Option<(Stats, Stats)>, + write: Option<(Stats, Stats)>, + ) -> Result<()> { + if let Some(read) = read { + self.read_weight = self.params.weight_params.calc_weight(&read.0)?; + self.read = Some(read); + } + if let Some(write) = write { + self.write_weight = self.params.weight_params.calc_weight(&write.0)?; + self.write = Some(write); + } + Ok(()) + } + + /// Sets the block id that was used. + pub fn set_block_number(&mut self, block_number: String) { + self.block_number = block_number + } + + /// Fills out the `weights.hbs` or specified HBS template with its own data. + /// Writes the result to `path` which can be a directory or file. + pub fn write(&self, path: &Option, hbs_template: &Option) -> Result<()> { + let mut handlebars = handlebars::Handlebars::new(); + // Format large integers with underscore. + handlebars.register_helper("underscore", Box::new(UnderscoreHelper)); + // Don't HTML escape any characters. + handlebars.register_escape_fn(|s| -> String { s.to_string() }); + // Use custom template if provided. + let template = match hbs_template { + Some(template) if template.is_file() => fs::read_to_string(template)?, + Some(_) => return Err("Handlebars template is not a valid file!".into()), + None => TEMPLATE.to_string(), + }; + + let out_path = self.build_path(path); + let mut fd = fs::File::create(&out_path)?; + info!("Writing weights to {:?}", fs::canonicalize(&out_path)?); + + handlebars + .render_template_to_write(&template, &self, &mut fd) + .map_err(|e| format!("HBS template write: {:?}", e).into()) + } + + /// Builds a path for the weight file. + fn build_path(&self, weight_out: &Option) -> PathBuf { + let mut path = match weight_out { + Some(p) => PathBuf::from(p), + None => PathBuf::new(), + }; + + if path.is_dir() || path.as_os_str().is_empty() { + path.push(format!("{}_weights", self.db_name.to_lowercase())); + path.set_extension("rs"); + } + path + } +} diff --git a/utils/frame/benchmarking-cli/src/storage/weights.hbs b/utils/frame/benchmarking-cli/src/storage/weights.hbs new file mode 100644 index 000000000000..63f896e1104b --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/weights.hbs @@ -0,0 +1,111 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION {{version}} +//! DATE: {{date}} +//! +//! DATABASE: `{{db_name}}`, RUNTIME: `{{runtime_name}}` +//! BLOCK-NUM: `{{block_number}}` +//! SKIP-WRITE: `{{params.skip_write}}`, SKIP-READ: `{{params.skip_read}}`, WARMUPS: `{{params.warmups}}` +//! STATE-VERSION: `V{{params.state_version}}`, STATE-CACHE-SIZE: `{{params.state_cache_size}}` +//! WEIGHT-PATH: `{{params.weight_params.weight_path}}` +//! METRIC: `{{params.weight_params.weight_metric}}`, WEIGHT-MUL: `{{params.weight_params.weight_mul}}`, WEIGHT-ADD: `{{params.weight_params.weight_add}}` + +// Executed Command: +{{#each args as |arg|}} +// {{arg}} +{{/each}} + +/// Storage DB weights for the `{{runtime_name}}` runtime and `{{db_name}}`. +pub mod constants { + use frame_support::{ + parameter_types, + weights::{constants, RuntimeDbWeight}, + }; + + parameter_types! { + {{#if (eq db_name "ParityDb")}} + /// `ParityDB` can be enabled with a feature flag, but is still experimental. These weights + /// are available for brave runtime engineers who may want to try this out as default. + {{else}} + /// By default, Substrate uses `RocksDB`, so this will be the weight used throughout + /// the runtime. + {{/if}} + pub const {{db_name}}Weight: RuntimeDbWeight = RuntimeDbWeight { + /// Time to read one storage item. + /// Calculated by multiplying the *{{params.weight_params.weight_metric}}* of all values with `{{params.weight_params.weight_mul}}` and adding `{{params.weight_params.weight_add}}`. + /// + /// Stats [NS]: + /// Min, Max: {{underscore read.0.min}}, {{underscore read.0.max}} + /// Average: {{underscore read.0.avg}} + /// Median: {{underscore read.0.median}} + /// Std-Dev: {{read.0.stddev}} + /// + /// Percentiles [NS]: + /// 99th: {{underscore read.0.p99}} + /// 95th: {{underscore read.0.p95}} + /// 75th: {{underscore read.0.p75}} + read: {{underscore read_weight}} * constants::WEIGHT_PER_NANOS, + + /// Time to write one storage item. + /// Calculated by multiplying the *{{params.weight_params.weight_metric}}* of all values with `{{params.weight_params.weight_mul}}` and adding `{{params.weight_params.weight_add}}`. + /// + /// Stats [NS]: + /// Min, Max: {{underscore write.0.min}}, {{underscore write.0.max}} + /// Average: {{underscore write.0.avg}} + /// Median: {{underscore write.0.median}} + /// Std-Dev: {{write.0.stddev}} + /// + /// Percentiles [NS]: + /// 99th: {{underscore write.0.p99}} + /// 95th: {{underscore write.0.p95}} + /// 75th: {{underscore write.0.p75}} + write: {{underscore write_weight}} * constants::WEIGHT_PER_NANOS, + }; + } + + #[cfg(test)] + mod test_db_weights { + use super::constants::{{db_name}}Weight as W; + use frame_support::weights::constants; + + /// Checks that all weights exist and have sane values. + // NOTE: If this test fails but you are sure that the generated values are fine, + // you can delete it. + #[test] + fn bound() { + // At least 1 µs. + assert!( + W::get().reads(1) >= constants::WEIGHT_PER_MICROS, + "Read weight should be at least 1 µs." + ); + assert!( + W::get().writes(1) >= constants::WEIGHT_PER_MICROS, + "Write weight should be at least 1 µs." + ); + // At most 1 ms. + assert!( + W::get().reads(1) <= constants::WEIGHT_PER_MILLIS, + "Read weight should be at most 1 ms." + ); + assert!( + W::get().writes(1) <= constants::WEIGHT_PER_MILLIS, + "Write weight should be at most 1 ms." + ); + } + } +} diff --git a/utils/frame/benchmarking-cli/src/storage/write.rs b/utils/frame/benchmarking-cli/src/storage/write.rs new file mode 100644 index 000000000000..d5d5bc2fffa5 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/write.rs @@ -0,0 +1,136 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sc_cli::Result; +use sc_client_api::UsageProvider; +use sc_client_db::{DbHash, DbState}; +use sp_api::StateBackend; +use sp_blockchain::HeaderBackend; +use sp_database::{ColumnId, Transaction}; +use sp_runtime::{ + generic::BlockId, + traits::{Block as BlockT, HashFor, Header as HeaderT}, +}; +use sp_trie::PrefixedMemoryDB; + +use log::{info, trace}; +use rand::prelude::*; +use std::{fmt::Debug, sync::Arc, time::Instant}; + +use super::cmd::StorageCmd; +use crate::shared::BenchRecord; + +impl StorageCmd { + /// Benchmarks the time it takes to write a single Storage item. + /// Uses the latest state that is available for the given client. + pub(crate) fn bench_write( + &self, + client: Arc, + (db, state_col): (Arc>, ColumnId), + storage: Arc>>, + ) -> Result + where + Block: BlockT
+ Debug, + H: HeaderT, + C: UsageProvider + HeaderBackend, + { + // Store the time that it took to write each value. + let mut record = BenchRecord::default(); + + let block = BlockId::Number(client.usage_info().chain.best_number); + let header = client.header(block)?.ok_or("Header not found")?; + let original_root = *header.state_root(); + let trie = DbState::::new(storage.clone(), original_root); + + info!("Preparing keys from block {}", block); + // Load all KV pairs and randomly shuffle them. + let mut kvs = trie.pairs(); + let mut rng = Self::setup_rng(); + kvs.shuffle(&mut rng); + + // Generate all random values first; Make sure there are no collisions with existing + // db entries, so we can rollback all additions without corrupting existing entries. + for (k, original_v) in kvs.iter_mut() { + 'retry: loop { + let mut new_v = vec![0; original_v.len()]; + // Create a random value to overwrite with. + // NOTE: We use a possibly higher entropy than the original value, + // could be improved but acts as an over-estimation which is fine for now. + rng.fill_bytes(&mut new_v[..]); + let new_kv = vec![(k.as_ref(), Some(new_v.as_ref()))]; + let (_, mut stx) = trie.storage_root(new_kv.iter().cloned(), self.state_version()); + for (mut k, (_, rc)) in stx.drain().into_iter() { + if rc > 0 { + db.sanitize_key(&mut k); + if db.get(state_col, &k).is_some() { + trace!("Benchmark-store key creation: Key collision detected, retry"); + continue 'retry + } + } + } + *original_v = new_v; + break + } + } + + info!("Writing {} keys", kvs.len()); + // Write each value in one commit. + for (k, new_v) in kvs.iter() { + // Interesting part here: + let start = Instant::now(); + // Create a TX that will modify the Trie in the DB and + // calculate the root hash of the Trie after the modification. + let replace = vec![(k.as_ref(), Some(new_v.as_ref()))]; + let (_, stx) = trie.storage_root(replace.iter().cloned(), self.state_version()); + // Only the keep the insertions, since we do not want to benchmark pruning. + let tx = convert_tx::(db.clone(), stx.clone(), false, state_col); + db.commit(tx).map_err(|e| format!("Writing to the Database: {}", e))?; + record.append(new_v.len(), start.elapsed())?; + + // Now undo the changes by removing what was added. + let tx = convert_tx::(db.clone(), stx.clone(), true, state_col); + db.commit(tx).map_err(|e| format!("Writing to the Database: {}", e))?; + } + Ok(record) + } +} + +/// Converts a Trie transaction into a DB transaction. +/// Removals are ignored and will not be included in the final tx. +/// `invert_inserts` replaces all inserts with removals. +fn convert_tx( + db: Arc>, + mut tx: PrefixedMemoryDB>, + invert_inserts: bool, + col: ColumnId, +) -> Transaction { + let mut ret = Transaction::::default(); + + for (mut k, (v, rc)) in tx.drain().into_iter() { + if rc > 0 { + db.sanitize_key(&mut k); + if invert_inserts { + ret.remove(col, &k); + } else { + ret.set(col, &k, &v); + } + } + // < 0 means removal - ignored. + // 0 means no modification. + } + ret +} diff --git a/utils/frame/frame-utilities-cli/Cargo.toml b/utils/frame/frame-utilities-cli/Cargo.toml index 1b6597fc9f2f..43c8b3189895 100644 --- a/utils/frame/frame-utilities-cli/Cargo.toml +++ b/utils/frame/frame-utilities-cli/Cargo.toml @@ -2,19 +2,20 @@ name = "substrate-frame-cli" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "cli interface for FRAME" documentation = "https://docs.rs/substrate-frame-cli" readme = "README.md" [dependencies] -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } +clap = { version = "3.1.6", features = ["derive"] } + +sp-core = { version = "6.0.0", path = "../../../primitives/core" } sc-cli = { version = "0.10.0-dev", path = "../../../client/cli" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -structopt = "0.3.8" +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } frame-system = { version = "4.0.0-dev", path = "../../../frame/system" } frame-support = { version = "4.0.0-dev", path = "../../../frame/support" } diff --git a/utils/frame/frame-utilities-cli/src/lib.rs b/utils/frame/frame-utilities-cli/src/lib.rs index 4f5b1da5766a..9a18d39263c9 100644 --- a/utils/frame/frame-utilities-cli/src/lib.rs +++ b/utils/frame/frame-utilities-cli/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/utils/frame/frame-utilities-cli/src/pallet_id.rs b/utils/frame/frame-utilities-cli/src/pallet_id.rs index 2caac7db588a..c39bee8b8746 100644 --- a/utils/frame/frame-utilities-cli/src/pallet_id.rs +++ b/utils/frame/frame-utilities-cli/src/pallet_id.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,43 +17,42 @@ //! Implementation of the `palletid` subcommand +use clap::Parser; use frame_support::PalletId; use sc_cli::{ utils::print_from_uri, with_crypto_scheme, CryptoSchemeFlag, Error, KeystoreParams, OutputTypeFlag, }; -use sp_core::crypto::{Ss58AddressFormat, Ss58Codec}; +use sp_core::crypto::{unwrap_or_default_ss58_version, Ss58AddressFormat, Ss58Codec}; use sp_runtime::traits::AccountIdConversion; -use std::convert::{TryFrom, TryInto}; -use structopt::StructOpt; /// The `palletid` command -#[derive(Debug, StructOpt)] -#[structopt(name = "palletid", about = "Inspect a module ID address")] +#[derive(Debug, Parser)] +#[clap(name = "palletid", about = "Inspect a module ID address")] pub struct PalletIdCmd { /// The module ID used to derive the account id: String, /// network address format - #[structopt( + #[clap( long, value_name = "NETWORK", possible_values = &Ss58AddressFormat::all_names()[..], parse(try_from_str = Ss58AddressFormat::try_from), - case_insensitive = true, + ignore_case = true, )] pub network: Option, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub output_scheme: OutputTypeFlag, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub crypto_scheme: CryptoSchemeFlag, #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub keystore_params: KeystoreParams, } @@ -78,7 +77,7 @@ impl PalletIdCmd { with_crypto_scheme!( self.crypto_scheme.scheme, print_from_uri( - &account_id.to_ss58check_with_version(self.network.clone().unwrap_or_default()), + &account_id.to_ss58check_with_version(unwrap_or_default_ss58_version(self.network)), password, self.network, self.output_scheme.output_type.clone() diff --git a/utils/frame/generate-bags/Cargo.toml b/utils/frame/generate-bags/Cargo.toml index 384307fbec9e..7c7fdea056b2 100644 --- a/utils/frame/generate-bags/Cargo.toml +++ b/utils/frame/generate-bags/Cargo.toml @@ -1,26 +1,25 @@ [package] name = "generate-bags" -version = "3.0.0" +version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Bag threshold generation script for pallet-bag-list" readme = "README.md" [dependencies] # FRAME -frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../frame/support" } +frame-support = { version = "4.0.0-dev", path = "../../../frame/support" } frame-election-provider-support = { version = "4.0.0-dev", path = "../../../frame/election-provider-support", features = ["runtime-benchmarks"] } -frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../frame/system" } -pallet-staking = { version = "4.0.0-dev", default-features = false, path = "../../../frame/staking" } +frame-system = { version = "4.0.0-dev", path = "../../../frame/system" } +pallet-staking = { version = "4.0.0-dev", path = "../../../frame/staking" } # primitives -sp-io = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/io" } +sp-io = { version = "6.0.0", path = "../../../primitives/io" } # third party chrono = { version = "0.4.19" } -git2 = { version = "0.13.20", default-features = false } +git2 = { version = "0.13.25", default-features = false } num-format = { version = "0.4.0" } -structopt = "0.3.21" diff --git a/utils/frame/generate-bags/node-runtime/Cargo.toml b/utils/frame/generate-bags/node-runtime/Cargo.toml index 7fcd981a6bbd..d5c8cab7ba0d 100644 --- a/utils/frame/generate-bags/node-runtime/Cargo.toml +++ b/utils/frame/generate-bags/node-runtime/Cargo.toml @@ -2,16 +2,16 @@ name = "node-runtime-generate-bags" version = "3.0.0" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Bag threshold generation script for pallet-bag-list and node-runtime." readme = "README.md" [dependencies] node-runtime = { version = "3.0.0-dev", path = "../../../../bin/node/runtime" } -generate-bags = { version = "3.0.0", path = "../" } +generate-bags = { version = "4.0.0-dev", path = "../" } # third-party -structopt = "0.3.21" +clap = { version = "3.1.6", features = ["derive"] } diff --git a/utils/frame/generate-bags/node-runtime/src/main.rs b/utils/frame/generate-bags/node-runtime/src/main.rs index 5d36b381a7d0..12bcf8d28cf2 100644 --- a/utils/frame/generate-bags/node-runtime/src/main.rs +++ b/utils/frame/generate-bags/node-runtime/src/main.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,30 +17,31 @@ //! Make the set of bag thresholds to be used with pallet-bags-list. +use clap::Parser; use generate_bags::generate_thresholds; use std::path::PathBuf; -use structopt::StructOpt; -#[derive(Debug, StructOpt)] +#[derive(Debug, Parser)] +// #[clap(author, version, about)] struct Opt { /// How many bags to generate. - #[structopt(long, default_value = "200")] + #[clap(long, default_value = "200")] n_bags: usize, /// Where to write the output. output: PathBuf, /// The total issuance of the currency used to create `VoteWeight`. - #[structopt(short, long)] + #[clap(short, long)] total_issuance: u128, /// The minimum account balance (i.e. existential deposit) for the currency used to create /// `VoteWeight`. - #[structopt(short, long)] + #[clap(short, long)] minimum_balance: u128, } fn main() -> Result<(), std::io::Error> { - let Opt { n_bags, output, total_issuance, minimum_balance } = Opt::from_args(); + let Opt { n_bags, output, total_issuance, minimum_balance } = Opt::parse(); generate_thresholds::(n_bags, &output, total_issuance, minimum_balance) } diff --git a/utils/frame/generate-bags/src/lib.rs b/utils/frame/generate-bags/src/lib.rs index af9df4435bca..d4507c3be33e 100644 --- a/utils/frame/generate-bags/src/lib.rs +++ b/utils/frame/generate-bags/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -71,7 +71,6 @@ fn existential_weight( minimum_balance: u128, ) -> VoteWeight { use frame_support::traits::CurrencyToVote; - use std::convert::TryInto; T::CurrencyToVote::to_vote( minimum_balance @@ -126,7 +125,7 @@ pub fn constant_ratio(existential_weight: VoteWeight, n_bags: usize) -> f64 { /// The last element is always `VoteWeight::MAX`. /// /// All other elements are computed from the previous according to the formula -/// `threshold[k + 1] = (threshold[k] * ratio).max(threshold[k] + 1); +/// `threshold[k + 1] = (threshold[k] * ratio).max(threshold[k] + 1);` pub fn thresholds( existential_weight: VoteWeight, constant_ratio: f64, diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index ce774679f94c..343dda689275 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -2,9 +2,9 @@ name = "remote-externalities" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "An externalities provided environemnt that can load itself from remote nodes or cache files" readme = "README.md" @@ -13,25 +13,24 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -jsonrpsee-ws-client = { version = "0.3.0", default-features = false, features = [ - "tokio1", -]} -jsonrpsee-proc-macros = "0.3.0" +jsonrpsee = { version = "0.10.1", features = ["ws-client", "macros"] } env_logger = "0.9" +frame-support = { path = "../../../frame/support", optional = true, version = "4.0.0-dev" } log = "0.4.11" -codec = { package = "parity-scale-codec", version = "2.0.0" } +codec = { package = "parity-scale-codec", version = "3.0.0" } serde_json = "1.0" -serde = "1.0.126" +serde = "1.0.136" -sp-io = { version = "4.0.0-dev", path = "../../../primitives/io" } -sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } -sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -sp-version = { version = "4.0.0-dev", path = "../../../primitives/version" } +sp-io = { version = "6.0.0", path = "../../../primitives/io" } +sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } +sp-version = { version = "5.0.0", path = "../../../primitives/version" } [dev-dependencies] -tokio = { version = "1.10", features = ["macros", "rt-multi-thread"] } +tokio = { version = "1.17.0", features = ["macros", "rt-multi-thread"] } pallet-elections-phragmen = { path = "../../../frame/elections-phragmen", version = "5.0.0-dev" } +frame-support = { path = "../../../frame/support", version = "4.0.0-dev" } [features] -remote-test = [] +remote-test = ["frame-support"] diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 2052780286c6..533d65c49c2e 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,51 +21,86 @@ //! based chain, or a local state snapshot file. use codec::{Decode, Encode}; -use jsonrpsee_ws_client::{types::v2::params::JsonRpcParams, WsClient, WsClientBuilder}; + +use jsonrpsee::{ + core::{client::ClientT, Error as RpcError}, + proc_macros::rpc, + rpc_params, + ws_client::{WsClient, WsClientBuilder}, +}; + use log::*; +use serde::de::DeserializeOwned; use sp_core::{ hashing::twox_128, hexdisplay::HexDisplay, - storage::{StorageData, StorageKey}, + storage::{ + well_known_keys::{is_default_child_storage_key, DEFAULT_CHILD_STORAGE_KEY_PREFIX}, + ChildInfo, ChildType, PrefixedStorageKey, StorageData, StorageKey, + }, }; pub use sp_io::TestExternalities; -use sp_runtime::traits::Block as BlockT; +use sp_runtime::{traits::Block as BlockT, StateVersion}; use std::{ fs, path::{Path, PathBuf}, + sync::Arc, }; pub mod rpc_api; -type KeyPair = (StorageKey, StorageData); +type KeyValue = (StorageKey, StorageData); +type TopKeyValues = Vec; +type ChildKeyValues = Vec<(ChildInfo, Vec)>; const LOG_TARGET: &str = "remote-ext"; -const DEFAULT_TARGET: &str = "wss://rpc.polkadot.io"; +const DEFAULT_TARGET: &str = "wss://rpc.polkadot.io:443"; const BATCH_SIZE: usize = 1000; +const PAGE: u32 = 512; -jsonrpsee_proc_macros::rpc_client_api! { - RpcApi { - #[rpc(method = "state_getStorage", positional_params)] - fn get_storage(prefix: StorageKey, hash: Option) -> StorageData; - #[rpc(method = "state_getKeysPaged", positional_params)] - fn get_keys_paged( - prefix: Option, - count: u32, - start_key: Option, - hash: Option, - ) -> Vec; - #[rpc(method = "chain_getFinalizedHead", positional_params)] - fn finalized_head() -> B::Hash; - } +#[rpc(client)] +pub trait RpcApi { + #[method(name = "childstate_getKeys")] + fn child_get_keys( + &self, + child_key: PrefixedStorageKey, + prefix: StorageKey, + hash: Option, + ) -> Result, RpcError>; + + #[method(name = "childstate_getStorage")] + fn child_get_storage( + &self, + child_key: PrefixedStorageKey, + prefix: StorageKey, + hash: Option, + ) -> Result; + + #[method(name = "state_getStorage")] + fn get_storage(&self, prefix: StorageKey, hash: Option) -> Result; + + #[method(name = "state_getKeysPaged")] + fn get_keys_paged( + &self, + prefix: Option, + count: u32, + start_key: Option, + hash: Option, + ) -> Result, RpcError>; + + #[method(name = "chain_getFinalizedHead")] + fn finalized_head(&self) -> Result; } /// The execution mode. #[derive(Clone)] pub enum Mode { - /// Online. + /// Online. Potentially writes to a cache file. Online(OnlineConfig), /// Offline. Uses a state snapshot file and needs not any client config. Offline(OfflineConfig), + /// Prefer using a cache file if it exists, else use a remote server. + OfflineOrElseOnline(OfflineConfig, OnlineConfig), } impl Default for Mode { @@ -84,21 +119,52 @@ pub struct OfflineConfig { } /// Description of the transport protocol (for online execution). -#[derive(Debug)] -pub struct Transport { - uri: String, - client: Option, +#[derive(Debug, Clone)] +pub enum Transport { + /// Use the `URI` to open a new WebSocket connection. + Uri(String), + /// Use existing WebSocket connection. + RemoteClient(Arc), } -impl Clone for Transport { - fn clone(&self) -> Self { - Self { uri: self.uri.clone(), client: None } +impl Transport { + fn as_client(&self) -> Option<&WsClient> { + match self { + Self::RemoteClient(client) => Some(&*client), + _ => None, + } + } + + // Open a new WebSocket connection if it's not connected. + async fn map_uri(&mut self) -> Result<(), &'static str> { + if let Self::Uri(uri) = self { + log::debug!(target: LOG_TARGET, "initializing remote client to {:?}", uri); + + let ws_client = WsClientBuilder::default() + .max_request_body_size(u32::MAX) + .build(&uri) + .await + .map_err(|e| { + log::error!(target: LOG_TARGET, "error: {:?}", e); + "failed to build ws client" + })?; + + *self = Self::RemoteClient(Arc::new(ws_client)) + } + + Ok(()) } } impl From for Transport { - fn from(t: String) -> Self { - Self { uri: t, client: None } + fn from(uri: String) -> Self { + Transport::Uri(uri) + } +} + +impl From> for Transport { + fn from(client: Arc) -> Self { + Transport::RemoteClient(client) } } @@ -116,14 +182,15 @@ pub struct OnlineConfig { pub pallets: Vec, /// Transport config. pub transport: Transport, + /// Lookout for child-keys, and scrape them as well if set to true. + pub scrape_children: bool, } impl OnlineConfig { /// Return rpc (ws) client. fn rpc_client(&self) -> &WsClient { self.transport - .client - .as_ref() + .as_client() .expect("ws client must have been initialized by now; qed.") } } @@ -131,14 +198,21 @@ impl OnlineConfig { impl Default for OnlineConfig { fn default() -> Self { Self { - transport: Transport { uri: DEFAULT_TARGET.to_owned(), client: None }, + transport: Transport::Uri(DEFAULT_TARGET.to_owned()), at: None, state_snapshot: None, pallets: vec![], + scrape_children: true, } } } +impl From for OnlineConfig { + fn from(s: String) -> Self { + Self { transport: s.into(), ..Default::default() } + } +} + /// Configuration of the state snapshot. #[derive(Clone)] pub struct SnapshotConfig { @@ -152,6 +226,12 @@ impl SnapshotConfig { } } +impl From for SnapshotConfig { + fn from(s: String) -> Self { + Self::new(s) + } +} + impl Default for SnapshotConfig { fn default() -> Self { Self { path: Path::new("SNAPSHOT").into() } @@ -160,35 +240,43 @@ impl Default for SnapshotConfig { /// Builder for remote-externalities. pub struct Builder { - /// Custom key-pairs to be injected into the externalities. - inject: Vec, + /// Custom key-pairs to be injected into the externalities. The *hashed* keys and values must + /// be given. + hashed_key_values: Vec, /// Storage entry key prefixes to be injected into the externalities. The *hashed* prefix must /// be given. hashed_prefixes: Vec>, /// Storage entry keys to be injected into the externalities. The *hashed* key must be given. hashed_keys: Vec>, + /// The keys that will be excluded from the final externality. The *hashed* key must be given. + hashed_blacklist: Vec>, /// connectivity mode, online or offline. mode: Mode, + /// The state version being used. + state_version: StateVersion, } // NOTE: ideally we would use `DefaultNoBound` here, but not worth bringing in frame-support for // that. -impl Default for Builder { +impl Default for Builder { fn default() -> Self { Self { - inject: Default::default(), mode: Default::default(), + hashed_key_values: Default::default(), hashed_prefixes: Default::default(), hashed_keys: Default::default(), + hashed_blacklist: Default::default(), + state_version: StateVersion::V1, } } } // Mode methods -impl Builder { +impl Builder { fn as_online(&self) -> &OnlineConfig { match &self.mode { Mode::Online(config) => &config, + Mode::OfflineOrElseOnline(_, config) => &config, _ => panic!("Unexpected mode: Online"), } } @@ -196,69 +284,66 @@ impl Builder { fn as_online_mut(&mut self) -> &mut OnlineConfig { match &mut self.mode { Mode::Online(config) => config, + Mode::OfflineOrElseOnline(_, config) => config, _ => panic!("Unexpected mode: Online"), } } } // RPC methods -impl Builder { +impl Builder { async fn rpc_get_storage( &self, key: StorageKey, maybe_at: Option, ) -> Result { trace!(target: LOG_TARGET, "rpc: get_storage"); - RpcApi::::get_storage(self.as_online().rpc_client(), key, maybe_at) - .await - .map_err(|e| { - error!("Error = {:?}", e); - "rpc get_storage failed." - }) + self.as_online().rpc_client().get_storage(key, maybe_at).await.map_err(|e| { + error!(target: LOG_TARGET, "Error = {:?}", e); + "rpc get_storage failed." + }) } + /// Get the latest finalized head. async fn rpc_get_head(&self) -> Result { trace!(target: LOG_TARGET, "rpc: finalized_head"); - RpcApi::::finalized_head(self.as_online().rpc_client()).await.map_err(|e| { - error!("Error = {:?}", e); + self.as_online().rpc_client().finalized_head().await.map_err(|e| { + error!(target: LOG_TARGET, "Error = {:?}", e); "rpc finalized_head failed." }) } /// Get all the keys at `prefix` at `hash` using the paged, safe RPC methods. - async fn get_keys_paged( + async fn rpc_get_keys_paged( &self, prefix: StorageKey, at: B::Hash, ) -> Result, &'static str> { - const PAGE: u32 = 512; let mut last_key: Option = None; let mut all_keys: Vec = vec![]; let keys = loop { - let page = RpcApi::::get_keys_paged( - self.as_online().rpc_client(), - Some(prefix.clone()), - PAGE, - last_key.clone(), - Some(at), - ) - .await - .map_err(|e| { - error!(target: LOG_TARGET, "Error = {:?}", e); - "rpc get_keys failed" - })?; + let page = self + .as_online() + .rpc_client() + .get_keys_paged(Some(prefix.clone()), PAGE, last_key.clone(), Some(at)) + .await + .map_err(|e| { + error!(target: LOG_TARGET, "Error = {:?}", e); + "rpc get_keys failed" + })?; let page_len = page.len(); + all_keys.extend(page); if page_len < PAGE as usize { - debug!(target: LOG_TARGET, "last page received: {}", page_len); + log::debug!(target: LOG_TARGET, "last page received: {}", page_len); break all_keys } else { let new_last_key = all_keys.last().expect("all_keys is populated; has .last(); qed"); - debug!( + log::debug!( target: LOG_TARGET, - "new total = {}, full page received: {:?}", + "new total = {}, full page received: {}", all_keys.len(), HexDisplay::from(new_last_key) ); @@ -277,39 +362,32 @@ impl Builder { &self, prefix: StorageKey, at: B::Hash, - ) -> Result, &'static str> { - use jsonrpsee_ws_client::types::traits::Client; - use serde_json::to_value; - let keys = self.get_keys_paged(prefix, at).await?; + ) -> Result, &'static str> { + let keys = self.rpc_get_keys_paged(prefix, at).await?; let keys_count = keys.len(); - debug!(target: LOG_TARGET, "Querying a total of {} keys", keys.len()); + log::debug!(target: LOG_TARGET, "Querying a total of {} keys", keys.len()); - let mut key_values: Vec = vec![]; + let mut key_values: Vec = vec![]; let client = self.as_online().rpc_client(); for chunk_keys in keys.chunks(BATCH_SIZE) { let batch = chunk_keys .iter() .cloned() - .map(|key| { - ( - "state_getStorage", - JsonRpcParams::Array(vec![ - to_value(key).expect("json serialization will work; qed."), - to_value(at).expect("json serialization will work; qed."), - ]), - ) - }) + .map(|key| ("state_getStorage", rpc_params![key, at])) .collect::>(); + let values = client.batch_request::>(batch).await.map_err(|e| { log::error!( target: LOG_TARGET, "failed to execute batch: {:?}. Error: {:?}", - chunk_keys, + chunk_keys.iter().map(|k| HexDisplay::from(k)).collect::>(), e ); "batch failed." })?; + assert_eq!(chunk_keys.len(), values.len()); + for (idx, key) in chunk_keys.into_iter().enumerate() { let maybe_value = values[idx].clone(); let value = maybe_value.unwrap_or_else(|| { @@ -319,7 +397,7 @@ impl Builder { key_values.push((key.clone(), value)); if key_values.len() % (10 * BATCH_SIZE) == 0 { let ratio: f64 = key_values.len() as f64 / keys_count as f64; - debug!( + log::debug!( target: LOG_TARGET, "progress = {:.2} [{} / {}]", ratio, @@ -332,56 +410,246 @@ impl Builder { Ok(key_values) } + + /// Get the values corresponding to `child_keys` at the given `prefixed_top_key`. + pub(crate) async fn rpc_child_get_storage_paged( + &self, + prefixed_top_key: &StorageKey, + child_keys: Vec, + at: B::Hash, + ) -> Result, &'static str> { + let mut child_kv_inner = vec![]; + for batch_child_key in child_keys.chunks(BATCH_SIZE) { + let batch_request = batch_child_key + .iter() + .cloned() + .map(|key| { + ( + "childstate_getStorage", + rpc_params![ + PrefixedStorageKey::new(prefixed_top_key.as_ref().to_vec()), + key, + at + ], + ) + }) + .collect::>(); + + let batch_response = self + .as_online() + .rpc_client() + .batch_request::>(batch_request) + .await + .map_err(|e| { + log::error!( + target: LOG_TARGET, + "failed to execute batch: {:?}. Error: {:?}", + batch_child_key, + e + ); + "batch failed." + })?; + + assert_eq!(batch_child_key.len(), batch_response.len()); + + for (idx, key) in batch_child_key.into_iter().enumerate() { + let maybe_value = batch_response[idx].clone(); + let value = maybe_value.unwrap_or_else(|| { + log::warn!(target: LOG_TARGET, "key {:?} had none corresponding value.", &key); + StorageData(vec![]) + }); + child_kv_inner.push((key.clone(), value)); + } + } + + Ok(child_kv_inner) + } + + pub(crate) async fn rpc_child_get_keys( + &self, + prefixed_top_key: &StorageKey, + child_prefix: StorageKey, + at: B::Hash, + ) -> Result, &'static str> { + let child_keys = self + .as_online() + .rpc_client() + .child_get_keys( + PrefixedStorageKey::new(prefixed_top_key.as_ref().to_vec()), + child_prefix, + Some(at), + ) + .await + .map_err(|e| { + error!(target: LOG_TARGET, "Error = {:?}", e); + "rpc child_get_keys failed." + })?; + + debug!( + target: LOG_TARGET, + "scraped {} child-keys of the child-bearing top key: {}", + child_keys.len(), + HexDisplay::from(prefixed_top_key) + ); + + Ok(child_keys) + } } // Internal methods -impl Builder { - /// Save the given data as state snapshot. - fn save_state_snapshot(&self, data: &[KeyPair], path: &Path) -> Result<(), &'static str> { - debug!(target: LOG_TARGET, "writing to state snapshot file {:?}", path); - fs::write(path, data.encode()).map_err(|_| "fs::write failed.")?; +impl Builder { + /// Save the given data to the top keys snapshot. + fn save_top_snapshot(&self, data: &[KeyValue], path: &PathBuf) -> Result<(), &'static str> { + let mut path = path.clone(); + let encoded = data.encode(); + path.set_extension("top"); + debug!( + target: LOG_TARGET, + "writing {} bytes to state snapshot file {:?}", + encoded.len(), + path + ); + fs::write(path, encoded).map_err(|_| "fs::write failed.")?; Ok(()) } - /// initialize `Self` from state snapshot. Panics if the file does not exist. - fn load_state_snapshot(&self, path: &Path) -> Result, &'static str> { - info!(target: LOG_TARGET, "scraping key-pairs from state snapshot {:?}", path); + /// Save the given data to the child keys snapshot. + fn save_child_snapshot( + &self, + data: &ChildKeyValues, + path: &PathBuf, + ) -> Result<(), &'static str> { + let mut path = path.clone(); + path.set_extension("child"); + let encoded = data.encode(); + debug!( + target: LOG_TARGET, + "writing {} bytes to state snapshot file {:?}", + encoded.len(), + path + ); + fs::write(path, encoded).map_err(|_| "fs::write failed.")?; + Ok(()) + } + + fn load_top_snapshot(&self, path: &PathBuf) -> Result { + let mut path = path.clone(); + path.set_extension("top"); + info!(target: LOG_TARGET, "loading top key-pairs from snapshot {:?}", path); let bytes = fs::read(path).map_err(|_| "fs::read failed.")?; - Decode::decode(&mut &*bytes).map_err(|_| "decode failed") + Decode::decode(&mut &*bytes).map_err(|e| { + log::error!(target: LOG_TARGET, "{:?}", e); + "decode failed" + }) + } + + fn load_child_snapshot(&self, path: &PathBuf) -> Result { + let mut path = path.clone(); + path.set_extension("child"); + info!(target: LOG_TARGET, "loading child key-pairs from snapshot {:?}", path); + let bytes = fs::read(path).map_err(|_| "fs::read failed.")?; + Decode::decode(&mut &*bytes).map_err(|e| { + log::error!(target: LOG_TARGET, "{:?}", e); + "decode failed" + }) + } + + /// Load all the `top` keys from the remote config, and maybe write then to cache. + async fn load_top_remote_and_maybe_save(&self) -> Result { + let top_kv = self.load_top_remote().await?; + if let Some(c) = &self.as_online().state_snapshot { + self.save_top_snapshot(&top_kv, &c.path)?; + } + Ok(top_kv) + } + + /// Load all of the child keys from the remote config, given the already scraped list of top key + /// pairs. + /// + /// Stores all values to cache as well, if provided. + async fn load_child_remote_and_maybe_save( + &self, + top_kv: &[KeyValue], + ) -> Result { + let child_kv = self.load_child_remote(&top_kv).await?; + if let Some(c) = &self.as_online().state_snapshot { + self.save_child_snapshot(&child_kv, &c.path)?; + } + Ok(child_kv) + } + + /// Load all of the child keys from the remote config, given the already scraped list of top key + /// pairs. + /// + /// `top_kv` need not be only child-bearing top keys. It should be all of the top keys that are + /// included thus far. + async fn load_child_remote(&self, top_kv: &[KeyValue]) -> Result { + let child_roots = top_kv + .iter() + .filter_map(|(k, _)| is_default_child_storage_key(k.as_ref()).then(|| k)) + .collect::>(); + + info!( + target: LOG_TARGET, + "👩‍👦 scraping child-tree data from {} top keys", + child_roots.len() + ); + + let mut child_kv = vec![]; + for prefixed_top_key in child_roots { + let at = self.as_online().at.expect("at must be initialized in online mode."); + let child_keys = + self.rpc_child_get_keys(prefixed_top_key, StorageKey(vec![]), at).await?; + let child_kv_inner = + self.rpc_child_get_storage_paged(prefixed_top_key, child_keys, at).await?; + + let prefixed_top_key = PrefixedStorageKey::new(prefixed_top_key.clone().0); + let un_prefixed = match ChildType::from_prefixed_key(&prefixed_top_key) { + Some((ChildType::ParentKeyId, storage_key)) => storage_key, + None => { + log::error!(target: LOG_TARGET, "invalid key: {:?}", prefixed_top_key); + return Err("Invalid child key") + }, + }; + + child_kv.push((ChildInfo::new_default(&un_prefixed), child_kv_inner)); + } + + Ok(child_kv) } /// Build `Self` from a network node denoted by `uri`. - async fn load_remote(&self) -> Result, &'static str> { + async fn load_top_remote(&self) -> Result { let config = self.as_online(); let at = self .as_online() .at .expect("online config must be initialized by this point; qed.") .clone(); - info!(target: LOG_TARGET, "scraping key-pairs from remote @ {:?}", at); + log::info!(target: LOG_TARGET, "scraping key-pairs from remote @ {:?}", at); let mut keys_and_values = if config.pallets.len() > 0 { let mut filtered_kv = vec![]; - for f in config.pallets.iter() { - let hashed_prefix = StorageKey(twox_128(f.as_bytes()).to_vec()); - let module_kv = self.rpc_get_pairs_paged(hashed_prefix.clone(), at).await?; - info!( + for p in config.pallets.iter() { + let hashed_prefix = StorageKey(twox_128(p.as_bytes()).to_vec()); + let pallet_kv = self.rpc_get_pairs_paged(hashed_prefix.clone(), at).await?; + log::info!( target: LOG_TARGET, - "downloaded data for module {} (count: {} / prefix: {:?}).", - f, - module_kv.len(), + "downloaded data for module {} (count: {} / prefix: {}).", + p, + pallet_kv.len(), HexDisplay::from(&hashed_prefix), ); - filtered_kv.extend(module_kv); + filtered_kv.extend(pallet_kv); } filtered_kv } else { - info!(target: LOG_TARGET, "downloading data for all pallets."); + log::info!(target: LOG_TARGET, "downloading data for all pallets."); self.rpc_get_pairs_paged(StorageKey(vec![]), at).await? }; for prefix in &self.hashed_prefixes { - debug!( + log::info!( target: LOG_TARGET, "adding data for hashed prefix: {:?}", HexDisplay::from(prefix) @@ -393,7 +661,11 @@ impl Builder { for key in &self.hashed_keys { let key = StorageKey(key.to_vec()); - debug!(target: LOG_TARGET, "adding data for hashed key: {:?}", HexDisplay::from(&key)); + log::info!( + target: LOG_TARGET, + "adding data for hashed key: {:?}", + HexDisplay::from(&key) + ); let value = self.rpc_get_storage(key.clone(), Some(at)).await?; keys_and_values.push((key, value)); } @@ -402,16 +674,8 @@ impl Builder { } pub(crate) async fn init_remote_client(&mut self) -> Result<(), &'static str> { - let mut online = self.as_online_mut(); - debug!(target: LOG_TARGET, "initializing remote client to {:?}", online.transport.uri); - // First, initialize the ws client. - let ws_client = WsClientBuilder::default() - .max_request_body_size(u32::MAX) - .build(&online.transport.uri) - .await - .map_err(|_| "failed to build ws client")?; - online.transport.client = Some(ws_client); + self.as_online_mut().transport.map_uri().await?; // Then, if `at` is not set, set it. if self.as_online().at.is_none() { @@ -422,40 +686,80 @@ impl Builder { Ok(()) } - pub(crate) async fn pre_build(mut self) -> Result, &'static str> { - let mut base_kv = match self.mode.clone() { - Mode::Offline(config) => self.load_state_snapshot(&config.state_snapshot.path)?, - Mode::Online(config) => { + pub(crate) async fn pre_build( + mut self, + ) -> Result<(TopKeyValues, ChildKeyValues), &'static str> { + let mut top_kv = match self.mode.clone() { + Mode::Offline(config) => self.load_top_snapshot(&config.state_snapshot.path)?, + Mode::Online(_) => { self.init_remote_client().await?; - let kp = self.load_remote().await?; - if let Some(c) = config.state_snapshot { - self.save_state_snapshot(&kp, &c.path)?; + self.load_top_remote_and_maybe_save().await? + }, + Mode::OfflineOrElseOnline(offline_config, _) => { + if let Ok(kv) = self.load_top_snapshot(&offline_config.state_snapshot.path) { + kv + } else { + self.init_remote_client().await?; + self.load_top_remote_and_maybe_save().await? } - kp }, }; - debug!( - target: LOG_TARGET, - "extending externalities with {} manually injected key-values", - self.inject.len() - ); - base_kv.extend(self.inject.clone()); - Ok(base_kv) + // inject manual key values. + if !self.hashed_key_values.is_empty() { + log::info!( + target: LOG_TARGET, + "extending externalities with {} manually injected key-values", + self.hashed_key_values.len() + ); + top_kv.extend(self.hashed_key_values.clone()); + } + + // exclude manual key values. + if !self.hashed_blacklist.is_empty() { + log::info!( + target: LOG_TARGET, + "excluding externalities from {} keys", + self.hashed_blacklist.len() + ); + top_kv.retain(|(k, _)| !self.hashed_blacklist.contains(&k.0)) + } + + let child_kv = match self.mode.clone() { + Mode::Online(_) => self.load_child_remote_and_maybe_save(&top_kv).await?, + Mode::OfflineOrElseOnline(offline_config, _) => + if let Ok(kv) = self.load_child_snapshot(&offline_config.state_snapshot.path) { + kv + } else { + self.load_child_remote_and_maybe_save(&top_kv).await? + }, + Mode::Offline(ref config) => self + .load_child_snapshot(&config.state_snapshot.path) + .map_err(|why| { + log::warn!( + target: LOG_TARGET, + "failed to load child-key file due to {:?}.", + why + ) + }) + .unwrap_or_default(), + }; + + Ok((top_kv, child_kv)) } } // Public methods -impl Builder { +impl Builder { /// Create a new builder. pub fn new() -> Self { Default::default() } /// Inject a manual list of key and values to the storage. - pub fn inject_key_value(mut self, injections: &[KeyPair]) -> Self { + pub fn inject_hashed_key_value(mut self, injections: &[KeyValue]) -> Self { for i in injections { - self.inject.push(i.clone()); + self.hashed_key_values.push(i.clone()); } self } @@ -468,6 +772,22 @@ impl Builder { self } + /// Just a utility wrapper of [`Self::inject_hashed_prefix`] that injects + /// [`DEFAULT_CHILD_STORAGE_KEY_PREFIX`] as a prefix. + /// + /// If set, this will guarantee that the child-tree data of ALL pallets will be downloaded. + /// + /// This is not needed if the entire state is being downloaded. + /// + /// Otherwise, the only other way to make sure a child-tree is manually included is to inject + /// its root (`DEFAULT_CHILD_STORAGE_KEY_PREFIX`, plus some other postfix) into + /// [`Self::inject_hashed_key`]. Unfortunately, there's no federated way of managing child tree + /// roots as of now and each pallet does its own thing. Therefore, it is not possible for this + /// library to automatically include child trees of pallet X, when its top keys are included. + pub fn inject_default_child_tree_prefix(self) -> Self { + self.inject_hashed_prefix(DEFAULT_CHILD_STORAGE_KEY_PREFIX) + } + /// Inject a hashed key to scrape. This is treated as-is, and should be pre-hashed. /// /// This should be used to inject a "KEY", like a storage value. @@ -476,12 +796,25 @@ impl Builder { self } + /// Blacklist this hashed key from the final externalities. This is treated as-is, and should be + /// pre-hashed. + pub fn blacklist_hashed_key(mut self, hashed: &[u8]) -> Self { + self.hashed_blacklist.push(hashed.to_vec()); + self + } + /// Configure a state snapshot to be used. pub fn mode(mut self, mode: Mode) -> Self { self.mode = mode; self } + /// The state version to use. + pub fn state_version(mut self, version: StateVersion) -> Self { + self.state_version = version; + self + } + /// overwrite the `at` value, if `mode` is set to [`Mode::Online`]. /// /// noop if `mode` is [`Mode::Offline`] @@ -495,16 +828,42 @@ impl Builder { /// Build the test externalities. pub async fn build(self) -> Result { - let kv = self.pre_build().await?; - let mut ext = TestExternalities::new_empty(); - - info!(target: LOG_TARGET, "injecting a total of {} keys", kv.len()); - for (k, v) in kv { - let (k, v) = (k.0, v.0); - // Insert the key,value pair into the test trie backend - ext.insert(k, v); + let state_version = self.state_version; + let (top_kv, child_kv) = self.pre_build().await?; + let mut ext = TestExternalities::new_with_code_and_state( + Default::default(), + Default::default(), + state_version, + ); + + info!(target: LOG_TARGET, "injecting a total of {} top keys", top_kv.len()); + for (k, v) in top_kv { + // skip writing the child root data. + if is_default_child_storage_key(k.as_ref()) { + continue + } + ext.insert(k.0, v.0); } + info!( + target: LOG_TARGET, + "injecting a total of {} child keys", + child_kv.iter().map(|(_, kv)| kv).flatten().count() + ); + + for (info, key_values) in child_kv { + for (k, v) in key_values { + ext.insert_child(info.clone(), k.0, v.0); + } + } + + ext.commit_all().unwrap(); + info!( + target: LOG_TARGET, + "initialized state externalities with storage root {:?}", + ext.as_backend().root() + ); + Ok(ext) } } @@ -541,14 +900,100 @@ mod tests { .expect("Can't read state snapshot file") .execute_with(|| {}); } + + #[tokio::test] + async fn can_exclude_from_cache() { + init_logger(); + + // get the first key from the cache file. + let some_key = Builder::::new() + .mode(Mode::Offline(OfflineConfig { + state_snapshot: SnapshotConfig::new("test_data/proxy_test"), + })) + .build() + .await + .expect("Can't read state snapshot file") + .execute_with(|| { + let key = + sp_io::storage::next_key(&[]).expect("some key must exist in the snapshot"); + assert!(sp_io::storage::get(&key).is_some()); + key + }); + + Builder::::new() + .mode(Mode::Offline(OfflineConfig { + state_snapshot: SnapshotConfig::new("test_data/proxy_test"), + })) + .blacklist_hashed_key(&some_key) + .build() + .await + .expect("Can't read state snapshot file") + .execute_with(|| assert!(sp_io::storage::get(&some_key).is_none())); + } } #[cfg(all(test, feature = "remote-test"))] mod remote_tests { use super::test_prelude::*; + const REMOTE_INACCESSIBLE: &'static str = "Can't reach the remote node. Is it running?"; #[tokio::test] - async fn can_build_one_pallet() { + async fn offline_else_online_works() { + init_logger(); + // this shows that in the second run, we use the remote and create a cache. + Builder::::new() + .mode(Mode::OfflineOrElseOnline( + OfflineConfig { + state_snapshot: SnapshotConfig::new("offline_else_online_works_data"), + }, + OnlineConfig { + pallets: vec!["Proxy".to_owned()], + state_snapshot: Some(SnapshotConfig::new("offline_else_online_works_data")), + ..Default::default() + }, + )) + .build() + .await + .expect(REMOTE_INACCESSIBLE) + .execute_with(|| {}); + + // this shows that in the second run, we are not using the remote + Builder::::new() + .mode(Mode::OfflineOrElseOnline( + OfflineConfig { + state_snapshot: SnapshotConfig::new("offline_else_online_works_data"), + }, + OnlineConfig { + pallets: vec!["Proxy".to_owned()], + state_snapshot: Some(SnapshotConfig::new("offline_else_online_works_data")), + transport: "ws://non-existent:666".to_owned().into(), + ..Default::default() + }, + )) + .build() + .await + .expect(REMOTE_INACCESSIBLE) + .execute_with(|| {}); + + let to_delete = std::fs::read_dir(Path::new(".")) + .unwrap() + .into_iter() + .map(|d| d.unwrap()) + .filter(|p| { + p.path().file_name().unwrap_or_default() == "offline_else_online_works_data" || + p.path().extension().unwrap_or_default() == "top" || + p.path().extension().unwrap_or_default() == "child" + }) + .collect::>(); + assert!(to_delete.len() > 0); + for d in to_delete { + std::fs::remove_file(d.path()).unwrap(); + } + } + + #[tokio::test] + #[ignore = "too slow"] + async fn can_build_one_big_pallet() { init_logger(); Builder::::new() .mode(Mode::Online(OnlineConfig { @@ -557,7 +1002,33 @@ mod remote_tests { })) .build() .await - .expect("Can't reach the remote node. Is it running?") + .expect(REMOTE_INACCESSIBLE) + .execute_with(|| {}); + } + + #[tokio::test] + async fn can_build_one_small_pallet() { + init_logger(); + Builder::::new() + .mode(Mode::Online(OnlineConfig { + transport: "wss://kusama-rpc.polkadot.io:443".to_owned().into(), + pallets: vec!["Council".to_owned()], + ..Default::default() + })) + .build() + .await + .expect(REMOTE_INACCESSIBLE) + .execute_with(|| {}); + + Builder::::new() + .mode(Mode::Online(OnlineConfig { + transport: "wss://rpc.polkadot.io:443".to_owned().into(), + pallets: vec!["Council".to_owned()], + ..Default::default() + })) + .build() + .await + .expect(REMOTE_INACCESSIBLE) .execute_with(|| {}); } @@ -566,78 +1037,118 @@ mod remote_tests { init_logger(); Builder::::new() .mode(Mode::Online(OnlineConfig { - pallets: vec![ - "Proxy".to_owned(), - "Multisig".to_owned(), - "PhragmenElection".to_owned(), - ], + transport: "wss://kusama-rpc.polkadot.io:443".to_owned().into(), + pallets: vec!["Proxy".to_owned(), "Multisig".to_owned()], ..Default::default() })) .build() .await - .expect("Can't reach the remote node. Is it running?") + .expect(REMOTE_INACCESSIBLE) + .execute_with(|| {}); + + Builder::::new() + .mode(Mode::Online(OnlineConfig { + transport: "wss://rpc.polkadot.io:443".to_owned().into(), + pallets: vec!["Proxy".to_owned(), "Multisig".to_owned()], + ..Default::default() + })) + .build() + .await + .expect(REMOTE_INACCESSIBLE) .execute_with(|| {}); } #[tokio::test] - async fn sanity_check_decoding() { - use pallet_elections_phragmen::SeatHolder; - use sp_core::crypto::Ss58Codec; - type AccountId = sp_runtime::AccountId32; - type Balance = u128; - frame_support::generate_storage_alias!( - PhragmenElection, - Members => - Value>> - ); + async fn can_create_top_snapshot() { + init_logger(); + Builder::::new() + .mode(Mode::Online(OnlineConfig { + state_snapshot: Some(SnapshotConfig::new("can_create_top_snapshot_data")), + pallets: vec!["Proxy".to_owned()], + ..Default::default() + })) + .build() + .await + .expect(REMOTE_INACCESSIBLE) + .execute_with(|| {}); + + let to_delete = std::fs::read_dir(Path::new(".")) + .unwrap() + .into_iter() + .map(|d| d.unwrap()) + .filter(|p| { + p.path().file_name().unwrap_or_default() == "can_create_top_snapshot_data" || + p.path().extension().unwrap_or_default() == "top" || + p.path().extension().unwrap_or_default() == "child" + }) + .collect::>(); + + assert!(to_delete.len() > 0); + for d in to_delete { + use std::os::unix::fs::MetadataExt; + if d.path().extension().unwrap_or_default() == "top" { + // if this is the top snapshot it must not be empty. + assert!(std::fs::metadata(d.path()).unwrap().size() > 1); + } else { + // the child is empty for this pallet. + assert!(std::fs::metadata(d.path()).unwrap().size() == 1); + } + std::fs::remove_file(d.path()).unwrap(); + } + } + + #[tokio::test] + async fn can_build_child_tree() { init_logger(); Builder::::new() .mode(Mode::Online(OnlineConfig { - pallets: vec!["PhragmenElection".to_owned()], + transport: "wss://rpc.polkadot.io:443".to_owned().into(), + pallets: vec!["Crowdloan".to_owned()], ..Default::default() })) .build() .await - .expect("Can't reach the remote node. Is it running?") - .execute_with(|| { - // Gav's polkadot account. 99% this will be in the council. - let gav_polkadot = - AccountId::from_ss58check("13RDY9nrJpyTDBSUdBw12dGwhk19sGwsrVZ2bxkzYHBSagP2") - .unwrap(); - let members = Members::get().unwrap(); - assert!(members - .iter() - .map(|s| s.who.clone()) - .find(|a| a == &gav_polkadot) - .is_some()); - }); + .expect(REMOTE_INACCESSIBLE) + .execute_with(|| {}); } #[tokio::test] - async fn can_create_state_snapshot() { + async fn can_create_child_snapshot() { init_logger(); Builder::::new() .mode(Mode::Online(OnlineConfig { - state_snapshot: Some(SnapshotConfig::new("test_snapshot_to_remove.bin")), - pallets: vec!["Balances".to_owned()], + state_snapshot: Some(SnapshotConfig::new("can_create_child_snapshot_data")), + pallets: vec!["Crowdloan".to_owned()], ..Default::default() })) + .inject_default_child_tree_prefix() .build() .await - .expect("Can't reach the remote node. Is it running?") + .expect(REMOTE_INACCESSIBLE) .execute_with(|| {}); - let to_delete = std::fs::read_dir(SnapshotConfig::default().path) + let to_delete = std::fs::read_dir(Path::new(".")) .unwrap() .into_iter() .map(|d| d.unwrap()) - .filter(|p| p.path().extension().unwrap_or_default() == "bin") + .filter(|p| { + p.path().file_name().unwrap_or_default() == "can_create_child_snapshot_data" || + p.path().extension().unwrap_or_default() == "top" || + p.path().extension().unwrap_or_default() == "child" + }) .collect::>(); assert!(to_delete.len() > 0); for d in to_delete { + use std::os::unix::fs::MetadataExt; + // if this is the top snapshot it must not be empty + if d.path().extension().unwrap_or_default() == "child" { + assert!(std::fs::metadata(d.path()).unwrap().size() > 1); + } else { + assert!(std::fs::metadata(d.path()).unwrap().size() > 1); + } std::fs::remove_file(d.path()).unwrap(); } } @@ -646,9 +1157,54 @@ mod remote_tests { async fn can_fetch_all() { init_logger(); Builder::::new() + .mode(Mode::Online(OnlineConfig { + state_snapshot: Some(SnapshotConfig::new("can_fetch_all_data")), + ..Default::default() + })) + .build() + .await + .expect(REMOTE_INACCESSIBLE) + .execute_with(|| {}); + + let to_delete = std::fs::read_dir(Path::new(".")) + .unwrap() + .into_iter() + .map(|d| d.unwrap()) + .filter(|p| { + p.path().file_name().unwrap_or_default() == "can_fetch_all_data" || + p.path().extension().unwrap_or_default() == "top" || + p.path().extension().unwrap_or_default() == "child" + }) + .collect::>(); + + assert!(to_delete.len() > 0); + + for d in to_delete { + use std::os::unix::fs::MetadataExt; + // if we download everything, child tree must also be filled. + if d.path().extension().unwrap_or_default() == "child" { + assert!(std::fs::metadata(d.path()).unwrap().size() > 1); + } else { + assert!(std::fs::metadata(d.path()).unwrap().size() > 1); + } + std::fs::remove_file(d.path()).unwrap(); + } + } + + #[tokio::test] + async fn can_build_child_tree() { + init_logger(); + Builder::::new() + .mode(Mode::Online(OnlineConfig { + // transport: "wss://kusama-rpc.polkadot.io".to_owned().into(), + transport: "ws://kianenigma-archive:9924".to_owned().into(), + // transport: "ws://localhost:9999".to_owned().into(), + pallets: vec!["Crowdloan".to_owned()], + ..Default::default() + })) .build() .await - .expect("Can't reach the remote node. Is it running?") + .expect(REMOTE_INACCESSIBLE) .execute_with(|| {}); } } diff --git a/utils/frame/remote-externalities/src/rpc_api.rs b/utils/frame/remote-externalities/src/rpc_api.rs index 24050856a96a..37555de480d4 100644 --- a/utils/frame/remote-externalities/src/rpc_api.rs +++ b/utils/frame/remote-externalities/src/rpc_api.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,9 +18,10 @@ //! WS RPC API for one off RPC calls to a substrate node. // TODO: Consolidate one off RPC calls https://github.com/paritytech/substrate/issues/8988 -use jsonrpsee_ws_client::{ - types::{traits::Client, v2::params::JsonRpcParams}, - WsClient, WsClientBuilder, +use jsonrpsee::{ + core::client::ClientT, + rpc_params, + ws_client::{WsClient, WsClientBuilder}, }; use sp_runtime::{ generic::SignedBlock, @@ -34,11 +35,10 @@ where Block::Header: serde::de::DeserializeOwned, S: AsRef, { - let params = vec![hash_to_json::(at)?]; let client = build_client(from).await?; client - .request::("chain_getHeader", JsonRpcParams::Array(params)) + .request::("chain_getHeader", rpc_params!(at)) .await .map_err(|e| format!("chain_getHeader request failed: {:?}", e)) } @@ -52,7 +52,7 @@ where let client = build_client(from).await?; client - .request::("chain_getFinalizedHead", JsonRpcParams::NoParams) + .request::("chain_getFinalizedHead", None) .await .map_err(|e| format!("chain_getFinalizedHead request failed: {:?}", e)) } @@ -64,23 +64,16 @@ where Block: BlockT + serde::de::DeserializeOwned, Block::Header: HeaderT, { - let params = vec![hash_to_json::(at)?]; let client = build_client(from).await?; let signed_block = client - .request::>("chain_getBlock", JsonRpcParams::Array(params)) + .request::>("chain_getBlock", rpc_params!(at)) .await .map_err(|e| format!("chain_getBlock request failed: {:?}", e))?; Ok(signed_block.block) } -/// Convert a block hash to a serde json value. -fn hash_to_json(hash: Block::Hash) -> Result { - serde_json::to_value(hash) - .map_err(|e| format!("Block hash could not be converted to JSON: {:?}", e)) -} - -/// Build a website client that connects to `from`. +/// Build a websocket client that connects to `from`. async fn build_client>(from: S) -> Result { WsClientBuilder::default() .max_request_body_size(u32::MAX) @@ -99,13 +92,9 @@ where Block: BlockT + serde::de::DeserializeOwned, Block::Header: HeaderT, { - let params = if let Some(at) = at { vec![hash_to_json::(at)?] } else { vec![] }; let client = build_client(from).await?; client - .request::( - "state_getRuntimeVersion", - JsonRpcParams::Array(params), - ) + .request::("state_getRuntimeVersion", rpc_params!(at)) .await .map_err(|e| format!("state_getRuntimeVersion request failed: {:?}", e)) } diff --git a/utils/frame/remote-externalities/test_data/proxy_test b/utils/frame/remote-externalities/test_data/proxy_test.top similarity index 100% rename from utils/frame/remote-externalities/test_data/proxy_test rename to utils/frame/remote-externalities/test_data/proxy_test.top diff --git a/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml b/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml new file mode 100644 index 000000000000..deb641a89a46 --- /dev/null +++ b/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "substrate-state-trie-migration-rpc" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "Node-specific RPC methods for interaction with state trie migration." +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +scale-info = { version = "2.0.1", default-features = false, features = ["derive"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false } +serde = { version = "1", features = ["derive"] } +log = { version = "0.4.14", default-features = false } + +sp-std = { path = "../../../../primitives/std" } +sp-io = { path = "../../../../primitives/io" } +sp-core = { path = "../../../../primitives/core" } +sp-state-machine = { path = "../../../../primitives/state-machine" } +sp-trie = { path = "../../../../primitives/trie" } +trie-db = { version = "0.23.1" } + +jsonrpc-core = "18.0.0" +jsonrpc-core-client = "18.0.0" +jsonrpc-derive = "18.0.0" + +# Substrate Dependencies +sc-client-api = { version = "4.0.0-dev", path = "../../../../client/api" } +sc-rpc-api = { version = "0.10.0-dev", path = "../../../../client/rpc-api" } +sp-runtime = { version = "6.0.0", path = "../../../../primitives/runtime" } + +[dev-dependencies] +serde_json = "1" diff --git a/utils/frame/rpc/state-trie-migration-rpc/README.md b/utils/frame/rpc/state-trie-migration-rpc/README.md new file mode 100644 index 000000000000..03bbfdf1b593 --- /dev/null +++ b/utils/frame/rpc/state-trie-migration-rpc/README.md @@ -0,0 +1,3 @@ +Node-specific RPC methods for interaction with trie migration. + +License: Apache-2.0 diff --git a/utils/frame/rpc/state-trie-migration-rpc/src/lib.rs b/utils/frame/rpc/state-trie-migration-rpc/src/lib.rs new file mode 100644 index 000000000000..98a3cf964843 --- /dev/null +++ b/utils/frame/rpc/state-trie-migration-rpc/src/lib.rs @@ -0,0 +1,164 @@ +// This file is part of Substrate. + +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Rpc for state migration. + +use jsonrpc_core::{Error, ErrorCode, Result}; +use jsonrpc_derive::rpc; +use sc_rpc_api::DenyUnsafe; +use serde::{Deserialize, Serialize}; +use sp_runtime::{generic::BlockId, traits::Block as BlockT}; +use std::sync::Arc; + +use sp_core::{ + storage::{ChildInfo, ChildType, PrefixedStorageKey}, + Hasher, +}; +use sp_state_machine::Backend; +use sp_trie::{trie_types::TrieDB, KeySpacedDB, Trie}; +use trie_db::{ + node::{NodePlan, ValuePlan}, + TrieDBNodeIterator, +}; + +fn count_migrate<'a, H: Hasher>( + storage: &'a dyn trie_db::HashDBRef>, + root: &'a H::Out, +) -> std::result::Result<(u64, TrieDB<'a, H>), String> { + let mut nb = 0u64; + let trie = TrieDB::new(storage, root).map_err(|e| format!("TrieDB creation error: {}", e))?; + let iter_node = + TrieDBNodeIterator::new(&trie).map_err(|e| format!("TrieDB node iterator error: {}", e))?; + for node in iter_node { + let node = node.map_err(|e| format!("TrieDB node iterator error: {}", e))?; + match node.2.node_plan() { + NodePlan::Leaf { value, .. } | NodePlan::NibbledBranch { value: Some(value), .. } => + if let ValuePlan::Inline(range) = value { + if (range.end - range.start) as u32 >= + sp_core::storage::TRIE_VALUE_NODE_THRESHOLD + { + nb += 1; + } + }, + _ => (), + } + } + Ok((nb, trie)) +} + +/// Check trie migration status. +pub fn migration_status(backend: &B) -> std::result::Result<(u64, u64), String> +where + H: Hasher, + H::Out: codec::Codec, + B: Backend, +{ + let trie_backend = if let Some(backend) = backend.as_trie_backend() { + backend + } else { + return Err("No access to trie from backend.".to_string()) + }; + let essence = trie_backend.essence(); + let (nb_to_migrate, trie) = count_migrate(essence, &essence.root())?; + + let mut nb_to_migrate_child = 0; + let mut child_roots: Vec<(ChildInfo, Vec)> = Vec::new(); + // get all child trie roots + for key_value in trie.iter().map_err(|e| format!("TrieDB node iterator error: {}", e))? { + let (key, value) = key_value.map_err(|e| format!("TrieDB node iterator error: {}", e))?; + if key[..].starts_with(sp_core::storage::well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX) + { + let prefixed_key = PrefixedStorageKey::new(key); + let (_type, unprefixed) = ChildType::from_prefixed_key(&prefixed_key).unwrap(); + child_roots.push((ChildInfo::new_default(unprefixed), value)); + } + } + for (child_info, root) in child_roots { + let mut child_root = H::Out::default(); + let storage = KeySpacedDB::new(essence, child_info.keyspace()); + + child_root.as_mut()[..].copy_from_slice(&root[..]); + nb_to_migrate_child += count_migrate(&storage, &child_root)?.0; + } + + Ok((nb_to_migrate, nb_to_migrate_child)) +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +pub struct MigrationStatusResult { + top_remaining_to_migrate: u64, + child_remaining_to_migrate: u64, +} + +/// Migration RPC methods. +#[rpc] +pub trait StateMigrationApi { + /// Check current migration state. + /// + /// This call is performed locally without submitting any transactions. Thus executing this + /// won't change any state. Nonetheless it is a VERY costy call that should be + /// only exposed to trusted peers. + #[rpc(name = "state_trieMigrationStatus")] + fn call(&self, at: Option) -> Result; +} + +/// An implementation of state migration specific RPC methods. +pub struct MigrationRpc { + client: Arc, + backend: Arc, + deny_unsafe: DenyUnsafe, + _marker: std::marker::PhantomData<(B, BA)>, +} + +impl MigrationRpc { + /// Create new state migration rpc for the given reference to the client. + pub fn new(client: Arc, backend: Arc, deny_unsafe: DenyUnsafe) -> Self { + MigrationRpc { client, backend, deny_unsafe, _marker: Default::default() } + } +} + +impl StateMigrationApi<::Hash> for MigrationRpc +where + B: BlockT, + C: Send + Sync + 'static + sc_client_api::HeaderBackend, + BA: 'static + sc_client_api::backend::Backend, +{ + fn call(&self, at: Option<::Hash>) -> Result { + if let Err(err) = self.deny_unsafe.check_if_safe() { + return Err(err.into()) + } + + let block_id = BlockId::hash(at.unwrap_or_else(|| self.client.info().best_hash)); + let state = self.backend.state_at(block_id).map_err(error_into_rpc_err)?; + let (top, child) = migration_status(&state).map_err(error_into_rpc_err)?; + + Ok(MigrationStatusResult { + top_remaining_to_migrate: top, + child_remaining_to_migrate: child, + }) + } +} + +fn error_into_rpc_err(err: impl std::fmt::Display) -> Error { + Error { + code: ErrorCode::InternalError, + message: "Error while checking migration state".into(), + data: Some(err.to_string().into()), + } +} diff --git a/utils/frame/rpc/support/Cargo.toml b/utils/frame/rpc/support/Cargo.toml index a94f18d0e892..fe304370727f 100644 --- a/utils/frame/rpc/support/Cargo.toml +++ b/utils/frame/rpc/support/Cargo.toml @@ -5,9 +5,9 @@ authors = [ "Parity Technologies ", "Andrew Dirksen ", ] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Substrate RPC for FRAME's support" @@ -15,15 +15,15 @@ description = "Substrate RPC for FRAME's support" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -futures = "0.3.16" +futures = "0.3.21" jsonrpc-client-transports = { version = "18.0.0", features = ["http"] } -codec = { package = "parity-scale-codec", version = "2.0.0" } +codec = { package = "parity-scale-codec", version = "3.0.0" } serde = "1" frame-support = { version = "4.0.0-dev", path = "../../../../frame/support" } -sp-storage = { version = "4.0.0-dev", path = "../../../../primitives/storage" } +sp-storage = { version = "6.0.0", path = "../../../../primitives/storage" } sc-rpc-api = { version = "0.10.0-dev", path = "../../../../client/rpc-api" } [dev-dependencies] frame-system = { version = "4.0.0-dev", path = "../../../../frame/system" } -scale-info = "1.0" -tokio = "1.10" +scale-info = "2.0.1" +tokio = "1.17.0" diff --git a/utils/frame/rpc/support/src/lib.rs b/utils/frame/rpc/support/src/lib.rs index 1b2453c361d9..5d7cba19f643 100644 --- a/utils/frame/rpc/support/src/lib.rs +++ b/utils/frame/rpc/support/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -126,7 +126,7 @@ impl StorageQuery { block_index: Option, ) -> Result, RpcError> { let opt: Option = state_client.storage(self.key, block_index).await?; - opt.map(|encoded| V::decode_all(&encoded.0)) + opt.map(|encoded| V::decode_all(&mut &encoded.0[..])) .transpose() .map_err(|decode_err| RpcError::Other(Box::new(decode_err))) } diff --git a/utils/frame/rpc/system/Cargo.toml b/utils/frame/rpc/system/Cargo.toml index e9ae506ef6b0..31a6f93e847b 100644 --- a/utils/frame/rpc/system/Cargo.toml +++ b/utils/frame/rpc/system/Cargo.toml @@ -2,9 +2,9 @@ name = "substrate-frame-rpc-system" version = "4.0.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "FRAME's system exposed over Substrate RPC" readme = "README.md" @@ -14,16 +14,16 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] sc-client-api = { version = "4.0.0-dev", path = "../../../../client/api" } -codec = { package = "parity-scale-codec", version = "2.0.0" } -futures = "0.3.16" +codec = { package = "parity-scale-codec", version = "3.0.0" } +futures = "0.3.21" jsonrpc-core = "18.0.0" jsonrpc-core-client = "18.0.0" jsonrpc-derive = "18.0.0" log = "0.4.8" -sp-runtime = { version = "4.0.0-dev", path = "../../../../primitives/runtime" } +sp-runtime = { version = "6.0.0", path = "../../../../primitives/runtime" } sp-api = { version = "4.0.0-dev", path = "../../../../primitives/api" } frame-system-rpc-runtime-api = { version = "4.0.0-dev", path = "../../../../frame/system/rpc/runtime-api" } -sp-core = { version = "4.0.0-dev", path = "../../../../primitives/core" } +sp-core = { version = "6.0.0", path = "../../../../primitives/core" } sp-blockchain = { version = "4.0.0-dev", path = "../../../../primitives/blockchain" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "../../../../client/transaction-pool/api" } sp-block-builder = { version = "4.0.0-dev", path = "../../../../primitives/block-builder" } @@ -31,5 +31,5 @@ sc-rpc-api = { version = "0.10.0-dev", path = "../../../../client/rpc-api" } [dev-dependencies] substrate-test-runtime-client = { version = "2.0.0", path = "../../../../test-utils/runtime/client" } -sp-tracing = { version = "4.0.0-dev", path = "../../../../primitives/tracing" } +sp-tracing = { version = "5.0.0", path = "../../../../primitives/tracing" } sc-transaction-pool = { version = "4.0.0-dev", path = "../../../../client/transaction-pool" } diff --git a/utils/frame/rpc/system/src/lib.rs b/utils/frame/rpc/system/src/lib.rs index f0f37f0b2067..0eae4d061afb 100644 --- a/utils/frame/rpc/system/src/lib.rs +++ b/utils/frame/rpc/system/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,16 +20,16 @@ use std::sync::Arc; use codec::{Codec, Decode, Encode}; -use futures::{future::ready, FutureExt, TryFutureExt}; +use futures::FutureExt; use jsonrpc_core::{Error as RpcError, ErrorCode}; use jsonrpc_derive::rpc; -use sc_client_api::light::{future_header, Fetcher, RemoteBlockchain, RemoteCallRequest}; use sc_rpc_api::DenyUnsafe; use sc_transaction_pool_api::{InPoolTransaction, TransactionPool}; +use sp_api::ApiExt; use sp_block_builder::BlockBuilder; -use sp_blockchain::{Error as ClientError, HeaderBackend}; +use sp_blockchain::HeaderBackend; use sp_core::{hexdisplay::HexDisplay, Bytes}; -use sp_runtime::{generic::BlockId, traits}; +use sp_runtime::{generic::BlockId, legacy, traits}; pub use self::gen_client::Client as SystemClient; pub use frame_system_rpc_runtime_api::AccountNonceApi; @@ -107,7 +107,7 @@ where let nonce = api.account_nonce(&at, account.clone()).map_err(|e| RpcError { code: ErrorCode::ServerError(Error::RuntimeError.into()), message: "Unable to query nonce.".into(), - data: Some(format!("{:?}", e).into()), + data: Some(e.to_string().into()), })?; Ok(adjust_nonce(&*self.pool, account, nonce)) @@ -136,14 +136,40 @@ where .map_err(|e| RpcError { code: ErrorCode::ServerError(Error::DecodeError.into()), message: "Unable to dry run extrinsic.".into(), - data: Some(format!("{:?}", e).into()), + data: Some(e.to_string().into()), })?; - let result = api.apply_extrinsic(&at, uxt).map_err(|e| RpcError { - code: ErrorCode::ServerError(Error::RuntimeError.into()), - message: "Unable to dry run extrinsic.".into(), - data: Some(format!("{:?}", e).into()), - })?; + let api_version = api + .api_version::>(&at) + .map_err(|e| RpcError { + code: ErrorCode::ServerError(Error::RuntimeError.into()), + message: "Unable to dry run extrinsic.".into(), + data: Some(e.to_string().into()), + })? + .ok_or_else(|| RpcError { + code: ErrorCode::ServerError(Error::RuntimeError.into()), + message: "Unable to dry run extrinsic.".into(), + data: Some( + format!("Could not find `BlockBuilder` api for block `{:?}`.", at).into(), + ), + })?; + + let result = if api_version < 6 { + #[allow(deprecated)] + api.apply_extrinsic_before_version_6(&at, uxt) + .map(legacy::byte_sized_error::convert_to_latest) + .map_err(|e| RpcError { + code: ErrorCode::ServerError(Error::RuntimeError.into()), + message: "Unable to dry run extrinsic.".into(), + data: Some(e.to_string().into()), + })? + } else { + api.apply_extrinsic(&at, uxt).map_err(|e| RpcError { + code: ErrorCode::ServerError(Error::RuntimeError.into()), + message: "Unable to dry run extrinsic.".into(), + data: Some(e.to_string().into()), + })? + }; Ok(Encode::encode(&result).into()) }; @@ -154,90 +180,6 @@ where } } -/// An implementation of System-specific RPC methods on light client. -pub struct LightSystem { - client: Arc, - remote_blockchain: Arc>, - fetcher: Arc, - pool: Arc

, -} - -impl LightSystem { - /// Create new `LightSystem`. - pub fn new( - client: Arc, - remote_blockchain: Arc>, - fetcher: Arc, - pool: Arc

, - ) -> Self { - LightSystem { client, remote_blockchain, fetcher, pool } - } -} - -impl SystemApi<::Hash, AccountId, Index> - for LightSystem -where - P: TransactionPool + 'static, - C: HeaderBackend, - C: Send + Sync + 'static, - F: Fetcher + 'static, - Block: traits::Block, - AccountId: Clone + std::fmt::Display + Codec + Send + 'static, - Index: Clone + std::fmt::Display + Codec + Send + traits::AtLeast32Bit + 'static, -{ - fn nonce(&self, account: AccountId) -> FutureResult { - let best_hash = self.client.info().best_hash; - let best_id = BlockId::hash(best_hash); - let future_best_header = future_header(&*self.remote_blockchain, &*self.fetcher, best_id); - let fetcher = self.fetcher.clone(); - let call_data = account.encode(); - let future_best_header = future_best_header.and_then(move |maybe_best_header| { - ready( - maybe_best_header - .ok_or_else(|| ClientError::UnknownBlock(format!("{}", best_hash))), - ) - }); - - let future_nonce = future_best_header.and_then(move |best_header| { - fetcher.remote_call(RemoteCallRequest { - block: best_hash, - header: best_header, - method: "AccountNonceApi_account_nonce".into(), - call_data, - retry_count: None, - }) - }); - - let future_nonce = future_nonce.and_then(|nonce| async move { - Index::decode(&mut &nonce[..]) - .map_err(|e| ClientError::CallResultDecode("Cannot decode account nonce", e)) - }); - let future_nonce = future_nonce.map_err(|e| RpcError { - code: ErrorCode::ServerError(Error::RuntimeError.into()), - message: "Unable to query nonce.".into(), - data: Some(format!("{:?}", e).into()), - }); - - let pool = self.pool.clone(); - future_nonce.map_ok(move |nonce| adjust_nonce(&*pool, account, nonce)).boxed() - } - - fn dry_run( - &self, - _extrinsic: Bytes, - _at: Option<::Hash>, - ) -> FutureResult { - async { - Err(RpcError { - code: ErrorCode::MethodNotFound, - message: "Unable to dry run extrinsic.".into(), - data: None, - }) - } - .boxed() - } -} - /// Adjust account nonce from state, so that tx with the nonce will be /// placed after all ready txpool transactions. fn adjust_nonce(pool: &P, account: AccountId, nonce: Index) -> Index diff --git a/utils/frame/try-runtime/cli/Cargo.toml b/utils/frame/try-runtime/cli/Cargo.toml index 11b899db4ca4..3f0ee257d975 100644 --- a/utils/frame/try-runtime/cli/Cargo.toml +++ b/utils/frame/try-runtime/cli/Cargo.toml @@ -2,9 +2,9 @@ name = "try-runtime-cli" version = "0.10.0-dev" authors = ["Parity Technologies "] -edition = "2018" +edition = "2021" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" description = "Cli command runtime testing and dry-running" readme = "README.md" @@ -13,24 +13,23 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] +clap = { version = "3.1.6", features = ["derive"] } log = "0.4.8" -parity-scale-codec = { version = "2.0.0" } -serde = "1.0.126" -structopt = "0.3.8" +parity-scale-codec = "3.0.0" +serde = "1.0.136" +zstd = "0.9.0" sc-service = { version = "0.10.0-dev", default-features = false, path = "../../../../client/service" } sc-cli = { version = "0.10.0-dev", path = "../../../../client/cli" } sc-executor = { version = "0.10.0-dev", path = "../../../../client/executor" } sc-chain-spec = { version = "4.0.0-dev", path = "../../../../client/chain-spec" } -sp-state-machine = { version = "0.10.0-dev", path = "../../../../primitives/state-machine" } -sp-runtime = { version = "4.0.0-dev", path = "../../../../primitives/runtime" } -sp-core = { version = "4.0.0-dev", path = "../../../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../../../primitives/io" } -sp-keystore = { version = "0.10.0-dev", path = "../../../../primitives/keystore" } -sp-externalities = { version = "0.10.0-dev", path = "../../../../primitives/externalities" } -sp-version = { version = "4.0.0-dev", path = "../../../../primitives/version" } +sp-state-machine = { version = "0.12.0", path = "../../../../primitives/state-machine" } +sp-runtime = { version = "6.0.0", path = "../../../../primitives/runtime" } +sp-core = { version = "6.0.0", path = "../../../../primitives/core" } +sp-io = { version = "6.0.0", path = "../../../../primitives/io" } +sp-keystore = { version = "0.12.0", path = "../../../../primitives/keystore" } +sp-externalities = { version = "0.12.0", path = "../../../../primitives/externalities" } +sp-version = { version = "5.0.0", path = "../../../../primitives/version" } remote-externalities = { version = "0.10.0-dev", path = "../../remote-externalities" } -jsonrpsee-ws-client = { version = "0.3.0", default-features = false, features = [ - "tokio1", -]} +jsonrpsee = { version = "0.10.1", default-features = false, features = ["ws-client"] } diff --git a/utils/frame/try-runtime/cli/src/commands/execute_block.rs b/utils/frame/try-runtime/cli/src/commands/execute_block.rs index 19422db90119..b1a56f7e8f8e 100644 --- a/utils/frame/try-runtime/cli/src/commands/execute_block.rs +++ b/utils/frame/try-runtime/cli/src/commands/execute_block.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +17,7 @@ use crate::{ build_executor, ensure_matching_spec, extract_code, full_extensions, hash_of, local_spec, - state_machine_call, SharedParams, State, LOG_TARGET, + state_machine_call_with_proof, SharedParams, State, LOG_TARGET, }; use remote_externalities::rpc_api; use sc_service::{Configuration, NativeExecutionDispatch}; @@ -26,25 +26,25 @@ use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; use std::{fmt::Debug, str::FromStr}; /// Configurations of the [`Command::ExecuteBlock`]. -#[derive(Debug, Clone, structopt::StructOpt)] +#[derive(Debug, Clone, clap::Parser)] pub struct ExecuteBlockCmd { /// Overwrite the wasm code in state or not. - #[structopt(long)] + #[clap(long)] overwrite_wasm_code: bool, /// If set, then the state root check is disabled by the virtue of calling into /// `TryRuntime_execute_block_no_check` instead of /// `Core_execute_block`. - #[structopt(long)] + #[clap(long)] no_check: bool, /// The block hash at which to fetch the block. /// /// If the `live` state type is being used, then this can be omitted, and is equal to whatever /// the `state::at` is. Only use this (with care) when combined with a snapshot. - #[structopt( + #[clap( long, - multiple = false, + multiple_values = false, parse(try_from_str = crate::parse::hash) )] block_at: Option, @@ -53,9 +53,9 @@ pub struct ExecuteBlockCmd { /// /// If the `live` state type is being used, then this can be omitted, and is equal to whatever /// the `state::uri` is. Only use this (with care) when combined with a snapshot. - #[structopt( + #[clap( long, - multiple = false, + multiple_values = false, parse(try_from_str = crate::parse::url) )] block_ws_uri: Option, @@ -65,7 +65,7 @@ pub struct ExecuteBlockCmd { /// For this command only, if the `live` is used, then state of the parent block is fetched. /// /// If `block_at` is provided, then the [`State::Live::at`] is being ignored. - #[structopt(subcommand)] + #[clap(subcommand)] state: State, } @@ -143,7 +143,7 @@ where let builder = if command.overwrite_wasm_code { let (code_key, code) = extract_code(&config.chain_spec)?; - builder.inject_key_value(&[(code_key, code)]) + builder.inject_hashed_key_value(&[(code_key, code)]) } else { builder.inject_hashed_key(well_known_keys::CODE) }; @@ -157,7 +157,7 @@ where header.digest_mut().pop(); let block = Block::new(header, extrinsics); - let (expected_spec_name, expected_spec_version) = + let (expected_spec_name, expected_spec_version, _) = local_spec::(&ext, &executor); ensure_matching_spec::( block_ws_uri.clone(), @@ -167,7 +167,7 @@ where ) .await; - let _ = state_machine_call::( + let _ = state_machine_call_with_proof::( &ext, &executor, execution, diff --git a/utils/frame/try-runtime/cli/src/commands/follow_chain.rs b/utils/frame/try-runtime/cli/src/commands/follow_chain.rs index 0526f5d327fb..db305fa59076 100644 --- a/utils/frame/try-runtime/cli/src/commands/follow_chain.rs +++ b/utils/frame/try-runtime/cli/src/commands/follow_chain.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,11 +17,11 @@ use crate::{ build_executor, ensure_matching_spec, extract_code, full_extensions, local_spec, parse, - state_machine_call, SharedParams, LOG_TARGET, + state_machine_call_with_proof, SharedParams, LOG_TARGET, }; -use jsonrpsee_ws_client::{ - types::{traits::SubscriptionClient, v2::params::JsonRpcParams, Subscription}, - WsClientBuilder, +use jsonrpsee::{ + core::client::{Subscription, SubscriptionClientT}, + ws_client::WsClientBuilder, }; use parity_scale_codec::Decode; use remote_externalities::{rpc_api, Builder, Mode, OnlineConfig}; @@ -35,14 +35,10 @@ const SUB: &'static str = "chain_subscribeFinalizedHeads"; const UN_SUB: &'static str = "chain_unsubscribeFinalizedHeads"; /// Configurations of the [`Command::FollowChain`]. -#[derive(Debug, Clone, structopt::StructOpt)] +#[derive(Debug, Clone, clap::Parser)] pub struct FollowChainCmd { /// The url to connect to. - #[structopt( - short, - long, - parse(try_from_str = parse::url), - )] + #[clap(short, long, parse(try_from_str = parse::url))] uri: String, } @@ -72,7 +68,7 @@ where log::info!(target: LOG_TARGET, "subscribing to {:?} / {:?}", SUB, UN_SUB); let mut subscription: Subscription = - client.subscribe(&SUB, JsonRpcParams::NoParams, &UN_SUB).await.unwrap(); + client.subscribe(&SUB, None, &UN_SUB).await.unwrap(); let (code_key, code) = extract_code(&config.chain_spec)?; let executor = build_executor::(&shared, &config); @@ -80,13 +76,13 @@ where loop { let header = match subscription.next().await { - Ok(Some(header)) => header, - Ok(None) => { - log::warn!("subscription returned `None`. Probably decoding has failed."); + Some(Ok(header)) => header, + None => { + log::warn!("subscription closed"); break }, - Err(why) => { - log::warn!("subscription returned error: {:?}.", why); + Some(Err(why)) => { + log::warn!("subscription returned error: {:?}. Probably decoding has failed.", why); continue }, }; @@ -112,8 +108,10 @@ where ..Default::default() })); - let new_ext = - builder.inject_key_value(&[(code_key.clone(), code.clone())]).build().await?; + let new_ext = builder + .inject_hashed_key_value(&[(code_key.clone(), code.clone())]) + .build() + .await?; log::info!( target: LOG_TARGET, "initialized state externalities at {:?}, storage root {:?}", @@ -121,7 +119,7 @@ where new_ext.as_backend().root() ); - let (expected_spec_name, expected_spec_version) = + let (expected_spec_name, expected_spec_version, spec_state_version) = local_spec::(&new_ext, &executor); ensure_matching_spec::( command.uri.clone(), @@ -131,13 +129,13 @@ where ) .await; - maybe_state_ext = Some(new_ext); + maybe_state_ext = Some((new_ext, spec_state_version)); } - let state_ext = + let (state_ext, spec_state_version) = maybe_state_ext.as_mut().expect("state_ext either existed or was just created"); - let (mut changes, encoded_result) = state_machine_call::( + let (mut changes, encoded_result) = state_machine_call_with_proof::( &state_ext, &executor, execution, @@ -150,11 +148,14 @@ where .map_err(|e| format!("failed to decode output: {:?}", e))?; let storage_changes = changes - .drain_storage_changes::<_, _, NumberFor>( + .drain_storage_changes( &state_ext.backend, - None, - Default::default(), &mut Default::default(), + // Note that in case a block contains a runtime upgrade, + // state version could potentially be incorrect here, + // this is very niche and would only result in unaligned + // roots, so this use case is ignored for now. + *spec_state_version, ) .unwrap(); state_ext.backend.apply_transaction( diff --git a/utils/frame/try-runtime/cli/src/commands/mod.rs b/utils/frame/try-runtime/cli/src/commands/mod.rs index bfd8290fb31c..4861d94f077c 100644 --- a/utils/frame/try-runtime/cli/src/commands/mod.rs +++ b/utils/frame/try-runtime/cli/src/commands/mod.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs b/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs index 6f37e4b3849f..72136e9236de 100644 --- a/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs +++ b/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,19 +28,19 @@ use sp_runtime::traits::{Block as BlockT, Header, NumberFor}; use std::{fmt::Debug, str::FromStr}; /// Configurations of the [`Command::OffchainWorker`]. -#[derive(Debug, Clone, structopt::StructOpt)] +#[derive(Debug, Clone, clap::Parser)] pub struct OffchainWorkerCmd { /// Overwrite the wasm code in state or not. - #[structopt(long)] + #[clap(long)] overwrite_wasm_code: bool, /// The block hash at which to fetch the header. /// /// If the `live` state type is being used, then this can be omitted, and is equal to whatever /// the `state::at` is. Only use this (with care) when combined with a snapshot. - #[structopt( + #[clap( long, - multiple = false, + multiple_values = false, parse(try_from_str = parse::hash) )] header_at: Option, @@ -49,15 +49,15 @@ pub struct OffchainWorkerCmd { /// /// If the `live` state type is being used, then this can be omitted, and is equal to whatever /// the `state::uri` is. Only use this (with care) when combined with a snapshot. - #[structopt( + #[clap( long, - multiple = false, + multiple_values = false, parse(try_from_str = parse::url) )] header_ws_uri: Option, /// The state type to use. - #[structopt(subcommand)] + #[clap(subcommand)] pub state: State, } @@ -132,7 +132,7 @@ where let builder = if command.overwrite_wasm_code { let (code_key, code) = extract_code(&config.chain_spec)?; - builder.inject_key_value(&[(code_key, code)]) + builder.inject_hashed_key_value(&[(code_key, code)]) } else { builder.inject_hashed_key(well_known_keys::CODE) }; @@ -140,7 +140,7 @@ where builder.build().await? }; - let (expected_spec_name, expected_spec_version) = + let (expected_spec_name, expected_spec_version, _) = local_spec::(&ext, &executor); ensure_matching_spec::( header_ws_uri, diff --git a/utils/frame/try-runtime/cli/src/commands/on_runtime_upgrade.rs b/utils/frame/try-runtime/cli/src/commands/on_runtime_upgrade.rs index 86f5548b8aaf..616498da0249 100644 --- a/utils/frame/try-runtime/cli/src/commands/on_runtime_upgrade.rs +++ b/utils/frame/try-runtime/cli/src/commands/on_runtime_upgrade.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,15 +23,15 @@ use sc_service::Configuration; use sp_runtime::traits::{Block as BlockT, NumberFor}; use crate::{ - build_executor, ensure_matching_spec, extract_code, local_spec, state_machine_call, + build_executor, ensure_matching_spec, extract_code, local_spec, state_machine_call_with_proof, SharedParams, State, LOG_TARGET, }; /// Configurations of the [`Command::OnRuntimeUpgrade`]. -#[derive(Debug, Clone, structopt::StructOpt)] +#[derive(Debug, Clone, clap::Parser)] pub struct OnRuntimeUpgradeCmd { /// The state type to use. - #[structopt(subcommand)] + #[clap(subcommand)] pub state: State, } @@ -54,11 +54,11 @@ where let ext = { let builder = command.state.builder::()?; let (code_key, code) = extract_code(&config.chain_spec)?; - builder.inject_key_value(&[(code_key, code)]).build().await? + builder.inject_hashed_key_value(&[(code_key, code)]).build().await? }; if let Some(uri) = command.state.live_uri() { - let (expected_spec_name, expected_spec_version) = + let (expected_spec_name, expected_spec_version, _) = local_spec::(&ext, &executor); ensure_matching_spec::( uri, @@ -69,7 +69,7 @@ where .await; } - let (_, encoded_result) = state_machine_call::( + let (_, encoded_result) = state_machine_call_with_proof::( &ext, &executor, execution, diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index d5ccca956025..ef8db7a1e5a0 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,9 +29,9 @@ //! //! Some resources about the above: //! -//! 1. +//! 1. //! 2. -//! 3. +//! 3. //! //! --- //! @@ -113,6 +113,8 @@ //! well with try-runtime's expensive RPC queries: //! //! - set `--rpc-max-payload 1000` to ensure large RPC queries can work. +//! - set `--ws-max-out-buffer-capacity 1000` to ensure the websocket connection can handle large +//! RPC queries. //! - set `--rpc-cors all` to ensure ws connections can come through. //! //! Note that *none* of the try-runtime operations need unsafe RPCs. @@ -125,7 +127,7 @@ //! #### Adding pre/post hooks //! //! One of the gems that come only in the `try-runtime` feature flag is the `pre_upgrade` and -//! `post_upgrade` hooks for [`OnRuntimeUpgrade`]. This trait is implemented either inside the +//! `post_upgrade` hooks for `OnRuntimeUpgrade`. This trait is implemented either inside the //! pallet, or manually in a runtime, to define a migration. In both cases, these functions can be //! added, given the right flag: //! @@ -141,7 +143,9 @@ //! //! These hooks allow you to execute some code, only within the `on-runtime-upgrade` command, before //! and after the migration. If any data needs to be temporarily stored between the pre/post -//! migration hooks, [`OnRuntimeUpgradeHelpersExt`] can help with that. +//! migration hooks, `OnRuntimeUpgradeHelpersExt` can help with that. Note that you should be +//! mindful with any mutable storage ops in the pre/post migration checks, as you almost certainly +//! will not want to mutate any of the storage that is to be migrated. //! //! #### Logging //! @@ -151,7 +155,7 @@ //! //! #### Guarding migrations //! -//! Always make sure that any migration code is guarded either by [`StorageVersion`], or by some +//! Always make sure that any migration code is guarded either by `StorageVersion`, or by some //! custom storage item, so that it is NEVER executed twice, even if the code lives in two //! consecutive runtimes. //! @@ -160,7 +164,7 @@ //! Run the migrations of the local runtime on the state of polkadot, from the polkadot repo where //! we have `--chain polkadot-dev`, on the latest finalized block's state //! -//! ```ignore +//! ```sh //! RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \ //! cargo run try-runtime \ //! --execution Native \ @@ -174,7 +178,7 @@ //! Same as previous one, but let's say we want to run this command from the substrate repo, where //! we don't have a matching spec name/version. //! -//! ```ignore +//! ```sh //! RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \ //! cargo run try-runtime \ //! --execution Native \ @@ -188,7 +192,7 @@ //! Same as the previous one, but run it at specific block number's state. This means that this //! block hash's state shall not yet have been pruned in `rpc.polkadot.io`. //! -//! ```ignore +//! ```sh //! RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \ //! cargo run try-runtime \ //! --execution Native \ @@ -206,7 +210,7 @@ //! First, let's assume you are in a branch that has the same spec name/version as the live polkadot //! network. //! -//! ```ignore +//! ```sh //! RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \ //! cargo run try-runtime \ //! --execution Wasm \ @@ -222,14 +226,14 @@ //! change `--execution Wasm` to `--execution Native` to achieve this. Your logs of `executor=trace` //! should show something among the lines of: //! -//! ```ignore +//! ```text //! Request for native execution succeeded (native: polkadot-9900 (parity-polkadot-0.tx7.au0), chain: polkadot-9900 (parity-polkadot-0.tx7.au0)) //! ``` //! //! If you don't have matching spec versions, then are doomed to execute wasm. In this case, you can //! manually overwrite the wasm code with your local runtime: //! -//! ```ignore +//! ```sh //! RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \ //! cargo run try-runtime \ //! --execution Wasm \ @@ -242,12 +246,12 @@ //! ``` //! //! For all of these blocks, the block with hash `` is being used, and the initial state -//! is the state of the parent hash. This is because by omitting [`ExecuteBlockCmd::block_at`], the +//! is the state of the parent hash. This is because by omitting `ExecuteBlockCmd::block_at`, the //! `--at` is used for both. This should be good enough for 99% of the cases. The only case where //! you need to specify `block-at` and `block-ws-uri` is with snapshots. Let's say you have a file //! `snap` and you know it corresponds to the state of the parent block of `X`. Then you'd do: //! -//! ```ignore +//! ```sh //! RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \ //! cargo run try-runtime \ //! --execution Wasm \ @@ -266,7 +270,9 @@ use remote_externalities::{ Builder, Mode, OfflineConfig, OnlineConfig, SnapshotConfig, TestExternalities, }; use sc_chain_spec::ChainSpec; -use sc_cli::{CliConfiguration, ExecutionStrategy, WasmExecutionMethod}; +use sc_cli::{ + CliConfiguration, ExecutionStrategy, WasmExecutionMethod, DEFAULT_WASM_EXECUTION_METHOD, +}; use sc_executor::NativeElseWasmExecutor; use sc_service::{Configuration, NativeExecutionDispatch}; use sp_core::{ @@ -281,8 +287,11 @@ use sp_core::{ }; use sp_externalities::Extensions; use sp_keystore::{testing::KeyStore, KeystoreExt}; -use sp_runtime::traits::{Block as BlockT, NumberFor}; -use sp_state_machine::{OverlayedChanges, StateMachine}; +use sp_runtime::{ + traits::{Block as BlockT, NumberFor}, + DeserializeOwned, +}; +use sp_state_machine::{InMemoryProvingBackend, OverlayedChanges, StateMachine}; use std::{fmt::Debug, path::PathBuf, str::FromStr}; mod commands; @@ -290,7 +299,7 @@ pub(crate) mod parse; pub(crate) const LOG_TARGET: &'static str = "try-runtime::cli"; /// Possible commands of `try-runtime`. -#[derive(Debug, Clone, structopt::StructOpt)] +#[derive(Debug, Clone, clap::Subcommand)] pub enum Command { /// Execute the migrations of the "local runtime". /// @@ -303,7 +312,7 @@ pub enum Command { /// Executes the given block against some state. /// - /// Unlike [`Command:::OnRuntimeUpgrade`], this command needs two inputs: the state, and the + /// Unlike [`Command::OnRuntimeUpgrade`], this command needs two inputs: the state, and the /// block data. Since the state could be cached (see [`State::Snap`]), different flags are /// provided for both. `--block-at` and `--block-uri`, if provided, are only used for fetching /// the block. For convenience, these flags can be both emitted, if the [`State::Live`] is @@ -311,7 +320,7 @@ pub enum Command { /// /// Note that by default, this command does not overwrite the code, so in wasm execution, the /// live chain's code is used. This can be disabled if desired, see - /// [`ExecuteBlockCmd::overwrite_wasm_code`]. + /// `ExecuteBlockCmd::overwrite_wasm_code`. /// /// Note that if you do overwrite the wasm code, or generally use the local runtime for this, /// you might @@ -323,7 +332,7 @@ pub enum Command { /// different state transition function. /// /// To make testing slightly more dynamic, you can disable the state root check by enabling - /// [`ExecuteBlockCmd::no_check`]. If you get signature verification errors, you should + /// `ExecuteBlockCmd::no_check`. If you get signature verification errors, you should /// manually tweak your local runtime's spec version to fix this. /// /// A subtle detail of execute block is that if you want to execute block 100 of a live chain @@ -332,19 +341,19 @@ pub enum Command { /// If [`State::Snap`] is being used, then this needs to be manually taken into consideration. /// /// This executes the same runtime api as normal block import, namely `Core_execute_block`. If - /// [`ExecuteBlockCmd::no_check`] is set, it uses a custom, try-runtime-only runtime + /// `ExecuteBlockCmd::no_check` is set, it uses a custom, try-runtime-only runtime /// api called `TryRuntime_execute_block_no_check`. ExecuteBlock(commands::execute_block::ExecuteBlockCmd), /// Executes *the offchain worker hooks* of a given block against some state. /// - /// Similar to [`Command:::ExecuteBlock`], this command needs two inputs: the state, and the + /// Similar to [`Command::ExecuteBlock`], this command needs two inputs: the state, and the /// header data. Likewise, `--header-at` and `--header-uri` can be filled, or omitted if /// [`State::Live`] is used. /// - /// Similar to [`Command:::ExecuteBlock`], this command does not overwrite the code, so in wasm + /// Similar to [`Command::ExecuteBlock`], this command does not overwrite the code, so in wasm /// execution, the live chain's code is used. This can be disabled if desired, see - /// [`OffchainWorkerCmd::overwrite_wasm_code`]. + /// `OffchainWorkerCmd::overwrite_wasm_code`. /// /// This executes the same runtime api as normal block import, namely /// `OffchainWorkerApi_offchain_worker`. @@ -370,70 +379,64 @@ pub enum Command { } /// Shared parameters of the `try-runtime` commands -#[derive(Debug, Clone, structopt::StructOpt)] +#[derive(Debug, Clone, clap::Parser)] pub struct SharedParams { /// Shared parameters of substrate cli. #[allow(missing_docs)] - #[structopt(flatten)] + #[clap(flatten)] pub shared_params: sc_cli::SharedParams, /// The execution strategy that should be used. - #[structopt( - long = "execution", - value_name = "STRATEGY", - possible_values = &ExecutionStrategy::variants(), - case_insensitive = true, - default_value = "Wasm", - )] + #[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true, default_value = "Wasm")] pub execution: ExecutionStrategy, /// Type of wasm execution used. - #[structopt( + #[clap( long = "wasm-execution", value_name = "METHOD", - possible_values = &WasmExecutionMethod::variants(), - case_insensitive = true, - default_value = "Compiled" + possible_values = WasmExecutionMethod::variants(), + ignore_case = true, + default_value = DEFAULT_WASM_EXECUTION_METHOD, )] pub wasm_method: WasmExecutionMethod, /// The number of 64KB pages to allocate for Wasm execution. Defaults to /// [`sc_service::Configuration.default_heap_pages`]. - #[structopt(long)] + #[clap(long)] pub heap_pages: Option, /// When enabled, the spec name check will not panic, and instead only show a warning. - #[structopt(long)] + #[clap(long)] pub no_spec_name_check: bool, } /// Our `try-runtime` command. /// /// See [`Command`] for more info. -#[derive(Debug, Clone, structopt::StructOpt)] +#[derive(Debug, Clone, clap::Parser)] pub struct TryRuntimeCmd { - #[structopt(flatten)] + #[clap(flatten)] pub shared: SharedParams, - #[structopt(subcommand)] + #[clap(subcommand)] pub command: Command, } /// The source of runtime *state* to use. -#[derive(Debug, Clone, structopt::StructOpt)] +#[derive(Debug, Clone, clap::Subcommand)] pub enum State { /// Use a state snapshot as the source of runtime state. /// /// This can be crated by passing a value to [`State::Live::snapshot_path`]. Snap { - #[structopt(short, long)] + #[clap(short, long)] snapshot_path: PathBuf, }, /// Use a live chain as the source of runtime state. Live { /// The url to connect to. - #[structopt( + #[clap( short, long, parse(try_from_str = parse::url), @@ -444,27 +447,35 @@ pub enum State { /// /// If non provided, then the latest finalized head is used. This is particularly useful /// for [`Command::OnRuntimeUpgrade`]. - #[structopt( + #[clap( short, long, - multiple = false, + multiple_values = false, parse(try_from_str = parse::hash), )] at: Option, /// An optional state snapshot file to WRITE to. Not written if set to `None`. - #[structopt(short, long)] + #[clap(short, long)] snapshot_path: Option, /// The pallets to scrape. If empty, entire chain state will be scraped. - #[structopt(short, long, require_delimiter = true)] - pallets: Option>, + #[clap(short, long, multiple_values = true)] + pallets: Vec, + + /// Fetch the child-keys as well. + /// + /// Default is `false`, if specific `--pallets` are specified, `true` otherwise. In other + /// words, if you scrape the whole state the child tree data is included out of the box. + /// Otherwise, it must be enabled explicitly using this flag. + #[clap(long)] + child_tree: bool, }, } impl State { /// Create the [`remote_externalities::Builder`] from self. - pub(crate) fn builder(&self) -> sc_cli::Result> + pub(crate) fn builder(&self) -> sc_cli::Result> where Block::Hash: FromStr, ::Err: Debug, @@ -474,21 +485,26 @@ impl State { Builder::::new().mode(Mode::Offline(OfflineConfig { state_snapshot: SnapshotConfig::new(snapshot_path), })), - State::Live { snapshot_path, pallets, uri, at } => { + State::Live { snapshot_path, pallets, uri, at, child_tree } => { let at = match at { Some(at_str) => Some(hash_of::(at_str)?), None => None, }; - Builder::::new() + let mut builder = Builder::::new() .mode(Mode::Online(OnlineConfig { transport: uri.to_owned().into(), state_snapshot: snapshot_path.as_ref().map(SnapshotConfig::new), - pallets: pallets.to_owned().unwrap_or_default(), + pallets: pallets.clone(), + scrape_children: true, at, })) .inject_hashed_key( &[twox_128(b"System"), twox_128(b"LastRuntimeUpgrade")].concat(), - ) + ); + if *child_tree { + builder = builder.inject_default_child_tree_prefix(); + } + builder }, }) } @@ -603,7 +619,7 @@ pub(crate) async fn ensure_matching_spec { // first, deal with spec name - if expected_spec_name == name { + if expected_spec_name.to_lowercase() == name { log::info!(target: LOG_TARGET, "found matching spec name: {:?}", name); } else { let msg = format!( @@ -662,8 +678,14 @@ pub(crate) fn build_executor( let wasm_method = shared.wasm_method; let heap_pages = shared.heap_pages.or(config.default_heap_pages); let max_runtime_instances = config.max_runtime_instances; + let runtime_cache_size = config.runtime_cache_size; - NativeElseWasmExecutor::::new(wasm_method.into(), heap_pages, max_runtime_instances) + NativeElseWasmExecutor::::new( + wasm_method.into(), + heap_pages, + max_runtime_instances, + runtime_cache_size, + ) } /// Execute the given `method` and `data` on top of `ext`, returning the results (encoded) and the @@ -677,9 +699,8 @@ pub(crate) fn state_machine_call sc_cli::Result<(OverlayedChanges, Vec)> { let mut changes = Default::default(); - let encoded_results = StateMachine::<_, _, NumberFor, _>::new( + let encoded_results = StateMachine::new( &ext.backend, - None, &mut changes, executor, method, @@ -689,9 +710,88 @@ pub(crate) fn state_machine_call(Into::into)?; + + Ok((changes, encoded_results)) +} + +/// Same as [`state_machine_call`], but it also computes and prints the storage proof in different +/// size and formats. +/// +/// Make sure [`LOG_TARGET`] is enabled in logging. +pub(crate) fn state_machine_call_with_proof( + ext: &TestExternalities, + executor: &NativeElseWasmExecutor, + execution: sc_cli::ExecutionStrategy, + method: &'static str, + data: &[u8], + extensions: Extensions, +) -> sc_cli::Result<(OverlayedChanges, Vec)> { + use parity_scale_codec::Encode; + use sp_core::hexdisplay::HexDisplay; + + let mut changes = Default::default(); + let backend = ext.backend.clone(); + let proving_backend = InMemoryProvingBackend::new(&backend); + + let runtime_code_backend = sp_state_machine::backend::BackendRuntimeCode::new(&proving_backend); + let runtime_code = runtime_code_backend.runtime_code()?; + + let pre_root = backend.root().clone(); + + let encoded_results = StateMachine::new( + &proving_backend, + &mut changes, + executor, + method, + data, + extensions, + &runtime_code, + sp_core::testing::TaskExecutor::new(), + ) + .execute(execution.into()) + .map_err(|e| format!("failed to execute {}: {}", method, e)) .map_err::(Into::into)?; + let proof = proving_backend.extract_proof(); + let proof_size = proof.encoded_size(); + let compact_proof = proof + .clone() + .into_compact_proof::(pre_root) + .map_err(|e| format!("failed to generate compact proof {}: {:?}", method, e))?; + + let compact_proof_size = compact_proof.encoded_size(); + let compressed_proof = zstd::stream::encode_all(&compact_proof.encode()[..], 0) + .map_err(|e| format!("failed to generate compact proof {}: {:?}", method, e))?; + + let proof_nodes = proof.into_nodes(); + + let humanize = |s| { + if s < 1024 * 1024 { + format!("{:.2} KB ({} bytes)", s as f64 / 1024f64, s) + } else { + format!( + "{:.2} MB ({} KB) ({} bytes)", + s as f64 / (1024f64 * 1024f64), + s as f64 / 1024f64, + s + ) + } + }; + log::info!( + target: LOG_TARGET, + "proof: {} / {} nodes", + HexDisplay::from(&proof_nodes.iter().flatten().cloned().collect::>()), + proof_nodes.len() + ); + log::info!(target: LOG_TARGET, "proof size: {}", humanize(proof_size)); + log::info!(target: LOG_TARGET, "compact proof size: {}", humanize(compact_proof_size),); + log::info!( + target: LOG_TARGET, + "zstd-compressed compact proof {}", + humanize(compressed_proof.len()), + ); Ok((changes, encoded_results)) } @@ -699,7 +799,7 @@ pub(crate) fn state_machine_call( ext: &TestExternalities, executor: &NativeElseWasmExecutor, -) -> (String, u32) { +) -> (String, u32, sp_core::storage::StateVersion) { let (_, encoded) = state_machine_call::( &ext, &executor, @@ -711,6 +811,9 @@ pub(crate) fn local_spec( .expect("all runtimes should have version; qed"); ::decode(&mut &*encoded) .map_err(|e| format!("failed to decode output: {:?}", e)) - .map(|v| (v.spec_name.into(), v.spec_version)) + .map(|v| { + let state_version = v.state_version(); + (v.spec_name.into(), v.spec_version, state_version) + }) .expect("all runtimes should have version; qed") } diff --git a/utils/frame/try-runtime/cli/src/parse.rs b/utils/frame/try-runtime/cli/src/parse.rs index 7f205fbacd31..15a0251ebc34 100644 --- a/utils/frame/try-runtime/cli/src/parse.rs +++ b/utils/frame/try-runtime/cli/src/parse.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,8 +18,11 @@ //! Utils for parsing user input pub(crate) fn hash(block_hash: &str) -> Result { - let (block_hash, offset) = - if block_hash.starts_with("0x") { (&block_hash[2..], 2) } else { (block_hash, 0) }; + let (block_hash, offset) = if let Some(block_hash) = block_hash.strip_prefix("0x") { + (block_hash, 2) + } else { + (block_hash, 0) + }; if let Some(pos) = block_hash.chars().position(|c| !c.is_ascii_hexdigit()) { Err(format!( diff --git a/utils/prometheus/Cargo.toml b/utils/prometheus/Cargo.toml index 8b647d6282fb..4940712f2f4d 100644 --- a/utils/prometheus/Cargo.toml +++ b/utils/prometheus/Cargo.toml @@ -1,11 +1,11 @@ [package] description = "Endpoint to expose Prometheus metrics" name = "substrate-prometheus-endpoint" -version = "0.9.0" +version = "0.10.0-dev" license = "Apache-2.0" authors = ["Parity Technologies "] -edition = "2018" -homepage = "https://substrate.dev" +edition = "2021" +homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" readme = "README.md" @@ -14,13 +14,12 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] log = "0.4.8" -prometheus = { version = "0.11.0", default-features = false } -futures-util = { version = "0.3.17", default-features = false, features = ["io"] } -derive_more = "0.99" -async-std = { version = "1.10.0", features = ["unstable"] } -tokio = "1.10" -hyper = { version = "0.14.11", default-features = false, features = ["http1", "server", "tcp"] } +prometheus = { version = "0.13.0", default-features = false } +futures-util = { version = "0.3.19", default-features = false, features = ["io"] } +thiserror = "1.0" +tokio = { version = "1.17.0", features = ["parking_lot"] } +hyper = { version = "0.14.16", default-features = false, features = ["http1", "server", "tcp"] } [dev-dependencies] -hyper = { version = "0.14.11", features = ["client"] } -tokio = { version = "1.10", features = ["rt-multi-thread"] } +hyper = { version = "0.14.16", features = ["client"] } +tokio = { version = "1.17.0", features = ["rt-multi-thread"] } diff --git a/utils/prometheus/src/lib.rs b/utils/prometheus/src/lib.rs index f81b82cb1764..3ea9d45d48b1 100644 --- a/utils/prometheus/src/lib.rs +++ b/utils/prometheus/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +15,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use futures_util::future::Future; use hyper::{ http::StatusCode, server::Server, @@ -34,7 +33,6 @@ pub use prometheus::{ use prometheus::{core::Collector, Encoder, TextEncoder}; use std::net::SocketAddr; -mod networking; mod sourced; pub use sourced::{MetricSource, SourcedCounter, SourcedGauge, SourcedMetric}; @@ -47,27 +45,22 @@ pub fn register( Ok(metric) } -#[derive(Debug, derive_more::Display, derive_more::From)] +#[derive(Debug, thiserror::Error)] pub enum Error { /// Hyper internal error. - Hyper(hyper::Error), + #[error(transparent)] + Hyper(#[from] hyper::Error), + /// Http request error. - Http(hyper::http::Error), + #[error(transparent)] + Http(#[from] hyper::http::Error), + /// i/o error. - Io(std::io::Error), - #[display(fmt = "Prometheus port {} already in use.", _0)] - PortInUse(SocketAddr), -} + #[error(transparent)] + Io(#[from] std::io::Error), -impl std::error::Error for Error { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - Error::Hyper(error) => Some(error), - Error::Http(error) => Some(error), - Error::Io(error) => Some(error), - Error::PortInUse(_) => None, - } - } + #[error("Prometheus port {0} already in use.")] + PortInUse(SocketAddr), } async fn request_metrics(req: Request, registry: Registry) -> Result, Error> { @@ -90,23 +83,10 @@ async fn request_metrics(req: Request, registry: Registry) -> Result hyper::rt::Executor for Executor -where - T: Future + Send + 'static, - T::Output: Send + 'static, -{ - fn execute(&self, future: T) { - async_std::task::spawn(future); - } -} - /// Initializes the metrics context, and starts an HTTP server /// to serve metrics. pub async fn init_prometheus(prometheus_addr: SocketAddr, registry: Registry) -> Result<(), Error> { - let listener = async_std::net::TcpListener::bind(&prometheus_addr) + let listener = tokio::net::TcpListener::bind(&prometheus_addr) .await .map_err(|_| Error::PortInUse(prometheus_addr))?; @@ -115,12 +95,11 @@ pub async fn init_prometheus(prometheus_addr: SocketAddr, registry: Registry) -> /// Init prometheus using the given listener. async fn init_prometheus_with_listener( - listener: async_std::net::TcpListener, + listener: tokio::net::TcpListener, registry: Registry, ) -> Result<(), Error> { - use networking::Incoming; - - log::info!("〽️ Prometheus exporter started at {}", listener.local_addr()?); + let listener = hyper::server::conn::AddrIncoming::from_listener(listener)?; + log::info!("〽️ Prometheus exporter started at {}", listener.local_addr()); let service = make_service_fn(move |_| { let registry = registry.clone(); @@ -132,7 +111,7 @@ async fn init_prometheus_with_listener( } }); - let server = Server::builder(Incoming(listener.incoming())).executor(Executor).serve(service); + let server = Server::builder(listener).serve(service); let result = server.await.map_err(Into::into); @@ -143,7 +122,6 @@ async fn init_prometheus_with_listener( mod tests { use super::*; use hyper::{Client, Uri}; - use std::convert::TryFrom; #[test] fn prometheus_works() { @@ -152,7 +130,7 @@ mod tests { let runtime = tokio::runtime::Runtime::new().expect("Creates the runtime"); let listener = runtime - .block_on(async_std::net::TcpListener::bind("127.0.0.1:0")) + .block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) .expect("Creates listener"); let local_addr = listener.local_addr().expect("Returns the local addr"); diff --git a/utils/prometheus/src/networking.rs b/utils/prometheus/src/networking.rs deleted file mode 100644 index a24216bd2362..000000000000 --- a/utils/prometheus/src/networking.rs +++ /dev/null @@ -1,71 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use async_std::pin::Pin; -use futures_util::{ - io::{AsyncRead, AsyncWrite}, - stream::Stream, -}; -use std::task::{Context, Poll}; - -pub struct Incoming<'a>(pub async_std::net::Incoming<'a>); - -impl hyper::server::accept::Accept for Incoming<'_> { - type Conn = TcpStream; - type Error = async_std::io::Error; - - fn poll_accept( - self: Pin<&mut Self>, - cx: &mut Context, - ) -> Poll>> { - Pin::new(&mut Pin::into_inner(self).0) - .poll_next(cx) - .map(|opt| opt.map(|res| res.map(TcpStream))) - } -} - -pub struct TcpStream(pub async_std::net::TcpStream); - -impl tokio::io::AsyncRead for TcpStream { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context, - buf: &mut tokio::io::ReadBuf<'_>, - ) -> Poll> { - Pin::new(&mut Pin::into_inner(self).0) - .poll_read(cx, buf.initialize_unfilled()) - .map_ok(|s| buf.set_filled(s)) - } -} - -impl tokio::io::AsyncWrite for TcpStream { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context, - buf: &[u8], - ) -> Poll> { - Pin::new(&mut Pin::into_inner(self).0).poll_write(cx, buf) - } - - fn poll_flush(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Pin::new(&mut Pin::into_inner(self).0).poll_flush(cx) - } - - fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Pin::new(&mut Pin::into_inner(self).0).poll_close(cx) - } -} diff --git a/utils/prometheus/src/sourced.rs b/utils/prometheus/src/sourced.rs index ca37eef021f6..9f52d1eff47c 100644 --- a/utils/prometheus/src/sourced.rs +++ b/utils/prometheus/src/sourced.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/utils/wasm-builder/Cargo.toml b/utils/wasm-builder/Cargo.toml index 721f332e130f..66c04432c5de 100644 --- a/utils/wasm-builder/Cargo.toml +++ b/utils/wasm-builder/Cargo.toml @@ -3,21 +3,22 @@ name = "substrate-wasm-builder" version = "5.0.0-dev" authors = ["Parity Technologies "] description = "Utility for building WASM binaries" -edition = "2018" +edition = "2021" readme = "README.md" repository = "https://github.com/paritytech/substrate/" license = "Apache-2.0" -homepage = "https://substrate.dev" +homepage = "https://substrate.io" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] [dependencies] build-helper = "0.1.1" -cargo_metadata = "0.13.1" +cargo_metadata = "0.14.2" tempfile = "3.1.0" toml = "0.5.4" walkdir = "2.3.2" wasm-gc-api = "0.1.11" ansi_term = "0.12.1" -sp-maybe-compressed-blob = { version = "4.0.0-dev", path = "../../primitives/maybe-compressed-blob" } +sp-maybe-compressed-blob = { version = "4.1.0-dev", path = "../../primitives/maybe-compressed-blob" } +strum = { version = "0.23.0", features = ["derive"] } diff --git a/utils/wasm-builder/src/builder.rs b/utils/wasm-builder/src/builder.rs index 113b5eb068da..81a869396818 100644 --- a/utils/wasm-builder/src/builder.rs +++ b/utils/wasm-builder/src/builder.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/utils/wasm-builder/src/lib.rs b/utils/wasm-builder/src/lib.rs index b13ecc4e4ab3..6a7f0d7ca3cd 100644 --- a/utils/wasm-builder/src/lib.rs +++ b/utils/wasm-builder/src/lib.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -83,6 +83,10 @@ //! needs to be absolute. //! - `WASM_BUILD_TOOLCHAIN` - The toolchain that should be used to build the Wasm binaries. The //! format needs to be the same as used by cargo, e.g. `nightly-2020-02-20`. +//! - `WASM_BUILD_WORKSPACE_HINT` - Hint the workspace that is being built. This is normally not +//! required as we walk up from the target directory until we find a `Cargo.toml`. If the target +//! directory is changed for the build, this environment variable can be used to point to the +//! actual workspace. //! //! Each project can be skipped individually by using the environment variable //! `SKIP_PROJECT_NAME_WASM_BUILD`. Where `PROJECT_NAME` needs to be replaced by the name of the @@ -116,9 +120,11 @@ pub use builder::{WasmBuilder, WasmBuilderSelectProject}; const SKIP_BUILD_ENV: &str = "SKIP_WASM_BUILD"; /// Environment variable to force a certain build type when building the wasm binary. -/// Expects "debug" or "release" as value. +/// Expects "debug", "release" or "production" as value. /// -/// By default the WASM binary uses the same build type as the main cargo build. +/// When unset the WASM binary uses the same build type as the main cargo build with +/// the exception of a debug build: In this case the wasm build defaults to `release` in +/// order to avoid a slowdown when not explicitly requested. const WASM_BUILD_TYPE_ENV: &str = "WASM_BUILD_TYPE"; /// Environment variable to extend the `RUSTFLAGS` variable given to the wasm build. @@ -138,6 +144,9 @@ const WASM_BUILD_TOOLCHAIN: &str = "WASM_BUILD_TOOLCHAIN"; /// Environment variable that makes sure the WASM build is triggered. const FORCE_WASM_BUILD_ENV: &str = "FORCE_WASM_BUILD"; +/// Environment variable that hints the workspace we are building. +const WASM_BUILD_WORKSPACE_HINT: &str = "WASM_BUILD_WORKSPACE_HINT"; + /// Write to the given `file` if the `content` is different. fn write_file_if_changed(file: impl AsRef, content: impl AsRef) { if fs::read_to_string(file.as_ref()).ok().as_deref() != Some(content.as_ref()) { diff --git a/utils/wasm-builder/src/prerequisites.rs b/utils/wasm-builder/src/prerequisites.rs index 0dad8b781ae5..fb04dc3c98fb 100644 --- a/utils/wasm-builder/src/prerequisites.rs +++ b/utils/wasm-builder/src/prerequisites.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -58,7 +58,7 @@ fn create_check_toolchain_project(project_dir: &Path) { [package] name = "wasm-test" version = "1.0.0" - edition = "2018" + edition = "2021" build = "build.rs" [lib] @@ -120,6 +120,9 @@ fn check_wasm_toolchain_installed( let manifest_path = temp.path().join("Cargo.toml").display().to_string(); let mut build_cmd = cargo_command.command(); + // Chdir to temp to avoid including project's .cargo/config.toml + // by accident - it can happen in some CI environments. + build_cmd.current_dir(&temp); build_cmd.args(&[ "build", "--target=wasm32-unknown-unknown", @@ -132,12 +135,22 @@ fn check_wasm_toolchain_installed( } let mut run_cmd = cargo_command.command(); + // Chdir to temp to avoid including project's .cargo/config.toml + // by accident - it can happen in some CI environments. + run_cmd.current_dir(&temp); run_cmd.args(&["run", "--manifest-path", &manifest_path]); // Unset the `CARGO_TARGET_DIR` to prevent a cargo deadlock build_cmd.env_remove("CARGO_TARGET_DIR"); run_cmd.env_remove("CARGO_TARGET_DIR"); + // Make sure the host's flags aren't used here, e.g. if an alternative linker is specified + // in the RUSTFLAGS then the check we do here will break unless we clear these. + build_cmd.env_remove("CARGO_ENCODED_RUSTFLAGS"); + run_cmd.env_remove("CARGO_ENCODED_RUSTFLAGS"); + build_cmd.env_remove("RUSTFLAGS"); + run_cmd.env_remove("RUSTFLAGS"); + build_cmd.output().map_err(|_| err_msg.clone()).and_then(|s| { if s.status.success() { let version = run_cmd.output().ok().and_then(|o| String::from_utf8(o.stdout).ok()); diff --git a/utils/wasm-builder/src/wasm_project.rs b/utils/wasm-builder/src/wasm_project.rs index 3806a890a106..e94703b610a4 100644 --- a/utils/wasm-builder/src/wasm_project.rs +++ b/utils/wasm-builder/src/wasm_project.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,6 +17,8 @@ use crate::{write_file_if_changed, CargoCommandVersioned}; +use build_helper::rerun_if_changed; +use cargo_metadata::{CargoOpt, Metadata, MetadataCommand}; use std::{ borrow::ToOwned, collections::HashSet, @@ -26,13 +28,8 @@ use std::{ path::{Path, PathBuf}, process, }; - +use strum::{EnumIter, IntoEnumIterator}; use toml::value::Table; - -use build_helper::rerun_if_changed; - -use cargo_metadata::{Metadata, MetadataCommand}; - use walkdir::WalkDir; /// Colorize an info message. @@ -77,8 +74,18 @@ fn crate_metadata(cargo_manifest: &Path) -> Metadata { let cargo_lock_existed = cargo_lock.exists(); + // If we can find a `Cargo.lock`, we assume that this is the workspace root and there exists a + // `Cargo.toml` that we can use for getting the metadata. + let cargo_manifest = if let Some(mut cargo_lock) = find_cargo_lock(cargo_manifest) { + cargo_lock.set_file_name("Cargo.toml"); + cargo_lock + } else { + cargo_manifest.to_path_buf() + }; + let crate_metadata = MetadataCommand::new() .manifest_path(cargo_manifest) + .features(CargoOpt::AllFeatures) .exec() .expect("`cargo metadata` can not fail on project `Cargo.toml`; qed"); @@ -118,9 +125,9 @@ pub(crate) fn create_and_compile( features_to_enable, ); - build_project(&project, default_rustflags, cargo_cmd); + let profile = build_project(&project, default_rustflags, cargo_cmd); let (wasm_binary, wasm_binary_compressed, bloaty) = - compact_wasm_file(&project, project_cargo_toml, wasm_binary_name); + compact_wasm_file(&project, profile, project_cargo_toml, wasm_binary_name); wasm_binary .as_ref() @@ -151,18 +158,29 @@ fn find_cargo_lock(cargo_manifest: &Path) -> Option { } } - if let Some(path) = find_impl(build_helper::out_dir()) { - return Some(path) + if let Ok(workspace) = env::var(crate::WASM_BUILD_WORKSPACE_HINT) { + let path = PathBuf::from(workspace); + + if path.join("Cargo.lock").exists() { + return Some(path.join("Cargo.lock")) + } else { + build_helper::warning!( + "`{}` env variable doesn't point to a directory that contains a `Cargo.lock`.", + crate::WASM_BUILD_WORKSPACE_HINT, + ); + } } - if let Some(path) = find_impl(cargo_manifest.to_path_buf()) { + if let Some(path) = find_impl(build_helper::out_dir()) { return Some(path) } build_helper::warning!( - "Could not find `Cargo.lock` for `{}`, while searching from `{}`.", + "Could not find `Cargo.lock` for `{}`, while searching from `{}`. \ + To fix this, point the `{}` env variable to the directory of the workspace being compiled.", cargo_manifest.display(), - build_helper::out_dir().display() + build_helper::out_dir().display(), + crate::WASM_BUILD_WORKSPACE_HINT, ); None @@ -225,16 +243,22 @@ fn create_project_cargo_toml( let mut wasm_workspace_toml = Table::new(); - // Add `profile` with release and dev + // Add different profiles which are selected by setting `WASM_BUILD_TYPE`. let mut release_profile = Table::new(); release_profile.insert("panic".into(), "abort".into()); - release_profile.insert("lto".into(), true.into()); + release_profile.insert("lto".into(), "thin".into()); + + let mut production_profile = Table::new(); + production_profile.insert("inherits".into(), "release".into()); + production_profile.insert("lto".into(), "fat".into()); + production_profile.insert("codegen-units".into(), 1.into()); let mut dev_profile = Table::new(); dev_profile.insert("panic".into(), "abort".into()); let mut profile = Table::new(); profile.insert("release".into(), release_profile.into()); + profile.insert("production".into(), production_profile.into()); profile.insert("dev".into(), dev_profile.into()); wasm_workspace_toml.insert("profile".into(), profile.into()); @@ -266,8 +290,7 @@ fn create_project_cargo_toml( let mut package = Table::new(); package.insert("name".into(), format!("{}-wasm", crate_name).into()); package.insert("version".into(), "1.0.0".into()); - package.insert("edition".into(), "2018".into()); - package.insert("resolver".into(), "2".into()); + package.insert("edition".into(), "2021".into()); wasm_workspace_toml.insert("package".into(), package.into()); @@ -318,13 +341,30 @@ fn project_enabled_features( ) -> Vec { let package = find_package_by_manifest_path(cargo_manifest, crate_metadata); + let std_enabled = package.features.get("std"); + let mut enabled_features = package .features - .keys() - .filter(|f| { + .iter() + .filter(|(f, v)| { let mut feature_env = f.replace("-", "_"); feature_env.make_ascii_uppercase(); + // If this is a feature that corresponds only to an optional dependency + // and this feature is enabled by the `std` feature, we assume that this + // is only done through the `std` feature. This is a bad heuristic and should + // be removed after namespaced features are landed: + // https://doc.rust-lang.org/cargo/reference/unstable.html#namespaced-features + // Then we can just express this directly in the `Cargo.toml` and do not require + // this heuristic anymore. However, for the transition phase between now and namespaced + // features already being present in nightly, we need this code to make + // runtimes compile with all the possible rustc versions. + if v.len() == 1 && v.get(0).map_or(false, |v| *v == format!("dep:{}", f)) { + if std_enabled.as_ref().map(|e| e.iter().any(|ef| ef == *f)).unwrap_or(false) { + return false + } + } + // We don't want to enable the `std`/`default` feature for the wasm build and // we need to check if the feature is enabled by checking the env variable. *f != "std" && @@ -332,7 +372,7 @@ fn project_enabled_features( .map(|v| v == "1") .unwrap_or_default() }) - .cloned() + .map(|d| d.0.clone()) .collect::>(); enabled_features.sort(); @@ -400,25 +440,119 @@ fn create_project( wasm_project_folder } -/// Returns if the project should be built as a release. -fn is_release_build() -> bool { - if let Ok(var) = env::var(crate::WASM_BUILD_TYPE_ENV) { - match var.as_str() { - "release" => true, - "debug" => false, - var => panic!( - "Unexpected value for `{}` env variable: {}\nOne of the following are expected: `debug` or `release`.", - crate::WASM_BUILD_TYPE_ENV, - var, - ), +/// The cargo profile that is used to build the wasm project. +#[derive(Debug, EnumIter)] +enum Profile { + /// The `--profile dev` profile. + Debug, + /// The `--profile release` profile. + Release, + /// The `--profile production` profile. + Production, +} + +impl Profile { + /// Create a profile by detecting which profile is used for the main build. + /// + /// We cannot easily determine the profile that is used by the main cargo invocation + /// because the `PROFILE` environment variable won't contain any custom profiles like + /// "production". It would only contain the builtin profile where the custom profile + /// inherits from. This is why we inspect the build path to learn which profile is used. + /// + /// # Note + /// + /// Can be overriden by setting [`crate::WASM_BUILD_TYPE_ENV`]. + fn detect(wasm_project: &Path) -> Profile { + let (name, overriden) = if let Ok(name) = env::var(crate::WASM_BUILD_TYPE_ENV) { + (name, true) + } else { + // First go backwards to the beginning of the target directory. + // Then go forwards to find the "wbuild" directory. + // We need to go backwards first because when starting from the root there + // might be a chance that someone has a "wbuild" directory somewhere in the path. + let name = wasm_project + .components() + .rev() + .take_while(|c| c.as_os_str() != "target") + .collect::>() + .iter() + .rev() + .take_while(|c| c.as_os_str() != "wbuild") + .last() + .expect("We put the wasm project within a `target/.../wbuild` path; qed") + .as_os_str() + .to_str() + .expect("All our profile directory names are ascii; qed") + .to_string(); + (name, false) + }; + match (Profile::iter().find(|p| p.directory() == name), overriden) { + // When not overriden by a env variable we default to using the `Release` profile + // for the wasm build even when the main build uses the debug build. This + // is because the `Debug` profile is too slow for normal development activities. + (Some(Profile::Debug), false) => Profile::Release, + // For any other profile or when overriden we take it at face value. + (Some(profile), _) => profile, + // For non overriden unknown profiles we fall back to `Release`. + // This allows us to continue building when a custom profile is used for the + // main builds cargo. When explicitly passing a profile via env variable we are + // not doing a fallback. + (None, false) => { + let profile = Profile::Release; + build_helper::warning!( + "Unknown cargo profile `{}`. Defaulted to `{:?}` for the runtime build.", + name, + profile, + ); + profile + }, + // Invalid profile specified. + (None, true) => { + // We use println! + exit instead of a panic in order to have a cleaner output. + println!( + "Unexpected profile name: `{}`. One of the following is expected: {:?}", + name, + Profile::iter().map(|p| p.directory()).collect::>(), + ); + process::exit(1); + }, } - } else { - true + } + + /// The name of the profile as supplied to the cargo `--profile` cli option. + fn name(&self) -> &'static str { + match self { + Self::Debug => "dev", + Self::Release => "release", + Self::Production => "production", + } + } + + /// The sub directory within `target` where cargo places the build output. + /// + /// # Note + /// + /// Usually this is the same as [`Self::name`] with the exception of the debug + /// profile which is called `dev`. + fn directory(&self) -> &'static str { + match self { + Self::Debug => "debug", + _ => self.name(), + } + } + + /// Whether the resulting binary should be compacted and compressed. + fn wants_compact(&self) -> bool { + !matches!(self, Self::Debug) } } /// Build the project to create the WASM binary. -fn build_project(project: &Path, default_rustflags: &str, cargo_cmd: CargoCommandVersioned) { +fn build_project( + project: &Path, + default_rustflags: &str, + cargo_cmd: CargoCommandVersioned, +) -> Profile { let manifest_path = project.join("Cargo.toml"); let mut build_cmd = cargo_cmd.command(); @@ -447,16 +581,16 @@ fn build_project(project: &Path, default_rustflags: &str, cargo_cmd: CargoComman build_cmd.arg("--color=always"); } - if is_release_build() { - build_cmd.arg("--release"); - }; + let profile = Profile::detect(project); + build_cmd.arg("--profile"); + build_cmd.arg(profile.name()); println!("{}", colorize_info_message("Information that should be included in a bug report.")); println!("{} {:?}", colorize_info_message("Executing build command:"), build_cmd); println!("{} {}", colorize_info_message("Using rustc version:"), cargo_cmd.rustc_version()); match build_cmd.status().map(|s| s.success()) { - Ok(true) => {}, + Ok(true) => profile, // Use `process.exit(1)` to have a clean error output. _ => process::exit(1), } @@ -465,18 +599,17 @@ fn build_project(project: &Path, default_rustflags: &str, cargo_cmd: CargoComman /// Compact the WASM binary using `wasm-gc` and compress it using zstd. fn compact_wasm_file( project: &Path, + profile: Profile, cargo_manifest: &Path, wasm_binary_name: Option, ) -> (Option, Option, WasmBinaryBloaty) { - let is_release_build = is_release_build(); - let target = if is_release_build { "release" } else { "debug" }; let default_wasm_binary_name = get_wasm_binary_name(cargo_manifest); let wasm_file = project .join("target/wasm32-unknown-unknown") - .join(target) + .join(profile.directory()) .join(format!("{}.wasm", default_wasm_binary_name)); - let wasm_compact_file = if is_release_build { + let wasm_compact_file = if profile.wants_compact() { let wasm_compact_file = project.join(format!( "{}.compact.wasm", wasm_binary_name.clone().unwrap_or_else(|| default_wasm_binary_name.clone()), @@ -524,8 +657,8 @@ fn compress_wasm(wasm_binary_path: &Path, compressed_binary_out_path: &Path) -> true } else { - println!( - "cargo:warning=Writing uncompressed wasm. Exceeded maximum size {}", + build_helper::warning!( + "Writing uncompressed wasm. Exceeded maximum size {}", CODE_BLOB_BOMB_LIMIT, ); @@ -667,11 +800,13 @@ fn copy_wasm_to_target_directory(cargo_manifest: &Path, wasm_binary: &WasmBinary }; if !target_dir.is_absolute() { - panic!( + // We use println! + exit instead of a panic in order to have a cleaner output. + println!( "Environment variable `{}` with `{}` is not an absolute path!", crate::WASM_TARGET_DIRECTORY, target_dir.display(), ); + process::exit(1); } fs::create_dir_all(&target_dir).expect("Creates `WASM_TARGET_DIRECTORY`.");